@loaders.gl/parquet 4.4.0-alpha.2 → 4.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +17 -10
- package/dist/index.cjs.map +3 -3
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -0
- package/dist/lib/arrow/convert-columns-to-row-group.js +1 -0
- package/dist/lib/arrow/convert-columns-to-row-group.js.map +1 -0
- package/dist/lib/arrow/convert-row-group-to-columns.js +1 -0
- package/dist/lib/arrow/convert-row-group-to-columns.js.map +1 -0
- package/dist/lib/arrow/convert-schema-from-parquet.js +1 -0
- package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -0
- package/dist/lib/arrow/convert-schema-to-parquet.js +1 -0
- package/dist/lib/arrow/convert-schema-to-parquet.js.map +1 -0
- package/dist/lib/constants.js +2 -1
- package/dist/lib/constants.js.map +1 -0
- package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -1
- package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -0
- package/dist/lib/parsers/get-parquet-schema.js +1 -0
- package/dist/lib/parsers/get-parquet-schema.js.map +1 -0
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js +1 -0
- package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow.js +1 -0
- package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-columns.js +1 -0
- package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -0
- package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
- package/dist/lib/parsers/parse-parquet-to-json.js +3 -2
- package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -0
- package/dist/lib/utils/load-wasm.js +1 -0
- package/dist/lib/utils/load-wasm.js.map +1 -0
- package/dist/lib/utils/make-stream-iterator.js +1 -0
- package/dist/lib/utils/make-stream-iterator.js.map +1 -0
- package/dist/parquet-arrow-loader.js +1 -0
- package/dist/parquet-arrow-loader.js.map +1 -0
- package/dist/parquet-arrow-writer.js +1 -0
- package/dist/parquet-arrow-writer.js.map +1 -0
- package/dist/parquet-format.js +1 -0
- package/dist/parquet-format.js.map +1 -0
- package/dist/parquet-json-loader.js +2 -1
- package/dist/parquet-json-loader.js.map +1 -0
- package/dist/parquet-json-writer.js +2 -1
- package/dist/parquet-json-writer.js.map +1 -0
- package/dist/parquetjs/codecs/declare.js +1 -0
- package/dist/parquetjs/codecs/declare.js.map +1 -0
- package/dist/parquetjs/codecs/dictionary.js +1 -0
- package/dist/parquetjs/codecs/dictionary.js.map +1 -0
- package/dist/parquetjs/codecs/index.js +1 -0
- package/dist/parquetjs/codecs/index.js.map +1 -0
- package/dist/parquetjs/codecs/plain.js +3 -2
- package/dist/parquetjs/codecs/plain.js.map +1 -0
- package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
- package/dist/parquetjs/codecs/rle.js +13 -6
- package/dist/parquetjs/codecs/rle.js.map +1 -0
- package/dist/parquetjs/compression.js +1 -0
- package/dist/parquetjs/compression.js.map +1 -0
- package/dist/parquetjs/encoder/parquet-encoder.js +1 -0
- package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +1 -0
- package/dist/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/BsonType.js +1 -0
- package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js +1 -0
- package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js +1 -0
- package/dist/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ConvertedType.js +1 -0
- package/dist/parquetjs/parquet-thrift/ConvertedType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js +1 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +1 -0
- package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DateType.js +1 -0
- package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DecimalType.js +1 -0
- package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +1 -0
- package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.js +1 -0
- package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/EnumType.js +1 -0
- package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -0
- package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js +1 -0
- package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +1 -0
- package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/IntType.js +1 -0
- package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/JsonType.js +1 -0
- package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/KeyValue.js +1 -0
- package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/ListType.js +1 -0
- package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/LogicalType.js +1 -0
- package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/MapType.js +1 -0
- package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js +1 -0
- package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js +1 -0
- package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/NullType.js +1 -0
- package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js +1 -0
- package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +1 -0
- package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageHeader.js +1 -0
- package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageLocation.js +1 -0
- package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/PageType.js +1 -0
- package/dist/parquetjs/parquet-thrift/PageType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/RowGroup.js +1 -0
- package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SchemaElement.js +1 -0
- package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/SortingColumn.js +1 -0
- package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Statistics.js +1 -0
- package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/StringType.js +1 -0
- package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/TimeType.js +1 -0
- package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/TimeUnit.js +1 -0
- package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/TimestampType.js +1 -0
- package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/Type.js +1 -0
- package/dist/parquetjs/parquet-thrift/Type.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -0
- package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/UUIDType.js +1 -0
- package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -0
- package/dist/parquetjs/parquet-thrift/index.js +1 -0
- package/dist/parquetjs/parquet-thrift/index.js.map +1 -0
- package/dist/parquetjs/parser/decoders.js +2 -1
- package/dist/parquetjs/parser/decoders.js.map +1 -0
- package/dist/parquetjs/parser/parquet-reader.js +1 -0
- package/dist/parquetjs/parser/parquet-reader.js.map +1 -0
- package/dist/parquetjs/schema/declare.js +1 -0
- package/dist/parquetjs/schema/declare.js.map +1 -0
- package/dist/parquetjs/schema/schema.js +1 -0
- package/dist/parquetjs/schema/schema.js.map +1 -0
- package/dist/parquetjs/schema/shred.js +1 -0
- package/dist/parquetjs/schema/shred.js.map +1 -0
- package/dist/parquetjs/schema/types.js +1 -0
- package/dist/parquetjs/schema/types.js.map +1 -0
- package/dist/parquetjs/utils/file-utils.js +1 -0
- package/dist/parquetjs/utils/file-utils.js.map +1 -0
- package/dist/parquetjs/utils/read-utils.js +1 -0
- package/dist/parquetjs/utils/read-utils.js.map +1 -0
- package/dist/polyfills/buffer/buffer-polyfill.browser.js +1 -0
- package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +1 -0
- package/dist/polyfills/buffer/buffer-polyfill.node.js +1 -0
- package/dist/polyfills/buffer/buffer-polyfill.node.js.map +1 -0
- package/dist/polyfills/buffer/buffer.js +1 -0
- package/dist/polyfills/buffer/buffer.js.map +1 -0
- package/dist/polyfills/buffer/index.d.ts +1 -1
- package/dist/polyfills/buffer/index.d.ts.map +1 -1
- package/dist/polyfills/buffer/index.js +2 -1
- package/dist/polyfills/buffer/index.js.map +1 -0
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +1 -28
- package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +1 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js +2 -1
- package/dist/polyfills/buffer/install-buffer-polyfill.js.map +1 -0
- package/dist/polyfills/util.js +1 -0
- package/dist/polyfills/util.js.map +1 -0
- package/dist/workers/parquet-json-worker.js +1 -0
- package/dist/workers/parquet-json-worker.js.map +1 -0
- package/package.json +16 -13
- package/src/lib/encoders/encode-arrow-to-parquet.ts +4 -2
- package/src/lib/parsers/parse-parquet-to-json.ts +4 -2
- package/src/parquetjs/codecs/plain.ts +4 -4
- package/src/parquetjs/codecs/rle.ts +17 -11
- package/src/parquetjs/encoder/parquet-encoder.ts +10 -10
- package/src/parquetjs/parser/decoders.ts +2 -2
- package/src/parquetjs/utils/read-utils.ts +1 -1
package/dist/index.cjs
CHANGED
|
@@ -1997,7 +1997,10 @@ function encodeValues2(type, values, opts) {
|
|
|
1997
1997
|
}
|
|
1998
1998
|
}
|
|
1999
1999
|
if (repeats) {
|
|
2000
|
-
buf = Buffer.concat([
|
|
2000
|
+
buf = Buffer.concat([
|
|
2001
|
+
buf,
|
|
2002
|
+
encodeRunRepeated(values[values.length - 1], repeats, opts)
|
|
2003
|
+
]);
|
|
2001
2004
|
} else if (run.length) {
|
|
2002
2005
|
buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
|
|
2003
2006
|
}
|
|
@@ -2073,7 +2076,10 @@ function encodeRunBitpacked(values, opts) {
|
|
|
2073
2076
|
buf[Math.floor(b / 8)] |= 1 << b % 8;
|
|
2074
2077
|
}
|
|
2075
2078
|
}
|
|
2076
|
-
return Buffer.concat([
|
|
2079
|
+
return Buffer.concat([
|
|
2080
|
+
Buffer.from(import_varint.default.encode(values.length / 8 << 1 | 1)),
|
|
2081
|
+
buf
|
|
2082
|
+
]);
|
|
2077
2083
|
}
|
|
2078
2084
|
function encodeRunRepeated(value, count, opts) {
|
|
2079
2085
|
const bitWidth = opts.bitWidth;
|
|
@@ -6245,7 +6251,7 @@ var FileMetaData = class {
|
|
|
6245
6251
|
};
|
|
6246
6252
|
|
|
6247
6253
|
// dist/lib/constants.js
|
|
6248
|
-
var VERSION = true ? "4.4.0
|
|
6254
|
+
var VERSION = true ? "4.4.0" : "latest";
|
|
6249
6255
|
var PARQUET_WASM_URL = "https://unpkg.com/parquet-wasm@0.6.1/esm/parquet_wasm_bg.wasm";
|
|
6250
6256
|
var PARQUET_MAGIC = "PAR1";
|
|
6251
6257
|
var PARQUET_MAGIC_ENCRYPTED = "PARE";
|
|
@@ -6590,7 +6596,7 @@ function preserveBinary(d) {
|
|
|
6590
6596
|
return d;
|
|
6591
6597
|
}
|
|
6592
6598
|
if (Buffer.isBuffer(d)) {
|
|
6593
|
-
return d.buffer
|
|
6599
|
+
return new Uint8Array(d.buffer, d.byteOffset, d.byteLength).slice().buffer;
|
|
6594
6600
|
}
|
|
6595
6601
|
return d.toString();
|
|
6596
6602
|
}
|
|
@@ -6878,7 +6884,7 @@ async function getSchemaFromParquetReader(reader) {
|
|
|
6878
6884
|
|
|
6879
6885
|
// dist/lib/parsers/parse-parquet-to-json.js
|
|
6880
6886
|
async function parseParquetFile(file, options) {
|
|
6881
|
-
var _a;
|
|
6887
|
+
var _a, _b, _c;
|
|
6882
6888
|
installBufferPolyfill();
|
|
6883
6889
|
await preloadCompressions(options);
|
|
6884
6890
|
const reader = new ParquetReader(file, {
|
|
@@ -6890,14 +6896,14 @@ async function parseParquetFile(file, options) {
|
|
|
6890
6896
|
for await (const rowBatch of rowBatches) {
|
|
6891
6897
|
let limitHasReached = false;
|
|
6892
6898
|
for (const row of rowBatch) {
|
|
6893
|
-
if ((options == null ? void 0 : options.limit) && rows.length >= (options == null ? void 0 : options.limit)) {
|
|
6899
|
+
if ((options == null ? void 0 : options.limit) && rows.length >= ((_b = options == null ? void 0 : options.core) == null ? void 0 : _b.limit)) {
|
|
6894
6900
|
limitHasReached = true;
|
|
6895
6901
|
break;
|
|
6896
6902
|
}
|
|
6897
6903
|
rows.push(row);
|
|
6898
6904
|
}
|
|
6899
6905
|
if (limitHasReached) {
|
|
6900
|
-
import_log.default.warn(`Rows number limit has been reached. Only first ${options == null ? void 0 : options.limit} are loaded`)();
|
|
6906
|
+
import_log.default.warn(`Rows number limit has been reached. Only first ${(_c = options == null ? void 0 : options.core) == null ? void 0 : _c.limit} are loaded`)();
|
|
6901
6907
|
break;
|
|
6902
6908
|
}
|
|
6903
6909
|
}
|
|
@@ -6945,7 +6951,7 @@ async function* parseGeoParquetFileInBatches(file, options) {
|
|
|
6945
6951
|
}
|
|
6946
6952
|
|
|
6947
6953
|
// dist/parquet-json-loader.js
|
|
6948
|
-
var VERSION2 = true ? "4.4.0
|
|
6954
|
+
var VERSION2 = true ? "4.4.0" : "latest";
|
|
6949
6955
|
var ParquetJSONWorkerLoader = {
|
|
6950
6956
|
...ParquetFormat,
|
|
6951
6957
|
dataType: null,
|
|
@@ -6999,7 +7005,7 @@ var GeoParquetLoader = {
|
|
|
6999
7005
|
};
|
|
7000
7006
|
|
|
7001
7007
|
// dist/parquet-json-writer.js
|
|
7002
|
-
var VERSION3 = true ? "4.4.0
|
|
7008
|
+
var VERSION3 = true ? "4.4.0" : "latest";
|
|
7003
7009
|
var ParquetJSONWriter = {
|
|
7004
7010
|
...ParquetFormat,
|
|
7005
7011
|
id: "parquet",
|
|
@@ -7167,7 +7173,8 @@ async function encodeArrowToParquet(table, options) {
|
|
|
7167
7173
|
const wasmProperties = new wasm.WriterPropertiesBuilder().build();
|
|
7168
7174
|
try {
|
|
7169
7175
|
const parquetBytes = wasm.writeParquet(wasmTable, wasmProperties);
|
|
7170
|
-
|
|
7176
|
+
const bytes = new Uint8Array(parquetBytes.buffer, parquetBytes.byteOffset, parquetBytes.byteLength);
|
|
7177
|
+
return bytes.slice().buffer;
|
|
7171
7178
|
} finally {
|
|
7172
7179
|
}
|
|
7173
7180
|
}
|