@loaders.gl/parquet 4.4.0-alpha.2 → 4.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. package/dist/index.cjs +17 -10
  2. package/dist/index.cjs.map +3 -3
  3. package/dist/index.js +1 -0
  4. package/dist/index.js.map +1 -0
  5. package/dist/lib/arrow/convert-columns-to-row-group.js +1 -0
  6. package/dist/lib/arrow/convert-columns-to-row-group.js.map +1 -0
  7. package/dist/lib/arrow/convert-row-group-to-columns.js +1 -0
  8. package/dist/lib/arrow/convert-row-group-to-columns.js.map +1 -0
  9. package/dist/lib/arrow/convert-schema-from-parquet.js +1 -0
  10. package/dist/lib/arrow/convert-schema-from-parquet.js.map +1 -0
  11. package/dist/lib/arrow/convert-schema-to-parquet.js +1 -0
  12. package/dist/lib/arrow/convert-schema-to-parquet.js.map +1 -0
  13. package/dist/lib/constants.js +2 -1
  14. package/dist/lib/constants.js.map +1 -0
  15. package/dist/lib/encoders/encode-arrow-to-parquet.d.ts.map +1 -1
  16. package/dist/lib/encoders/encode-arrow-to-parquet.js +3 -1
  17. package/dist/lib/encoders/encode-arrow-to-parquet.js.map +1 -0
  18. package/dist/lib/parsers/get-parquet-schema.js +1 -0
  19. package/dist/lib/parsers/get-parquet-schema.js.map +1 -0
  20. package/dist/lib/parsers/parse-geoparquet-to-geojson.js +1 -0
  21. package/dist/lib/parsers/parse-geoparquet-to-geojson.js.map +1 -0
  22. package/dist/lib/parsers/parse-parquet-to-arrow.js +1 -0
  23. package/dist/lib/parsers/parse-parquet-to-arrow.js.map +1 -0
  24. package/dist/lib/parsers/parse-parquet-to-columns.js +1 -0
  25. package/dist/lib/parsers/parse-parquet-to-columns.js.map +1 -0
  26. package/dist/lib/parsers/parse-parquet-to-json.d.ts.map +1 -1
  27. package/dist/lib/parsers/parse-parquet-to-json.js +3 -2
  28. package/dist/lib/parsers/parse-parquet-to-json.js.map +1 -0
  29. package/dist/lib/utils/load-wasm.js +1 -0
  30. package/dist/lib/utils/load-wasm.js.map +1 -0
  31. package/dist/lib/utils/make-stream-iterator.js +1 -0
  32. package/dist/lib/utils/make-stream-iterator.js.map +1 -0
  33. package/dist/parquet-arrow-loader.js +1 -0
  34. package/dist/parquet-arrow-loader.js.map +1 -0
  35. package/dist/parquet-arrow-writer.js +1 -0
  36. package/dist/parquet-arrow-writer.js.map +1 -0
  37. package/dist/parquet-format.js +1 -0
  38. package/dist/parquet-format.js.map +1 -0
  39. package/dist/parquet-json-loader.js +2 -1
  40. package/dist/parquet-json-loader.js.map +1 -0
  41. package/dist/parquet-json-writer.js +2 -1
  42. package/dist/parquet-json-writer.js.map +1 -0
  43. package/dist/parquetjs/codecs/declare.js +1 -0
  44. package/dist/parquetjs/codecs/declare.js.map +1 -0
  45. package/dist/parquetjs/codecs/dictionary.js +1 -0
  46. package/dist/parquetjs/codecs/dictionary.js.map +1 -0
  47. package/dist/parquetjs/codecs/index.js +1 -0
  48. package/dist/parquetjs/codecs/index.js.map +1 -0
  49. package/dist/parquetjs/codecs/plain.js +3 -2
  50. package/dist/parquetjs/codecs/plain.js.map +1 -0
  51. package/dist/parquetjs/codecs/rle.d.ts.map +1 -1
  52. package/dist/parquetjs/codecs/rle.js +13 -6
  53. package/dist/parquetjs/codecs/rle.js.map +1 -0
  54. package/dist/parquetjs/compression.js +1 -0
  55. package/dist/parquetjs/compression.js.map +1 -0
  56. package/dist/parquetjs/encoder/parquet-encoder.js +1 -0
  57. package/dist/parquetjs/encoder/parquet-encoder.js.map +1 -0
  58. package/dist/parquetjs/parquet-thrift/BoundaryOrder.js +1 -0
  59. package/dist/parquetjs/parquet-thrift/BoundaryOrder.js.map +1 -0
  60. package/dist/parquetjs/parquet-thrift/BsonType.js +1 -0
  61. package/dist/parquetjs/parquet-thrift/BsonType.js.map +1 -0
  62. package/dist/parquetjs/parquet-thrift/ColumnChunk.js +1 -0
  63. package/dist/parquetjs/parquet-thrift/ColumnChunk.js.map +1 -0
  64. package/dist/parquetjs/parquet-thrift/ColumnIndex.js +1 -0
  65. package/dist/parquetjs/parquet-thrift/ColumnIndex.js.map +1 -0
  66. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js +1 -0
  67. package/dist/parquetjs/parquet-thrift/ColumnMetaData.js.map +1 -0
  68. package/dist/parquetjs/parquet-thrift/ColumnOrder.js +1 -0
  69. package/dist/parquetjs/parquet-thrift/ColumnOrder.js.map +1 -0
  70. package/dist/parquetjs/parquet-thrift/CompressionCodec.js +1 -0
  71. package/dist/parquetjs/parquet-thrift/CompressionCodec.js.map +1 -0
  72. package/dist/parquetjs/parquet-thrift/ConvertedType.js +1 -0
  73. package/dist/parquetjs/parquet-thrift/ConvertedType.js.map +1 -0
  74. package/dist/parquetjs/parquet-thrift/DataPageHeader.js +1 -0
  75. package/dist/parquetjs/parquet-thrift/DataPageHeader.js.map +1 -0
  76. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js +1 -0
  77. package/dist/parquetjs/parquet-thrift/DataPageHeaderV2.js.map +1 -0
  78. package/dist/parquetjs/parquet-thrift/DateType.js +1 -0
  79. package/dist/parquetjs/parquet-thrift/DateType.js.map +1 -0
  80. package/dist/parquetjs/parquet-thrift/DecimalType.js +1 -0
  81. package/dist/parquetjs/parquet-thrift/DecimalType.js.map +1 -0
  82. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js +1 -0
  83. package/dist/parquetjs/parquet-thrift/DictionaryPageHeader.js.map +1 -0
  84. package/dist/parquetjs/parquet-thrift/Encoding.js +1 -0
  85. package/dist/parquetjs/parquet-thrift/Encoding.js.map +1 -0
  86. package/dist/parquetjs/parquet-thrift/EnumType.js +1 -0
  87. package/dist/parquetjs/parquet-thrift/EnumType.js.map +1 -0
  88. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js +1 -0
  89. package/dist/parquetjs/parquet-thrift/FieldRepetitionType.js.map +1 -0
  90. package/dist/parquetjs/parquet-thrift/FileMetaData.js +1 -0
  91. package/dist/parquetjs/parquet-thrift/FileMetaData.js.map +1 -0
  92. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js +1 -0
  93. package/dist/parquetjs/parquet-thrift/IndexPageHeader.js.map +1 -0
  94. package/dist/parquetjs/parquet-thrift/IntType.js +1 -0
  95. package/dist/parquetjs/parquet-thrift/IntType.js.map +1 -0
  96. package/dist/parquetjs/parquet-thrift/JsonType.js +1 -0
  97. package/dist/parquetjs/parquet-thrift/JsonType.js.map +1 -0
  98. package/dist/parquetjs/parquet-thrift/KeyValue.js +1 -0
  99. package/dist/parquetjs/parquet-thrift/KeyValue.js.map +1 -0
  100. package/dist/parquetjs/parquet-thrift/ListType.js +1 -0
  101. package/dist/parquetjs/parquet-thrift/ListType.js.map +1 -0
  102. package/dist/parquetjs/parquet-thrift/LogicalType.js +1 -0
  103. package/dist/parquetjs/parquet-thrift/LogicalType.js.map +1 -0
  104. package/dist/parquetjs/parquet-thrift/MapType.js +1 -0
  105. package/dist/parquetjs/parquet-thrift/MapType.js.map +1 -0
  106. package/dist/parquetjs/parquet-thrift/MicroSeconds.js +1 -0
  107. package/dist/parquetjs/parquet-thrift/MicroSeconds.js.map +1 -0
  108. package/dist/parquetjs/parquet-thrift/MilliSeconds.js +1 -0
  109. package/dist/parquetjs/parquet-thrift/MilliSeconds.js.map +1 -0
  110. package/dist/parquetjs/parquet-thrift/NullType.js +1 -0
  111. package/dist/parquetjs/parquet-thrift/NullType.js.map +1 -0
  112. package/dist/parquetjs/parquet-thrift/OffsetIndex.js +1 -0
  113. package/dist/parquetjs/parquet-thrift/OffsetIndex.js.map +1 -0
  114. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js +1 -0
  115. package/dist/parquetjs/parquet-thrift/PageEncodingStats.js.map +1 -0
  116. package/dist/parquetjs/parquet-thrift/PageHeader.js +1 -0
  117. package/dist/parquetjs/parquet-thrift/PageHeader.js.map +1 -0
  118. package/dist/parquetjs/parquet-thrift/PageLocation.js +1 -0
  119. package/dist/parquetjs/parquet-thrift/PageLocation.js.map +1 -0
  120. package/dist/parquetjs/parquet-thrift/PageType.js +1 -0
  121. package/dist/parquetjs/parquet-thrift/PageType.js.map +1 -0
  122. package/dist/parquetjs/parquet-thrift/RowGroup.js +1 -0
  123. package/dist/parquetjs/parquet-thrift/RowGroup.js.map +1 -0
  124. package/dist/parquetjs/parquet-thrift/SchemaElement.js +1 -0
  125. package/dist/parquetjs/parquet-thrift/SchemaElement.js.map +1 -0
  126. package/dist/parquetjs/parquet-thrift/SortingColumn.js +1 -0
  127. package/dist/parquetjs/parquet-thrift/SortingColumn.js.map +1 -0
  128. package/dist/parquetjs/parquet-thrift/Statistics.js +1 -0
  129. package/dist/parquetjs/parquet-thrift/Statistics.js.map +1 -0
  130. package/dist/parquetjs/parquet-thrift/StringType.js +1 -0
  131. package/dist/parquetjs/parquet-thrift/StringType.js.map +1 -0
  132. package/dist/parquetjs/parquet-thrift/TimeType.js +1 -0
  133. package/dist/parquetjs/parquet-thrift/TimeType.js.map +1 -0
  134. package/dist/parquetjs/parquet-thrift/TimeUnit.js +1 -0
  135. package/dist/parquetjs/parquet-thrift/TimeUnit.js.map +1 -0
  136. package/dist/parquetjs/parquet-thrift/TimestampType.js +1 -0
  137. package/dist/parquetjs/parquet-thrift/TimestampType.js.map +1 -0
  138. package/dist/parquetjs/parquet-thrift/Type.js +1 -0
  139. package/dist/parquetjs/parquet-thrift/Type.js.map +1 -0
  140. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js +1 -0
  141. package/dist/parquetjs/parquet-thrift/TypeDefinedOrder.js.map +1 -0
  142. package/dist/parquetjs/parquet-thrift/UUIDType.js +1 -0
  143. package/dist/parquetjs/parquet-thrift/UUIDType.js.map +1 -0
  144. package/dist/parquetjs/parquet-thrift/index.js +1 -0
  145. package/dist/parquetjs/parquet-thrift/index.js.map +1 -0
  146. package/dist/parquetjs/parser/decoders.js +2 -1
  147. package/dist/parquetjs/parser/decoders.js.map +1 -0
  148. package/dist/parquetjs/parser/parquet-reader.js +1 -0
  149. package/dist/parquetjs/parser/parquet-reader.js.map +1 -0
  150. package/dist/parquetjs/schema/declare.js +1 -0
  151. package/dist/parquetjs/schema/declare.js.map +1 -0
  152. package/dist/parquetjs/schema/schema.js +1 -0
  153. package/dist/parquetjs/schema/schema.js.map +1 -0
  154. package/dist/parquetjs/schema/shred.js +1 -0
  155. package/dist/parquetjs/schema/shred.js.map +1 -0
  156. package/dist/parquetjs/schema/types.js +1 -0
  157. package/dist/parquetjs/schema/types.js.map +1 -0
  158. package/dist/parquetjs/utils/file-utils.js +1 -0
  159. package/dist/parquetjs/utils/file-utils.js.map +1 -0
  160. package/dist/parquetjs/utils/read-utils.js +1 -0
  161. package/dist/parquetjs/utils/read-utils.js.map +1 -0
  162. package/dist/polyfills/buffer/buffer-polyfill.browser.js +1 -0
  163. package/dist/polyfills/buffer/buffer-polyfill.browser.js.map +1 -0
  164. package/dist/polyfills/buffer/buffer-polyfill.node.js +1 -0
  165. package/dist/polyfills/buffer/buffer-polyfill.node.js.map +1 -0
  166. package/dist/polyfills/buffer/buffer.js +1 -0
  167. package/dist/polyfills/buffer/buffer.js.map +1 -0
  168. package/dist/polyfills/buffer/index.d.ts +1 -1
  169. package/dist/polyfills/buffer/index.d.ts.map +1 -1
  170. package/dist/polyfills/buffer/index.js +2 -1
  171. package/dist/polyfills/buffer/index.js.map +1 -0
  172. package/dist/polyfills/buffer/install-buffer-polyfill.d.ts +1 -28
  173. package/dist/polyfills/buffer/install-buffer-polyfill.d.ts.map +1 -1
  174. package/dist/polyfills/buffer/install-buffer-polyfill.js +2 -1
  175. package/dist/polyfills/buffer/install-buffer-polyfill.js.map +1 -0
  176. package/dist/polyfills/util.js +1 -0
  177. package/dist/polyfills/util.js.map +1 -0
  178. package/dist/workers/parquet-json-worker.js +1 -0
  179. package/dist/workers/parquet-json-worker.js.map +1 -0
  180. package/package.json +16 -13
  181. package/src/lib/encoders/encode-arrow-to-parquet.ts +4 -2
  182. package/src/lib/parsers/parse-parquet-to-json.ts +4 -2
  183. package/src/parquetjs/codecs/plain.ts +4 -4
  184. package/src/parquetjs/codecs/rle.ts +17 -11
  185. package/src/parquetjs/encoder/parquet-encoder.ts +10 -10
  186. package/src/parquetjs/parser/decoders.ts +2 -2
  187. package/src/parquetjs/utils/read-utils.ts +1 -1
package/dist/index.cjs CHANGED
@@ -1997,7 +1997,10 @@ function encodeValues2(type, values, opts) {
1997
1997
  }
1998
1998
  }
1999
1999
  if (repeats) {
2000
- buf = Buffer.concat([buf, encodeRunRepeated(values[values.length - 1], repeats, opts)]);
2000
+ buf = Buffer.concat([
2001
+ buf,
2002
+ encodeRunRepeated(values[values.length - 1], repeats, opts)
2003
+ ]);
2001
2004
  } else if (run.length) {
2002
2005
  buf = Buffer.concat([buf, encodeRunBitpacked(run, opts)]);
2003
2006
  }
@@ -2073,7 +2076,10 @@ function encodeRunBitpacked(values, opts) {
2073
2076
  buf[Math.floor(b / 8)] |= 1 << b % 8;
2074
2077
  }
2075
2078
  }
2076
- return Buffer.concat([Buffer.from(import_varint.default.encode(values.length / 8 << 1 | 1)), buf]);
2079
+ return Buffer.concat([
2080
+ Buffer.from(import_varint.default.encode(values.length / 8 << 1 | 1)),
2081
+ buf
2082
+ ]);
2077
2083
  }
2078
2084
  function encodeRunRepeated(value, count, opts) {
2079
2085
  const bitWidth = opts.bitWidth;
@@ -6245,7 +6251,7 @@ var FileMetaData = class {
6245
6251
  };
6246
6252
 
6247
6253
  // dist/lib/constants.js
6248
- var VERSION = true ? "4.4.0-alpha.1" : "latest";
6254
+ var VERSION = true ? "4.4.0" : "latest";
6249
6255
  var PARQUET_WASM_URL = "https://unpkg.com/parquet-wasm@0.6.1/esm/parquet_wasm_bg.wasm";
6250
6256
  var PARQUET_MAGIC = "PAR1";
6251
6257
  var PARQUET_MAGIC_ENCRYPTED = "PARE";
@@ -6590,7 +6596,7 @@ function preserveBinary(d) {
6590
6596
  return d;
6591
6597
  }
6592
6598
  if (Buffer.isBuffer(d)) {
6593
- return d.buffer.slice(d.byteOffset, d.byteLength);
6599
+ return new Uint8Array(d.buffer, d.byteOffset, d.byteLength).slice().buffer;
6594
6600
  }
6595
6601
  return d.toString();
6596
6602
  }
@@ -6878,7 +6884,7 @@ async function getSchemaFromParquetReader(reader) {
6878
6884
 
6879
6885
  // dist/lib/parsers/parse-parquet-to-json.js
6880
6886
  async function parseParquetFile(file, options) {
6881
- var _a;
6887
+ var _a, _b, _c;
6882
6888
  installBufferPolyfill();
6883
6889
  await preloadCompressions(options);
6884
6890
  const reader = new ParquetReader(file, {
@@ -6890,14 +6896,14 @@ async function parseParquetFile(file, options) {
6890
6896
  for await (const rowBatch of rowBatches) {
6891
6897
  let limitHasReached = false;
6892
6898
  for (const row of rowBatch) {
6893
- if ((options == null ? void 0 : options.limit) && rows.length >= (options == null ? void 0 : options.limit)) {
6899
+ if ((options == null ? void 0 : options.limit) && rows.length >= ((_b = options == null ? void 0 : options.core) == null ? void 0 : _b.limit)) {
6894
6900
  limitHasReached = true;
6895
6901
  break;
6896
6902
  }
6897
6903
  rows.push(row);
6898
6904
  }
6899
6905
  if (limitHasReached) {
6900
- import_log.default.warn(`Rows number limit has been reached. Only first ${options == null ? void 0 : options.limit} are loaded`)();
6906
+ import_log.default.warn(`Rows number limit has been reached. Only first ${(_c = options == null ? void 0 : options.core) == null ? void 0 : _c.limit} are loaded`)();
6901
6907
  break;
6902
6908
  }
6903
6909
  }
@@ -6945,7 +6951,7 @@ async function* parseGeoParquetFileInBatches(file, options) {
6945
6951
  }
6946
6952
 
6947
6953
  // dist/parquet-json-loader.js
6948
- var VERSION2 = true ? "4.4.0-alpha.1" : "latest";
6954
+ var VERSION2 = true ? "4.4.0" : "latest";
6949
6955
  var ParquetJSONWorkerLoader = {
6950
6956
  ...ParquetFormat,
6951
6957
  dataType: null,
@@ -6999,7 +7005,7 @@ var GeoParquetLoader = {
6999
7005
  };
7000
7006
 
7001
7007
  // dist/parquet-json-writer.js
7002
- var VERSION3 = true ? "4.4.0-alpha.1" : "latest";
7008
+ var VERSION3 = true ? "4.4.0" : "latest";
7003
7009
  var ParquetJSONWriter = {
7004
7010
  ...ParquetFormat,
7005
7011
  id: "parquet",
@@ -7167,7 +7173,8 @@ async function encodeArrowToParquet(table, options) {
7167
7173
  const wasmProperties = new wasm.WriterPropertiesBuilder().build();
7168
7174
  try {
7169
7175
  const parquetBytes = wasm.writeParquet(wasmTable, wasmProperties);
7170
- return parquetBytes.buffer.slice(parquetBytes.byteOffset, parquetBytes.byteLength + parquetBytes.byteOffset);
7176
+ const bytes = new Uint8Array(parquetBytes.buffer, parquetBytes.byteOffset, parquetBytes.byteLength);
7177
+ return bytes.slice().buffer;
7171
7178
  } finally {
7172
7179
  }
7173
7180
  }