@loaders.gl/parquet 4.2.0-alpha.4 → 4.2.0-alpha.5

package/dist/polyfills/buffer/index.d.ts

@@ -1,4 +1,4 @@
-export { Buffer as BufferPolyfill } from './buffer';
-export { Buffer } from './install-buffer-polyfill';
-export { installBufferPolyfill } from './buffer-polyfill.node';
+export { Buffer as BufferPolyfill } from "./buffer.js";
+export { Buffer } from "./install-buffer-polyfill.js";
+export { installBufferPolyfill } from "./buffer-polyfill.node.js";
 //# sourceMappingURL=index.d.ts.map

package/dist/polyfills/buffer/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/polyfills/buffer/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,MAAM,IAAI,cAAc,EAAC,MAAM,UAAU,CAAC;AAClD,OAAO,EAAC,MAAM,EAAC,MAAM,2BAA2B,CAAC;AACjD,OAAO,EAAC,qBAAqB,EAAC,MAAM,wBAAwB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/polyfills/buffer/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAC,MAAM,IAAI,cAAc,EAAC,oBAAiB;AAClD,OAAO,EAAC,MAAM,EAAC,qCAAkC;AACjD,OAAO,EAAC,qBAAqB,EAAC,kCAA+B"}

package/dist/polyfills/buffer/index.js

@@ -1,4 +1,8 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 export { Buffer as BufferPolyfill } from "./buffer.js";
 export { Buffer } from "./install-buffer-polyfill.js";
 export { installBufferPolyfill } from "./buffer-polyfill.node.js";
-//# sourceMappingURL=index.js.map
+// import { installBufferPolyfill } from "./buffer-polyfill.node";
+// installBufferPolyfill();

package/dist/polyfills/buffer/install-buffer-polyfill.js

@@ -1,5 +1,6 @@
 import { installBufferPolyfill } from "./buffer-polyfill.node.js";
+// @ts-ignore
 globalThis.process = globalThis.process || {};
+// @ts-ignore
 globalThis.process.env = globalThis.process.env || {};
 export const Buffer = installBufferPolyfill();
-//# sourceMappingURL=install-buffer-polyfill.js.map

package/dist/polyfills/util.js

@@ -1,3 +1,6 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+// Polyfill for Node.js util library
 export const TextEncoder = globalThis.TextEncoder;
 export const TextDecoder = globalThis.TextDecoder;
-//# sourceMappingURL=util.js.map

package/dist/workers/parquet-worker.js

@@ -1,4 +1,6 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
 import { createLoaderWorker } from '@loaders.gl/loader-utils';
 import { ParquetLoader } from "../parquet-loader.js";
 createLoaderWorker(ParquetLoader);
-//# sourceMappingURL=parquet-worker.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "4.2.0-alpha.4",
+  "version": "4.2.0-alpha.5",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "type": "module",
@@ -40,7 +40,8 @@
   ],
   "scripts": {
     "pre-build": "echo npm run build-worker && yarn run copy-wasm",
-    "build-bundle": "ocular-bundle ./src/index.ts",
+    "build-bundle": "ocular-bundle ./bundle.ts --output=dist/dist.min.js",
+    "build-bundle-dev": "ocular-bundle ./bundle.ts --env=dev --output=dist/dist.dev.js",
     "build-worker": "esbuild src/workers/parquet-worker.ts --outfile=dist/parquet-worker.js --bundle --minify --sourcemap --external:{util,fs,path,crypto,events} --define:__VERSION__=\\\"$npm_package_version\\\"",
     "copy-wasm": "cp ../../node_modules/parquet-wasm/esm2/arrow1_bg.wasm dist/arrow1_bg.wasm"
   },
@@ -59,13 +60,13 @@
     "base64-js and ieee754 are used by buffer polyfill"
   ],
   "dependencies": {
-    "@loaders.gl/arrow": "4.2.0-alpha.4",
-    "@loaders.gl/bson": "4.2.0-alpha.4",
-    "@loaders.gl/compression": "4.2.0-alpha.4",
-    "@loaders.gl/gis": "4.2.0-alpha.4",
-    "@loaders.gl/loader-utils": "4.2.0-alpha.4",
-    "@loaders.gl/schema": "4.2.0-alpha.4",
-    "@loaders.gl/wkt": "4.2.0-alpha.4",
+    "@loaders.gl/arrow": "4.2.0-alpha.5",
+    "@loaders.gl/bson": "4.2.0-alpha.5",
+    "@loaders.gl/compression": "4.2.0-alpha.5",
+    "@loaders.gl/gis": "4.2.0-alpha.5",
+    "@loaders.gl/loader-utils": "4.2.0-alpha.5",
+    "@loaders.gl/schema": "4.2.0-alpha.5",
+    "@loaders.gl/wkt": "4.2.0-alpha.5",
     "async-mutex": "^0.2.2",
     "base64-js": "^1.3.1",
     "brotli": "^1.3.2",
@@ -81,9 +82,6 @@
     "varint": "^5.0.0",
     "zstd-codec": "^0.1"
   },
-  "peerDependencies": {
-    "apache-arrow": ">= 15.0.0"
-  },
   "devDependencies": {
     "@types/node": "^10.14.15",
     "@types/node-int64": "^0.4.29",
@@ -91,5 +89,9 @@
     "@types/varint": "^5.0.0",
     "apache-arrow": "^15.0.0"
   },
-  "gitHead": "6c52dee5c3f005648a394cc4aee7fc37005c8e83"
+  "peerDependencies": {
+    "@loaders.gl/core": "^4.0.0",
+    "apache-arrow": ">= 15.0.0"
+  },
+  "gitHead": "32d95a81971f104e4dfeb88ab57065f05321a76a"
 }

package/src/index.ts

@@ -37,4 +37,4 @@ export {
 } from './lib/arrow/convert-schema-from-parquet';
 
 // Experimental
-export {BufferPolyfill, installBufferPolyfill} from './polyfills/buffer';
+export {BufferPolyfill, installBufferPolyfill} from './polyfills/buffer/index';

package/src/lib/parsers/parse-parquet-to-columns.ts

@@ -10,7 +10,7 @@ import {ParquetRowGroup} from '../../parquetjs/schema/declare';
 import {ParquetSchema} from '../../parquetjs/schema/schema';
 import {materializeColumns} from '../../parquetjs/schema/shred';
 import {getSchemaFromParquetReader} from './get-parquet-schema';
-import {installBufferPolyfill} from '../../polyfills/buffer';
+import {installBufferPolyfill} from '../../polyfills/buffer/index';
 
 /**
  * @deprecated

package/src/lib/parsers/parse-parquet.ts

@@ -9,7 +9,7 @@ import type {ParquetLoaderOptions} from '../../parquet-loader';
 import type {ParquetRow} from '../../parquetjs/schema/declare';
 import {ParquetReader} from '../../parquetjs/parser/parquet-reader';
 import {getSchemaFromParquetReader} from './get-parquet-schema';
-import {installBufferPolyfill} from '../../polyfills/buffer';
+import {installBufferPolyfill} from '../../polyfills/buffer/index';
 
 /**
  *  * Parse a parquet file using parquetjs

package/src/parquetjs/encoder/parquet-encoder.ts

@@ -1,7 +1,7 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 /* eslint-disable camelcase */
 import {stream} from '@loaders.gl/loader-utils';
-import {ParquetCodecOptions, PARQUET_CODECS} from '../codecs';
+import {ParquetCodecOptions, PARQUET_CODECS} from '../codecs/index';
 import * as Compression from '../compression';
 import {
   ParquetRowGroup,
@@ -29,7 +29,7 @@ import {
   RowGroup,
   SchemaElement,
   Type
-} from '../parquet-thrift';
+} from '../parquet-thrift/index';
 import {osopen, oswrite, osclose} from '../utils/file-utils';
 import {getBitWidth, serializeThrift} from '../utils/read-utils';
 import Int64 from 'node-int64';

package/src/parquetjs/parser/decoders.ts

@@ -8,7 +8,7 @@ import {
   PrimitiveType,
   SchemaDefinition
 } from '../schema/declare';
-import {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs';
+import {CursorBuffer, ParquetCodecOptions, PARQUET_CODECS} from '../codecs/index';
 import {
   ConvertedType,
   Encoding,
@@ -17,7 +17,7 @@ import {
   PageType,
   SchemaElement,
   Type
-} from '../parquet-thrift';
+} from '../parquet-thrift/index';
 import {decompress} from '../compression';
 import {PARQUET_RDLVL_TYPE, PARQUET_RDLVL_ENCODING} from '../../lib/constants';
 import {decodePageHeader, getThriftEnum, getBitWidth} from '../utils/read-utils';

package/src/parquetjs/parser/parquet-reader.ts

@@ -6,7 +6,7 @@ import {decodeSchema} from './decoders';
 import {materializeRows} from '../schema/shred';
 
 import {PARQUET_MAGIC, PARQUET_MAGIC_ENCRYPTED} from '../../lib/constants';
-import {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift';
+import {ColumnChunk, CompressionCodec, FileMetaData, RowGroup, Type} from '../parquet-thrift/index';
 import {
   ParquetRowGroup,
   ParquetCompression,

package/src/parquetjs/schema/schema.ts

@@ -1,6 +1,6 @@
 // Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)
 
-import {PARQUET_CODECS} from '../codecs';
+import {PARQUET_CODECS} from '../codecs/index';
 import {PARQUET_COMPRESSION_METHODS} from '../compression';
 import {
   FieldDefinition,

package/src/parquetjs/utils/read-utils.ts

@@ -1,5 +1,5 @@
-import {TBufferedTransport, TCompactProtocol, TFramedTransport} from '../parquet-thrift';
-import {FileMetaData, PageHeader} from '../parquet-thrift';
+import {TBufferedTransport, TCompactProtocol, TFramedTransport} from '../parquet-thrift/index';
+import {FileMetaData, PageHeader} from '../parquet-thrift/index';
 
 class UFramedTransport extends TFramedTransport {
   public readPos: number = 0;

package/dist/index.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"index.js","names":["Buffer","ParquetWorkerLoader","ParquetLoader","GeoParquetWorkerLoader","GeoParquetLoader","ParquetColumnarWorkerLoader","ParquetColumnarLoader","ParquetWriter","_ParquetWriter","ParquetWasmLoader","ParquetWasmWorkerLoader","ParquetWasmWriter","preloadCompressions","ParquetSchema","ParquetReader","ParquetEncoder","convertParquetSchema","convertParquetToArrowSchema","BufferPolyfill","installBufferPolyfill"],"sources":["../src/index.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nexport {Buffer} from './polyfills/buffer/install-buffer-polyfill';\n\n// import {ArrowTable, ArrowTableBatch} from '@loaders.gl/arrow';\n\nexport {\n  ParquetWorkerLoader,\n  ParquetLoader,\n  GeoParquetWorkerLoader,\n  GeoParquetLoader,\n  ParquetColumnarWorkerLoader,\n  ParquetColumnarLoader\n} from './parquet-loader';\n\nexport {ParquetWriter as _ParquetWriter} from './parquet-writer';\n\n// EXPERIMENTAL - expose Parquet WASM loaders/writer\n\nexport type {ParquetWasmLoaderOptions} from './parquet-wasm-loader';\nexport {ParquetWasmLoader, ParquetWasmWorkerLoader} from './parquet-wasm-loader';\nexport {ParquetWasmWriter} from './parquet-wasm-writer';\n\n// EXPERIMENTAL - expose the internal parquetjs API\n\nexport {preloadCompressions} from './parquetjs/compression';\n\nexport {ParquetSchema} from './parquetjs/schema/schema';\nexport {ParquetReader} from './parquetjs/parser/parquet-reader';\nexport {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';\n\nexport {\n  convertParquetSchema,\n  convertParquetSchema as convertParquetToArrowSchema\n} from './lib/arrow/convert-schema-from-parquet';\n\n// Experimental\nexport {BufferPolyfill, installBufferPolyfill} from './polyfills/buffer';\n"],"mappings":"SAIQA,MAAM;AAAA,SAKZC,mBAAmB,EACnBC,aAAa,EACbC,sBAAsB,EACtBC,gBAAgB,EAChBC,2BAA2B,EAC3BC,qBAAqB;AAAA,SAGfC,aAAa,IAAIC,cAAc;AAAA,SAK/BC,iBAAiB,EAAEC,uBAAuB;AAAA,SAC1CC,iBAAiB;AAAA,SAIjBC,mBAAmB;AAAA,SAEnBC,aAAa;AAAA,SACbC,aAAa;AAAA,SACbC,cAAc;AAAA,SAGpBC,oBAAoB,EACpBA,oBAAoB,IAAIC,2BAA2B;AAAA,SAI7CC,cAAc,EAAEC,qBAAqB"}

package/dist/lib/arrow/convert-columns-to-row-group.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-columns-to-row-group.js","names":[],"sources":["../../../src/lib/arrow/convert-columns-to-row-group.ts"],"sourcesContent":[""],"mappings":""}

package/dist/lib/arrow/convert-row-group-to-columns.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-row-group-to-columns.js","names":["convertParquetRowGroupToColumns","schema","rowGroup","columns","columnName","data","Object","entries","columnData","values"],"sources":["../../../src/lib/arrow/convert-row-group-to-columns.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport {Schema} from '@loaders.gl/schema';\nimport {ParquetRowGroup} from '@loaders.gl/parquet/parquetjs/schema/declare';\n\nexport function convertParquetRowGroupToColumns(\n  schema: Schema,\n  rowGroup: ParquetRowGroup\n): Record<string, any[]> {\n  const columns: Record<string, any[]> = {};\n  for (const [columnName, data] of Object.entries(rowGroup.columnData)) {\n    columns[columnName] = columns[columnName] || data.values;\n  }\n  return columns;\n}\n"],"mappings":"AAOA,OAAO,SAASA,+BAA+BA,CAC7CC,MAAc,EACdC,QAAyB,EACF;EACvB,MAAMC,OAA8B,GAAG,CAAC,CAAC;EACzC,KAAK,MAAM,CAACC,UAAU,EAAEC,IAAI,CAAC,IAAIC,MAAM,CAACC,OAAO,CAACL,QAAQ,CAACM,UAAU,CAAC,EAAE;IACpEL,OAAO,CAACC,UAAU,CAAC,GAAGD,OAAO,CAACC,UAAU,CAAC,IAAIC,IAAI,CAACI,MAAM;EAC1D;EACA,OAAON,OAAO;AAChB"}

package/dist/lib/arrow/convert-schema-from-parquet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-schema-from-parquet.js","names":["PARQUET_TYPE_MAPPING","BOOLEAN","INT32","INT64","INT96","FLOAT","DOUBLE","BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","DATE","TIME_MILLIS","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","UINT_32","UINT_64","INT_8","INT_16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertParquetSchema","parquetSchema","parquetMetadata","fields","getFields","schema","metadata","getSchemaMetadata","name","field","children","push","type","nullable","optional","getFieldMetadata","arrowField","key","value","stringify","keyValueList","key_value_metadata"],"sources":["../../../src/lib/arrow/convert-schema-from-parquet.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport {Schema, Field, DataType} from '@loaders.gl/schema';\n\nimport type {ParquetSchema} from '../../parquetjs/schema/schema';\nimport type {FieldDefinition, ParquetField, ParquetType} from '../../parquetjs/schema/declare';\nimport {FileMetaData} from '../../parquetjs/parquet-thrift';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {\n  BOOLEAN: 'bool',\n  INT32: 'int32',\n  INT64: 'float64',\n  INT96: 'float64',\n  FLOAT: 'float32',\n  DOUBLE: 'float64',\n  BYTE_ARRAY: 'binary',\n  FIXED_LEN_BYTE_ARRAY: 'binary',\n  UTF8: 'utf8',\n  DATE: 'int32',\n  TIME_MILLIS: 'int64',\n  TIME_MICROS: 'int64',\n  TIMESTAMP_MILLIS: 'int64',\n  TIMESTAMP_MICROS: 'int64',\n  UINT_8: 'int32',\n  UINT_16: 'uint16',\n  UINT_32: 'uint32',\n  UINT_64: 'uint64',\n  INT_8: 'int8',\n  INT_16: 'int16',\n  INT_32: 'int32',\n  INT_64: 'int64',\n  JSON: 'binary',\n  BSON: 'binary',\n  // TODO check interal type\n  INTERVAL: 'binary',\n  DECIMAL_INT32: 'float32',\n  DECIMAL_INT64: 'float64',\n  DECIMAL_BYTE_ARRAY: 'float64',\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'\n};\n\nexport function convertParquetSchema(\n  parquetSchema: ParquetSchema,\n  parquetMetadata: FileMetaData | null\n): Schema {\n  const fields = getFields(parquetSchema.schema);\n  const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);\n\n  const schema: Schema = {\n    fields,\n    metadata: metadata || {}\n  };\n\n  return schema;\n}\n\nfunction getFields(schema: FieldDefinition): Field[] {\n  const fields: Field[] = [];\n\n  for (const name in schema) {\n    const field = schema[name];\n\n    if (field.fields) {\n      const children = getFields(field.fields);\n      fields.push({name, type: {type: 'struct', children}, nullable: field.optional});\n    } else {\n      const type = PARQUET_TYPE_MAPPING[field.type];\n      const metadata = getFieldMetadata(field);\n      const arrowField = {name, type, nullable: field.optional, metadata};\n      fields.push(arrowField);\n    }\n  }\n\n  return fields;\n}\n\nfunction getFieldMetadata(field: ParquetField): Record<string, string> | undefined {\n  let metadata: Record<string, string> | undefined;\n\n  for (const key in field) {\n    if (key !== 'name') {\n      let value = field[key] || '';\n      value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n      metadata = metadata || {};\n      metadata[key] = value;\n    }\n  }\n\n  return metadata;\n}\n\nfunction getSchemaMetadata(parquetMetadata: FileMetaData): Record<string, string> | undefined {\n  let metadata: Record<string, string> | undefined;\n\n  const keyValueList = parquetMetadata.key_value_metadata || [];\n  for (const {key, value} of keyValueList) {\n    if (typeof value === 'string') {\n      metadata = metadata || {};\n      metadata[key] = value;\n    }\n  }\n\n  return metadata;\n}\n"],"mappings":"AAUA,OAAO,MAAMA,oBAAuD,GAAG;EACrEC,OAAO,EAAE,MAAM;EACfC,KAAK,EAAE,OAAO;EACdC,KAAK,EAAE,SAAS;EAChBC,KAAK,EAAE,SAAS;EAChBC,KAAK,EAAE,SAAS;EAChBC,MAAM,EAAE,SAAS;EACjBC,UAAU,EAAE,QAAQ;EACpBC,oBAAoB,EAAE,QAAQ;EAC9BC,IAAI,EAAE,MAAM;EACZC,IAAI,EAAE,OAAO;EACbC,WAAW,EAAE,OAAO;EACpBC,WAAW,EAAE,OAAO;EACpBC,gBAAgB,EAAE,OAAO;EACzBC,gBAAgB,EAAE,OAAO;EACzBC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAE,QAAQ;EACjBC,OAAO,EAAE,QAAQ;EACjBC,OAAO,EAAE,QAAQ;EACjBC,KAAK,EAAE,MAAM;EACbC,MAAM,EAAE,OAAO;EACfC,MAAM,EAAE,OAAO;EACfC,MAAM,EAAE,OAAO;EACfC,IAAI,EAAE,QAAQ;EACdC,IAAI,EAAE,QAAQ;EAEdC,QAAQ,EAAE,QAAQ;EAClBC,aAAa,EAAE,SAAS;EACxBC,aAAa,EAAE,SAAS;EACxBC,kBAAkB,EAAE,SAAS;EAC7BC,4BAA4B,EAAE;AAChC,CAAC;AAED,OAAO,SAASC,oBAAoBA,CAClCC,aAA4B,EAC5BC,eAAoC,EAC5B;EACR,MAAMC,MAAM,GAAGC,SAAS,CAACH,aAAa,CAACI,MAAM,CAAC;EAC9C,MAAMC,QAAQ,GAAGJ,eAAe,IAAIK,iBAAiB,CAACL,eAAe,CAAC;EAEtE,MAAMG,MAAc,GAAG;IACrBF,MAAM;IACNG,QAAQ,EAAEA,QAAQ,IAAI,CAAC;EACzB,CAAC;EAED,OAAOD,MAAM;AACf;AAEA,SAASD,SAASA,CAACC,MAAuB,EAAW;EACnD,MAAMF,MAAe,GAAG,EAAE;EAE1B,KAAK,MAAMK,IAAI,IAAIH,MAAM,EAAE;IACzB,MAAMI,KAAK,GAAGJ,MAAM,CAACG,IAAI,CAAC;IAE1B,IAAIC,KAAK,CAACN,MAAM,EAAE;MAChB,MAAMO,QAAQ,GAAGN,SAAS,CAACK,KAAK,CAACN,MAAM,CAAC;MACxCA,MAAM,CAACQ,IAAI,CAAC;QAACH,IAAI;QAAEI,IAAI,EAAE;UAACA,IAAI,EAAE,QAAQ;UAAEF;QAAQ,CAAC;QAAEG,QAAQ,EAAEJ,KAAK,CAACK;MAAQ,CAAC,CAAC;IACjF,CAAC,MAAM;MACL,MAAMF,IAAI,GAAG1C,oBAAoB,CAACuC,KAAK,CAACG,IAAI,CAAC;MAC7C,MAAMN,QAAQ,GAAGS,gBAAgB,CAACN,KAAK,CAAC;MACxC,MAAMO,UAAU,GAAG;QAACR,IAAI;QAAEI,IAAI;QAAEC,QAAQ,EAAEJ,KAAK,CAACK,QAAQ;QAAER;MAAQ,CAAC;MACnEH,MAAM,CAACQ,IAAI,CAACK,UAAU,CAAC;IACzB;EACF;EAEA,OAAOb,MAAM;AACf;AAEA,SAASY,gBAAgBA,CAACN,KAAmB,EAAsC;EACjF,IAAIH,QAA4C;EAEhD,KAAK,MAAMW,GAAG,IAAIR,KAAK,EAAE;IACvB,IAAIQ,GAAG,KAAK,MAAM,EAAE;MAClB,IAAIC,KAAK,GAAGT,KAAK,CAACQ,GAAG,CAAC,IAAI,EAAE;MAC5BC,KAAK,GAAG,OAAOT,KAAK,CAACQ,GAAG,CAAC,KAAK,QAAQ,GAAGxB,IAAI,CAAC0B,SAAS,CAACV,KAAK,CAACQ,GAAG,CAAC,CAAC,GAAGR,KAAK,CAACQ,GAAG,CAAC;MAChFX,QAAQ,GAAGA,QAAQ,IAAI,CAAC,CAAC;MACzBA,QAAQ,CAACW,GAAG,CAAC,GAAGC,KAAK;IACvB;EACF;EAEA,OAAOZ,QAAQ;AACjB;AAEA,SAASC,iBAAiBA,CAACL,eAA6B,EAAsC;EAC5F,IAAII,QAA4C;EAEhD,MAAMc,YAAY,GAAGlB,eAAe,CAACmB,kBAAkB,IAAI,EAAE;EAC7D,KAAK,MAAM;IAACJ,GAAG;IAAEC;EAAK,CAAC,IAAIE,YAAY,EAAE;IACvC,IAAI,OAAOF,KAAK,KAAK,QAAQ,EAAE;MAC7BZ,QAAQ,GAAGA,QAAQ,IAAI,CAAC,CAAC;MACzBA,QAAQ,CAACW,GAAG,CAAC,GAAGC,KAAK;IACvB;EACF;EAEA,OAAOZ,QAAQ;AACjB"}

package/dist/lib/arrow/convert-schema-to-parquet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-schema-to-parquet.js","names":["PARQUET_TYPE_MAPPING","BOOLEAN","INT32","INT64","INT96","FLOAT","DOUBLE","BYTE_ARRAY","FIXED_LEN_BYTE_ARRAY","UTF8","DATE","TIME_MILLIS","TIME_MICROS","TIMESTAMP_MILLIS","TIMESTAMP_MICROS","UINT_8","UINT_16","UINT_32","UINT_64","INT_8","INT_16","INT_32","INT_64","JSON","BSON","INTERVAL","DECIMAL_INT32","DECIMAL_INT64","DECIMAL_BYTE_ARRAY","DECIMAL_FIXED_LEN_BYTE_ARRAY","convertToParquetSchema","schema","fields","metadata"],"sources":["../../../src/lib/arrow/convert-schema-to-parquet.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// import type {ParquetSchema} from '../../parquetjs/schema/schema';\nimport type {\n  // FieldDefinition, ParquetField,\n  ParquetType\n} from '../../parquetjs/schema/declare';\n\nimport {\n  Schema,\n  // Struct,\n  // Field,\n  DataType\n} from '@loaders.gl/schema';\n\nexport const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {\n  BOOLEAN: 'bool',\n  INT32: 'int32',\n  INT64: 'float64',\n  INT96: 'float64',\n  FLOAT: 'float32',\n  DOUBLE: 'float64',\n  BYTE_ARRAY: 'binary',\n  FIXED_LEN_BYTE_ARRAY: 'binary',\n  UTF8: 'utf8',\n  DATE: 'int32',\n  TIME_MILLIS: 'int64',\n  TIME_MICROS: 'int64',\n  TIMESTAMP_MILLIS: 'int64',\n  TIMESTAMP_MICROS: 'int64',\n  UINT_8: 'int32',\n  UINT_16: 'uint16',\n  UINT_32: 'uint32',\n  UINT_64: 'uint64',\n  INT_8: 'int8',\n  INT_16: 'int16',\n  INT_32: 'int32',\n  INT_64: 'int64',\n  JSON: 'binary',\n  BSON: 'binary',\n  // TODO check interval type\n  INTERVAL: 'binary',\n  DECIMAL_INT32: 'float32',\n  DECIMAL_INT64: 'float64',\n  DECIMAL_BYTE_ARRAY: 'float64',\n  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'\n};\n\nexport function convertToParquetSchema(schema: Schema): Schema {\n  const fields = []; // getFields(schema.fields);\n\n  // TODO add metadata if needed.\n  return {fields, metadata: {}};\n}\n\n// function getFields(schema: Field[]): Definition[] {\n//   const fields: Field[] = [];\n\n//   for (const name in schema) {\n//     const field = schema[name];\n\n//     // @ts-ignore\n//     const children = field.children as DataType[];\n//     if (children) {\n//       const childField = getFields(field.fields);\n//       const nestedField = new Field(name, new Struct(childField), field.optional);\n//       fields.push(nestedField);\n//     } else {\n//       const FieldType = PARQUET_TYPE_MAPPING[field.type];\n//       const metadata = getFieldMetadata(field);\n//       const arrowField = new Field(name, new FieldType(), field.optional, metadata);\n//       fields.push(arrowField);\n//     }\n//   }\n\n//   return fields;\n// }\n\n// function getFieldMetadata(field: ParquetField): Map<string, string> {\n//   const metadata = new Map();\n\n//   for (const key in field) {\n//     if (key !== 'name') {\n//       const value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];\n//       metadata.set(key, value);\n//     }\n//   }\n\n//   return metadata;\n// }\n"],"mappings":"AAiBA,OAAO,MAAMA,oBAAuD,GAAG;EACrEC,OAAO,EAAE,MAAM;EACfC,KAAK,EAAE,OAAO;EACdC,KAAK,EAAE,SAAS;EAChBC,KAAK,EAAE,SAAS;EAChBC,KAAK,EAAE,SAAS;EAChBC,MAAM,EAAE,SAAS;EACjBC,UAAU,EAAE,QAAQ;EACpBC,oBAAoB,EAAE,QAAQ;EAC9BC,IAAI,EAAE,MAAM;EACZC,IAAI,EAAE,OAAO;EACbC,WAAW,EAAE,OAAO;EACpBC,WAAW,EAAE,OAAO;EACpBC,gBAAgB,EAAE,OAAO;EACzBC,gBAAgB,EAAE,OAAO;EACzBC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAE,QAAQ;EACjBC,OAAO,EAAE,QAAQ;EACjBC,OAAO,EAAE,QAAQ;EACjBC,KAAK,EAAE,MAAM;EACbC,MAAM,EAAE,OAAO;EACfC,MAAM,EAAE,OAAO;EACfC,MAAM,EAAE,OAAO;EACfC,IAAI,EAAE,QAAQ;EACdC,IAAI,EAAE,QAAQ;EAEdC,QAAQ,EAAE,QAAQ;EAClBC,aAAa,EAAE,SAAS;EACxBC,aAAa,EAAE,SAAS;EACxBC,kBAAkB,EAAE,SAAS;EAC7BC,4BAA4B,EAAE;AAChC,CAAC;AAED,OAAO,SAASC,sBAAsBA,CAACC,MAAc,EAAU;EAC7D,MAAMC,MAAM,GAAG,EAAE;EAGjB,OAAO;IAACA,MAAM;IAAEC,QAAQ,EAAE,CAAC;EAAC,CAAC;AAC/B"}

package/dist/lib/constants.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"constants.js","names":["VERSION","PARQUET_WASM_URL","PARQUET_MAGIC","PARQUET_MAGIC_ENCRYPTED","PARQUET_VERSION","PARQUET_RDLVL_TYPE","PARQUET_RDLVL_ENCODING"],"sources":["../../src/lib/constants.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nexport const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\nexport const PARQUET_WASM_URL = 'https://unpkg.com/parquet-wasm@0.6.0-beta.1/esm/arrow1_bg.wasm';\n\n/**\n * Parquet File Magic String\n */\nexport const PARQUET_MAGIC = 'PAR1';\nexport const PARQUET_MAGIC_ENCRYPTED = 'PARE';\n\n/**\n * Parquet File Format Version\n */\nexport const PARQUET_VERSION = 1;\n\n/**\n * Internal type used for repetition/definition levels\n */\nexport const PARQUET_RDLVL_TYPE = 'INT32';\nexport const PARQUET_RDLVL_ENCODING = 'RLE';\n"],"mappings":"AAMA,OAAO,MAAMA,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAClF,OAAO,MAAMC,gBAAgB,GAAG,gEAAgE;AAKhG,OAAO,MAAMC,aAAa,GAAG,MAAM;AACnC,OAAO,MAAMC,uBAAuB,GAAG,MAAM;AAK7C,OAAO,MAAMC,eAAe,GAAG,CAAC;AAKhC,OAAO,MAAMC,kBAAkB,GAAG,OAAO;AACzC,OAAO,MAAMC,sBAAsB,GAAG,KAAK"}

package/dist/lib/parsers/get-parquet-schema.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"get-parquet-schema.js","names":["convertParquetSchema","unpackGeoMetadata","unpackJSONStringMetadata","getSchemaFromParquetReader","reader","parquetSchema","getSchema","parquetMetadata","getFileMetadata","schema"],"sources":["../../../src/lib/parsers/get-parquet-schema.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// loaders.gl\nimport {Schema} from '@loaders.gl/schema';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {convertParquetSchema} from '../arrow/convert-schema-from-parquet';\nimport {unpackGeoMetadata, unpackJSONStringMetadata} from '@loaders.gl/gis';\n\nexport async function getSchemaFromParquetReader(reader: ParquetReader): Promise<Schema> {\n  const parquetSchema = await reader.getSchema();\n  const parquetMetadata = await reader.getFileMetadata();\n  const schema = convertParquetSchema(parquetSchema, parquetMetadata);\n  unpackGeoMetadata(schema);\n  unpackJSONStringMetadata(schema, 'pandas');\n  return schema;\n}\n"],"mappings":"SAOQA,oBAAoB;AAC5B,SAAQC,iBAAiB,EAAEC,wBAAwB,QAAO,iBAAiB;AAE3E,OAAO,eAAeC,0BAA0BA,CAACC,MAAqB,EAAmB;EACvF,MAAMC,aAAa,GAAG,MAAMD,MAAM,CAACE,SAAS,CAAC,CAAC;EAC9C,MAAMC,eAAe,GAAG,MAAMH,MAAM,CAACI,eAAe,CAAC,CAAC;EACtD,MAAMC,MAAM,GAAGT,oBAAoB,CAACK,aAAa,EAAEE,eAAe,CAAC;EACnEN,iBAAiB,CAACQ,MAAM,CAAC;EACzBP,wBAAwB,CAACO,MAAM,EAAE,QAAQ,CAAC;EAC1C,OAAOA,MAAM;AACf"}

package/dist/lib/parsers/parse-geoparquet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-geoparquet.js","names":["convertWKBTableToGeoJSON","WKTLoader","WKBLoader","parseParquetFile","parseParquetFileInBatches","parseGeoParquetFile","file","options","_options$parquet","table","shape","parquet","convertTable","parseGeoParquetFileInBatches","tableBatches","batch","_options$parquet2","convertBatch","objectRowTable","schema","error","Error","objectRowBatch","geojsonTable"],"sources":["../../../src/lib/parsers/parse-geoparquet.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\nimport type {\n  GeoJSONTable,\n  GeoJSONTableBatch,\n  ObjectRowTable,\n  ObjectRowTableBatch\n} from '@loaders.gl/schema';\nimport {convertWKBTableToGeoJSON} from '@loaders.gl/gis';\nimport {WKTLoader, WKBLoader} from '@loaders.gl/wkt';\n\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\n\nimport {parseParquetFile, parseParquetFileInBatches} from './parse-parquet';\n\nexport async function parseGeoParquetFile(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): Promise<ObjectRowTable | GeoJSONTable> {\n  const table = await parseParquetFile(file, {...options, shape: 'object-row-table'});\n  const shape = options?.parquet?.shape;\n  return convertTable(table, shape);\n}\n\nexport async function* parseGeoParquetFileInBatches(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch> {\n  const tableBatches = parseParquetFileInBatches(file, {...options, shape: 'object-row-table'});\n\n  for await (const batch of tableBatches) {\n    const shape = options?.parquet?.shape;\n    yield convertBatch(batch, shape);\n  }\n}\n\nfunction convertTable(\n  objectRowTable: ObjectRowTable,\n  shape?: 'object-row-table' | 'geojson-table'\n): ObjectRowTable | GeoJSONTable {\n  switch (shape) {\n    case 'object-row-table':\n      return objectRowTable;\n\n    case 'geojson-table':\n      try {\n        return convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema!, [\n          WKTLoader,\n          WKBLoader\n        ]);\n      } catch (error) {\n        return objectRowTable;\n      }\n\n    default:\n      throw new Error(shape);\n  }\n}\n\nfunction convertBatch(\n  objectRowBatch: ObjectRowTableBatch,\n  shape?: 'object-row-table' | 'geojson-table'\n): ObjectRowTableBatch | GeoJSONTableBatch {\n  switch (shape) {\n    case 'object-row-table':\n      return objectRowBatch;\n\n    case 'geojson-table':\n      try {\n        const geojsonTable = convertWKBTableToGeoJSON(objectRowBatch, objectRowBatch.schema!, [\n          WKTLoader,\n          WKBLoader\n        ]);\n        return {\n          ...objectRowBatch,\n          ...geojsonTable\n        };\n      } catch (error) {\n        return objectRowBatch;\n      }\n\n    default:\n      throw new Error(shape);\n  }\n}\n"],"mappings":"AAWA,SAAQA,wBAAwB,QAAO,iBAAiB;AACxD,SAAQC,SAAS,EAAEC,SAAS,QAAO,iBAAiB;AAAC,SAI7CC,gBAAgB,EAAEC,yBAAyB;AAEnD,OAAO,eAAeC,mBAAmBA,CACvCC,IAAkB,EAClBC,OAA8B,EACU;EAAA,IAAAC,gBAAA;EACxC,MAAMC,KAAK,GAAG,MAAMN,gBAAgB,CAACG,IAAI,EAAE;IAAC,GAAGC,OAAO;IAAEG,KAAK,EAAE;EAAkB,CAAC,CAAC;EACnF,MAAMA,KAAK,GAAGH,OAAO,aAAPA,OAAO,wBAAAC,gBAAA,GAAPD,OAAO,CAAEI,OAAO,cAAAH,gBAAA,uBAAhBA,gBAAA,CAAkBE,KAAK;EACrC,OAAOE,YAAY,CAACH,KAAK,EAAEC,KAAK,CAAC;AACnC;AAEA,OAAO,gBAAgBG,4BAA4BA,CACjDP,IAAkB,EAClBC,OAA8B,EAC0B;EACxD,MAAMO,YAAY,GAAGV,yBAAyB,CAACE,IAAI,EAAE;IAAC,GAAGC,OAAO;IAAEG,KAAK,EAAE;EAAkB,CAAC,CAAC;EAE7F,WAAW,MAAMK,KAAK,IAAID,YAAY,EAAE;IAAA,IAAAE,iBAAA;IACtC,MAAMN,KAAK,GAAGH,OAAO,aAAPA,OAAO,wBAAAS,iBAAA,GAAPT,OAAO,CAAEI,OAAO,cAAAK,iBAAA,uBAAhBA,iBAAA,CAAkBN,KAAK;IACrC,MAAMO,YAAY,CAACF,KAAK,EAAEL,KAAK,CAAC;EAClC;AACF;AAEA,SAASE,YAAYA,CACnBM,cAA8B,EAC9BR,KAA4C,EACb;EAC/B,QAAQA,KAAK;IACX,KAAK,kBAAkB;MACrB,OAAOQ,cAAc;IAEvB,KAAK,eAAe;MAClB,IAAI;QACF,OAAOlB,wBAAwB,CAACkB,cAAc,EAAEA,cAAc,CAACC,MAAM,EAAG,CACtElB,SAAS,EACTC,SAAS,CACV,CAAC;MACJ,CAAC,CAAC,OAAOkB,KAAK,EAAE;QACd,OAAOF,cAAc;MACvB;IAEF;MACE,MAAM,IAAIG,KAAK,CAACX,KAAK,CAAC;EAC1B;AACF;AAEA,SAASO,YAAYA,CACnBK,cAAmC,EACnCZ,KAA4C,EACH;EACzC,QAAQA,KAAK;IACX,KAAK,kBAAkB;MACrB,OAAOY,cAAc;IAEvB,KAAK,eAAe;MAClB,IAAI;QACF,MAAMC,YAAY,GAAGvB,wBAAwB,CAACsB,cAAc,EAAEA,cAAc,CAACH,MAAM,EAAG,CACpFlB,SAAS,EACTC,SAAS,CACV,CAAC;QACF,OAAO;UACL,GAAGoB,cAAc;UACjB,GAAGC;QACL,CAAC;MACH,CAAC,CAAC,OAAOH,KAAK,EAAE;QACd,OAAOE,cAAc;MACvB;IAEF;MACE,MAAM,IAAID,KAAK,CAACX,KAAK,CAAC;EAC1B;AACF"}

package/dist/lib/parsers/parse-parquet-to-columns.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet-to-columns.js","names":["ParquetReader","materializeColumns","getSchemaFromParquetReader","installBufferPolyfill","parseParquetFileInColumns","file","options","batch","parseParquetFileInColumnarBatches","shape","schema","data","Error","reader","parquetSchema","getSchema","rowGroups","rowGroupIterator","parquet","rowGroup","convertRowGroupToTableBatch","batchType","length","rowCount"],"sources":["../../../src/lib/parsers/parse-parquet-to-columns.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {ColumnarTable, ColumnarTableBatch, Schema} from '@loaders.gl/schema';\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {ParquetRowGroup} from '../../parquetjs/schema/declare';\nimport {ParquetSchema} from '../../parquetjs/schema/schema';\nimport {materializeColumns} from '../../parquetjs/schema/shred';\nimport {getSchemaFromParquetReader} from './get-parquet-schema';\nimport {installBufferPolyfill} from '../../polyfills/buffer';\n\n/**\n * @deprecated\n */\nexport async function parseParquetFileInColumns(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): Promise<ColumnarTable> {\n  installBufferPolyfill();\n  for await (const batch of parseParquetFileInColumnarBatches(file, options)) {\n    return {\n      shape: 'columnar-table',\n      schema: batch.schema,\n      data: batch.data\n    };\n  }\n  throw new Error('empty table');\n}\n\n/**\n * @deprecated\n */\nexport async function* parseParquetFileInColumnarBatches(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ColumnarTableBatch> {\n  const reader = new ParquetReader(file);\n\n  // Extract schema and geo metadata\n  const schema = await getSchemaFromParquetReader(reader);\n\n  const parquetSchema = await reader.getSchema();\n\n  // Iterate over row batches\n  const rowGroups = reader.rowGroupIterator(options?.parquet);\n  for await (const rowGroup of rowGroups) {\n    yield convertRowGroupToTableBatch(rowGroup, parquetSchema, schema);\n  }\n}\n\nfunction convertRowGroupToTableBatch(\n  rowGroup: ParquetRowGroup,\n  parquetSchema: ParquetSchema,\n  schema: Schema\n): ColumnarTableBatch {\n  // const data = convertParquetRowGroupToColumns(schema, rowGroup);\n  const data = materializeColumns(parquetSchema, rowGroup);\n  return {\n    shape: 'columnar-table',\n    batchType: 'data',\n    schema,\n    data,\n    length: rowGroup.rowCount\n  };\n}\n"],"mappings":"SAOQA,aAAa;AAAA,SAGbC,kBAAkB;AAAA,SAClBC,0BAA0B;AAAA,SAC1BC,qBAAqB;AAK7B,OAAO,eAAeC,yBAAyBA,CAC7CC,IAAkB,EAClBC,OAA8B,EACN;EACxBH,qBAAqB,CAAC,CAAC;EACvB,WAAW,MAAMI,KAAK,IAAIC,iCAAiC,CAACH,IAAI,EAAEC,OAAO,CAAC,EAAE;IAC1E,OAAO;MACLG,KAAK,EAAE,gBAAgB;MACvBC,MAAM,EAAEH,KAAK,CAACG,MAAM;MACpBC,IAAI,EAAEJ,KAAK,CAACI;IACd,CAAC;EACH;EACA,MAAM,IAAIC,KAAK,CAAC,aAAa,CAAC;AAChC;AAKA,OAAO,gBAAgBJ,iCAAiCA,CACtDH,IAAkB,EAClBC,OAA8B,EACK;EACnC,MAAMO,MAAM,GAAG,IAAIb,aAAa,CAACK,IAAI,CAAC;EAGtC,MAAMK,MAAM,GAAG,MAAMR,0BAA0B,CAACW,MAAM,CAAC;EAEvD,MAAMC,aAAa,GAAG,MAAMD,MAAM,CAACE,SAAS,CAAC,CAAC;EAG9C,MAAMC,SAAS,GAAGH,MAAM,CAACI,gBAAgB,CAACX,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEY,OAAO,CAAC;EAC3D,WAAW,MAAMC,QAAQ,IAAIH,SAAS,EAAE;IACtC,MAAMI,2BAA2B,CAACD,QAAQ,EAAEL,aAAa,EAAEJ,MAAM,CAAC;EACpE;AACF;AAEA,SAASU,2BAA2BA,CAClCD,QAAyB,EACzBL,aAA4B,EAC5BJ,MAAc,EACM;EAEpB,MAAMC,IAAI,GAAGV,kBAAkB,CAACa,aAAa,EAAEK,QAAQ,CAAC;EACxD,OAAO;IACLV,KAAK,EAAE,gBAAgB;IACvBY,SAAS,EAAE,MAAM;IACjBX,MAAM;IACNC,IAAI;IACJW,MAAM,EAAEH,QAAQ,CAACI;EACnB,CAAC;AACH"}

package/dist/lib/parsers/parse-parquet.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet.js","names":["ParquetReader","getSchemaFromParquetReader","installBufferPolyfill","parseParquetFile","file","options","_options$parquet","_options$parquet2","reader","preserveBinary","parquet","schema","rows","rowBatches","rowBatchIterator","rowBatch","row","push","objectRowTable","shape","data","convertTable","parseParquetFileInBatches","_options$parquet3","_options$parquet4","table","batchType","length","Error"],"sources":["../../../src/lib/parsers/parse-parquet.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\nimport type {ObjectRowTable, ObjectRowTableBatch} from '@loaders.gl/schema';\n\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport type {ParquetRow} from '../../parquetjs/schema/declare';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {getSchemaFromParquetReader} from './get-parquet-schema';\nimport {installBufferPolyfill} from '../../polyfills/buffer';\n\n/**\n *  * Parse a parquet file using parquetjs\n * @param file\n * @param options\n * @returns\n */\nexport async function parseParquetFile(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): Promise<ObjectRowTable> {\n  installBufferPolyfill();\n\n  const reader = new ParquetReader(file, {\n    preserveBinary: options?.parquet?.preserveBinary\n  });\n\n  const schema = await getSchemaFromParquetReader(reader);\n\n  const rows: ParquetRow[] = [];\n\n  const rowBatches = reader.rowBatchIterator(options?.parquet);\n  for await (const rowBatch of rowBatches) {\n    // we have only one input batch so return\n    for (const row of rowBatch) {\n      rows.push(row);\n    }\n  }\n  const objectRowTable: ObjectRowTable = {\n    shape: 'object-row-table',\n    schema,\n    data: rows\n  };\n\n  const shape = options?.parquet?.shape;\n  return convertTable(objectRowTable, shape);\n}\n\n/**\n * Parse a parquet file in batches using parquetjs\n * @param file\n * @param options\n */\nexport async function* parseParquetFileInBatches(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ObjectRowTableBatch> {\n  const reader = new ParquetReader(file, {\n    preserveBinary: options?.parquet?.preserveBinary\n  });\n\n  const schema = await getSchemaFromParquetReader(reader);\n  const rowBatches = reader.rowBatchIterator(options?.parquet);\n  for await (const rows of rowBatches) {\n    const objectRowTable: ObjectRowTable = {\n      shape: 'object-row-table',\n      schema,\n      data: rows\n    };\n    const shape = options?.parquet?.shape;\n    const table = convertTable(objectRowTable, shape);\n\n    yield {\n      batchType: 'data',\n      schema,\n      ...table,\n      length: rows.length\n    };\n  }\n}\n\nfunction convertTable(\n  objectRowTable: ObjectRowTable,\n  shape?: 'object-row-table' | 'geojson-table'\n): ObjectRowTable {\n  switch (shape) {\n    case 'object-row-table':\n      return objectRowTable;\n\n    // Hack until geoparquet fixes up forwarded shape\n    case 'geojson-table':\n      return objectRowTable;\n\n    default:\n      throw new Error(shape);\n  }\n}\n"],"mappings":"SASQA,aAAa;AAAA,SACbC,0BAA0B;AAAA,SAC1BC,qBAAqB;AAQ7B,OAAO,eAAeC,gBAAgBA,CACpCC,IAAkB,EAClBC,OAA8B,EACL;EAAA,IAAAC,gBAAA,EAAAC,iBAAA;EACzBL,qBAAqB,CAAC,CAAC;EAEvB,MAAMM,MAAM,GAAG,IAAIR,aAAa,CAACI,IAAI,EAAE;IACrCK,cAAc,EAAEJ,OAAO,aAAPA,OAAO,wBAAAC,gBAAA,GAAPD,OAAO,CAAEK,OAAO,cAAAJ,gBAAA,uBAAhBA,gBAAA,CAAkBG;EACpC,CAAC,CAAC;EAEF,MAAME,MAAM,GAAG,MAAMV,0BAA0B,CAACO,MAAM,CAAC;EAEvD,MAAMI,IAAkB,GAAG,EAAE;EAE7B,MAAMC,UAAU,GAAGL,MAAM,CAACM,gBAAgB,CAACT,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEK,OAAO,CAAC;EAC5D,WAAW,MAAMK,QAAQ,IAAIF,UAAU,EAAE;IAEvC,KAAK,MAAMG,GAAG,IAAID,QAAQ,EAAE;MAC1BH,IAAI,CAACK,IAAI,CAACD,GAAG,CAAC;IAChB;EACF;EACA,MAAME,cAA8B,GAAG;IACrCC,KAAK,EAAE,kBAAkB;IACzBR,MAAM;IACNS,IAAI,EAAER;EACR,CAAC;EAED,MAAMO,KAAK,GAAGd,OAAO,aAAPA,OAAO,wBAAAE,iBAAA,GAAPF,OAAO,CAAEK,OAAO,cAAAH,iBAAA,uBAAhBA,iBAAA,CAAkBY,KAAK;EACrC,OAAOE,YAAY,CAACH,cAAc,EAAEC,KAAK,CAAC;AAC5C;AAOA,OAAO,gBAAgBG,yBAAyBA,CAC9ClB,IAAkB,EAClBC,OAA8B,EACM;EAAA,IAAAkB,iBAAA;EACpC,MAAMf,MAAM,GAAG,IAAIR,aAAa,CAACI,IAAI,EAAE;IACrCK,cAAc,EAAEJ,OAAO,aAAPA,OAAO,wBAAAkB,iBAAA,GAAPlB,OAAO,CAAEK,OAAO,cAAAa,iBAAA,uBAAhBA,iBAAA,CAAkBd;EACpC,CAAC,CAAC;EAEF,MAAME,MAAM,GAAG,MAAMV,0BAA0B,CAACO,MAAM,CAAC;EACvD,MAAMK,UAAU,GAAGL,MAAM,CAACM,gBAAgB,CAACT,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEK,OAAO,CAAC;EAC5D,WAAW,MAAME,IAAI,IAAIC,UAAU,EAAE;IAAA,IAAAW,iBAAA;IACnC,MAAMN,cAA8B,GAAG;MACrCC,KAAK,EAAE,kBAAkB;MACzBR,MAAM;MACNS,IAAI,EAAER;IACR,CAAC;IACD,MAAMO,KAAK,GAAGd,OAAO,aAAPA,OAAO,wBAAAmB,iBAAA,GAAPnB,OAAO,CAAEK,OAAO,cAAAc,iBAAA,uBAAhBA,iBAAA,CAAkBL,KAAK;IACrC,MAAMM,KAAK,GAAGJ,YAAY,CAACH,cAAc,EAAEC,KAAK,CAAC;IAEjD,MAAM;MACJO,SAAS,EAAE,MAAM;MACjBf,MAAM;MACN,GAAGc,KAAK;MACRE,MAAM,EAAEf,IAAI,CAACe;IACf,CAAC;EACH;AACF;AAEA,SAASN,YAAYA,CACnBH,cAA8B,EAC9BC,KAA4C,EAC5B;EAChB,QAAQA,KAAK;IACX,KAAK,kBAAkB;MACrB,OAAOD,cAAc;IAGvB,KAAK,eAAe;MAClB,OAAOA,cAAc;IAEvB;MACE,MAAM,IAAIU,KAAK,CAACT,KAAK,CAAC;EAC1B;AACF"}

package/dist/lib/wasm/encode-parquet-wasm.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"encode-parquet-wasm.js","names":["arrow","loadWasm","encode","table","options","_options$parquet","wasmUrl","parquet","wasm","arrowTable","data","ipcStream","tableToIPC","wasmTable","Table","fromIPCStream","wasmProperties","WriterPropertiesBuilder","build","parquetBytes","writeParquet","buffer","slice","byteOffset","byteLength"],"sources":["../../../src/lib/wasm/encode-parquet-wasm.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\nimport type {ArrowTable} from '@loaders.gl/arrow';\n\nimport * as arrow from 'apache-arrow';\nimport {loadWasm} from './load-wasm';\n\nimport type {ParquetWriterOptions} from '../../parquet-wasm-writer';\n\n/**\n * Encode Arrow arrow.Table to Parquet buffer\n */\nexport async function encode(\n  table: ArrowTable,\n  options: ParquetWriterOptions\n): Promise<ArrayBuffer> {\n  const wasmUrl = options.parquet?.wasmUrl!;\n  const wasm = await loadWasm(wasmUrl);\n\n  // Serialize the table to the IPC format.\n  const arrowTable: arrow.Table = table.data;\n  const ipcStream = arrow.tableToIPC(arrowTable);\n\n  // Pass the IPC stream to the Parquet writer.\n  const wasmTable = wasm.Table.fromIPCStream(ipcStream);\n  const wasmProperties = new wasm.WriterPropertiesBuilder().build();\n  try {\n    const parquetBytes = wasm.writeParquet(wasmTable, wasmProperties);\n    // const parquetBytes = wasm.writeParquet(wasmTable, wasmProperties);\n    return parquetBytes.buffer.slice(\n      parquetBytes.byteOffset,\n      parquetBytes.byteLength + parquetBytes.byteOffset\n    );\n  } finally {\n    // wasmTable.free();\n    // wasmProperties.free();\n  }\n}\n\n// type WriteOptions = {\n//   compression?: number;\n//   dictionaryEnabled?: boolean;\n//   encoding?: number;\n//   maxRowGroupSize?: number;\n//   maxStatisticsSize?: number;\n//   statisticsEnabled?: boolean;\n//   writeBatchSize?: number;\n//   dataPageSizeLimit?: number;\n//   dictionaryPageSizeLimit?: number;\n// };\n\n// columnCompression: Record<string, number>;\n// columnDictionaryEnabled: Record<string, boolean>;\n// columnEncoding: Record<string, number>;\n// columnMaxStatisticsSize\n// compression:Record<string, number>;\n// setCreatedBy\n// setDataPageSizeLimit\n// setDictionaryEnabled\n// setDictionaryPageSizeLimit\n// setEncoding\n// setMaxRowGroupSize\n// setMaxStatisticsSize\n// setStatisticsEnabled\n// setWriteBatchSize\n// setWriterVersion\n"],"mappings":"AAMA,OAAO,KAAKA,KAAK,MAAM,cAAc;AAAC,SAC9BC,QAAQ;AAOhB,OAAO,eAAeC,MAAMA,CAC1BC,KAAiB,EACjBC,OAA6B,EACP;EAAA,IAAAC,gBAAA;EACtB,MAAMC,OAAO,IAAAD,gBAAA,GAAGD,OAAO,CAACG,OAAO,cAAAF,gBAAA,uBAAfA,gBAAA,CAAiBC,OAAQ;EACzC,MAAME,IAAI,GAAG,MAAMP,QAAQ,CAACK,OAAO,CAAC;EAGpC,MAAMG,UAAuB,GAAGN,KAAK,CAACO,IAAI;EAC1C,MAAMC,SAAS,GAAGX,KAAK,CAACY,UAAU,CAACH,UAAU,CAAC;EAG9C,MAAMI,SAAS,GAAGL,IAAI,CAACM,KAAK,CAACC,aAAa,CAACJ,SAAS,CAAC;EACrD,MAAMK,cAAc,GAAG,IAAIR,IAAI,CAACS,uBAAuB,CAAC,CAAC,CAACC,KAAK,CAAC,CAAC;EACjE,IAAI;IACF,MAAMC,YAAY,GAAGX,IAAI,CAACY,YAAY,CAACP,SAAS,EAAEG,cAAc,CAAC;IAEjE,OAAOG,YAAY,CAACE,MAAM,CAACC,KAAK,CAC9BH,YAAY,CAACI,UAAU,EACvBJ,YAAY,CAACK,UAAU,GAAGL,YAAY,CAACI,UACzC,CAAC;EACH,CAAC,SAAS,CAGV;AACF"}

package/dist/lib/wasm/load-wasm.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"load-wasm.js","names":["initWasm","parquetWasm","PARQUET_WASM_URL","initializePromise","loadWasm","wasmUrl","arguments","length","undefined","Error"],"sources":["../../../src/lib/wasm/load-wasm.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// eslint-disable-next-line import/default\nimport initWasm from 'parquet-wasm';\nimport * as parquetWasm from 'parquet-wasm';\nimport {PARQUET_WASM_URL} from '../constants';\n\nlet initializePromise: any;\n\nexport async function loadWasm(wasmUrl: string = PARQUET_WASM_URL) {\n  if (!initializePromise && typeof initWasm === 'function') {\n    if (!wasmUrl) {\n      throw new Error('ParquetLoader: No wasmUrl provided');\n    }\n    // @ts-ignore\n    initializePromise = initWasm(wasmUrl);\n  }\n  await initializePromise;\n  return parquetWasm;\n}\n"],"mappings":"AAKA,OAAOA,QAAQ,MAAM,cAAc;AACnC,OAAO,KAAKC,WAAW,MAAM,cAAc;AAAC,SACpCC,gBAAgB;AAExB,IAAIC,iBAAsB;AAE1B,OAAO,eAAeC,QAAQA,CAAA,EAAqC;EAAA,IAApCC,OAAe,GAAAC,SAAA,CAAAC,MAAA,QAAAD,SAAA,QAAAE,SAAA,GAAAF,SAAA,MAAGJ,gBAAgB;EAC/D,IAAI,CAACC,iBAAiB,IAAI,OAAOH,QAAQ,KAAK,UAAU,EAAE;IACxD,IAAI,CAACK,OAAO,EAAE;MACZ,MAAM,IAAII,KAAK,CAAC,oCAAoC,CAAC;IACvD;IAEAN,iBAAiB,GAAGH,QAAQ,CAACK,OAAO,CAAC;EACvC;EACA,MAAMF,iBAAiB;EACvB,OAAOF,WAAW;AACpB"}

package/dist/lib/wasm/parse-parquet-wasm.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet-wasm.js","names":["serializeArrowSchema","loadWasm","arrow","parseParquetWasm","arrayBuffer","options","_options$parquet","arr","Uint8Array","wasmUrl","parquet","wasm","wasmTable","readParquet","ipcStream","intoIPCStream","arrowTable","tableFromIPC","shape","schema","data"],"sources":["../../../src/lib/wasm/parse-parquet-wasm.ts"],"sourcesContent":["// loaders.gl\n// SPDX-License-Identifier: MIT\n// Copyright (c) vis.gl contributors\n\n// eslint-disable\nimport type {ArrowTable} from '@loaders.gl/arrow';\nimport {serializeArrowSchema} from '@loaders.gl/arrow';\nimport type {ParquetWasmLoaderOptions} from '../../parquet-wasm-loader';\nimport {loadWasm} from './load-wasm';\nimport * as arrow from 'apache-arrow';\n\nexport async function parseParquetWasm(\n  arrayBuffer: ArrayBuffer,\n  options: ParquetWasmLoaderOptions\n): Promise<ArrowTable> {\n  const arr = new Uint8Array(arrayBuffer);\n\n  const wasmUrl = options?.parquet?.wasmUrl;\n  const wasm = await loadWasm(wasmUrl);\n  const wasmTable = wasm.readParquet(arr);\n  try {\n    const ipcStream = wasmTable.intoIPCStream();\n    const arrowTable = arrow.tableFromIPC(ipcStream);\n\n    return {\n      shape: 'arrow-table',\n      schema: serializeArrowSchema(arrowTable.schema),\n      data: arrowTable\n    };\n  } finally {\n    // wasmTable.free();\n  }\n}\n"],"mappings":"AAMA,SAAQA,oBAAoB,QAAO,mBAAmB;AAAC,SAE/CC,QAAQ;AAChB,OAAO,KAAKC,KAAK,MAAM,cAAc;AAErC,OAAO,eAAeC,gBAAgBA,CACpCC,WAAwB,EACxBC,OAAiC,EACZ;EAAA,IAAAC,gBAAA;EACrB,MAAMC,GAAG,GAAG,IAAIC,UAAU,CAACJ,WAAW,CAAC;EAEvC,MAAMK,OAAO,GAAGJ,OAAO,aAAPA,OAAO,wBAAAC,gBAAA,GAAPD,OAAO,CAAEK,OAAO,cAAAJ,gBAAA,uBAAhBA,gBAAA,CAAkBG,OAAO;EACzC,MAAME,IAAI,GAAG,MAAMV,QAAQ,CAACQ,OAAO,CAAC;EACpC,MAAMG,SAAS,GAAGD,IAAI,CAACE,WAAW,CAACN,GAAG,CAAC;EACvC,IAAI;IACF,MAAMO,SAAS,GAAGF,SAAS,CAACG,aAAa,CAAC,CAAC;IAC3C,MAAMC,UAAU,GAAGd,KAAK,CAACe,YAAY,CAACH,SAAS,CAAC;IAEhD,OAAO;MACLI,KAAK,EAAE,aAAa;MACpBC,MAAM,EAAEnB,oBAAoB,CAACgB,UAAU,CAACG,MAAM,CAAC;MAC/CC,IAAI,EAAEJ;IACR,CAAC;EACH,CAAC,SAAS,CAEV;AACF"}