@loaders.gl/parquet 4.0.4 → 4.1.0-alpha.10

package/src/polyfills/buffer/buffer.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 // This file is forked from https://github.com/feross/buffer under MIT license
 // The code has primarily been converted to TypeScript.

package/src/polyfills/buffer/index.ts

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 export {Buffer as BufferPolyfill} from './buffer';
 export {Buffer} from './install-buffer-polyfill';

package/src/polyfills/util.js

@@ -1,4 +1,5 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 // Polyfill for Node.js util library

package/src/workers/parquet-worker.ts

@@ -1,7 +1,8 @@
-// loaders.gl, MIT license
+// loaders.gl
+// SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
 import {createLoaderWorker} from '@loaders.gl/loader-utils';
-import {ParquetLoader} from '../index';
+import {ParquetLoader} from '../parquet-loader';
 
 createLoaderWorker(ParquetLoader);

package/dist/lib/parsers/parse-parquet-to-rows.d.ts

@@ -1,6 +0,0 @@
-import type { ReadableFile } from '@loaders.gl/loader-utils';
-import type { GeoJSONTable, GeoJSONTableBatch, ObjectRowTable, ObjectRowTableBatch } from '@loaders.gl/schema';
-import type { ParquetLoaderOptions } from '../../parquet-loader';
-export declare function parseParquetFile(file: ReadableFile, options?: ParquetLoaderOptions): Promise<ObjectRowTable | GeoJSONTable>;
-export declare function parseParquetFileInBatches(file: ReadableFile, options?: ParquetLoaderOptions): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch>;
-//# sourceMappingURL=parse-parquet-to-rows.d.ts.map

package/dist/lib/parsers/parse-parquet-to-rows.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet-to-rows.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-rows.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EACV,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,sBAAsB,CAAC;AAM/D,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,GAAG,YAAY,CAAC,CA0BxC;AAED,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,GAAG,iBAAiB,CAAC,CAuBxD"}

package/dist/lib/parsers/parse-parquet-to-rows.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet-to-rows.js","names":["convertWKBTableToGeoJSON","WKTLoader","WKBLoader","ParquetReader","getSchemaFromParquetReader","installBufferPolyfill","parseParquetFile","file","options","_options$parquet","_options$parquet2","reader","preserveBinary","parquet","schema","rows","rowBatches","rowBatchIterator","rowBatch","row","push","objectRowTable","shape","data","convertTable","parseParquetFileInBatches","_options$parquet3","_options$parquet4","table","batchType","length","error","Error"],"sources":["../../../src/lib/parsers/parse-parquet-to-rows.ts"],"sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n// import {ColumnarTableBatch} from '@loaders.gl/schema';\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\nimport type {\n  GeoJSONTable,\n  GeoJSONTableBatch,\n  ObjectRowTable,\n  ObjectRowTableBatch\n} from '@loaders.gl/schema';\nimport {convertWKBTableToGeoJSON} from '@loaders.gl/gis';\nimport {WKTLoader, WKBLoader} from '@loaders.gl/wkt';\n\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport type {ParquetRow} from '../../parquetjs/schema/declare';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {getSchemaFromParquetReader} from './get-parquet-schema';\nimport {installBufferPolyfill} from '../../polyfills/buffer';\n\nexport async function parseParquetFile(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): Promise<ObjectRowTable | GeoJSONTable> {\n  installBufferPolyfill();\n\n  const reader = new ParquetReader(file, {\n    preserveBinary: options?.parquet?.preserveBinary\n  });\n\n  const schema = await getSchemaFromParquetReader(reader);\n\n  const rows: ParquetRow[] = [];\n\n  const rowBatches = reader.rowBatchIterator(options?.parquet);\n  for await (const rowBatch of rowBatches) {\n    // we have only one input batch so return\n    for (const row of rowBatch) {\n      rows.push(row);\n    }\n  }\n  const objectRowTable: ObjectRowTable = {\n    shape: 'object-row-table',\n    schema,\n    data: rows\n  };\n\n  const shape = options?.parquet?.shape;\n  return convertTable(objectRowTable, shape);\n}\n\nexport async function* parseParquetFileInBatches(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch> {\n  const reader = new ParquetReader(file, {\n    preserveBinary: options?.parquet?.preserveBinary\n  });\n\n  const schema = await getSchemaFromParquetReader(reader);\n  const rowBatches = reader.rowBatchIterator(options?.parquet);\n  for await (const rows of rowBatches) {\n    const objectRowTable: ObjectRowTable = {\n      shape: 'object-row-table',\n      schema,\n      data: rows\n    };\n    const shape = options?.parquet?.shape;\n    const table = convertTable(objectRowTable, shape);\n\n    yield {\n      batchType: 'data',\n      schema,\n      ...table,\n      length: rows.length\n    };\n  }\n}\n\nfunction convertTable(\n  objectRowTable: ObjectRowTable,\n  shape?: 'object-row-table' | 'geojson-table'\n): ObjectRowTable | GeoJSONTable {\n  switch (shape) {\n    case 'object-row-table':\n      return objectRowTable;\n\n    case 'geojson-table':\n      try {\n        return convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema!, [\n          WKTLoader,\n          WKBLoader\n        ]);\n      } catch (error) {\n        return objectRowTable;\n      }\n\n    default:\n      throw new Error(shape);\n  }\n}\n"],"mappings":"AASA,SAAQA,wBAAwB,QAAO,iBAAiB;AACxD,SAAQC,SAAS,EAAEC,SAAS,QAAO,iBAAiB;AAAC,SAI7CC,aAAa;AAAA,SACbC,0BAA0B;AAAA,SAC1BC,qBAAqB;AAE7B,OAAO,eAAeC,gBAAgBA,CACpCC,IAAkB,EAClBC,OAA8B,EACU;EAAA,IAAAC,gBAAA,EAAAC,iBAAA;EACxCL,qBAAqB,CAAC,CAAC;EAEvB,MAAMM,MAAM,GAAG,IAAIR,aAAa,CAACI,IAAI,EAAE;IACrCK,cAAc,EAAEJ,OAAO,aAAPA,OAAO,wBAAAC,gBAAA,GAAPD,OAAO,CAAEK,OAAO,cAAAJ,gBAAA,uBAAhBA,gBAAA,CAAkBG;EACpC,CAAC,CAAC;EAEF,MAAME,MAAM,GAAG,MAAMV,0BAA0B,CAACO,MAAM,CAAC;EAEvD,MAAMI,IAAkB,GAAG,EAAE;EAE7B,MAAMC,UAAU,GAAGL,MAAM,CAACM,gBAAgB,CAACT,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEK,OAAO,CAAC;EAC5D,WAAW,MAAMK,QAAQ,IAAIF,UAAU,EAAE;IAEvC,KAAK,MAAMG,GAAG,IAAID,QAAQ,EAAE;MAC1BH,IAAI,CAACK,IAAI,CAACD,GAAG,CAAC;IAChB;EACF;EACA,MAAME,cAA8B,GAAG;IACrCC,KAAK,EAAE,kBAAkB;IACzBR,MAAM;IACNS,IAAI,EAAER;EACR,CAAC;EAED,MAAMO,KAAK,GAAGd,OAAO,aAAPA,OAAO,wBAAAE,iBAAA,GAAPF,OAAO,CAAEK,OAAO,cAAAH,iBAAA,uBAAhBA,iBAAA,CAAkBY,KAAK;EACrC,OAAOE,YAAY,CAACH,cAAc,EAAEC,KAAK,CAAC;AAC5C;AAEA,OAAO,gBAAgBG,yBAAyBA,CAC9ClB,IAAkB,EAClBC,OAA8B,EAC0B;EAAA,IAAAkB,iBAAA;EACxD,MAAMf,MAAM,GAAG,IAAIR,aAAa,CAACI,IAAI,EAAE;IACrCK,cAAc,EAAEJ,OAAO,aAAPA,OAAO,wBAAAkB,iBAAA,GAAPlB,OAAO,CAAEK,OAAO,cAAAa,iBAAA,uBAAhBA,iBAAA,CAAkBd;EACpC,CAAC,CAAC;EAEF,MAAME,MAAM,GAAG,MAAMV,0BAA0B,CAACO,MAAM,CAAC;EACvD,MAAMK,UAAU,GAAGL,MAAM,CAACM,gBAAgB,CAACT,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEK,OAAO,CAAC;EAC5D,WAAW,MAAME,IAAI,IAAIC,UAAU,EAAE;IAAA,IAAAW,iBAAA;IACnC,MAAMN,cAA8B,GAAG;MACrCC,KAAK,EAAE,kBAAkB;MACzBR,MAAM;MACNS,IAAI,EAAER;IACR,CAAC;IACD,MAAMO,KAAK,GAAGd,OAAO,aAAPA,OAAO,wBAAAmB,iBAAA,GAAPnB,OAAO,CAAEK,OAAO,cAAAc,iBAAA,uBAAhBA,iBAAA,CAAkBL,KAAK;IACrC,MAAMM,KAAK,GAAGJ,YAAY,CAACH,cAAc,EAAEC,KAAK,CAAC;IAEjD,MAAM;MACJO,SAAS,EAAE,MAAM;MACjBf,MAAM;MACN,GAAGc,KAAK;MACRE,MAAM,EAAEf,IAAI,CAACe;IACf,CAAC;EACH;AACF;AAEA,SAASN,YAAYA,CACnBH,cAA8B,EAC9BC,KAA4C,EACb;EAC/B,QAAQA,KAAK;IACX,KAAK,kBAAkB;MACrB,OAAOD,cAAc;IAEvB,KAAK,eAAe;MAClB,IAAI;QACF,OAAOrB,wBAAwB,CAACqB,cAAc,EAAEA,cAAc,CAACP,MAAM,EAAG,CACtEb,SAAS,EACTC,SAAS,CACV,CAAC;MACJ,CAAC,CAAC,OAAO6B,KAAK,EAAE;QACd,OAAOV,cAAc;MACvB;IAEF;MACE,MAAM,IAAIW,KAAK,CAACV,KAAK,CAAC;EAC1B;AACF"}

package/dist/parquetjs/modules.d.js

@@ -1,2 +0,0 @@
-
-//# sourceMappingURL=modules.d.js.map

package/dist/parquetjs/modules.d.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"modules.d.js","names":[],"sources":["../../src/parquetjs/modules.d.ts"],"sourcesContent":["// Forked from https://github.com/kbajalc/parquets under MIT license (Copyright (c) 2017 ironSource Ltd.)\n\ndeclare module 'int53' {\n  declare function readInt64BE(buffer: Buffer, offset?: number): number;\n  declare function readInt64LE(buffer: Buffer, offset?: number): number;\n  declare function readUInt64BE(buffer: Buffer, offset?: number): number;\n  declare function readUInt64LE(buffer: Buffer, offset?: number): number;\n  declare function writeInt64BE(value: number, buffer: Buffer, offset?: number): void;\n  declare function writeInt64LE(value: number, buffer: Buffer, offset?: number): void;\n  declare function writeUInt64BE(value: number, buffer: Buffer, offset?: number): void;\n  declare function writeUInt64LE(value: number, buffer: Buffer, offset?: number): void;\n}\n\n// declare module 'snappyjs' {\n//   declare function compress(uncompressed: Buffer): Buffer;\n//   declare function compress(uncompressed: ArrayBuffer): ArrayBuffer;\n//   declare function compress(uncompressed: Uint8Array): Uint8Array;\n//   declare function uncompress(compressed: Buffer): Buffer;\n//   declare function uncompress(compressed: ArrayBuffer): ArrayBuffer;\n//   declare function uncompress(compressed: Uint8Array): Uint8Array;\n// }\n"],"mappings":""}