@loaders.gl/parquet 4.0.3 → 4.1.0-alpha.1

package/src/parquet-loader.ts

@@ -1,13 +1,23 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
 import type {
   ObjectRowTable,
   ObjectRowTableBatch,
+  GeoJSONTable,
+  GeoJSONTableBatch,
   ColumnarTable,
   ColumnarTableBatch
 } from '@loaders.gl/schema';
+import {BlobFile} from '@loaders.gl/loader-utils';
+
+import {parseParquetFile, parseParquetFileInBatches} from './lib/parsers/parse-parquet';
+import {parseGeoParquetFile, parseGeoParquetFileInBatches} from './lib/parsers/parse-geoparquet';
+import {
+  parseParquetFileInColumns,
+  parseParquetFileInColumnarBatches
+} from './lib/parsers/parse-parquet-to-columns';
 
 export {Buffer} from './polyfills/buffer/install-buffer-polyfill';
 
@@ -32,8 +42,14 @@ export type ParquetLoaderOptions = LoaderOptions & {
   };
 };
 
-/** ParquetJS table loader */
-export const ParquetLoader: Loader<ObjectRowTable, ObjectRowTableBatch, ParquetLoaderOptions> = {
+/**
+ * ParquetJS table loader
+ */
+export const ParquetWorkerLoader: Loader<
+  ObjectRowTable,
+  ObjectRowTableBatch,
+  ParquetLoaderOptions
+> = {
   name: 'Apache Parquet',
   id: 'parquet',
   module: 'parquet',
@@ -55,7 +71,63 @@ export const ParquetLoader: Loader<ObjectRowTable, ObjectRowTableBatch, ParquetL
   }
 };
 
-export const ParquetColumnarLoader: Loader<
+/** ParquetJS table loader */
+export const ParquetLoader: LoaderWithParser<
+  ObjectRowTable | GeoJSONTable,
+  ObjectRowTableBatch | GeoJSONTableBatch,
+  ParquetLoaderOptions
+> = {
+  ...ParquetWorkerLoader,
+  parse: (arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) =>
+    parseParquetFile(new BlobFile(arrayBuffer), options),
+
+  parseFile: parseParquetFile,
+  parseFileInBatches: parseParquetFileInBatches
+};
+
+// Defeat tree shaking
+// @ts-ignore
+ParquetLoader.Buffer = Buffer;
+
+export const GeoParquetWorkerLoader: Loader<GeoJSONTable, GeoJSONTableBatch, ParquetLoaderOptions> =
+  {
+    name: 'Apache Parquet',
+    id: 'parquet',
+    module: 'parquet',
+    version: VERSION,
+    worker: true,
+    category: 'table',
+    extensions: ['parquet'],
+    mimeTypes: ['application/octet-stream'],
+    binary: true,
+    tests: ['PAR1', 'PARE'],
+    options: {
+      parquet: {
+        shape: 'geojson-table',
+        columnList: [],
+        geoparquet: true,
+        url: undefined,
+        preserveBinary: false
+      }
+    }
+  };
+
+/** ParquetJS table loader */
+export const GeoParquetLoader: LoaderWithParser<
+  ObjectRowTable | GeoJSONTable,
+  ObjectRowTableBatch | GeoJSONTableBatch,
+  ParquetLoaderOptions
+> = {
+  ...GeoParquetWorkerLoader,
+  parse(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
+    return parseGeoParquetFile(new BlobFile(arrayBuffer), options);
+  },
+  parseFile: parseGeoParquetFile,
+  parseFileInBatches: parseGeoParquetFileInBatches
+};
+
+/** @deprecated Test to see if we can improve perf of parquetjs loader */
+export const ParquetColumnarWorkerLoader: Loader<
   ColumnarTable,
   ColumnarTableBatch,
   ParquetLoaderOptions
@@ -73,8 +145,16 @@ export const ParquetColumnarLoader: Loader<
   options: ParquetLoader.options
 };
 
-// Defeat tree shaking
-// @ts-ignore
-ParquetLoader.Buffer = Buffer;
-// @ts-ignore
-ParquetColumnarLoader.Buffer = Buffer;
+/** @deprecated Test to see if we can improve perf of parquetjs loader */
+export const ParquetColumnarLoader: LoaderWithParser<
+  ColumnarTable,
+  ColumnarTableBatch,
+  ParquetLoaderOptions
+> = {
+  ...ParquetColumnarWorkerLoader,
+  parse(arrayBuffer: ArrayBuffer, options?: ParquetLoaderOptions) {
+    return parseParquetFileInColumns(new BlobFile(arrayBuffer), options);
+  },
+  parseFile: parseParquetFileInColumns,
+  parseFileInBatches: parseParquetFileInColumnarBatches
+};

package/src/parquet-wasm-loader.ts

@@ -1,9 +1,11 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
 
-import type {Loader, LoaderOptions} from '@loaders.gl/loader-utils';
+import type {Loader, LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
 import type {ArrowTable} from '@loaders.gl/arrow';
 
+import {parseParquetWasm} from './lib/wasm/parse-parquet-wasm';
+
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
@@ -17,7 +19,7 @@ export type ParquetWasmLoaderOptions = LoaderOptions & {
 };
 
 /** Parquet WASM table loader */
-export const ParquetWasmLoader: Loader<ArrowTable, never, ParquetWasmLoaderOptions> = {
+export const ParquetWasmWorkerLoader: Loader<ArrowTable, never, ParquetWasmLoaderOptions> = {
   name: 'Apache Parquet',
   id: 'parquet-wasm',
   module: 'parquet',
@@ -35,3 +37,9 @@ export const ParquetWasmLoader: Loader<ArrowTable, never, ParquetWasmLoaderOptio
     }
   }
 };
+
+/** Parquet WASM table loader */
+export const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions> = {
+  ...ParquetWasmWorkerLoader,
+  parse: parseParquetWasm
+};

package/src/workers/parquet-worker.ts

@@ -2,6 +2,6 @@
 // Copyright (c) vis.gl contributors
 
 import {createLoaderWorker} from '@loaders.gl/loader-utils';
-import {ParquetLoader} from '../index';
+import {ParquetLoader} from '../parquet-loader';
 
 createLoaderWorker(ParquetLoader);

package/dist/lib/parsers/parse-parquet-to-rows.d.ts

@@ -1,6 +0,0 @@
-import type { ReadableFile } from '@loaders.gl/loader-utils';
-import type { GeoJSONTable, GeoJSONTableBatch, ObjectRowTable, ObjectRowTableBatch } from '@loaders.gl/schema';
-import type { ParquetLoaderOptions } from '../../parquet-loader';
-export declare function parseParquetFile(file: ReadableFile, options?: ParquetLoaderOptions): Promise<ObjectRowTable | GeoJSONTable>;
-export declare function parseParquetFileInBatches(file: ReadableFile, options?: ParquetLoaderOptions): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch>;
-//# sourceMappingURL=parse-parquet-to-rows.d.ts.map

package/dist/lib/parsers/parse-parquet-to-rows.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet-to-rows.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-parquet-to-rows.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,YAAY,EAAC,MAAM,0BAA0B,CAAC;AAC3D,OAAO,KAAK,EACV,YAAY,EACZ,iBAAiB,EACjB,cAAc,EACd,mBAAmB,EACpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,KAAK,EAAC,oBAAoB,EAAC,MAAM,sBAAsB,CAAC;AAM/D,wBAAsB,gBAAgB,CACpC,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,OAAO,CAAC,cAAc,GAAG,YAAY,CAAC,CA0BxC;AAED,wBAAuB,yBAAyB,CAC9C,IAAI,EAAE,YAAY,EAClB,OAAO,CAAC,EAAE,oBAAoB,GAC7B,aAAa,CAAC,mBAAmB,GAAG,iBAAiB,CAAC,CAuBxD"}

package/dist/lib/parsers/parse-parquet-to-rows.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"parse-parquet-to-rows.js","names":["convertWKBTableToGeoJSON","WKTLoader","WKBLoader","ParquetReader","getSchemaFromParquetReader","installBufferPolyfill","parseParquetFile","file","options","_options$parquet","_options$parquet2","reader","preserveBinary","parquet","schema","rows","rowBatches","rowBatchIterator","rowBatch","row","push","objectRowTable","shape","data","convertTable","parseParquetFileInBatches","_options$parquet3","_options$parquet4","table","batchType","length","error","Error"],"sources":["../../../src/lib/parsers/parse-parquet-to-rows.ts"],"sourcesContent":["// import type {LoaderWithParser, Loader, LoaderOptions} from '@loaders.gl/loader-utils';\n// import {ColumnarTableBatch} from '@loaders.gl/schema';\nimport type {ReadableFile} from '@loaders.gl/loader-utils';\nimport type {\n  GeoJSONTable,\n  GeoJSONTableBatch,\n  ObjectRowTable,\n  ObjectRowTableBatch\n} from '@loaders.gl/schema';\nimport {convertWKBTableToGeoJSON} from '@loaders.gl/gis';\nimport {WKTLoader, WKBLoader} from '@loaders.gl/wkt';\n\nimport type {ParquetLoaderOptions} from '../../parquet-loader';\nimport type {ParquetRow} from '../../parquetjs/schema/declare';\nimport {ParquetReader} from '../../parquetjs/parser/parquet-reader';\nimport {getSchemaFromParquetReader} from './get-parquet-schema';\nimport {installBufferPolyfill} from '../../polyfills/buffer';\n\nexport async function parseParquetFile(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): Promise<ObjectRowTable | GeoJSONTable> {\n  installBufferPolyfill();\n\n  const reader = new ParquetReader(file, {\n    preserveBinary: options?.parquet?.preserveBinary\n  });\n\n  const schema = await getSchemaFromParquetReader(reader);\n\n  const rows: ParquetRow[] = [];\n\n  const rowBatches = reader.rowBatchIterator(options?.parquet);\n  for await (const rowBatch of rowBatches) {\n    // we have only one input batch so return\n    for (const row of rowBatch) {\n      rows.push(row);\n    }\n  }\n  const objectRowTable: ObjectRowTable = {\n    shape: 'object-row-table',\n    schema,\n    data: rows\n  };\n\n  const shape = options?.parquet?.shape;\n  return convertTable(objectRowTable, shape);\n}\n\nexport async function* parseParquetFileInBatches(\n  file: ReadableFile,\n  options?: ParquetLoaderOptions\n): AsyncIterable<ObjectRowTableBatch | GeoJSONTableBatch> {\n  const reader = new ParquetReader(file, {\n    preserveBinary: options?.parquet?.preserveBinary\n  });\n\n  const schema = await getSchemaFromParquetReader(reader);\n  const rowBatches = reader.rowBatchIterator(options?.parquet);\n  for await (const rows of rowBatches) {\n    const objectRowTable: ObjectRowTable = {\n      shape: 'object-row-table',\n      schema,\n      data: rows\n    };\n    const shape = options?.parquet?.shape;\n    const table = convertTable(objectRowTable, shape);\n\n    yield {\n      batchType: 'data',\n      schema,\n      ...table,\n      length: rows.length\n    };\n  }\n}\n\nfunction convertTable(\n  objectRowTable: ObjectRowTable,\n  shape?: 'object-row-table' | 'geojson-table'\n): ObjectRowTable | GeoJSONTable {\n  switch (shape) {\n    case 'object-row-table':\n      return objectRowTable;\n\n    case 'geojson-table':\n      try {\n        return convertWKBTableToGeoJSON(objectRowTable, objectRowTable.schema!, [\n          WKTLoader,\n          WKBLoader\n        ]);\n      } catch (error) {\n        return objectRowTable;\n      }\n\n    default:\n      throw new Error(shape);\n  }\n}\n"],"mappings":"AASA,SAAQA,wBAAwB,QAAO,iBAAiB;AACxD,SAAQC,SAAS,EAAEC,SAAS,QAAO,iBAAiB;AAAC,SAI7CC,aAAa;AAAA,SACbC,0BAA0B;AAAA,SAC1BC,qBAAqB;AAE7B,OAAO,eAAeC,gBAAgBA,CACpCC,IAAkB,EAClBC,OAA8B,EACU;EAAA,IAAAC,gBAAA,EAAAC,iBAAA;EACxCL,qBAAqB,CAAC,CAAC;EAEvB,MAAMM,MAAM,GAAG,IAAIR,aAAa,CAACI,IAAI,EAAE;IACrCK,cAAc,EAAEJ,OAAO,aAAPA,OAAO,wBAAAC,gBAAA,GAAPD,OAAO,CAAEK,OAAO,cAAAJ,gBAAA,uBAAhBA,gBAAA,CAAkBG;EACpC,CAAC,CAAC;EAEF,MAAME,MAAM,GAAG,MAAMV,0BAA0B,CAACO,MAAM,CAAC;EAEvD,MAAMI,IAAkB,GAAG,EAAE;EAE7B,MAAMC,UAAU,GAAGL,MAAM,CAACM,gBAAgB,CAACT,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEK,OAAO,CAAC;EAC5D,WAAW,MAAMK,QAAQ,IAAIF,UAAU,EAAE;IAEvC,KAAK,MAAMG,GAAG,IAAID,QAAQ,EAAE;MAC1BH,IAAI,CAACK,IAAI,CAACD,GAAG,CAAC;IAChB;EACF;EACA,MAAME,cAA8B,GAAG;IACrCC,KAAK,EAAE,kBAAkB;IACzBR,MAAM;IACNS,IAAI,EAAER;EACR,CAAC;EAED,MAAMO,KAAK,GAAGd,OAAO,aAAPA,OAAO,wBAAAE,iBAAA,GAAPF,OAAO,CAAEK,OAAO,cAAAH,iBAAA,uBAAhBA,iBAAA,CAAkBY,KAAK;EACrC,OAAOE,YAAY,CAACH,cAAc,EAAEC,KAAK,CAAC;AAC5C;AAEA,OAAO,gBAAgBG,yBAAyBA,CAC9ClB,IAAkB,EAClBC,OAA8B,EAC0B;EAAA,IAAAkB,iBAAA;EACxD,MAAMf,MAAM,GAAG,IAAIR,aAAa,CAACI,IAAI,EAAE;IACrCK,cAAc,EAAEJ,OAAO,aAAPA,OAAO,wBAAAkB,iBAAA,GAAPlB,OAAO,CAAEK,OAAO,cAAAa,iBAAA,uBAAhBA,iBAAA,CAAkBd;EACpC,CAAC,CAAC;EAEF,MAAME,MAAM,GAAG,MAAMV,0BAA0B,CAACO,MAAM,CAAC;EACvD,MAAMK,UAAU,GAAGL,MAAM,CAACM,gBAAgB,CAACT,OAAO,aAAPA,OAAO,uBAAPA,OAAO,CAAEK,OAAO,CAAC;EAC5D,WAAW,MAAME,IAAI,IAAIC,UAAU,EAAE;IAAA,IAAAW,iBAAA;IACnC,MAAMN,cAA8B,GAAG;MACrCC,KAAK,EAAE,kBAAkB;MACzBR,MAAM;MACNS,IAAI,EAAER;IACR,CAAC;IACD,MAAMO,KAAK,GAAGd,OAAO,aAAPA,OAAO,wBAAAmB,iBAAA,GAAPnB,OAAO,CAAEK,OAAO,cAAAc,iBAAA,uBAAhBA,iBAAA,CAAkBL,KAAK;IACrC,MAAMM,KAAK,GAAGJ,YAAY,CAACH,cAAc,EAAEC,KAAK,CAAC;IAEjD,MAAM;MACJO,SAAS,EAAE,MAAM;MACjBf,MAAM;MACN,GAAGc,KAAK;MACRE,MAAM,EAAEf,IAAI,CAACe;IACf,CAAC;EACH;AACF;AAEA,SAASN,YAAYA,CACnBH,cAA8B,EAC9BC,KAA4C,EACb;EAC/B,QAAQA,KAAK;IACX,KAAK,kBAAkB;MACrB,OAAOD,cAAc;IAEvB,KAAK,eAAe;MAClB,IAAI;QACF,OAAOrB,wBAAwB,CAACqB,cAAc,EAAEA,cAAc,CAACP,MAAM,EAAG,CACtEb,SAAS,EACTC,SAAS,CACV,CAAC;MACJ,CAAC,CAAC,OAAO6B,KAAK,EAAE;QACd,OAAOV,cAAc;MACvB;IAEF;MACE,MAAM,IAAIW,KAAK,CAACV,KAAK,CAAC;EAC1B;AACF"}