@loaders.gl/parquet 4.3.1 → 4.4.0-alpha.1

package/dist/index.cjs

@@ -39,17 +39,19 @@ __export(dist_exports, {
   BufferPolyfill: () => Buffer2,
   GeoParquetLoader: () => GeoParquetLoader,
   GeoParquetWorkerLoader: () => GeoParquetWorkerLoader,
-  ParquetColumnarLoader: () => ParquetColumnarLoader,
-  ParquetColumnarWorkerLoader: () => ParquetColumnarWorkerLoader,
+  ParquetArrowLoader: () => ParquetArrowLoader,
+  ParquetArrowWorkerLoader: () => ParquetArrowWorkerLoader,
+  ParquetArrowWriter: () => ParquetArrowWriter,
   ParquetEncoder: () => ParquetEncoder,
-  ParquetLoader: () => ParquetLoader,
+  ParquetFormat: () => ParquetFormat,
+  ParquetJSONLoader: () => ParquetJSONLoader,
+  ParquetJSONWorkerLoader: () => ParquetJSONWorkerLoader,
+  ParquetLoader: () => ParquetJSONLoader,
   ParquetReader: () => ParquetReader,
   ParquetSchema: () => ParquetSchema,
-  ParquetWasmLoader: () => ParquetWasmLoader,
-  ParquetWasmWorkerLoader: () => ParquetWasmWorkerLoader,
-  ParquetWasmWriter: () => ParquetWasmWriter,
-  ParquetWorkerLoader: () => ParquetWorkerLoader,
-  _ParquetWriter: () => ParquetWriter,
+  ParquetWorkerLoader: () => ParquetJSONWorkerLoader,
+  ParquetWriter: () => ParquetJSONWriter,
+  _ParquetJSONWriter: () => ParquetJSONWriter,
   convertParquetSchema: () => convertParquetSchema,
   convertParquetToArrowSchema: () => convertParquetSchema,
   installBufferPolyfill: () => installBufferPolyfill,
@@ -1742,10 +1744,20 @@ globalThis.process = globalThis.process || {};
 globalThis.process.env = globalThis.process.env || {};
 var Buffer3 = installBufferPolyfill();
 
-// dist/parquet-loader.js
+// dist/parquet-format.js
+var ParquetFormat = {
+  name: "Apache Parquet",
+  category: "table",
+  extensions: ["parquet"],
+  mimeTypes: ["application/octet-stream"],
+  binary: true,
+  tests: ["PAR1", "PARE"]
+};
+
+// dist/parquet-json-loader.js
 var import_loader_utils2 = require("@loaders.gl/loader-utils");
 
-// dist/lib/parsers/parse-parquet.js
+// dist/lib/parsers/parse-parquet-to-json.js
 var import_log = __toESM(require("@probe.gl/log"), 1);
 
 // dist/parquetjs/codecs/plain.js
@@ -2703,96 +2715,6 @@ function materializeColumnAsRows(schema, columnData, key, rows) {
     }
   }
 }
-function materializeColumns(schema, rowGroup) {
-  const columns = {};
-  for (const key in rowGroup.columnData) {
-    const columnData = rowGroup.columnData[key];
-    if (columnData.count) {
-      materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);
-    }
-  }
-  return columns;
-}
-function materializeColumnAsColumnarArray(schema, columnData, rowCount, key, columns) {
-  if (columnData.count <= 0) {
-    return;
-  }
-  const field = schema.findField(key);
-  const branch = schema.findFieldBranch(key);
-  const columnName = branch[0].name;
-  let column;
-  const { values } = columnData;
-  if (values.length === rowCount && branch[0].primitiveType) {
-    column = values;
-  }
-  if (column) {
-    columns[columnName] = column;
-    return;
-  }
-  column = new Array(rowCount);
-  for (let i = 0; i < rowCount; i++) {
-    column[i] = {};
-  }
-  columns[columnName] = column;
-  const rLevels = new Array(field.rLevelMax + 1).fill(0);
-  let vIndex = 0;
-  for (let i = 0; i < columnData.count; i++) {
-    const dLevel = columnData.dlevels[i];
-    const rLevel = columnData.rlevels[i];
-    rLevels[rLevel]++;
-    rLevels.fill(0, rLevel + 1);
-    let rIndex = 0;
-    let record = column[rLevels[rIndex++] - 1];
-    for (const step of branch) {
-      if (step === field || dLevel < step.dLevelMax) {
-        break;
-      }
-      switch (step.repetitionType) {
-        case "REPEATED":
-          if (!(step.name in record)) {
-            record[step.name] = [];
-          }
-          const ix = rLevels[rIndex++];
-          while (record[step.name].length <= ix) {
-            record[step.name].push({});
-          }
-          record = record[step.name][ix];
-          break;
-        default:
-          record[step.name] = record[step.name] || {};
-          record = record[step.name];
-      }
-    }
-    if (dLevel === field.dLevelMax) {
-      const value = fromPrimitive(
-        // @ts-ignore
-        field.originalType || field.primitiveType,
-        columnData.values[vIndex],
-        field
-      );
-      vIndex++;
-      switch (field.repetitionType) {
-        case "REPEATED":
-          if (!(field.name in record)) {
-            record[field.name] = [];
-          }
-          const ix = rLevels[rIndex];
-          while (record[field.name].length <= ix) {
-            record[field.name].push(null);
-          }
-          record[field.name][ix] = value;
-          break;
-        default:
-          record[field.name] = value;
-      }
-    }
-  }
-  for (let i = 0; i < rowCount; ++i) {
-    if (columnName in column[i]) {
-      column[i] = column[i][columnName];
-    }
-  }
-}
 
 // dist/parquetjs/schema/schema.js
 var ParquetSchema = class {
@@ -6323,7 +6245,7 @@ var FileMetaData = class {
 };
 
 // dist/lib/constants.js
-var VERSION = true ? "4.3.0" : "latest";
+var VERSION = true ? "4.4.0-alpha.0" : "latest";
 var PARQUET_WASM_URL = "https://unpkg.com/parquet-wasm@0.6.1/esm/parquet_wasm_bg.wasm";
 var PARQUET_MAGIC = "PAR1";
 var PARQUET_MAGIC_ENCRYPTED = "PARE";
@@ -6859,6 +6781,9 @@ __publicField(ParquetReader, "defaultProps", {
   preserveBinary: false
 });
 
+// dist/lib/parsers/get-parquet-schema.js
+var import_geoarrow = require("@loaders.gl/geoarrow");
+
 // dist/lib/arrow/convert-schema-from-parquet.js
 var PARQUET_TYPE_MAPPING = {
   BOOLEAN: "bool",
@@ -6942,19 +6867,18 @@ function getSchemaMetadata(parquetMetadata) {
 }
 
 // dist/lib/parsers/get-parquet-schema.js
-var import_gis = require("@loaders.gl/gis");
 async function getSchemaFromParquetReader(reader) {
   const parquetSchema = await reader.getSchema();
   const parquetMetadata = await reader.getFileMetadata();
   const schema = convertParquetSchema(parquetSchema, parquetMetadata);
-  (0, import_gis.unpackGeoMetadata)(schema);
-  (0, import_gis.unpackJSONStringMetadata)(schema, "pandas");
+  (0, import_geoarrow.unpackGeoMetadata)(schema.metadata);
+  (0, import_geoarrow.unpackJSONStringMetadata)(schema.metadata, "pandas");
   return schema;
 }
 
-// dist/lib/parsers/parse-parquet.js
+// dist/lib/parsers/parse-parquet-to-json.js
 async function parseParquetFile(file, options) {
-  var _a, _b;
+  var _a;
   installBufferPolyfill();
   await preloadCompressions(options);
   const reader = new ParquetReader(file, {
@@ -6982,11 +6906,10 @@ async function parseParquetFile(file, options) {
     schema,
     data: rows
   };
-  const shape = (_b = options == null ? void 0 : options.parquet) == null ? void 0 : _b.shape;
-  return convertTable(objectRowTable, shape);
+  return objectRowTable;
 }
 async function* parseParquetFileInBatches(file, options) {
-  var _a, _b;
+  var _a;
   installBufferPolyfill();
   await preloadCompressions(options);
   const reader = new ParquetReader(file, {
@@ -7000,136 +6923,39 @@ async function* parseParquetFileInBatches(file, options) {
       schema,
       data: rows
     };
-    const shape = (_b = options == null ? void 0 : options.parquet) == null ? void 0 : _b.shape;
-    const table = convertTable(objectRowTable, shape);
     yield {
       batchType: "data",
       schema,
-      ...table,
+      ...objectRowTable,
       length: rows.length
     };
   }
 }
-function convertTable(objectRowTable, shape) {
-  switch (shape) {
-    case "object-row-table":
-      return objectRowTable;
-    case "geojson-table":
-      return objectRowTable;
-    default:
-      throw new Error(shape);
-  }
-}
 
-// dist/lib/parsers/parse-geoparquet.js
-var import_gis2 = require("@loaders.gl/gis");
-var import_wkt = require("@loaders.gl/wkt");
+// dist/lib/parsers/parse-geoparquet-to-geojson.js
 async function parseGeoParquetFile(file, options) {
-  var _a;
   const table = await parseParquetFile(file, { ...options, shape: "object-row-table" });
-  const shape = (_a = options == null ? void 0 : options.parquet) == null ? void 0 : _a.shape;
-  return convertTable2(table, shape);
+  return table;
 }
 async function* parseGeoParquetFileInBatches(file, options) {
-  var _a;
   const tableBatches = parseParquetFileInBatches(file, { ...options, shape: "object-row-table" });
   for await (const batch of tableBatches) {
-    const shape = (_a = options == null ? void 0 : options.parquet) == null ? void 0 : _a.shape;
-    yield convertBatch(batch, shape);
-  }
-}
-function convertTable2(objectRowTable, shape) {
-  switch (shape) {
-    case "object-row-table":
-      return objectRowTable;
-    case "geojson-table":
-      try {
-        return (0, import_gis2.convertWKBTableToGeoJSON)(objectRowTable, objectRowTable.schema, [
-          import_wkt.WKTLoader,
-          import_wkt.WKBLoader
-        ]);
-      } catch (error) {
-        return objectRowTable;
-      }
-    default:
-      throw new Error(shape);
-  }
-}
-function convertBatch(objectRowBatch, shape) {
-  switch (shape) {
-    case "object-row-table":
-      return objectRowBatch;
-    case "geojson-table":
-      try {
-        const geojsonTable = (0, import_gis2.convertWKBTableToGeoJSON)(objectRowBatch, objectRowBatch.schema, [
-          import_wkt.WKTLoader,
-          import_wkt.WKBLoader
-        ]);
-        return {
-          ...objectRowBatch,
-          ...geojsonTable
-        };
-      } catch (error) {
-        return objectRowBatch;
-      }
-    default:
-      throw new Error(shape);
+    yield batch;
   }
 }
 
-// dist/lib/parsers/parse-parquet-to-columns.js
-async function parseParquetFileInColumns(file, options) {
-  installBufferPolyfill();
-  await preloadCompressions(options);
-  for await (const batch of parseParquetFileInColumnarBatches(file, options)) {
-    return {
-      shape: "columnar-table",
-      schema: batch.schema,
-      data: batch.data
-    };
-  }
-  throw new Error("empty table");
-}
-async function* parseParquetFileInColumnarBatches(file, options) {
-  installBufferPolyfill();
-  await preloadCompressions(options);
-  const reader = new ParquetReader(file);
-  const schema = await getSchemaFromParquetReader(reader);
-  const parquetSchema = await reader.getSchema();
-  const rowGroups = reader.rowGroupIterator(options == null ? void 0 : options.parquet);
-  for await (const rowGroup of rowGroups) {
-    yield convertRowGroupToTableBatch(rowGroup, parquetSchema, schema);
-  }
-}
-function convertRowGroupToTableBatch(rowGroup, parquetSchema, schema) {
-  const data = materializeColumns(parquetSchema, rowGroup);
-  return {
-    shape: "columnar-table",
-    batchType: "data",
-    schema,
-    data,
-    length: rowGroup.rowCount
-  };
-}
-
-// dist/parquet-loader.js
-var VERSION2 = true ? "4.3.0" : "latest";
-var ParquetWorkerLoader = {
+// dist/parquet-json-loader.js
+var VERSION2 = true ? "4.4.0-alpha.0" : "latest";
+var ParquetJSONWorkerLoader = {
+  ...ParquetFormat,
   dataType: null,
   batchType: null,
-  name: "Apache Parquet",
   id: "parquet",
   module: "parquet",
   version: VERSION2,
   worker: false,
-  category: "table",
-  extensions: ["parquet"],
-  mimeTypes: ["application/octet-stream"],
-  binary: true,
-  tests: ["PAR1", "PARE"],
   options: {
     parquet: {
-      shape: "object-row-table",
       columnList: [],
       geoparquet: true,
       url: void 0,
@@ -7137,31 +6963,25 @@ var ParquetWorkerLoader = {
     }
   }
 };
-var ParquetLoader = {
-  ...ParquetWorkerLoader,
+var ParquetJSONLoader = {
+  ...ParquetJSONWorkerLoader,
   dataType: null,
   batchType: null,
   parse: (arrayBuffer, options) => parseParquetFile(new import_loader_utils2.BlobFile(arrayBuffer), options),
   parseFile: parseParquetFile,
   parseFileInBatches: parseParquetFileInBatches
 };
-ParquetLoader.Buffer = Buffer3;
+ParquetJSONLoader.Buffer = Buffer3;
 var GeoParquetWorkerLoader = {
+  ...ParquetFormat,
   dataType: null,
   batchType: null,
-  name: "Apache Parquet",
   id: "parquet",
   module: "parquet",
   version: VERSION2,
   worker: true,
-  category: "table",
-  extensions: ["parquet"],
-  mimeTypes: ["application/octet-stream"],
-  binary: true,
-  tests: ["PAR1", "PARE"],
   options: {
     parquet: {
-      shape: "geojson-table",
       columnList: [],
       geoparquet: true,
       url: void 0,
@@ -7177,40 +6997,14 @@ var GeoParquetLoader = {
   parseFile: parseGeoParquetFile,
   parseFileInBatches: parseGeoParquetFileInBatches
 };
-var ParquetColumnarWorkerLoader = {
-  dataType: null,
-  batchType: null,
-  name: "Apache Parquet",
-  id: "parquet",
-  module: "parquet",
-  version: VERSION2,
-  worker: true,
-  category: "table",
-  extensions: ["parquet"],
-  mimeTypes: ["application/octet-stream"],
-  binary: true,
-  tests: ["PAR1", "PARE"],
-  options: ParquetLoader.options
-};
-var ParquetColumnarLoader = {
-  ...ParquetColumnarWorkerLoader,
-  parse(arrayBuffer, options) {
-    return parseParquetFileInColumns(new import_loader_utils2.BlobFile(arrayBuffer), options);
-  },
-  parseFile: parseParquetFileInColumns,
-  parseFileInBatches: parseParquetFileInColumnarBatches
-};
 
-// dist/parquet-writer.js
-var VERSION3 = true ? "4.3.0" : "latest";
-var ParquetWriter = {
-  name: "Apache Parquet",
+// dist/parquet-json-writer.js
+var VERSION3 = true ? "4.4.0-alpha.0" : "latest";
+var ParquetJSONWriter = {
+  ...ParquetFormat,
   id: "parquet",
   module: "parquet",
   version: VERSION3,
-  extensions: ["parquet"],
-  mimeTypes: ["application/octet-stream"],
-  binary: true,
   options: {},
   encode: async (data, options) => encodeSync(data, options),
   encodeSync
@@ -7219,11 +7013,11 @@ function encodeSync(data, options) {
   return new ArrayBuffer(0);
 }
 
-// dist/parquet-wasm-loader.js
+// dist/parquet-arrow-loader.js
 var import_loader_utils4 = require("@loaders.gl/loader-utils");
 
-// dist/lib/parsers/parse-parquet-wasm.js
-var import_arrow = require("@loaders.gl/arrow");
+// dist/lib/parsers/parse-parquet-to-arrow.js
+var import_schema_utils = require("@loaders.gl/schema-utils");
 var arrow = __toESM(require("apache-arrow"), 1);
 
 // dist/lib/utils/load-wasm.js
@@ -7271,8 +7065,8 @@ async function* makeNodeStreamIterator(stream, options) {
   yield* stream;
 }
 
-// dist/lib/parsers/parse-parquet-wasm.js
-async function parseParquetFileWasm(file, options) {
+// dist/lib/parsers/parse-parquet-to-arrow.js
+async function parseParquetFileToArrow(file, options) {
   const wasmUrl = options == null ? void 0 : options.wasmUrl;
   const wasm = await loadWasm(wasmUrl);
   let parquetFile;
@@ -7286,11 +7080,11 @@ async function parseParquetFileWasm(file, options) {
   const arrowTable = arrow.tableFromIPC(ipcStream);
   return {
     shape: "arrow-table",
-    schema: (0, import_arrow.serializeArrowSchema)(arrowTable.schema),
+    schema: (0, import_schema_utils.convertArrowToSchema)(arrowTable.schema),
     data: arrowTable
   };
 }
-async function* parseParquetFileInBatchesWasm(file, options) {
+async function* parseParquetFileToArrowInBatches(file, options) {
   const wasmUrl = options == null ? void 0 : options.wasmUrl;
   const wasm = await loadWasm(wasmUrl);
   let parquetFile;
@@ -7301,20 +7095,20 @@ async function* parseParquetFileInBatchesWasm(file, options) {
   }
   const stream = await parquetFile.stream(options);
   let schema;
-  for await (const table of makeStreamIterator(stream)) {
-    schema ||= (0, import_arrow.serializeArrowSchema)(table.schema);
+  for await (const recordBatch of makeStreamIterator(stream)) {
+    schema ||= (0, import_schema_utils.convertArrowToSchema)(recordBatch.schema);
     yield {
       batchType: "data",
       shape: "arrow-table",
       schema,
-      data: table.batches[0],
-      length: table.numRows
+      data: new arrow.Table(recordBatch),
+      length: recordBatch.numRows
     };
   }
 }
 
-// dist/parquet-wasm-loader.js
-var ParquetWasmWorkerLoader = {
+// dist/parquet-arrow-loader.js
+var ParquetArrowWorkerLoader = {
   dataType: null,
   batchType: null,
   name: "Apache Parquet",
@@ -7329,7 +7123,6 @@ var ParquetWasmWorkerLoader = {
   tests: ["PAR1", "PARE"],
   options: {
     parquet: {
-      shape: "arrow-table",
       limit: void 0,
       // Provide a limit to the number of rows to be read.
       offset: 0,
@@ -7346,25 +7139,25 @@ var ParquetWasmWorkerLoader = {
     }
   }
 };
-var ParquetWasmLoader = {
-  ...ParquetWasmWorkerLoader,
+var ParquetArrowLoader = {
+  ...ParquetArrowWorkerLoader,
   parse(arrayBuffer, options) {
-    const wasmOptions = { ...ParquetWasmLoader.options.parquet, ...options == null ? void 0 : options.parquet };
-    return parseParquetFileWasm(new import_loader_utils4.BlobFile(arrayBuffer), wasmOptions);
+    const wasmOptions = { ...ParquetArrowLoader.options.parquet, ...options == null ? void 0 : options.parquet };
+    return parseParquetFileToArrow(new import_loader_utils4.BlobFile(arrayBuffer), wasmOptions);
   },
   parseFile(file, options) {
-    const wasmOptions = { ...ParquetWasmLoader.options.parquet, ...options == null ? void 0 : options.parquet };
-    return parseParquetFileWasm(file, wasmOptions);
+    const wasmOptions = { ...ParquetArrowLoader.options.parquet, ...options == null ? void 0 : options.parquet };
+    return parseParquetFileToArrow(file, wasmOptions);
   },
   parseFileInBatches(file, options) {
-    const wasmOptions = { ...ParquetWasmLoader.options.parquet, ...options == null ? void 0 : options.parquet };
-    return parseParquetFileInBatchesWasm(file, wasmOptions);
+    const wasmOptions = { ...ParquetArrowLoader.options.parquet, ...options == null ? void 0 : options.parquet };
+    return parseParquetFileToArrowInBatches(file, wasmOptions);
   }
 };
 
-// dist/lib/encoders/encode-parquet-wasm.js
+// dist/lib/encoders/encode-arrow-to-parquet.js
 var arrow2 = __toESM(require("apache-arrow"), 1);
-async function encode(table, options) {
+async function encodeArrowToParquet(table, options) {
   var _a;
   const wasmUrl = (_a = options.parquet) == null ? void 0 : _a.wasmUrl;
   const wasm = await loadWasm(wasmUrl);
@@ -7379,23 +7172,20 @@ async function encode(table, options) {
   }
 }
 
-// dist/parquet-wasm-writer.js
-var ParquetWasmWriter = {
-  name: "Apache Parquet",
+// dist/parquet-arrow-writer.js
+var ParquetArrowWriter = {
+  ...ParquetFormat,
   id: "parquet-wasm",
   module: "parquet",
   version: VERSION,
-  extensions: ["parquet"],
-  mimeTypes: ["application/octet-stream"],
-  binary: true,
   options: {
     parquet: {
       wasmUrl: PARQUET_WASM_URL
     }
   },
   encode(arrowTable, options) {
-    options = { parquet: { ...ParquetWasmWriter.options.parquet, ...options == null ? void 0 : options.parquet }, ...options };
-    return encode(arrowTable, options);
+    options = { parquet: { ...ParquetArrowWriter.options.parquet, ...options == null ? void 0 : options.parquet }, ...options };
+    return encodeArrowToParquet(arrowTable, options);
   }
 };