@loaders.gl/parquet 3.4.6 → 4.0.0-alpha.10

package/dist/parquetjs/schema/shred.js

@@ -24,9 +24,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.materializeRecords = exports.shredRecord = exports.shredBuffer = exports.ParquetBuffer = void 0;
+exports.materializeColumns = exports.materializeRows = exports.shredRecord = exports.shredBuffer = exports.ParquetRowGroup = void 0;
 const declare_1 = require("./declare");
-Object.defineProperty(exports, "ParquetBuffer", { enumerable: true, get: function () { return declare_1.ParquetBuffer; } });
+Object.defineProperty(exports, "ParquetRowGroup", { enumerable: true, get: function () { return declare_1.ParquetRowGroup; } });
 const Types = __importStar(require("./types"));
 function shredBuffer(schema) {
     const columnData = {};
@@ -46,14 +46,14 @@ exports.shredBuffer = shredBuffer;
  * 'Shred' a record into a list of <value, repetition_level, definition_level>
  * tuples per column using the Google Dremel Algorithm..
  *
- * The buffer argument must point to an object into which the shredded record
- * will be returned. You may re-use the buffer for repeated calls to this function
- * to append to an existing buffer, as long as the schema is unchanged.
+ * The rowGroup argument must point to an object into which the shredded record
+ * will be returned. You may re-use the rowGroup for repeated calls to this function
+ * to append to an existing rowGroup, as long as the schema is unchanged.
  *
- * The format in which the shredded records will be stored in the buffer is as
+ * The format in which the shredded records will be stored in the rowGroup is as
  * follows:
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -64,22 +64,22 @@ exports.shredBuffer = shredBuffer;
  *      rowCount: X,
  *   }
  */
-function shredRecord(schema, record, buffer) {
+function shredRecord(schema, record, rowGroup) {
     /* shred the record, this may raise an exception */
     const data = shredBuffer(schema).columnData;
     shredRecordFields(schema.fields, record, data, 0, 0);
-    /* if no error during shredding, add the shredded record to the buffer */
-    if (buffer.rowCount === 0) {
-        buffer.rowCount = 1;
-        buffer.columnData = data;
+    /* if no error during shredding, add the shredded record to the rowGroup */
+    if (rowGroup.rowCount === 0) {
+        rowGroup.rowCount = 1;
+        rowGroup.columnData = data;
         return;
     }
-    buffer.rowCount += 1;
+    rowGroup.rowCount += 1;
     for (const field of schema.fieldList) {
-        Array.prototype.push.apply(buffer.columnData[field.key].rlevels, data[field.key].rlevels);
-        Array.prototype.push.apply(buffer.columnData[field.key].dlevels, data[field.key].dlevels);
-        Array.prototype.push.apply(buffer.columnData[field.key].values, data[field.key].values);
-        buffer.columnData[field.key].count += data[field.key].count;
+        Array.prototype.push.apply(rowGroup.columnData[field.key].rlevels, data[field.key].rlevels);
+        Array.prototype.push.apply(rowGroup.columnData[field.key].dlevels, data[field.key].dlevels);
+        Array.prototype.push.apply(rowGroup.columnData[field.key].values, data[field.key].values);
+        rowGroup.columnData[field.key].count += data[field.key].count;
     }
 }
 exports.shredRecord = shredRecord;
@@ -139,10 +139,10 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
  * tuples back to nested records (objects/arrays) using the Google Dremel
  * Algorithm..
  *
- * The buffer argument must point to an object with the following structure (i.e.
+ * The rowGroup argument must point to an object with the following structure (i.e.
  * the same structure that is returned by shredRecords):
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -153,22 +153,24 @@ function shredRecordFields(fields, record, data, rLevel, dLevel) {
  *      rowCount: X,
  *   }
  */
-function materializeRecords(schema, buffer) {
-    const records = [];
-    for (let i = 0; i < buffer.rowCount; i++) {
-        records.push({});
+function materializeRows(schema, rowGroup) {
+    const rows = [];
+    // rows = new Array(rowGroup.rowCount).fill({})'
+    for (let i = 0; i < rowGroup.rowCount; i++) {
+        rows.push({});
     }
-    for (const key in buffer.columnData) {
-        const columnData = buffer.columnData[key];
+    for (const key in rowGroup.columnData) {
+        const columnData = rowGroup.columnData[key];
         if (columnData.count) {
-            materializeColumn(schema, columnData, key, records);
+            materializeColumnAsRows(schema, columnData, key, rows);
         }
     }
-    return records;
+    return rows;
 }
-exports.materializeRecords = materializeRecords;
+exports.materializeRows = materializeRows;
+/** Populate record fields for one column */
 // eslint-disable-next-line max-statements, complexity
-function materializeColumn(schema, columnData, key, records) {
+function materializeColumnAsRows(schema, columnData, key, rows) {
     const field = schema.findField(key);
     const branch = schema.findFieldBranch(key);
     // tslint:disable-next-line:prefer-array-literal
@@ -180,7 +182,7 @@ function materializeColumn(schema, columnData, key, records) {
         rLevels[rLevel]++;
         rLevels.fill(0, rLevel + 1);
         let rIndex = 0;
-        let record = records[rLevels[rIndex++] - 1];
+        let record = rows[rLevels[rIndex++] - 1];
         // Internal nodes - Build a nested row object
         for (const step of branch) {
             if (step === field || dLevel < step.dLevelMax) {
@@ -235,10 +237,10 @@ function materializeColumn(schema, columnData, key, records) {
  * tuples back to nested records (objects/arrays) using the Google Dremel
  * Algorithm..
  *
- * The buffer argument must point to an object with the following structure (i.e.
+ * The rowGroup argument must point to an object with the following structure (i.e.
  * the same structure that is returned by shredRecords):
  *
- *   buffer = {
+ *   rowGroup = {
  *     columnData: [
  *       'my_col': {
  *          dlevels: [d1, d2, .. dN],
@@ -248,100 +250,106 @@ function materializeColumn(schema, columnData, key, records) {
  *      ],
  *      rowCount: X,
  *   }
- *
-export function extractColumns(schema: ParquetSchema, buffer: ParquetBuffer): Record<string, unknown> {
-  const columns: ParquetRecord = {};
-  for (const key in buffer.columnData) {
-    const columnData = buffer.columnData[key];
-    if (columnData.count) {
-      extractColumn(schema, columnData, key, columns);
+ */
+function materializeColumns(schema, rowGroup) {
+    const columns = {};
+    for (const key in rowGroup.columnData) {
+        const columnData = rowGroup.columnData[key];
+        if (columnData.count) {
+            materializeColumnAsColumnarArray(schema, columnData, rowGroup.rowCount, key, columns);
+        }
     }
-  }
-  return columns;
+    return columns;
 }
-
+exports.materializeColumns = materializeColumns;
 // eslint-disable-next-line max-statements, complexity
-function extractColumn(
-  schema: ParquetSchema,
-  columnData: ParquetData,
-  key: string,
-  columns: Record<string, unknown>
-) {
-  if (columnData.count <= 0) {
-    return;
-  }
-
-  const record = columns;
-
-  const field = schema.findField(key);
-  const branch = schema.findFieldBranch(key);
-
-  // tslint:disable-next-line:prefer-array-literal
-  const rLevels: number[] = new Array(field.rLevelMax + 1).fill(0);
-  let vIndex = 0;
-
-  let i = 0;
-  const dLevel = columnData.dlevels[i];
-  const rLevel = columnData.rlevels[i];
-  rLevels[rLevel]++;
-  rLevels.fill(0, rLevel + 1);
-
-  let rIndex = 0;
-  let record = records[rLevels[rIndex++] - 1];
-
-  // Internal nodes
-  for (const step of branch) {
-    if (step === field || dLevel < step.dLevelMax) {
-      break;
+function materializeColumnAsColumnarArray(schema, columnData, rowCount, key, columns) {
+    if (columnData.count <= 0) {
+        return;
     }
-
-    switch (step.repetitionType) {
-      case 'REPEATED':
-        if (!(step.name in record)) {
-          // eslint-disable max-depth
-          record[step.name] = [];
+    const field = schema.findField(key);
+    const branch = schema.findFieldBranch(key);
+    const columnName = branch[0].name;
+    let column;
+    const { values } = columnData;
+    if (values.length === rowCount && branch[0].primitiveType) {
+        // if (branch[0].repetitionType === `REQUIRED`) {
+        //   switch (branch[0].primitiveType) {
+        //     case 'INT32': return values instanceof Int32Array ? values : new Int32Array(values);
+        //   }
+        // }
+        column = values;
+    }
+    if (column) {
+        columns[columnName] = column;
+        return;
+    }
+    column = new Array(rowCount);
+    for (let i = 0; i < rowCount; i++) {
+        column[i] = {};
+    }
+    columns[columnName] = column;
+    // tslint:disable-next-line:prefer-array-literal
+    const rLevels = new Array(field.rLevelMax + 1).fill(0);
+    let vIndex = 0;
+    for (let i = 0; i < columnData.count; i++) {
+        const dLevel = columnData.dlevels[i];
+        const rLevel = columnData.rlevels[i];
+        rLevels[rLevel]++;
+        rLevels.fill(0, rLevel + 1);
+        let rIndex = 0;
+        let record = column[rLevels[rIndex++] - 1];
+        // Internal nodes - Build a nested row object
+        for (const step of branch) {
+            if (step === field || dLevel < step.dLevelMax) {
+                break;
+            }
+            switch (step.repetitionType) {
+                case 'REPEATED':
+                    if (!(step.name in record)) {
+                        // eslint-disable max-depth
+                        record[step.name] = [];
+                    }
+                    const ix = rLevels[rIndex++];
+                    while (record[step.name].length <= ix) {
+                        // eslint-disable max-depth
+                        record[step.name].push({});
+                    }
+                    record = record[step.name][ix];
+                    break;
+                default:
+                    record[step.name] = record[step.name] || {};
+                    record = record[step.name];
+            }
         }
-        const ix = rLevels[rIndex++];
-        while (record[step.name].length <= ix) {
-          // eslint-disable max-depth
-          record[step.name].push({});
+        // Leaf node - Add the value
+        if (dLevel === field.dLevelMax) {
+            const value = Types.fromPrimitive(
+            // @ts-ignore
+            field.originalType || field.primitiveType, columnData.values[vIndex], field);
+            vIndex++;
+            switch (field.repetitionType) {
+                case 'REPEATED':
+                    if (!(field.name in record)) {
+                        // eslint-disable max-depth
+                        record[field.name] = [];
+                    }
+                    const ix = rLevels[rIndex];
+                    while (record[field.name].length <= ix) {
+                        // eslint-disable max-depth
+                        record[field.name].push(null);
+                    }
+                    record[field.name][ix] = value;
+                    break;
+                default:
+                    record[field.name] = value;
+            }
         }
-        record = record[step.name][ix];
-        break;
-
-      default:
-        record[step.name] = record[step.name] || {};
-        record = record[step.name];
     }
-  }
-
-  // Leaf node
-  if (dLevel === field.dLevelMax) {
-    const value = Types.fromPrimitive(
-      // @ts-ignore
-      field.originalType || field.primitiveType,
-      columnData.values[vIndex],
-      field
-    );
-    vIndex++;
-
-    switch (field.repetitionType) {
-      case 'REPEATED':
-        if (!(field.name in record)) {
-          // eslint-disable max-depth
-          record[field.name] = [];
-        }
-        const ix = rLevels[rIndex];
-        while (record[field.name].length <= ix) {
-          // eslint-disable max-depth
-          record[field.name].push(null);
+    // Remove one level of nesting
+    for (let i = 0; i < rowCount; ++i) {
+        if (columnName in column[i]) {
+            column[i] = column[i][columnName];
         }
-        record[field.name][ix] = value;
-        break;
-
-      default:
-        record[field.name] = value;
     }
-  }
 }
-*/

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/parquet",
-  "version": "3.4.6",
+  "version": "4.0.0-alpha.10",
   "description": "Framework-independent loader for Apache Parquet files",
   "license": "MIT",
   "publishConfig": {
@@ -41,10 +41,10 @@
     "./src/lib/wasm/load-wasm/load-wasm-node.ts": "./src/lib/wasm/load-wasm/load-wasm-browser.ts"
   },
   "dependencies": {
-    "@loaders.gl/bson": "3.4.6",
-    "@loaders.gl/compression": "3.4.6",
-    "@loaders.gl/loader-utils": "3.4.6",
-    "@loaders.gl/schema": "3.4.6",
+    "@loaders.gl/bson": "4.0.0-alpha.10",
+    "@loaders.gl/compression": "4.0.0-alpha.10",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.10",
+    "@loaders.gl/schema": "4.0.0-alpha.10",
     "async-mutex": "^0.2.2",
     "brotli": "^1.3.2",
     "int53": "^0.2.4",
@@ -58,14 +58,14 @@
     "zstd-codec": "^0.1"
   },
   "peerDependencies": {
-    "apache-arrow": "^4.0.0"
+    "apache-arrow": "^9.0.0"
   },
   "devDependencies": {
     "@types/node": "^10.14.15",
     "@types/node-int64": "^0.4.29",
     "@types/thrift": "^0.10.8",
     "@types/varint": "^5.0.0",
-    "apache-arrow": "^4.0.0"
+    "apache-arrow": "^9.0.0"
   },
-  "gitHead": "f878cbf97013ab99866390ef58e6ca26717af6cb"
+  "gitHead": "7efdbe09e02098aad6d985e4d6465d08806e19a9"
 }

package/src/index.ts

@@ -1,33 +1,56 @@
+// loaders.gl, MIT license
+
 import type {LoaderWithParser} from '@loaders.gl/loader-utils';
+import type {
+  ObjectRowTable,
+  ObjectRowTableBatch,
+  ColumnarTable,
+  ColumnarTableBatch
+} from '@loaders.gl/schema';
+import type {Table as ArrowTable} from 'apache-arrow';
 
 // ParquetLoader
 
-import {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';
-import {ParquetLoader as ParquetWorkerLoader} from './parquet-loader';
+import {
+  ParquetLoader as ParquetWorkerLoader,
+  ParquetLoader as ParquetColumnarWorkerLoader,
+  ParquetLoaderOptions
+} from './parquet-loader';
 import {parseParquet, parseParquetFileInBatches} from './lib/parsers/parse-parquet-to-rows';
 import {
   parseParquetInColumns,
   parseParquetFileInColumnarBatches
 } from './lib/parsers/parse-parquet-to-columns';
-import {parseParquet as parseParquetWasm} from './lib/wasm/parse-parquet-wasm';
+
+import {parseParquetWasm, ParquetWasmLoaderOptions} from './lib/wasm/parse-parquet-wasm';
+import {ParquetWasmLoader as ParquetWasmWorkerLoader} from './parquet-wasm-loader';
 
 export {ParquetWorkerLoader, ParquetWasmWorkerLoader};
 
 /** ParquetJS table loader */
-export const ParquetLoader = {
+export const ParquetLoader: LoaderWithParser<
+  ObjectRowTable,
+  ObjectRowTableBatch,
+  ParquetLoaderOptions
+> = {
   ...ParquetWorkerLoader,
   parse: parseParquet,
   parseFileInBatches: parseParquetFileInBatches
 };
 
 /** ParquetJS table loader */
-export const ParquetColumnarLoader = {
-  ...ParquetWorkerLoader,
+// @ts-expect-error
+export const ParquetColumnarLoader: LoaderWithParser<
+  ColumnarTable,
+  ColumnarTableBatch,
+  ParquetLoaderOptions
+> = {
+  ...ParquetColumnarWorkerLoader,
   parse: parseParquetInColumns,
   parseFileInBatches: parseParquetFileInColumnarBatches
 };
 
-export const ParquetWasmLoader = {
+export const ParquetWasmLoader: LoaderWithParser<ArrowTable, never, ParquetWasmLoaderOptions> = {
   ...ParquetWasmWorkerLoader,
   parse: parseParquetWasm
 };
@@ -46,8 +69,8 @@ export {ParquetReader} from './parquetjs/parser/parquet-reader';
 export {ParquetEncoder} from './parquetjs/encoder/parquet-encoder';
 
 export {
-  convertSchemaFromParquet,
-  convertSchemaFromParquet as convertParquetToArrowSchema
+  convertParquetSchema,
+  convertParquetSchema as convertParquetToArrowSchema
 } from './lib/arrow/convert-schema-from-parquet';
 
 // TESTS

package/src/lib/arrow/convert-row-group-to-columns.ts

@@ -1,11 +1,11 @@
 // loaders.gl, MIT license
 
 import {Schema} from '@loaders.gl/schema';
-import {ParquetBuffer} from '@loaders.gl/parquet/parquetjs/schema/declare';
+import {ParquetRowGroup} from '@loaders.gl/parquet/parquetjs/schema/declare';
 
 export function convertParquetRowGroupToColumns(
   schema: Schema,
-  rowGroup: ParquetBuffer
+  rowGroup: ParquetRowGroup
 ): Record<string, any[]> {
   const columns: Record<string, any[]> = {};
   for (const [columnName, data] of Object.entries(rowGroup.columnData)) {

package/src/lib/arrow/convert-schema-from-parquet.ts

@@ -1,68 +1,57 @@
 // loaders.gl, MIT license
 
+import {Schema, Field, DataType} from '@loaders.gl/schema';
+
 import type {ParquetSchema} from '../../parquetjs/schema/schema';
 import type {FieldDefinition, ParquetField, ParquetType} from '../../parquetjs/schema/declare';
-import {FileMetaData} from '@loaders.gl/parquet/parquetjs/parquet-thrift';
-
-import {
-  Schema,
-  Struct,
-  Field,
-  DataType,
-  Bool,
-  Float64,
-  Int32,
-  Float32,
-  Binary,
-  Utf8,
-  Int64,
-  Uint16,
-  Uint32,
-  Uint64,
-  Int8,
-  Int16
-} from '@loaders.gl/schema';
-
-export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: typeof DataType} = {
-  BOOLEAN: Bool,
-  INT32: Int32,
-  INT64: Float64,
-  INT96: Float64,
-  FLOAT: Float32,
-  DOUBLE: Float64,
-  BYTE_ARRAY: Binary,
-  FIXED_LEN_BYTE_ARRAY: Binary,
-  UTF8: Utf8,
-  DATE: Int32,
-  TIME_MILLIS: Int64,
-  TIME_MICROS: Int64,
-  TIMESTAMP_MILLIS: Int64,
-  TIMESTAMP_MICROS: Int64,
-  UINT_8: Int32,
-  UINT_16: Uint16,
-  UINT_32: Uint32,
-  UINT_64: Uint64,
-  INT_8: Int8,
-  INT_16: Int16,
-  INT_32: Int32,
-  INT_64: Int64,
-  JSON: Binary,
-  BSON: Binary,
-  // TODO check interval type
-  INTERVAL: Binary,
-  DECIMAL_INT32: Float32,
-  DECIMAL_INT64: Float64,
-  DECIMAL_BYTE_ARRAY: Float64,
-  DECIMAL_FIXED_LEN_BYTE_ARRAY: Float64
+import {FileMetaData} from '../../parquetjs/parquet-thrift';
+
+export const PARQUET_TYPE_MAPPING: {[type in ParquetType]: DataType} = {
+  BOOLEAN: 'bool',
+  INT32: 'int32',
+  INT64: 'float64',
+  INT96: 'float64',
+  FLOAT: 'float32',
+  DOUBLE: 'float64',
+  BYTE_ARRAY: 'binary',
+  FIXED_LEN_BYTE_ARRAY: 'binary',
+  UTF8: 'utf8',
+  DATE: 'int32',
+  TIME_MILLIS: 'int64',
+  TIME_MICROS: 'int64',
+  TIMESTAMP_MILLIS: 'int64',
+  TIMESTAMP_MICROS: 'int64',
+  UINT_8: 'int32',
+  UINT_16: 'uint16',
+  UINT_32: 'uint32',
+  UINT_64: 'uint64',
+  INT_8: 'int8',
+  INT_16: 'int16',
+  INT_32: 'int32',
+  INT_64: 'int64',
+  JSON: 'binary',
+  BSON: 'binary',
+  // TODO check interal type
+  INTERVAL: 'binary',
+  DECIMAL_INT32: 'float32',
+  DECIMAL_INT64: 'float64',
+  DECIMAL_BYTE_ARRAY: 'float64',
+  DECIMAL_FIXED_LEN_BYTE_ARRAY: 'float64'
 };
 
-export function convertSchemaFromParquet(
+export function convertParquetSchema(
   parquetSchema: ParquetSchema,
-  parquetMetadata?: FileMetaData
+  parquetMetadata: FileMetaData | null
 ): Schema {
   const fields = getFields(parquetSchema.schema);
   const metadata = parquetMetadata && getSchemaMetadata(parquetMetadata);
-  return new Schema(fields, metadata);
+
+  const schema: Schema = {
+    fields,
+    metadata: metadata || {}
+  };
+
+  return schema;
 }
 
 function getFields(schema: FieldDefinition): Field[] {
@@ -72,13 +61,12 @@ function getFields(schema: FieldDefinition): Field[] {
     const field = schema[name];
 
     if (field.fields) {
-      const childFields = getFields(field.fields);
-      const nestedField = new Field(name, new Struct(childFields), field.optional);
-      fields.push(nestedField);
+      const children = getFields(field.fields);
+      fields.push({name, type: {type: 'struct', children}, nullable: field.optional});
     } else {
-      const FieldType = PARQUET_TYPE_MAPPING[field.type];
+      const type = PARQUET_TYPE_MAPPING[field.type];
       const metadata = getFieldMetadata(field);
-      const arrowField = new Field(name, new FieldType(), field.optional, metadata);
+      const arrowField = {name, type, nullable: field.optional, metadata};
       fields.push(arrowField);
     }
   }
@@ -86,27 +74,29 @@ function getFields(schema: FieldDefinition): Field[] {
   return fields;
 }
 
-function getFieldMetadata(field: ParquetField): Map<string, string> {
-  const metadata = new Map();
+function getFieldMetadata(field: ParquetField): Record<string, string> | undefined {
+  let metadata: Record<string, string> | undefined;
 
   for (const key in field) {
     if (key !== 'name') {
       let value = field[key] || '';
       value = typeof field[key] !== 'string' ? JSON.stringify(field[key]) : field[key];
-      metadata.set(key, value);
+      metadata = metadata || {};
+      metadata[key] = value;
     }
   }
 
   return metadata;
 }
 
-function getSchemaMetadata(parquetMetadata: FileMetaData): Map<string, string> {
-  const metadata = new Map();
+function getSchemaMetadata(parquetMetadata: FileMetaData): Record<string, string> | undefined {
+  let metadata: Record<string, string> | undefined;
 
   const keyValueList = parquetMetadata.key_value_metadata || [];
   for (const {key, value} of keyValueList) {
     if (typeof value === 'string') {
-      metadata.set(key, value);
+      metadata = metadata || {};
+      metadata[key] = value;
     }
   }