@loaders.gl/arrow 4.0.0-alpha.8 → 4.0.0-beta.1

package/src/lib/parse-arrow-in-batches.ts

@@ -1,12 +1,13 @@
 // TODO - this import defeats the sophisticated typescript checking in ArrowJS
-import {RecordBatchReader} from 'apache-arrow';
+import type {ArrowTableBatch} from './arrow-table';
+import {RecordBatchReader, Table as ApacheArrowTable} from 'apache-arrow';
 // import {isIterable} from '@loaders.gl/core';
 
 /**
  */
 export function parseArrowInBatches(
   asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
-): AsyncIterable<any> {
+): AsyncIterable<ArrowTableBatch> {
   // Creates the appropriate RecordBatchReader subclasses from the input
   // This will also close the underlying source in case of early termination or errors
 
@@ -25,25 +26,32 @@ export function parseArrowInBatches(
   }
   */
 
-  async function* makeArrowAsyncIterator() {
+  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {
+    // @ts-ignore
     const readers = RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
-      for await (const batch of reader) {
-        yield processBatch(batch);
+      for await (const recordBatch of reader) {
+        const arrowTabledBatch: ArrowTableBatch = {
+          shape: 'arrow-table',
+          batchType: 'data',
+          data: new ApacheArrowTable([recordBatch]),
+          length: recordBatch.data.length
+        };
+        // processBatch(recordBatch);
+        yield arrowTabledBatch;
       }
       break; // only processing one stream of batches
     }
   }
-  return makeArrowAsyncIterator();
-}
 
-function processBatch(batch) {
-  const values = {
-    metadata: batch.schema.metadata,
-    length: batch.length
-  };
-  batch.schema.fields.forEach(({name}, index) => {
-    values[name] = batch.getChildAt(index).toArray();
-  });
-  return values;
+  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;
 }
+
+// function processBatch(batch: RecordBatch): ArrowTableBatch {
+//   const values = {};
+//   batch.schema.fields.forEach(({name}, index) => {
+//     values[name] = batch.getChildAt(index)?.toArray();
+//   });
+//   return {
+//   };
+// }

package/src/lib/parse-arrow-sync.ts

@@ -1,46 +1,32 @@
-import type {ArrowLoaderOptions} from '../arrow-loader';
+import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
+import type {ArrowTable} from './arrow-table';
+import {convertTable} from '@loaders.gl/schema';
 import {tableFromIPC} from 'apache-arrow';
+import type {ArrowLoaderOptions} from '../arrow-loader';
+import {convertApacheArrowToArrowTable, convertArrowToColumnarTable} from './convert-table';
 
 // Parses arrow to a columnar table
-export default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {
-  const arrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
-
-  // Extract columns
-
-  // TODO - avoid calling `getColumn` on columns we are not interested in?
-  // Add options object?
-  const columnarTable = {};
+export default function parseArrowSync(
+  arrayBuffer,
+  options?: ArrowLoaderOptions
+): ArrowTable | ColumnarTable | ObjectRowTable {
+  const apacheArrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
+  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);
 
-  for (const field of arrowTable.schema.fields) {
-    // This (is intended to) coalesce all record batches into a single typed array
-    const arrowColumn = arrowTable.getChild(field.name);
-    const values = arrowColumn?.toArray();
-    columnarTable[field.name] = values;
-  }
-
-  switch (options?.arrow?.shape) {
+  const shape = options?.arrow?.shape || 'arrow-table';
+  switch (shape) {
     case 'arrow-table':
       return arrowTable;
-    case 'object-row-table':
-      return convertColumnarToRowFormatTable(columnarTable);
+
     case 'columnar-table':
-    default:
-      return columnarTable;
-  }
-}
+      return convertArrowToColumnarTable(arrowTable);
 
-function convertColumnarToRowFormatTable(columnarTable) {
-  const tableKeys = Object.keys(columnarTable);
-  const tableRowsCount = columnarTable[tableKeys[0]].length;
-  const rowFormatTable: {}[] = [];
+    case 'object-row-table':
+      const columnarTable = convertArrowToColumnarTable(arrowTable);
+      return convertTable(columnarTable, 'object-row-table');
 
-  for (let index = 0; index < tableRowsCount; index++) {
-    const tableItem = {};
-    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      const fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-    rowFormatTable.push(tableItem);
+    default:
+      // TODO
+      throw new Error(shape);
   }
-  return rowFormatTable;
 }

package/dist/arrow-loader.js

@@ -1,29 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports._typecheckArrowLoader = exports.ArrowLoader = void 0;
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-/** ArrowJS table loader */
-exports.ArrowLoader = {
-    name: 'Apache Arrow',
-    id: 'arrow',
-    module: 'arrow',
-    version: VERSION,
-    // worker: true,
-    category: 'table',
-    extensions: ['arrow', 'feather'],
-    mimeTypes: [
-        'application/vnd.apache.arrow.file',
-        'application/vnd.apache.arrow.stream',
-        'application/octet-stream'
-    ],
-    binary: true,
-    tests: ['ARROW'],
-    options: {
-        arrow: {
-            shape: 'columnar-table'
-        }
-    }
-};
-exports._typecheckArrowLoader = exports.ArrowLoader;

package/dist/arrow-writer.js

@@ -1,25 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ArrowWriter = void 0;
-const encode_arrow_1 = require("./lib/encode-arrow");
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-/** Apache Arrow writer */
-exports.ArrowWriter = {
-    name: 'Apache Arrow',
-    id: 'arrow',
-    module: 'arrow',
-    version: VERSION,
-    extensions: ['arrow', 'feather'],
-    mimeTypes: [
-        'application/vnd.apache.arrow.file',
-        'application/vnd.apache.arrow.stream',
-        'application/octet-stream'
-    ],
-    encodeSync(data, options) {
-        return (0, encode_arrow_1.encodeArrowSync)(data);
-    },
-    binary: true,
-    options: {}
-};

package/dist/bundle.js

@@ -1,5 +0,0 @@
-"use strict";
-// @ts-nocheck
-const moduleExports = require('./index');
-globalThis.loaders = globalThis.loaders || {};
-module.exports = Object.assign(globalThis.loaders, moduleExports);

package/dist/index.js

@@ -1,28 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports._typecheckArrowLoader = exports.ArrowLoader = exports.ArrowWorkerLoader = exports.ArrowWriter = exports.VECTOR_TYPES = void 0;
-const arrow_loader_1 = require("./arrow-loader");
-Object.defineProperty(exports, "ArrowWorkerLoader", { enumerable: true, get: function () { return arrow_loader_1.ArrowLoader; } });
-const parse_arrow_sync_1 = __importDefault(require("./lib/parse-arrow-sync"));
-const parse_arrow_in_batches_1 = require("./lib/parse-arrow-in-batches");
-const schema_1 = require("@loaders.gl/schema");
-const arrow_table_batch_1 = __importDefault(require("./lib/arrow-table-batch"));
-// Make the ArrowBatch type available
-schema_1.TableBatchBuilder.ArrowBatch = arrow_table_batch_1.default;
-// Types
-var types_1 = require("./types");
-Object.defineProperty(exports, "VECTOR_TYPES", { enumerable: true, get: function () { return types_1.VECTOR_TYPES; } });
-// Arrow writer
-var arrow_writer_1 = require("./arrow-writer");
-Object.defineProperty(exports, "ArrowWriter", { enumerable: true, get: function () { return arrow_writer_1.ArrowWriter; } });
-/** ArrowJS table loader */
-exports.ArrowLoader = {
-    ...arrow_loader_1.ArrowLoader,
-    parse: async (arraybuffer, options) => (0, parse_arrow_sync_1.default)(arraybuffer, options),
-    parseSync: parse_arrow_sync_1.default,
-    parseInBatches: parse_arrow_in_batches_1.parseArrowInBatches
-};
-exports._typecheckArrowLoader = exports.ArrowLoader;

package/dist/lib/arrow-table-batch.js

@@ -1,65 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const apache_arrow_1 = require("apache-arrow");
-const schema_1 = require("@loaders.gl/schema");
-class ArrowTableBatchAggregator extends schema_1.ColumnarTableBatchAggregator {
-    constructor(schema, options) {
-        super(schema, options);
-        this.arrowSchema = null;
-    }
-    getBatch() {
-        const batch = super.getBatch();
-        if (batch) {
-            // Get the arrow schema
-            this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
-            // Get arrow format vectors
-            const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
-            // Create the record batch
-            const recordBatch = new apache_arrow_1.RecordBatch(this.arrowSchema, (0, apache_arrow_1.makeData)({
-                type: new apache_arrow_1.Struct(this.arrowSchema.fields),
-                children: arrowVectors.map(({ data }) => data[0])
-            }));
-            return {
-                shape: 'arrow-table',
-                batchType: 'data',
-                data: recordBatch,
-                length: batch.length
-            };
-        }
-        return null;
-    }
-}
-exports.default = ArrowTableBatchAggregator;
-// Convert from a simple loaders.gl schema to an Arrow schema
-function getArrowSchema(schema) {
-    const arrowFields = [];
-    for (const key in schema) {
-        const field = schema[key];
-        if (field.type === Float32Array) {
-            // TODO - just store the original field as metadata?
-            const metadata = new Map(); // field;
-            // arrow: new Field(name, nullable, metadata)
-            const arrowField = new apache_arrow_1.Field(field.name, new apache_arrow_1.Float32(), field.nullable, metadata);
-            arrowFields.push(arrowField);
-        }
-    }
-    if (arrowFields.length === 0) {
-        throw new Error('No arrow convertible fields');
-    }
-    return new apache_arrow_1.Schema(arrowFields);
-}
-// Convert from simple loaders.gl arrays to arrow vectors
-function getArrowVectors(arrowSchema, data) {
-    const arrowVectors = [];
-    for (const field of arrowSchema.fields) {
-        const vector = data[field.name];
-        if (vector instanceof Float32Array) {
-            const arrowVector = (0, apache_arrow_1.makeVector)(vector);
-            arrowVectors.push(arrowVector);
-        }
-    }
-    if (arrowSchema.fields.length !== arrowVectors.length) {
-        throw new Error('Some columns not arrow convertible');
-    }
-    return arrowVectors;
-}

package/dist/lib/encode-arrow.js

@@ -1,38 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.encodeArrowSync = void 0;
-const apache_arrow_1 = require("apache-arrow");
-const types_1 = require("../types");
-/**
- * Encodes set of arrays into the Apache Arrow columnar format
- * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
- * @param data - columns data
- * @param options - the writer options
- * @returns - encoded ArrayBuffer
- */
-function encodeArrowSync(data) {
-    const vectors = {};
-    for (const arrayData of data) {
-        const arrayVector = createVector(arrayData.array, arrayData.type);
-        vectors[arrayData.name] = arrayVector;
-    }
-    const table = new apache_arrow_1.Table(vectors);
-    const arrowBuffer = (0, apache_arrow_1.tableToIPC)(table);
-    return arrowBuffer;
-}
-exports.encodeArrowSync = encodeArrowSync;
-/**
- * Create Arrow Vector from given data and vector type
- * @param array {import('../types').AnyArrayType} - columns data
- * @param type {number} - the writer options
- * @return a vector of one of vector's types defined in the Apache Arrow library
- */
-function createVector(array, type) {
-    switch (type) {
-        case types_1.VECTOR_TYPES.DATE:
-            return (0, apache_arrow_1.vectorFromArray)(array);
-        case types_1.VECTOR_TYPES.FLOAT:
-        default:
-            return (0, apache_arrow_1.vectorFromArray)(array);
-    }
-}

package/dist/lib/parse-arrow-in-batches.js

@@ -1,47 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseArrowInBatches = void 0;
-// TODO - this import defeats the sophisticated typescript checking in ArrowJS
-const apache_arrow_1 = require("apache-arrow");
-// import {isIterable} from '@loaders.gl/core';
-/**
- */
-function parseArrowInBatches(asyncIterator) {
-    // Creates the appropriate RecordBatchReader subclasses from the input
-    // This will also close the underlying source in case of early termination or errors
-    // As an optimization, return a non-async iterator
-    /*
-    if (isIterable(readers)) {
-      function* makeArrowIterator() {
-        for (const reader of readers) {
-          for (const batch of reader) {
-            yield processBatch(batch, reader);
-          }
-          break; // only processing one stream of batches
-        }
-      }
-      const arrowIterator = makeArrowIterator();
-    }
-    */
-    async function* makeArrowAsyncIterator() {
-        const readers = apache_arrow_1.RecordBatchReader.readAll(asyncIterator);
-        for await (const reader of readers) {
-            for await (const batch of reader) {
-                yield processBatch(batch);
-            }
-            break; // only processing one stream of batches
-        }
-    }
-    return makeArrowAsyncIterator();
-}
-exports.parseArrowInBatches = parseArrowInBatches;
-function processBatch(batch) {
-    const values = {
-        metadata: batch.schema.metadata,
-        length: batch.length
-    };
-    batch.schema.fields.forEach(({ name }, index) => {
-        values[name] = batch.getChildAt(index).toArray();
-    });
-    return values;
-}

package/dist/lib/parse-arrow-sync.js

@@ -1,41 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const apache_arrow_1 = require("apache-arrow");
-// Parses arrow to a columnar table
-function parseArrowSync(arrayBuffer, options) {
-    const arrowTable = (0, apache_arrow_1.tableFromIPC)([new Uint8Array(arrayBuffer)]);
-    // Extract columns
-    // TODO - avoid calling `getColumn` on columns we are not interested in?
-    // Add options object?
-    const columnarTable = {};
-    for (const field of arrowTable.schema.fields) {
-        // This (is intended to) coalesce all record batches into a single typed array
-        const arrowColumn = arrowTable.getChild(field.name);
-        const values = arrowColumn?.toArray();
-        columnarTable[field.name] = values;
-    }
-    switch (options?.arrow?.shape) {
-        case 'arrow-table':
-            return arrowTable;
-        case 'object-row-table':
-            return convertColumnarToRowFormatTable(columnarTable);
-        case 'columnar-table':
-        default:
-            return columnarTable;
-    }
-}
-exports.default = parseArrowSync;
-function convertColumnarToRowFormatTable(columnarTable) {
-    const tableKeys = Object.keys(columnarTable);
-    const tableRowsCount = columnarTable[tableKeys[0]].length;
-    const rowFormatTable = [];
-    for (let index = 0; index < tableRowsCount; index++) {
-        const tableItem = {};
-        for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-            const fieldName = tableKeys[keyIndex];
-            tableItem[fieldName] = columnarTable[fieldName][index];
-        }
-        rowFormatTable.push(tableItem);
-    }
-    return rowFormatTable;
-}

package/dist/types.js

@@ -1,8 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.VECTOR_TYPES = void 0;
-var VECTOR_TYPES;
-(function (VECTOR_TYPES) {
-    VECTOR_TYPES[VECTOR_TYPES["FLOAT"] = 0] = "FLOAT";
-    VECTOR_TYPES[VECTOR_TYPES["DATE"] = 1] = "DATE";
-})(VECTOR_TYPES = exports.VECTOR_TYPES || (exports.VECTOR_TYPES = {}));

package/dist/workers/arrow-worker.js

@@ -1,5 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const index_1 = require("../index");
-(0, loader_utils_1.createLoaderWorker)(index_1.ArrowLoader);