@loaders.gl/arrow 3.4.11 → 3.4.12

package/dist/arrow-worker.js

@@ -14956,7 +14956,7 @@ return true;`);
   }
 
   // src/arrow-loader.ts
-  var VERSION = true ? "3.4.11" : "latest";
+  var VERSION = true ? "3.4.12" : "latest";
   var DEFAULT_ARROW_LOADER_OPTIONS = {
     arrow: {
       shape: "columnar-table"

package/dist/es5/arrow-loader.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports._typecheckArrowLoader = exports.ArrowLoader = void 0;
-var VERSION = typeof "3.4.11" !== 'undefined' ? "3.4.11" : 'latest';
+var VERSION = typeof "3.4.12" !== 'undefined' ? "3.4.12" : 'latest';
 var DEFAULT_ARROW_LOADER_OPTIONS = {
   arrow: {
     shape: 'columnar-table'

package/dist/es5/arrow-writer.js

@@ -5,7 +5,7 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.ArrowWriter = void 0;
 var _encodeArrow = require("./lib/encode-arrow");
-var VERSION = typeof "3.4.11" !== 'undefined' ? "3.4.11" : 'latest';
+var VERSION = typeof "3.4.12" !== 'undefined' ? "3.4.12" : 'latest';
 var ArrowWriter = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/esm/arrow-loader.js

@@ -1,4 +1,4 @@
-const VERSION = typeof "3.4.11" !== 'undefined' ? "3.4.11" : 'latest';
+const VERSION = typeof "3.4.12" !== 'undefined' ? "3.4.12" : 'latest';
 const DEFAULT_ARROW_LOADER_OPTIONS = {
   arrow: {
     shape: 'columnar-table'

package/dist/esm/arrow-writer.js

@@ -1,5 +1,5 @@
 import { encodeArrowSync } from './lib/encode-arrow';
-const VERSION = typeof "3.4.11" !== 'undefined' ? "3.4.11" : 'latest';
+const VERSION = typeof "3.4.12" !== 'undefined' ? "3.4.12" : 'latest';
 export const ArrowWriter = {
   name: 'Apache Arrow',
   id: 'arrow',

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/arrow",
-  "version": "3.4.11",
+  "version": "3.4.12",
   "description": "Simple columnar table loader for the Apache Arrow format",
   "license": "MIT",
   "publishConfig": {
@@ -36,9 +36,9 @@
     "build-worker": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "3.4.11",
-    "@loaders.gl/schema": "3.4.11",
+    "@loaders.gl/loader-utils": "3.4.12",
+    "@loaders.gl/schema": "3.4.12",
     "apache-arrow": "^4.0.0"
   },
-  "gitHead": "2b8d490d2372e70dd89eef8f6b215a5205c16621"
+  "gitHead": "85254762ae3082f608a863291deef09f2b40c6cf"
 }

package/dist/arrow-loader.js

@@ -1,30 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports._typecheckArrowLoader = exports.ArrowLoader = void 0;
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-const DEFAULT_ARROW_LOADER_OPTIONS = {
-    arrow: {
-        shape: 'columnar-table'
-    }
-};
-/** ArrowJS table loader */
-exports.ArrowLoader = {
-    name: 'Apache Arrow',
-    id: 'arrow',
-    module: 'arrow',
-    version: VERSION,
-    worker: true,
-    category: 'table',
-    extensions: ['arrow', 'feather'],
-    mimeTypes: [
-        'application/vnd.apache.arrow.file',
-        'application/vnd.apache.arrow.stream',
-        'application/octet-stream'
-    ],
-    binary: true,
-    tests: ['ARROW'],
-    options: DEFAULT_ARROW_LOADER_OPTIONS
-};
-exports._typecheckArrowLoader = exports.ArrowLoader;

package/dist/arrow-writer.js

@@ -1,26 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ArrowWriter = void 0;
-const encode_arrow_1 = require("./lib/encode-arrow");
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-/** Apache Arrow writer */
-exports.ArrowWriter = {
-    name: 'Apache Arrow',
-    id: 'arrow',
-    module: 'arrow',
-    version: VERSION,
-    extensions: ['arrow', 'feather'],
-    mimeTypes: [
-        'application/vnd.apache.arrow.file',
-        'application/vnd.apache.arrow.stream',
-        'application/octet-stream'
-    ],
-    encodeSync,
-    binary: true,
-    options: {}
-};
-function encodeSync(data, options) {
-    return (0, encode_arrow_1.encodeArrowSync)(data);
-}

package/dist/bundle.js

@@ -1,5 +0,0 @@
-"use strict";
-// @ts-nocheck
-const moduleExports = require('./index');
-globalThis.loaders = globalThis.loaders || {};
-module.exports = Object.assign(globalThis.loaders, moduleExports);

package/dist/index.js

@@ -1,28 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports._typecheckArrowLoader = exports.ArrowLoader = exports.ArrowWorkerLoader = exports.ArrowWriter = exports.VECTOR_TYPES = void 0;
-const arrow_loader_1 = require("./arrow-loader");
-Object.defineProperty(exports, "ArrowWorkerLoader", { enumerable: true, get: function () { return arrow_loader_1.ArrowLoader; } });
-const parse_arrow_sync_1 = __importDefault(require("./lib/parse-arrow-sync"));
-const parse_arrow_in_batches_1 = require("./lib/parse-arrow-in-batches");
-const schema_1 = require("@loaders.gl/schema");
-const arrow_table_batch_1 = __importDefault(require("./lib/arrow-table-batch"));
-// Make the ArrowBatch type available
-schema_1.TableBatchBuilder.ArrowBatch = arrow_table_batch_1.default;
-// Types
-var types_1 = require("./types");
-Object.defineProperty(exports, "VECTOR_TYPES", { enumerable: true, get: function () { return types_1.VECTOR_TYPES; } });
-// Arrow writer
-var arrow_writer_1 = require("./arrow-writer");
-Object.defineProperty(exports, "ArrowWriter", { enumerable: true, get: function () { return arrow_writer_1.ArrowWriter; } });
-/** ArrowJS table loader */
-exports.ArrowLoader = {
-    ...arrow_loader_1.ArrowLoader,
-    parse: async (arraybuffer, options) => (0, parse_arrow_sync_1.default)(arraybuffer, options),
-    parseSync: parse_arrow_sync_1.default,
-    parseInBatches: parse_arrow_in_batches_1.parseArrowInBatches
-};
-exports._typecheckArrowLoader = exports.ArrowLoader;

package/dist/lib/arrow-table-batch.js

@@ -1,62 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const apache_arrow_1 = require("apache-arrow");
-const schema_1 = require("@loaders.gl/schema");
-class ArrowTableBatchAggregator extends schema_1.ColumnarTableBatchAggregator {
-    constructor(schema, options) {
-        super(schema, options);
-        this.arrowSchema = null;
-    }
-    getBatch() {
-        const batch = super.getBatch();
-        if (batch) {
-            // Get the arrow schema
-            this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
-            // Get arrow format vectors
-            const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
-            // Create the record batch
-            // new RecordBatch(schema, numRows, vectors, ...);
-            const recordBatch = new apache_arrow_1.RecordBatch(this.arrowSchema, batch.length, arrowVectors);
-            return {
-                shape: 'arrow-table',
-                batchType: 'data',
-                data: recordBatch,
-                length: batch.length
-            };
-        }
-        return null;
-    }
-}
-exports.default = ArrowTableBatchAggregator;
-// Convert from a simple loaders.gl schema to an Arrow schema
-function getArrowSchema(schema) {
-    const arrowFields = [];
-    for (const key in schema) {
-        const field = schema[key];
-        if (field.type === Float32Array) {
-            const metadata = field; // just store the original field as metadata
-            // arrow: new Field(name, nullable, metadata)
-            const arrowField = new apache_arrow_1.Field(field.name, new apache_arrow_1.Float32(), field.nullable, metadata);
-            arrowFields.push(arrowField);
-        }
-    }
-    if (arrowFields.length === 0) {
-        throw new Error('No arrow convertible fields');
-    }
-    return new apache_arrow_1.Schema(arrowFields);
-}
-// Convert from simple loaders.gl arrays to arrow vectors
-function getArrowVectors(arrowSchema, data) {
-    const arrowVectors = [];
-    for (const field of arrowSchema.fields) {
-        const vector = data[field.name];
-        if (vector instanceof Float32Array) {
-            const arrowVector = apache_arrow_1.Float32Vector.from(vector);
-            arrowVectors.push(arrowVector);
-        }
-    }
-    if (arrowSchema.fields.length !== arrowVectors.length) {
-        throw new Error('Some columns not arrow convertible');
-    }
-    return arrowVectors;
-}

package/dist/lib/encode-arrow.js

@@ -1,40 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.encodeArrowSync = void 0;
-const apache_arrow_1 = require("apache-arrow");
-const types_1 = require("../types");
-/**
- * Encodes set of arrays into the Apache Arrow columnar format
- * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
- * @param data - columns data
- * @param options - the writer options
- * @returns - encoded ArrayBuffer
- */
-function encodeArrowSync(data) {
-    const vectors = [];
-    const arrayNames = [];
-    for (const arrayData of data) {
-        arrayNames.push(arrayData.name);
-        const arrayVector = createVector(arrayData.array, arrayData.type);
-        vectors.push(arrayVector);
-    }
-    const table = apache_arrow_1.Table.new(vectors, arrayNames);
-    const arrowBuffer = table.serialize();
-    return arrowBuffer;
-}
-exports.encodeArrowSync = encodeArrowSync;
-/**
- * Create Arrow Vector from given data and vector type
- * @param array {import('../types').AnyArrayType} - columns data
- * @param type {number} - the writer options
- * @return a vector of one of vector's types defined in the Apache Arrow library
- */
-function createVector(array, type) {
-    switch (type) {
-        case types_1.VECTOR_TYPES.DATE:
-            return apache_arrow_1.DateVector.from(array);
-        case types_1.VECTOR_TYPES.FLOAT:
-        default:
-            return apache_arrow_1.FloatVector.from(array);
-    }
-}

package/dist/lib/parse-arrow-in-batches.js

@@ -1,48 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.parseArrowInBatches = void 0;
-// TODO - this import defeats the sophisticated typescript checking in ArrowJS
-const apache_arrow_1 = require("apache-arrow");
-// import {isIterable} from '@loaders.gl/core';
-/**
- */
-function parseArrowInBatches(asyncIterator) {
-    // Creates the appropriate RecordBatchReader subclasses from the input
-    // This will also close the underlying source in case of early termination or errors
-    // As an optimization, return a non-async iterator
-    /*
-    if (isIterable(readers)) {
-      function* makeArrowIterator() {
-        for (const reader of readers) {
-          for (const batch of reader) {
-            yield processBatch(batch, reader);
-          }
-          break; // only processing one stream of batches
-        }
-      }
-      const arrowIterator = makeArrowIterator();
-    }
-    */
-    async function* makeArrowAsyncIterator() {
-        // @ts-expect-error Fixed in arrow 9
-        const readers = apache_arrow_1.RecordBatchReader.readAll(asyncIterator);
-        for await (const reader of readers) {
-            for await (const batch of reader) {
-                yield processBatch(batch);
-            }
-            break; // only processing one stream of batches
-        }
-    }
-    return makeArrowAsyncIterator();
-}
-exports.parseArrowInBatches = parseArrowInBatches;
-function processBatch(batch) {
-    const values = {
-        metadata: batch.schema.metadata,
-        length: batch.length
-    };
-    batch.schema.fields.forEach(({ name }, index) => {
-        values[name] = batch.getChildAt(index).toArray();
-    });
-    return values;
-}

package/dist/lib/parse-arrow-sync.js

@@ -1,41 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const apache_arrow_1 = require("apache-arrow");
-// Parses arrow to a columnar table
-function parseArrowSync(arrayBuffer, options) {
-    const arrowTable = apache_arrow_1.Table.from([new Uint8Array(arrayBuffer)]);
-    // Extract columns
-    // TODO - avoid calling `getColumn` on columns we are not interested in?
-    // Add options object?
-    const columnarTable = {};
-    arrowTable.schema.fields.forEach((field) => {
-        // This (is intended to) coalesce all record batches into a single typed array
-        const arrowColumn = arrowTable.getColumn(field.name);
-        const values = arrowColumn.toArray();
-        columnarTable[field.name] = values;
-    });
-    switch (options?.arrow?.shape) {
-        case 'arrow-table':
-            return arrowTable;
-        case 'object-row-table':
-            return convertColumnarToRowFormatTable(columnarTable);
-        case 'columnar-table':
-        default:
-            return columnarTable;
-    }
-}
-exports.default = parseArrowSync;
-function convertColumnarToRowFormatTable(columnarTable) {
-    const tableKeys = Object.keys(columnarTable);
-    const tableRowsCount = columnarTable[tableKeys[0]].length;
-    const rowFormatTable = [];
-    for (let index = 0; index < tableRowsCount; index++) {
-        const tableItem = {};
-        for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-            const fieldName = tableKeys[keyIndex];
-            tableItem[fieldName] = columnarTable[fieldName][index];
-        }
-        rowFormatTable.push(tableItem);
-    }
-    return rowFormatTable;
-}

package/dist/types.js

@@ -1,8 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.VECTOR_TYPES = void 0;
-var VECTOR_TYPES;
-(function (VECTOR_TYPES) {
-    VECTOR_TYPES[VECTOR_TYPES["FLOAT"] = 0] = "FLOAT";
-    VECTOR_TYPES[VECTOR_TYPES["DATE"] = 1] = "DATE";
-})(VECTOR_TYPES = exports.VECTOR_TYPES || (exports.VECTOR_TYPES = {}));

package/dist/workers/arrow-worker.js

@@ -1,5 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const index_1 = require("../index");
-(0, loader_utils_1.createLoaderWorker)(index_1.ArrowLoader);