@loaders.gl/arrow 4.0.0-alpha.4 → 4.0.0-alpha.6

package/dist/esm/arrow-writer.js

@@ -0,0 +1,16 @@
+import { encodeArrowSync } from './lib/encode-arrow';
+const VERSION = typeof "4.0.0-alpha.6" !== 'undefined' ? "4.0.0-alpha.6" : 'latest';
+export const ArrowWriter = {
+  name: 'Apache Arrow',
+  id: 'arrow',
+  module: 'arrow',
+  version: VERSION,
+  extensions: ['arrow', 'feather'],
+  mimeTypes: ['application/vnd.apache.arrow.file', 'application/vnd.apache.arrow.stream', 'application/octet-stream'],
+  encodeSync(data, options) {
+    return encodeArrowSync(data);
+  },
+  binary: true,
+  options: {}
+};
+//# sourceMappingURL=arrow-writer.js.map

package/dist/esm/arrow-writer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-writer.js","names":["encodeArrowSync","VERSION","ArrowWriter","name","id","module","version","extensions","mimeTypes","encodeSync","data","options","binary"],"sources":["../../src/arrow-writer.ts"],"sourcesContent":["// import type {} from '@loaders.gl/loader-utils';\nimport type {Writer, WriterOptions} from '@loaders.gl/loader-utils';\nimport {ColumnarTable} from './lib/encode-arrow';\nimport {encodeArrowSync} from './lib/encode-arrow';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\ntype ArrowWriterOptions = WriterOptions & {\n  arrow?: {};\n};\n\n/** Apache Arrow writer */\nexport const ArrowWriter: Writer<ColumnarTable, never, ArrowWriterOptions> = {\n  name: 'Apache Arrow',\n  id: 'arrow',\n  module: 'arrow',\n  version: VERSION,\n  extensions: ['arrow', 'feather'],\n  mimeTypes: [\n    'application/vnd.apache.arrow.file',\n    'application/vnd.apache.arrow.stream',\n    'application/octet-stream'\n  ],\n  encodeSync(data, options?) {\n    return encodeArrowSync(data);\n  },\n  binary: true,\n  options: {}\n};\n"],"mappings":"AAGA,SAAQA,eAAe,QAAO,oBAAoB;AAIlD,MAAMC,OAAO,GAAG,sBAAkB,KAAK,WAAW,qBAAiB,QAAQ;AAO3E,OAAO,MAAMC,WAA6D,GAAG;EAC3EC,IAAI,EAAE,cAAc;EACpBC,EAAE,EAAE,OAAO;EACXC,MAAM,EAAE,OAAO;EACfC,OAAO,EAAEL,OAAO;EAChBM,UAAU,EAAE,CAAC,OAAO,EAAE,SAAS,CAAC;EAChCC,SAAS,EAAE,CACT,mCAAmC,EACnC,qCAAqC,EACrC,0BAA0B,CAC3B;EACDC,UAAUA,CAACC,IAAI,EAAEC,OAAQ,EAAE;IACzB,OAAOX,eAAe,CAACU,IAAI,CAAC;EAC9B,CAAC;EACDE,MAAM,EAAE,IAAI;EACZD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/esm/bundle.js

@@ -0,0 +1,4 @@
+const moduleExports = require('./index');
+globalThis.loaders = globalThis.loaders || {};
+module.exports = Object.assign(globalThis.loaders, moduleExports);
+//# sourceMappingURL=bundle.js.map

package/dist/esm/bundle.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"bundle.js","names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"sources":["../../src/bundle.ts"],"sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAS,CAAC;AACxCC,UAAU,CAACC,OAAO,GAAGD,UAAU,CAACC,OAAO,IAAI,CAAC,CAAC;AAC7CC,MAAM,CAACC,OAAO,GAAGC,MAAM,CAACC,MAAM,CAACL,UAAU,CAACC,OAAO,EAAEH,aAAa,CAAC"}

package/dist/esm/index.js

@@ -0,0 +1,17 @@
+import { ArrowLoader as ArrowWorkerLoader } from './arrow-loader';
+import parseSync from './lib/parse-arrow-sync';
+import { parseArrowInBatches } from './lib/parse-arrow-in-batches';
+import { TableBatchBuilder } from '@loaders.gl/schema';
+import ArrowTableBatchAggregator from './lib/arrow-table-batch';
+TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
+export { VECTOR_TYPES } from './types';
+export { ArrowWriter } from './arrow-writer';
+export { ArrowWorkerLoader };
+export const ArrowLoader = {
+  ...ArrowWorkerLoader,
+  parse: async (arraybuffer, options) => parseSync(arraybuffer, options),
+  parseSync,
+  parseInBatches: parseArrowInBatches
+};
+export const _typecheckArrowLoader = ArrowLoader;
+//# sourceMappingURL=index.js.map

package/dist/esm/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","names":["ArrowLoader","ArrowWorkerLoader","parseSync","parseArrowInBatches","TableBatchBuilder","ArrowTableBatchAggregator","ArrowBatch","VECTOR_TYPES","ArrowWriter","parse","arraybuffer","options","parseInBatches","_typecheckArrowLoader"],"sources":["../../src/index.ts"],"sourcesContent":["import type {LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {ArrowLoaderOptions} from './arrow-loader';\nimport {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';\nimport parseSync from './lib/parse-arrow-sync';\nimport {parseArrowInBatches} from './lib/parse-arrow-in-batches';\n\nimport {TableBatchBuilder} from '@loaders.gl/schema';\nimport ArrowTableBatchAggregator from './lib/arrow-table-batch';\n\n// Make the ArrowBatch type available\nTableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;\n\n// Types\nexport {VECTOR_TYPES} from './types';\n\n// Arrow writer\n\nexport {ArrowWriter} from './arrow-writer';\n\n// Arrow loader\n\nexport type {ArrowLoaderOptions};\nexport {ArrowWorkerLoader};\n\n/** ArrowJS table loader */\nexport const ArrowLoader: LoaderWithParser = {\n  ...ArrowWorkerLoader,\n  parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>\n    parseSync(arraybuffer, options),\n  parseSync,\n  parseInBatches: parseArrowInBatches\n};\n\nexport const _typecheckArrowLoader: LoaderWithParser = ArrowLoader;\n"],"mappings":"AAEA,SAAQA,WAAW,IAAIC,iBAAiB,QAAO,gBAAgB;AAC/D,OAAOC,SAAS,MAAM,wBAAwB;AAC9C,SAAQC,mBAAmB,QAAO,8BAA8B;AAEhE,SAAQC,iBAAiB,QAAO,oBAAoB;AACpD,OAAOC,yBAAyB,MAAM,yBAAyB;AAG/DD,iBAAiB,CAACE,UAAU,GAAGD,yBAAyB;AAGxD,SAAQE,YAAY,QAAO,SAAS;AAIpC,SAAQC,WAAW,QAAO,gBAAgB;AAK1C,SAAQP,iBAAiB;AAGzB,OAAO,MAAMD,WAA6B,GAAG;EAC3C,GAAGC,iBAAiB;EACpBQ,KAAK,EAAE,MAAAA,CAAOC,WAAwB,EAAEC,OAA4B,KAClET,SAAS,CAACQ,WAAW,EAAEC,OAAO,CAAC;EACjCT,SAAS;EACTU,cAAc,EAAET;AAClB,CAAC;AAED,OAAO,MAAMU,qBAAuC,GAAGb,WAAW"}

package/dist/esm/lib/arrow-table-batch.js

@@ -0,0 +1,63 @@
+import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
+import { Schema, Field, RecordBatch, Struct, makeVector, makeData, Float32 } from 'apache-arrow';
+import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
+export default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+  constructor(schema, options) {
+    super(schema, options);
+    _defineProperty(this, "arrowSchema", void 0);
+    this.arrowSchema = null;
+  }
+  getBatch() {
+    const batch = super.getBatch();
+    if (batch) {
+      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
+      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
+      const recordBatch = new RecordBatch(this.arrowSchema, makeData({
+        type: new Struct(this.arrowSchema.fields),
+        children: arrowVectors.map(_ref => {
+          let {
+            data
+          } = _ref;
+          return data[0];
+        })
+      }));
+      return {
+        shape: 'arrow-table',
+        batchType: 'data',
+        data: recordBatch,
+        length: batch.length
+      };
+    }
+    return null;
+  }
+}
+function getArrowSchema(schema) {
+  const arrowFields = [];
+  for (const key in schema) {
+    const field = schema[key];
+    if (field.type === Float32Array) {
+      const metadata = new Map();
+      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);
+      arrowFields.push(arrowField);
+    }
+  }
+  if (arrowFields.length === 0) {
+    throw new Error('No arrow convertible fields');
+  }
+  return new Schema(arrowFields);
+}
+function getArrowVectors(arrowSchema, data) {
+  const arrowVectors = [];
+  for (const field of arrowSchema.fields) {
+    const vector = data[field.name];
+    if (vector instanceof Float32Array) {
+      const arrowVector = makeVector(vector);
+      arrowVectors.push(arrowVector);
+    }
+  }
+  if (arrowSchema.fields.length !== arrowVectors.length) {
+    throw new Error('Some columns not arrow convertible');
+  }
+  return arrowVectors;
+}
+//# sourceMappingURL=arrow-table-batch.js.map

package/dist/esm/lib/arrow-table-batch.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-table-batch.js","names":["Schema","Field","RecordBatch","Struct","makeVector","makeData","Float32","ColumnarTableBatchAggregator","ArrowTableBatchAggregator","constructor","schema","options","_defineProperty","arrowSchema","getBatch","batch","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","type","fields","children","map","_ref","shape","batchType","length","arrowFields","key","field","Float32Array","metadata","Map","arrowField","name","nullable","push","Error","vector","arrowVector"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {\n  Schema,\n  Field,\n  RecordBatch,\n  Struct,\n  makeVector,\n  makeData,\n  Vector,\n  Float32\n} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n\n      // Create the record batch\n      const recordBatch = new RecordBatch(\n        this.arrowSchema,\n        makeData({\n          type: new Struct(this.arrowSchema.fields),\n          children: arrowVectors.map(({data}) => data[0])\n        })\n      );\n\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: recordBatch,\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema): Schema {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      // TODO - just store the original field as metadata?\n      const metadata = new Map(); // field;\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data): Vector[] {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = makeVector(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"mappings":";AACA,SACEA,MAAM,EACNC,KAAK,EACLC,WAAW,EACXC,MAAM,EACNC,UAAU,EACVC,QAAQ,EAERC,OAAO,QACF,cAAc;AACrB,SAAQC,4BAA4B,QAAO,oBAAoB;AAE/D,eAAe,MAAMC,yBAAyB,SAASD,4BAA4B,CAAC;EAGlFE,WAAWA,CAACC,MAAM,EAAEC,OAAO,EAAE;IAC3B,KAAK,CAACD,MAAM,EAAEC,OAAO,CAAC;IAACC,eAAA;IACvB,IAAI,CAACC,WAAW,GAAG,IAAI;EACzB;EAEAC,QAAQA,CAAA,EAA2B;IACjC,MAAMC,KAAK,GAAG,KAAK,CAACD,QAAQ,CAAC,CAAC;IAC9B,IAAIC,KAAK,EAAE;MAET,IAAI,CAACF,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIG,cAAc,CAACD,KAAK,CAACL,MAAM,CAAC;MAGnE,MAAMO,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACL,WAAW,EAAEE,KAAK,CAACI,IAAI,CAAC;MAGlE,MAAMC,WAAW,GAAG,IAAIlB,WAAW,CACjC,IAAI,CAACW,WAAW,EAChBR,QAAQ,CAAC;QACPgB,IAAI,EAAE,IAAIlB,MAAM,CAAC,IAAI,CAACU,WAAW,CAACS,MAAM,CAAC;QACzCC,QAAQ,EAAEN,YAAY,CAACO,GAAG,CAACC,IAAA;UAAA,IAAC;YAACN;UAAI,CAAC,GAAAM,IAAA;UAAA,OAAKN,IAAI,CAAC,CAAC,CAAC;QAAA;MAChD,CAAC,CACH,CAAC;MAED,OAAO;QACLO,KAAK,EAAE,aAAa;QACpBC,SAAS,EAAE,MAAM;QACjBR,IAAI,EAAEC,WAAW;QACjBQ,MAAM,EAAEb,KAAK,CAACa;MAChB,CAAC;IACH;IAEA,OAAO,IAAI;EACb;AACF;AAGA,SAASZ,cAAcA,CAACN,MAAM,EAAU;EACtC,MAAMmB,WAAoB,GAAG,EAAE;EAC/B,KAAK,MAAMC,GAAG,IAAIpB,MAAM,EAAE;IACxB,MAAMqB,KAAK,GAAGrB,MAAM,CAACoB,GAAG,CAAC;IACzB,IAAIC,KAAK,CAACV,IAAI,KAAKW,YAAY,EAAE;MAE/B,MAAMC,QAAQ,GAAG,IAAIC,GAAG,CAAC,CAAC;MAE1B,MAAMC,UAAU,GAAG,IAAIlC,KAAK,CAAC8B,KAAK,CAACK,IAAI,EAAE,IAAI9B,OAAO,CAAC,CAAC,EAAEyB,KAAK,CAACM,QAAQ,EAAEJ,QAAQ,CAAC;MACjFJ,WAAW,CAACS,IAAI,CAACH,UAAU,CAAC;IAC9B;EACF;EACA,IAAIN,WAAW,CAACD,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAIW,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIvC,MAAM,CAAC6B,WAAW,CAAC;AAChC;AAGA,SAASX,eAAeA,CAACL,WAAW,EAAEM,IAAI,EAAY;EACpD,MAAMF,YAAmB,GAAG,EAAE;EAC9B,KAAK,MAAMc,KAAK,IAAIlB,WAAW,CAACS,MAAM,EAAE;IACtC,MAAMkB,MAAM,GAAGrB,IAAI,CAACY,KAAK,CAACK,IAAI,CAAC;IAC/B,IAAII,MAAM,YAAYR,YAAY,EAAE;MAClC,MAAMS,WAAW,GAAGrC,UAAU,CAACoC,MAAM,CAAC;MACtCvB,YAAY,CAACqB,IAAI,CAACG,WAAW,CAAC;IAChC;EACF;EACA,IAAI5B,WAAW,CAACS,MAAM,CAACM,MAAM,KAAKX,YAAY,CAACW,MAAM,EAAE;IACrD,MAAM,IAAIW,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAOtB,YAAY;AACrB"}

package/dist/esm/lib/encode-arrow.js

@@ -0,0 +1,22 @@
+import { Table, tableToIPC, vectorFromArray } from 'apache-arrow';
+import { VECTOR_TYPES } from '../types';
+export function encodeArrowSync(data) {
+  const vectors = {};
+  for (const arrayData of data) {
+    const arrayVector = createVector(arrayData.array, arrayData.type);
+    vectors[arrayData.name] = arrayVector;
+  }
+  const table = new Table(vectors);
+  const arrowBuffer = tableToIPC(table);
+  return arrowBuffer;
+}
+function createVector(array, type) {
+  switch (type) {
+    case VECTOR_TYPES.DATE:
+      return vectorFromArray(array);
+    case VECTOR_TYPES.FLOAT:
+    default:
+      return vectorFromArray(array);
+  }
+}
+//# sourceMappingURL=encode-arrow.js.map

package/dist/esm/lib/encode-arrow.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"encode-arrow.js","names":["Table","tableToIPC","vectorFromArray","VECTOR_TYPES","encodeArrowSync","data","vectors","arrayData","arrayVector","createVector","array","type","name","table","arrowBuffer","DATE","FLOAT"],"sources":["../../../src/lib/encode-arrow.ts"],"sourcesContent":["import {Table, Vector, tableToIPC, vectorFromArray} from 'apache-arrow';\nimport {AnyArrayType, VECTOR_TYPES} from '../types';\n\nexport type ColumnarTable = {\n  name: string;\n  array: AnyArrayType;\n  type: number;\n}[];\n\n/**\n * Encodes set of arrays into the Apache Arrow columnar format\n * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format\n * @param data - columns data\n * @param options - the writer options\n * @returns - encoded ArrayBuffer\n */\nexport function encodeArrowSync(data: ColumnarTable): ArrayBuffer {\n  const vectors: Record<string, Vector> = {};\n  for (const arrayData of data) {\n    const arrayVector = createVector(arrayData.array, arrayData.type);\n    vectors[arrayData.name] = arrayVector;\n  }\n  const table = new Table(vectors);\n  const arrowBuffer = tableToIPC(table);\n  return arrowBuffer;\n}\n\n/**\n * Create Arrow Vector from given data and vector type\n * @param array {import('../types').AnyArrayType} - columns data\n * @param type {number} - the writer options\n * @return a vector of one of vector's types defined in the Apache Arrow library\n */\nfunction createVector(array, type): Vector {\n  switch (type) {\n    case VECTOR_TYPES.DATE:\n      return vectorFromArray(array);\n    case VECTOR_TYPES.FLOAT:\n    default:\n      return vectorFromArray(array);\n  }\n}\n"],"mappings":"AAAA,SAAQA,KAAK,EAAUC,UAAU,EAAEC,eAAe,QAAO,cAAc;AACvE,SAAsBC,YAAY,QAAO,UAAU;AAenD,OAAO,SAASC,eAAeA,CAACC,IAAmB,EAAe;EAChE,MAAMC,OAA+B,GAAG,CAAC,CAAC;EAC1C,KAAK,MAAMC,SAAS,IAAIF,IAAI,EAAE;IAC5B,MAAMG,WAAW,GAAGC,YAAY,CAACF,SAAS,CAACG,KAAK,EAAEH,SAAS,CAACI,IAAI,CAAC;IACjEL,OAAO,CAACC,SAAS,CAACK,IAAI,CAAC,GAAGJ,WAAW;EACvC;EACA,MAAMK,KAAK,GAAG,IAAIb,KAAK,CAACM,OAAO,CAAC;EAChC,MAAMQ,WAAW,GAAGb,UAAU,CAACY,KAAK,CAAC;EACrC,OAAOC,WAAW;AACpB;AAQA,SAASL,YAAYA,CAACC,KAAK,EAAEC,IAAI,EAAU;EACzC,QAAQA,IAAI;IACV,KAAKR,YAAY,CAACY,IAAI;MACpB,OAAOb,eAAe,CAACQ,KAAK,CAAC;IAC/B,KAAKP,YAAY,CAACa,KAAK;IACvB;MACE,OAAOd,eAAe,CAACQ,KAAK,CAAC;EACjC;AACF"}

package/dist/esm/lib/parse-arrow-in-batches.js

@@ -0,0 +1,27 @@
+import { RecordBatchReader } from 'apache-arrow';
+export function parseArrowInBatches(asyncIterator) {
+  async function* makeArrowAsyncIterator() {
+    const readers = RecordBatchReader.readAll(asyncIterator);
+    for await (const reader of readers) {
+      for await (const batch of reader) {
+        yield processBatch(batch);
+      }
+      break;
+    }
+  }
+  return makeArrowAsyncIterator();
+}
+function processBatch(batch) {
+  const values = {
+    metadata: batch.schema.metadata,
+    length: batch.length
+  };
+  batch.schema.fields.forEach((_ref, index) => {
+    let {
+      name
+    } = _ref;
+    values[name] = batch.getChildAt(index).toArray();
+  });
+  return values;
+}
+//# sourceMappingURL=parse-arrow-in-batches.js.map

package/dist/esm/lib/parse-arrow-in-batches.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-arrow-in-batches.js","names":["RecordBatchReader","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","readAll","reader","batch","processBatch","values","metadata","schema","length","fields","forEach","_ref","index","name","getChildAt","toArray"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<any> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator() {\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const batch of reader) {\n        yield processBatch(batch);\n      }\n      break; // only processing one stream of batches\n    }\n  }\n  return makeArrowAsyncIterator();\n}\n\nfunction processBatch(batch) {\n  const values = {\n    metadata: batch.schema.metadata,\n    length: batch.length\n  };\n  batch.schema.fields.forEach(({name}, index) => {\n    values[name] = batch.getChildAt(index).toArray();\n  });\n  return values;\n}\n"],"mappings":"AACA,SAAQA,iBAAiB,QAAO,cAAc;AAK9C,OAAO,SAASC,mBAAmBA,CACjCC,aAAiE,EAC7C;EAmBpB,gBAAgBC,sBAAsBA,CAAA,EAAG;IACvC,MAAMC,OAAO,GAAGJ,iBAAiB,CAACK,OAAO,CAACH,aAAa,CAAC;IACxD,WAAW,MAAMI,MAAM,IAAIF,OAAO,EAAE;MAClC,WAAW,MAAMG,KAAK,IAAID,MAAM,EAAE;QAChC,MAAME,YAAY,CAACD,KAAK,CAAC;MAC3B;MACA;IACF;EACF;EACA,OAAOJ,sBAAsB,CAAC,CAAC;AACjC;AAEA,SAASK,YAAYA,CAACD,KAAK,EAAE;EAC3B,MAAME,MAAM,GAAG;IACbC,QAAQ,EAAEH,KAAK,CAACI,MAAM,CAACD,QAAQ;IAC/BE,MAAM,EAAEL,KAAK,CAACK;EAChB,CAAC;EACDL,KAAK,CAACI,MAAM,CAACE,MAAM,CAACC,OAAO,CAAC,CAAAC,IAAA,EAASC,KAAK,KAAK;IAAA,IAAlB;MAACC;IAAI,CAAC,GAAAF,IAAA;IACjCN,MAAM,CAACQ,IAAI,CAAC,GAAGV,KAAK,CAACW,UAAU,CAACF,KAAK,CAAC,CAACG,OAAO,CAAC,CAAC;EAClD,CAAC,CAAC;EACF,OAAOV,MAAM;AACf"}

package/dist/esm/lib/parse-arrow-sync.js

@@ -0,0 +1,35 @@
+import { tableFromIPC } from 'apache-arrow';
+export default function parseArrowSync(arrayBuffer, options) {
+  var _options$arrow;
+  const arrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
+  const columnarTable = {};
+  for (const field of arrowTable.schema.fields) {
+    const arrowColumn = arrowTable.getChild(field.name);
+    const values = arrowColumn === null || arrowColumn === void 0 ? void 0 : arrowColumn.toArray();
+    columnarTable[field.name] = values;
+  }
+  switch (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) {
+    case 'arrow-table':
+      return arrowTable;
+    case 'object-row-table':
+      return convertColumnarToRowFormatTable(columnarTable);
+    case 'columnar-table':
+    default:
+      return columnarTable;
+  }
+}
+function convertColumnarToRowFormatTable(columnarTable) {
+  const tableKeys = Object.keys(columnarTable);
+  const tableRowsCount = columnarTable[tableKeys[0]].length;
+  const rowFormatTable = [];
+  for (let index = 0; index < tableRowsCount; index++) {
+    const tableItem = {};
+    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
+      const fieldName = tableKeys[keyIndex];
+      tableItem[fieldName] = columnarTable[fieldName][index];
+    }
+    rowFormatTable.push(tableItem);
+  }
+  return rowFormatTable;
+}
+//# sourceMappingURL=parse-arrow-sync.js.map

package/dist/esm/lib/parse-arrow-sync.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-arrow-sync.js","names":["tableFromIPC","parseArrowSync","arrayBuffer","options","_options$arrow","arrowTable","Uint8Array","columnarTable","field","schema","fields","arrowColumn","getChild","name","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../../src/lib/parse-arrow-sync.ts"],"sourcesContent":["import type {ArrowLoaderOptions} from '../arrow-loader';\nimport {tableFromIPC} from 'apache-arrow';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {\n  const arrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);\n\n  // Extract columns\n\n  // TODO - avoid calling `getColumn` on columns we are not interested in?\n  // Add options object?\n  const columnarTable = {};\n\n  for (const field of arrowTable.schema.fields) {\n    // This (is intended to) coalesce all record batches into a single typed array\n    const arrowColumn = arrowTable.getChild(field.name);\n    const values = arrowColumn?.toArray();\n    columnarTable[field.name] = values;\n  }\n\n  switch (options?.arrow?.shape) {\n    case 'arrow-table':\n      return arrowTable;\n    case 'object-row-table':\n      return convertColumnarToRowFormatTable(columnarTable);\n    case 'columnar-table':\n    default:\n      return columnarTable;\n  }\n}\n\nfunction convertColumnarToRowFormatTable(columnarTable) {\n  const tableKeys = Object.keys(columnarTable);\n  const tableRowsCount = columnarTable[tableKeys[0]].length;\n  const rowFormatTable: {}[] = [];\n\n  for (let index = 0; index < tableRowsCount; index++) {\n    const tableItem = {};\n    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n      const fieldName = tableKeys[keyIndex];\n      tableItem[fieldName] = columnarTable[fieldName][index];\n    }\n    rowFormatTable.push(tableItem);\n  }\n  return rowFormatTable;\n}\n"],"mappings":"AACA,SAAQA,YAAY,QAAO,cAAc;AAGzC,eAAe,SAASC,cAAcA,CAACC,WAAW,EAAEC,OAA4B,EAAE;EAAA,IAAAC,cAAA;EAChF,MAAMC,UAAU,GAAGL,YAAY,CAAC,CAAC,IAAIM,UAAU,CAACJ,WAAW,CAAC,CAAC,CAAC;EAM9D,MAAMK,aAAa,GAAG,CAAC,CAAC;EAExB,KAAK,MAAMC,KAAK,IAAIH,UAAU,CAACI,MAAM,CAACC,MAAM,EAAE;IAE5C,MAAMC,WAAW,GAAGN,UAAU,CAACO,QAAQ,CAACJ,KAAK,CAACK,IAAI,CAAC;IACnD,MAAMC,MAAM,GAAGH,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEI,OAAO,CAAC,CAAC;IACrCR,aAAa,CAACC,KAAK,CAACK,IAAI,CAAC,GAAGC,MAAM;EACpC;EAEA,QAAQX,OAAO,aAAPA,OAAO,wBAAAC,cAAA,GAAPD,OAAO,CAAEa,KAAK,cAAAZ,cAAA,uBAAdA,cAAA,CAAgBa,KAAK;IAC3B,KAAK,aAAa;MAChB,OAAOZ,UAAU;IACnB,KAAK,kBAAkB;MACrB,OAAOa,+BAA+B,CAACX,aAAa,CAAC;IACvD,KAAK,gBAAgB;IACrB;MACE,OAAOA,aAAa;EACxB;AACF;AAEA,SAASW,+BAA+BA,CAACX,aAAa,EAAE;EACtD,MAAMY,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACd,aAAa,CAAC;EAC5C,MAAMe,cAAc,GAAGf,aAAa,CAACY,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EACzD,MAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,MAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,MAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGrB,aAAa,CAACqB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EACA,OAAOF,cAAc;AACvB"}

package/dist/esm/types.js

@@ -0,0 +1,6 @@
+export let VECTOR_TYPES = function (VECTOR_TYPES) {
+  VECTOR_TYPES[VECTOR_TYPES["FLOAT"] = 0] = "FLOAT";
+  VECTOR_TYPES[VECTOR_TYPES["DATE"] = 1] = "DATE";
+  return VECTOR_TYPES;
+}({});
+//# sourceMappingURL=types.js.map

package/dist/esm/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","names":["VECTOR_TYPES"],"sources":["../../src/types.ts"],"sourcesContent":["type TypedIntArray =\n  | Int8Array\n  | Uint8Array\n  | Int16Array\n  | Uint16Array\n  | Int32Array\n  | Uint32Array\n  | Int32Array\n  | Uint32Array;\n\ntype TypedFloatArray = Float32Array | Float64Array;\n\ntype TypedArray = TypedIntArray | TypedFloatArray;\n\nexport type AnyArrayType = Array<any> | TypedArray;\n\nexport enum VECTOR_TYPES {\n  FLOAT,\n  DATE\n}\n"],"mappings":"AAgBA,WAAYA,YAAY,aAAZA,YAAY;EAAZA,YAAY,CAAZA,YAAY;EAAZA,YAAY,CAAZA,YAAY;EAAA,OAAZA,YAAY;AAAA"}

package/dist/esm/workers/arrow-worker.js

@@ -0,0 +1,4 @@
+import { createLoaderWorker } from '@loaders.gl/loader-utils';
+import { ArrowLoader } from '../index';
+createLoaderWorker(ArrowLoader);
+//# sourceMappingURL=arrow-worker.js.map

package/dist/esm/workers/arrow-worker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-worker.js","names":["createLoaderWorker","ArrowLoader"],"sources":["../../../src/workers/arrow-worker.ts"],"sourcesContent":["import {createLoaderWorker} from '@loaders.gl/loader-utils';\nimport {ArrowLoader} from '../index';\n\ncreateLoaderWorker(ArrowLoader);\n"],"mappings":"AAAA,SAAQA,kBAAkB,QAAO,0BAA0B;AAC3D,SAAQC,WAAW,QAAO,UAAU;AAEpCD,kBAAkB,CAACC,WAAW,CAAC"}

package/dist/index.d.ts

@@ -0,0 +1,11 @@
+import type { LoaderWithParser } from '@loaders.gl/loader-utils';
+import type { ArrowLoaderOptions } from './arrow-loader';
+import { ArrowLoader as ArrowWorkerLoader } from './arrow-loader';
+export { VECTOR_TYPES } from './types';
+export { ArrowWriter } from './arrow-writer';
+export type { ArrowLoaderOptions };
+export { ArrowWorkerLoader };
+/** ArrowJS table loader */
+export declare const ArrowLoader: LoaderWithParser;
+export declare const _typecheckArrowLoader: LoaderWithParser;
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAC,WAAW,IAAI,iBAAiB,EAAC,MAAM,gBAAgB,CAAC;AAWhE,OAAO,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAIrC,OAAO,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAI3C,YAAY,EAAC,kBAAkB,EAAC,CAAC;AACjC,OAAO,EAAC,iBAAiB,EAAC,CAAC;AAE3B,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,gBAMzB,CAAC;AAEF,eAAO,MAAM,qBAAqB,EAAE,gBAA8B,CAAC"}

package/dist/index.js

@@ -1,16 +1,28 @@
-import { ArrowLoader as ArrowWorkerLoader } from './arrow-loader';
-import parseSync from './lib/parse-arrow-sync';
-import { parseArrowInBatches } from './lib/parse-arrow-in-batches';
-import { TableBatchBuilder } from '@loaders.gl/schema';
-import ArrowTableBatchAggregator from './lib/arrow-table-batch';
-TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
-export { VECTOR_TYPES } from './types';
-export { ArrowWriter } from './arrow-writer';
-export { ArrowWorkerLoader };
-export const ArrowLoader = { ...ArrowWorkerLoader,
-  parse: async (arraybuffer, options) => parseSync(arraybuffer, options),
-  parseSync,
-  parseInBatches: parseArrowInBatches
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
 };
-export const _typecheckArrowLoader = ArrowLoader;
-//# sourceMappingURL=index.js.map
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._typecheckArrowLoader = exports.ArrowLoader = exports.ArrowWorkerLoader = exports.ArrowWriter = exports.VECTOR_TYPES = void 0;
+const arrow_loader_1 = require("./arrow-loader");
+Object.defineProperty(exports, "ArrowWorkerLoader", { enumerable: true, get: function () { return arrow_loader_1.ArrowLoader; } });
+const parse_arrow_sync_1 = __importDefault(require("./lib/parse-arrow-sync"));
+const parse_arrow_in_batches_1 = require("./lib/parse-arrow-in-batches");
+const schema_1 = require("@loaders.gl/schema");
+const arrow_table_batch_1 = __importDefault(require("./lib/arrow-table-batch"));
+// Make the ArrowBatch type available
+schema_1.TableBatchBuilder.ArrowBatch = arrow_table_batch_1.default;
+// Types
+var types_1 = require("./types");
+Object.defineProperty(exports, "VECTOR_TYPES", { enumerable: true, get: function () { return types_1.VECTOR_TYPES; } });
+// Arrow writer
+var arrow_writer_1 = require("./arrow-writer");
+Object.defineProperty(exports, "ArrowWriter", { enumerable: true, get: function () { return arrow_writer_1.ArrowWriter; } });
+/** ArrowJS table loader */
+exports.ArrowLoader = {
+    ...arrow_loader_1.ArrowLoader,
+    parse: async (arraybuffer, options) => (0, parse_arrow_sync_1.default)(arraybuffer, options),
+    parseSync: parse_arrow_sync_1.default,
+    parseInBatches: parse_arrow_in_batches_1.parseArrowInBatches
+};
+exports._typecheckArrowLoader = exports.ArrowLoader;

package/dist/lib/arrow-table-batch.d.ts

@@ -0,0 +1,9 @@
+import type { ArrowTableBatch } from '@loaders.gl/schema';
+import { Schema } from 'apache-arrow';
+import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
+export default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+    arrowSchema: Schema | null;
+    constructor(schema: any, options: any);
+    getBatch(): ArrowTableBatch | null;
+}
+//# sourceMappingURL=arrow-table-batch.d.ts.map

package/dist/lib/arrow-table-batch.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"arrow-table-batch.d.ts","sourceRoot":"","sources":["../../src/lib/arrow-table-batch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,MAAM,EAQP,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,4BAA4B,EAAC,MAAM,oBAAoB,CAAC;AAEhE,MAAM,CAAC,OAAO,OAAO,yBAA0B,SAAQ,4BAA4B;IACjF,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEf,MAAM,KAAA,EAAE,OAAO,KAAA;IAK3B,QAAQ,IAAI,eAAe,GAAG,IAAI;CA4BnC"}

package/dist/lib/arrow-table-batch.js

@@ -1,71 +1,65 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { Schema, Field, RecordBatch, Float32Vector, Float32 } from 'apache-arrow';
-import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
-export default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
-  constructor(schema, options) {
-    super(schema, options);
-
-    _defineProperty(this, "arrowSchema", void 0);
-
-    this.arrowSchema = null;
-  }
-
-  getBatch() {
-    const batch = super.getBatch();
-
-    if (batch) {
-      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
-      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
-      const recordBatch = new RecordBatch(this.arrowSchema, batch.length, arrowVectors);
-      return {
-        shape: 'arrow-table',
-        batchType: 'data',
-        data: recordBatch,
-        length: batch.length
-      };
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const apache_arrow_1 = require("apache-arrow");
+const schema_1 = require("@loaders.gl/schema");
+class ArrowTableBatchAggregator extends schema_1.ColumnarTableBatchAggregator {
+    constructor(schema, options) {
+        super(schema, options);
+        this.arrowSchema = null;
+    }
+    getBatch() {
+        const batch = super.getBatch();
+        if (batch) {
+            // Get the arrow schema
+            this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
+            // Get arrow format vectors
+            const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
+            // Create the record batch
+            const recordBatch = new apache_arrow_1.RecordBatch(this.arrowSchema, (0, apache_arrow_1.makeData)({
+                type: new apache_arrow_1.Struct(this.arrowSchema.fields),
+                children: arrowVectors.map(({ data }) => data[0])
+            }));
+            return {
+                shape: 'arrow-table',
+                batchType: 'data',
+                data: recordBatch,
+                length: batch.length
+            };
+        }
+        return null;
     }
-
-    return null;
-  }
-
 }
-
+exports.default = ArrowTableBatchAggregator;
+// Convert from a simple loaders.gl schema to an Arrow schema
 function getArrowSchema(schema) {
-  const arrowFields = [];
-
-  for (const key in schema) {
-    const field = schema[key];
-
-    if (field.type === Float32Array) {
-      const metadata = field;
-      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);
-      arrowFields.push(arrowField);
+    const arrowFields = [];
+    for (const key in schema) {
+        const field = schema[key];
+        if (field.type === Float32Array) {
+            // TODO - just store the original field as metadata?
+            const metadata = new Map(); // field;
+            // arrow: new Field(name, nullable, metadata)
+            const arrowField = new apache_arrow_1.Field(field.name, new apache_arrow_1.Float32(), field.nullable, metadata);
+            arrowFields.push(arrowField);
+        }
+    }
+    if (arrowFields.length === 0) {
+        throw new Error('No arrow convertible fields');
     }
-  }
-
-  if (arrowFields.length === 0) {
-    throw new Error('No arrow convertible fields');
-  }
-
-  return new Schema(arrowFields);
+    return new apache_arrow_1.Schema(arrowFields);
 }
-
+// Convert from simple loaders.gl arrays to arrow vectors
 function getArrowVectors(arrowSchema, data) {
-  const arrowVectors = [];
-
-  for (const field of arrowSchema.fields) {
-    const vector = data[field.name];
-
-    if (vector instanceof Float32Array) {
-      const arrowVector = Float32Vector.from(vector);
-      arrowVectors.push(arrowVector);
+    const arrowVectors = [];
+    for (const field of arrowSchema.fields) {
+        const vector = data[field.name];
+        if (vector instanceof Float32Array) {
+            const arrowVector = (0, apache_arrow_1.makeVector)(vector);
+            arrowVectors.push(arrowVector);
+        }
+    }
+    if (arrowSchema.fields.length !== arrowVectors.length) {
+        throw new Error('Some columns not arrow convertible');
     }
-  }
-
-  if (arrowSchema.fields.length !== arrowVectors.length) {
-    throw new Error('Some columns not arrow convertible');
-  }
-
-  return arrowVectors;
+    return arrowVectors;
 }
-//# sourceMappingURL=arrow-table-batch.js.map

package/dist/lib/encode-arrow.d.ts

@@ -0,0 +1,15 @@
+import { AnyArrayType } from '../types';
+export type ColumnarTable = {
+    name: string;
+    array: AnyArrayType;
+    type: number;
+}[];
+/**
+ * Encodes set of arrays into the Apache Arrow columnar format
+ * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
+ * @param data - columns data
+ * @param options - the writer options
+ * @returns - encoded ArrayBuffer
+ */
+export declare function encodeArrowSync(data: ColumnarTable): ArrayBuffer;
+//# sourceMappingURL=encode-arrow.d.ts.map

package/dist/lib/encode-arrow.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"encode-arrow.d.ts","sourceRoot":"","sources":["../../src/lib/encode-arrow.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,YAAY,EAAe,MAAM,UAAU,CAAC;AAEpD,MAAM,MAAM,aAAa,GAAG;IAC1B,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,IAAI,EAAE,MAAM,CAAC;CACd,EAAE,CAAC;AAEJ;;;;;;GAMG;AACH,wBAAgB,eAAe,CAAC,IAAI,EAAE,aAAa,GAAG,WAAW,CAShE"}

package/dist/lib/encode-arrow.js

@@ -1,28 +1,38 @@
-import { Table, FloatVector, DateVector } from 'apache-arrow';
-import { VECTOR_TYPES } from '../types';
-export function encodeArrowSync(data) {
-  const vectors = [];
-  const arrayNames = [];
-
-  for (const arrayData of data) {
-    arrayNames.push(arrayData.name);
-    const arrayVector = createVector(arrayData.array, arrayData.type);
-    vectors.push(arrayVector);
-  }
-
-  const table = Table.new(vectors, arrayNames);
-  const arrowBuffer = table.serialize();
-  return arrowBuffer;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.encodeArrowSync = void 0;
+const apache_arrow_1 = require("apache-arrow");
+const types_1 = require("../types");
+/**
+ * Encodes set of arrays into the Apache Arrow columnar format
+ * https://arrow.apache.org/docs/format/Columnar.html#ipc-file-format
+ * @param data - columns data
+ * @param options - the writer options
+ * @returns - encoded ArrayBuffer
+ */
+function encodeArrowSync(data) {
+    const vectors = {};
+    for (const arrayData of data) {
+        const arrayVector = createVector(arrayData.array, arrayData.type);
+        vectors[arrayData.name] = arrayVector;
+    }
+    const table = new apache_arrow_1.Table(vectors);
+    const arrowBuffer = (0, apache_arrow_1.tableToIPC)(table);
+    return arrowBuffer;
 }
-
+exports.encodeArrowSync = encodeArrowSync;
+/**
+ * Create Arrow Vector from given data and vector type
+ * @param array {import('../types').AnyArrayType} - columns data
+ * @param type {number} - the writer options
+ * @return a vector of one of vector's types defined in the Apache Arrow library
+ */
 function createVector(array, type) {
-  switch (type) {
-    case VECTOR_TYPES.DATE:
-      return DateVector.from(array);
-
-    case VECTOR_TYPES.FLOAT:
-    default:
-      return FloatVector.from(array);
-  }
+    switch (type) {
+        case types_1.VECTOR_TYPES.DATE:
+            return (0, apache_arrow_1.vectorFromArray)(array);
+        case types_1.VECTOR_TYPES.FLOAT:
+        default:
+            return (0, apache_arrow_1.vectorFromArray)(array);
+    }
 }
-//# sourceMappingURL=encode-arrow.js.map

package/dist/lib/parse-arrow-in-batches.d.ts

@@ -0,0 +1,4 @@
+/**
+ */
+export declare function parseArrowInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>): AsyncIterable<any>;
+//# sourceMappingURL=parse-arrow-in-batches.d.ts.map

package/dist/lib/parse-arrow-in-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,GAAG,CAAC,CA6BpB"}