@loaders.gl/arrow 4.0.0-alpha.23 → 4.0.0-alpha.24

package/dist/esm/lib/convert-table.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-table.js","names":["convertApacheArrowToArrowTable","arrowTable","shape","data","convertArrowToColumnarTable","table","columnarTable","field","schema","fields","arrowColumn","getChild","name","values","toArray","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../../src/lib/convert-table.ts"],"sourcesContent":["// loaders.gl, MIT license\nimport type {ColumnarTable, ObjectRowTable, ArrowTable} from '@loaders.gl/schema';\nimport type {Table as ApacheArrowTable} from 'apache-arrow';\n\n/**\n * Wrap an apache arrow table in a loaders.gl table wrapper.\n * From this additional conversions are available.\n * @param arrowTable\n * @returns\n */\nexport function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable {\n  return {\n    shape: 'arrow-table',\n    data: arrowTable\n  };\n}\n\n/**\n * Convert an Apache Arrow table to a ColumnarTable\n * @note Currently does not convert schema\n */\nexport function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {\n  // TODO - avoid calling `getColumn` on columns we are not interested in?\n  // Add options object?\n\n  const arrowTable = table.data;\n  const columnarTable = {};\n\n  for (const field of arrowTable.schema.fields) {\n    // This (is intended to) coalesce all record batches into a single typed array\n    const arrowColumn = arrowTable.getChild(field.name);\n    const values = arrowColumn?.toArray();\n    columnarTable[field.name] = values;\n  }\n\n  return {\n    shape: 'columnar-table',\n    data: columnarTable\n  };\n}\n\n/**\n *\n * @note - should be part of schema module\n */\nexport function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {\n  const tableKeys = Object.keys(columnarTable);\n  const tableRowsCount = columnarTable[tableKeys[0]].length;\n\n  const rowFormatTable: {}[] = [];\n\n  for (let index = 0; index < tableRowsCount; index++) {\n    const tableItem = {};\n    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n      const fieldName = tableKeys[keyIndex];\n      tableItem[fieldName] = columnarTable[fieldName][index];\n    }\n    rowFormatTable.push(tableItem);\n  }\n\n  return {\n    shape: 'object-row-table',\n    data: rowFormatTable\n  };\n}\n"],"mappings":"AAUA,OAAO,SAASA,8BAA8BA,CAACC,UAA4B,EAAc;EACvF,OAAO;IACLC,KAAK,EAAE,aAAa;IACpBC,IAAI,EAAEF;EACR,CAAC;AACH;AAMA,OAAO,SAASG,2BAA2BA,CAACC,KAAiB,EAAiB;EAI5E,MAAMJ,UAAU,GAAGI,KAAK,CAACF,IAAI;EAC7B,MAAMG,aAAa,GAAG,CAAC,CAAC;EAExB,KAAK,MAAMC,KAAK,IAAIN,UAAU,CAACO,MAAM,CAACC,MAAM,EAAE;IAE5C,MAAMC,WAAW,GAAGT,UAAU,CAACU,QAAQ,CAACJ,KAAK,CAACK,IAAI,CAAC;IACnD,MAAMC,MAAM,GAAGH,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEI,OAAO,CAAC,CAAC;IACrCR,aAAa,CAACC,KAAK,CAACK,IAAI,CAAC,GAAGC,MAAM;EACpC;EAEA,OAAO;IACLX,KAAK,EAAE,gBAAgB;IACvBC,IAAI,EAAEG;EACR,CAAC;AACH;AAMA,OAAO,SAASS,+BAA+BA,CAACT,aAA4B,EAAkB;EAC5F,MAAMU,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACZ,aAAa,CAAC;EAC5C,MAAMa,cAAc,GAAGb,aAAa,CAACU,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EAEzD,MAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,MAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,MAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGnB,aAAa,CAACmB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EAEA,OAAO;IACLrB,KAAK,EAAE,kBAAkB;IACzBC,IAAI,EAAEkB;EACR,CAAC;AACH"}

package/dist/esm/lib/parse-arrow-in-batches.js

@@ -3,25 +3,18 @@ export function parseArrowInBatches(asyncIterator) {
   async function* makeArrowAsyncIterator() {
     const readers = RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
-      for await (const batch of reader) {
-        yield processBatch(batch);
+      for await (const recordBatch of reader) {
+        const arrowTabledBatch = {
+          shape: 'arrow-table',
+          batchType: 'data',
+          data: recordBatch,
+          length: recordBatch.data.length
+        };
+        yield arrowTabledBatch;
       }
       break;
     }
   }
   return makeArrowAsyncIterator();
 }
-function processBatch(batch) {
-  const values = {
-    metadata: batch.schema.metadata,
-    length: batch.length
-  };
-  batch.schema.fields.forEach((_ref, index) => {
-    let {
-      name
-    } = _ref;
-    values[name] = batch.getChildAt(index).toArray();
-  });
-  return values;
-}
 //# sourceMappingURL=parse-arrow-in-batches.js.map

package/dist/esm/lib/parse-arrow-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-in-batches.js","names":["RecordBatchReader","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","readAll","reader","batch","processBatch","values","metadata","schema","length","fields","forEach","_ref","index","name","getChildAt","toArray"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<any> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator() {\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const batch of reader) {\n        yield processBatch(batch);\n      }\n      break; // only processing one stream of batches\n    }\n  }\n  return makeArrowAsyncIterator();\n}\n\nfunction processBatch(batch) {\n  const values = {\n    metadata: batch.schema.metadata,\n    length: batch.length\n  };\n  batch.schema.fields.forEach(({name}, index) => {\n    values[name] = batch.getChildAt(index).toArray();\n  });\n  return values;\n}\n"],"mappings":"AACA,SAAQA,iBAAiB,QAAO,cAAc;AAK9C,OAAO,SAASC,mBAAmBA,CACjCC,aAAiE,EAC7C;EAmBpB,gBAAgBC,sBAAsBA,CAAA,EAAG;IACvC,MAAMC,OAAO,GAAGJ,iBAAiB,CAACK,OAAO,CAACH,aAAa,CAAC;IACxD,WAAW,MAAMI,MAAM,IAAIF,OAAO,EAAE;MAClC,WAAW,MAAMG,KAAK,IAAID,MAAM,EAAE;QAChC,MAAME,YAAY,CAACD,KAAK,CAAC;MAC3B;MACA;IACF;EACF;EACA,OAAOJ,sBAAsB,CAAC,CAAC;AACjC;AAEA,SAASK,YAAYA,CAACD,KAAK,EAAE;EAC3B,MAAME,MAAM,GAAG;IACbC,QAAQ,EAAEH,KAAK,CAACI,MAAM,CAACD,QAAQ;IAC/BE,MAAM,EAAEL,KAAK,CAACK;EAChB,CAAC;EACDL,KAAK,CAACI,MAAM,CAACE,MAAM,CAACC,OAAO,CAAC,CAAAC,IAAA,EAASC,KAAK,KAAK;IAAA,IAAlB;MAACC;IAAI,CAAC,GAAAF,IAAA;IACjCN,MAAM,CAACQ,IAAI,CAAC,GAAGV,KAAK,CAACW,UAAU,CAACF,KAAK,CAAC,CAACG,OAAO,CAAC,CAAC;EAClD,CAAC,CAAC;EACF,OAAOV,MAAM;AACf"}
+{"version":3,"file":"parse-arrow-in-batches.js","names":["RecordBatchReader","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","readAll","reader","recordBatch","arrowTabledBatch","shape","batchType","data","length"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {ArrowTableBatch} from '@loaders.gl/schema';\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: recordBatch,\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n\n//   };\n// }\n"],"mappings":"AAEA,SAAQA,iBAAiB,QAAO,cAAc;AAK9C,OAAO,SAASC,mBAAmBA,CACjCC,aAAiE,EACjC;EAmBhC,gBAAgBC,sBAAsBA,CAAA,EAAmC;IAEvE,MAAMC,OAAO,GAAGJ,iBAAiB,CAACK,OAAO,CAACH,aAAa,CAAC;IACxD,WAAW,MAAMI,MAAM,IAAIF,OAAO,EAAE;MAClC,WAAW,MAAMG,WAAW,IAAID,MAAM,EAAE;QACtC,MAAME,gBAAiC,GAAG;UACxCC,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBC,IAAI,EAAEJ,WAAW;UACjBK,MAAM,EAAEL,WAAW,CAACI,IAAI,CAACC;QAC3B,CAAC;QAED,MAAMJ,gBAAgB;MACxB;MACA;IACF;EACF;EAEA,OAAOL,sBAAsB,CAAC,CAAC;AACjC"}

package/dist/esm/lib/parse-arrow-sync.js

@@ -1,35 +1,21 @@
+import { convertTable } from '@loaders.gl/schema';
 import { tableFromIPC } from 'apache-arrow';
+import { convertApacheArrowToArrowTable, convertArrowToColumnarTable } from './convert-table';
 export default function parseArrowSync(arrayBuffer, options) {
   var _options$arrow;
-  const arrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
-  const columnarTable = {};
-  for (const field of arrowTable.schema.fields) {
-    const arrowColumn = arrowTable.getChild(field.name);
-    const values = arrowColumn === null || arrowColumn === void 0 ? void 0 : arrowColumn.toArray();
-    columnarTable[field.name] = values;
-  }
-  switch (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) {
+  const apacheArrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
+  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);
+  const shape = (options === null || options === void 0 ? void 0 : (_options$arrow = options.arrow) === null || _options$arrow === void 0 ? void 0 : _options$arrow.shape) || 'arrow-table';
+  switch (shape) {
     case 'arrow-table':
       return arrowTable;
-    case 'object-row-table':
-      return convertColumnarToRowFormatTable(columnarTable);
     case 'columnar-table':
+      return convertArrowToColumnarTable(arrowTable);
+    case 'object-row-table':
+      const columnarTable = convertArrowToColumnarTable(arrowTable);
+      return convertTable(columnarTable, 'object-row-table');
     default:
-      return columnarTable;
-  }
-}
-function convertColumnarToRowFormatTable(columnarTable) {
-  const tableKeys = Object.keys(columnarTable);
-  const tableRowsCount = columnarTable[tableKeys[0]].length;
-  const rowFormatTable = [];
-  for (let index = 0; index < tableRowsCount; index++) {
-    const tableItem = {};
-    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      const fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-    rowFormatTable.push(tableItem);
+      throw new Error(shape);
   }
-  return rowFormatTable;
 }
 //# sourceMappingURL=parse-arrow-sync.js.map

package/dist/esm/lib/parse-arrow-sync.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-sync.js","names":["tableFromIPC","parseArrowSync","arrayBuffer","options","_options$arrow","arrowTable","Uint8Array","columnarTable","field","schema","fields","arrowColumn","getChild","name","values","toArray","arrow","shape","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../../src/lib/parse-arrow-sync.ts"],"sourcesContent":["import type {ArrowLoaderOptions} from '../arrow-loader';\nimport {tableFromIPC} from 'apache-arrow';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {\n  const arrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);\n\n  // Extract columns\n\n  // TODO - avoid calling `getColumn` on columns we are not interested in?\n  // Add options object?\n  const columnarTable = {};\n\n  for (const field of arrowTable.schema.fields) {\n    // This (is intended to) coalesce all record batches into a single typed array\n    const arrowColumn = arrowTable.getChild(field.name);\n    const values = arrowColumn?.toArray();\n    columnarTable[field.name] = values;\n  }\n\n  switch (options?.arrow?.shape) {\n    case 'arrow-table':\n      return arrowTable;\n    case 'object-row-table':\n      return convertColumnarToRowFormatTable(columnarTable);\n    case 'columnar-table':\n    default:\n      return columnarTable;\n  }\n}\n\nfunction convertColumnarToRowFormatTable(columnarTable) {\n  const tableKeys = Object.keys(columnarTable);\n  const tableRowsCount = columnarTable[tableKeys[0]].length;\n  const rowFormatTable: {}[] = [];\n\n  for (let index = 0; index < tableRowsCount; index++) {\n    const tableItem = {};\n    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n      const fieldName = tableKeys[keyIndex];\n      tableItem[fieldName] = columnarTable[fieldName][index];\n    }\n    rowFormatTable.push(tableItem);\n  }\n  return rowFormatTable;\n}\n"],"mappings":"AACA,SAAQA,YAAY,QAAO,cAAc;AAGzC,eAAe,SAASC,cAAcA,CAACC,WAAW,EAAEC,OAA4B,EAAE;EAAA,IAAAC,cAAA;EAChF,MAAMC,UAAU,GAAGL,YAAY,CAAC,CAAC,IAAIM,UAAU,CAACJ,WAAW,CAAC,CAAC,CAAC;EAM9D,MAAMK,aAAa,GAAG,CAAC,CAAC;EAExB,KAAK,MAAMC,KAAK,IAAIH,UAAU,CAACI,MAAM,CAACC,MAAM,EAAE;IAE5C,MAAMC,WAAW,GAAGN,UAAU,CAACO,QAAQ,CAACJ,KAAK,CAACK,IAAI,CAAC;IACnD,MAAMC,MAAM,GAAGH,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEI,OAAO,CAAC,CAAC;IACrCR,aAAa,CAACC,KAAK,CAACK,IAAI,CAAC,GAAGC,MAAM;EACpC;EAEA,QAAQX,OAAO,aAAPA,OAAO,wBAAAC,cAAA,GAAPD,OAAO,CAAEa,KAAK,cAAAZ,cAAA,uBAAdA,cAAA,CAAgBa,KAAK;IAC3B,KAAK,aAAa;MAChB,OAAOZ,UAAU;IACnB,KAAK,kBAAkB;MACrB,OAAOa,+BAA+B,CAACX,aAAa,CAAC;IACvD,KAAK,gBAAgB;IACrB;MACE,OAAOA,aAAa;EACxB;AACF;AAEA,SAASW,+BAA+BA,CAACX,aAAa,EAAE;EACtD,MAAMY,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACd,aAAa,CAAC;EAC5C,MAAMe,cAAc,GAAGf,aAAa,CAACY,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EACzD,MAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,MAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,MAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGrB,aAAa,CAACqB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EACA,OAAOF,cAAc;AACvB"}
+{"version":3,"file":"parse-arrow-sync.js","names":["convertTable","tableFromIPC","convertApacheArrowToArrowTable","convertArrowToColumnarTable","parseArrowSync","arrayBuffer","options","_options$arrow","apacheArrowTable","Uint8Array","arrowTable","shape","arrow","columnarTable","Error"],"sources":["../../../src/lib/parse-arrow-sync.ts"],"sourcesContent":["import type {ArrowTable, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport {convertTable} from '@loaders.gl/schema';\nimport {tableFromIPC} from 'apache-arrow';\nimport type {ArrowLoaderOptions} from '../arrow-loader';\nimport {convertApacheArrowToArrowTable, convertArrowToColumnarTable} from './convert-table';\n\n// Parses arrow to a columnar table\nexport default function parseArrowSync(\n  arrayBuffer,\n  options?: ArrowLoaderOptions\n): ArrowTable | ColumnarTable | ObjectRowTable {\n  const apacheArrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);\n  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);\n\n  const shape = options?.arrow?.shape || 'arrow-table';\n  switch (shape) {\n    case 'arrow-table':\n      return arrowTable;\n\n    case 'columnar-table':\n      return convertArrowToColumnarTable(arrowTable);\n\n    case 'object-row-table':\n      const columnarTable = convertArrowToColumnarTable(arrowTable);\n      return convertTable(columnarTable, 'object-row-table');\n\n    default:\n      // TODO\n      throw new Error(shape);\n  }\n}\n"],"mappings":"AACA,SAAQA,YAAY,QAAO,oBAAoB;AAC/C,SAAQC,YAAY,QAAO,cAAc;AAEzC,SAAQC,8BAA8B,EAAEC,2BAA2B,QAAO,iBAAiB;AAG3F,eAAe,SAASC,cAAcA,CACpCC,WAAW,EACXC,OAA4B,EACiB;EAAA,IAAAC,cAAA;EAC7C,MAAMC,gBAAgB,GAAGP,YAAY,CAAC,CAAC,IAAIQ,UAAU,CAACJ,WAAW,CAAC,CAAC,CAAC;EACpE,MAAMK,UAAU,GAAGR,8BAA8B,CAACM,gBAAgB,CAAC;EAEnE,MAAMG,KAAK,GAAG,CAAAL,OAAO,aAAPA,OAAO,wBAAAC,cAAA,GAAPD,OAAO,CAAEM,KAAK,cAAAL,cAAA,uBAAdA,cAAA,CAAgBI,KAAK,KAAI,aAAa;EACpD,QAAQA,KAAK;IACX,KAAK,aAAa;MAChB,OAAOD,UAAU;IAEnB,KAAK,gBAAgB;MACnB,OAAOP,2BAA2B,CAACO,UAAU,CAAC;IAEhD,KAAK,kBAAkB;MACrB,MAAMG,aAAa,GAAGV,2BAA2B,CAACO,UAAU,CAAC;MAC7D,OAAOV,YAAY,CAACa,aAAa,EAAE,kBAAkB,CAAC;IAExD;MAEE,MAAM,IAAIC,KAAK,CAACH,KAAK,CAAC;EAC1B;AACF"}

package/dist/index.d.ts

@@ -1,10 +1,11 @@
 import type { LoaderWithParser } from '@loaders.gl/loader-utils';
 import type { ArrowLoaderOptions } from './arrow-loader';
+import { ArrowTable, ArrowTableBatch, ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
 import { ArrowLoader as ArrowWorkerLoader } from './arrow-loader';
 export { VECTOR_TYPES } from './types';
 export { ArrowWriter } from './arrow-writer';
 export type { ArrowLoaderOptions };
 export { ArrowWorkerLoader };
 /** ArrowJS table loader */
-export declare const ArrowLoader: LoaderWithParser;
+export declare const ArrowLoader: LoaderWithParser<ArrowTable | ColumnarTable | ObjectRowTable, ArrowTableBatch, ArrowLoaderOptions>;
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAC,WAAW,IAAI,iBAAiB,EAAC,MAAM,gBAAgB,CAAC;AAWhE,OAAO,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAIrC,OAAO,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAI3C,YAAY,EAAC,kBAAkB,EAAC,CAAC;AACjC,OAAO,EAAC,iBAAiB,EAAC,CAAC;AAE3B,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,gBAMzB,CAAC"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAC,MAAM,0BAA0B,CAAC;AAC/D,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,gBAAgB,CAAC;AACvD,OAAO,EAAC,UAAU,EAAE,eAAe,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAE9F,OAAO,EAAC,WAAW,IAAI,iBAAiB,EAAC,MAAM,gBAAgB,CAAC;AAUhE,OAAO,EAAC,YAAY,EAAC,MAAM,SAAS,CAAC;AAIrC,OAAO,EAAC,WAAW,EAAC,MAAM,gBAAgB,CAAC;AAI3C,YAAY,EAAC,kBAAkB,EAAC,CAAC;AACjC,OAAO,EAAC,iBAAiB,EAAC,CAAC;AAE3B,2BAA2B;AAC3B,eAAO,MAAM,WAAW,EAAE,gBAAgB,CACxC,UAAU,GAAG,aAAa,GAAG,cAAc,EAC3C,eAAe,EACf,kBAAkB,CAOnB,CAAC"}

package/dist/lib/arrow-table-batch.d.ts

@@ -1,7 +1,7 @@
 import type { ArrowTableBatch } from '@loaders.gl/schema';
 import { Schema } from 'apache-arrow';
 import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
-export default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+export declare class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
     arrowSchema: Schema | null;
     constructor(schema: any, options: any);
     getBatch(): ArrowTableBatch | null;

package/dist/lib/arrow-table-batch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-table-batch.d.ts","sourceRoot":"","sources":["../../src/lib/arrow-table-batch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,MAAM,EAQP,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,4BAA4B,EAAC,MAAM,oBAAoB,CAAC;AAEhE,MAAM,CAAC,OAAO,OAAO,yBAA0B,SAAQ,4BAA4B;IACjF,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEf,MAAM,KAAA,EAAE,OAAO,KAAA;IAK3B,QAAQ,IAAI,eAAe,GAAG,IAAI;CA4BnC"}
+{"version":3,"file":"arrow-table-batch.d.ts","sourceRoot":"","sources":["../../src/lib/arrow-table-batch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,MAAM,EAQP,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,4BAA4B,EAAC,MAAM,oBAAoB,CAAC;AAEhE,qBAAa,yBAA0B,SAAQ,4BAA4B;IACzE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEf,MAAM,KAAA,EAAE,OAAO,KAAA;IAK3B,QAAQ,IAAI,eAAe,GAAG,IAAI;CA4BnC"}

package/dist/lib/convert-table.d.ts

@@ -0,0 +1,20 @@
+import type { ColumnarTable, ObjectRowTable, ArrowTable } from '@loaders.gl/schema';
+import type { Table as ApacheArrowTable } from 'apache-arrow';
+/**
+ * Wrap an apache arrow table in a loaders.gl table wrapper.
+ * From this additional conversions are available.
+ * @param arrowTable
+ * @returns
+ */
+export declare function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable;
+/**
+ * Convert an Apache Arrow table to a ColumnarTable
+ * @note Currently does not convert schema
+ */
+export declare function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable;
+/**
+ *
+ * @note - should be part of schema module
+ */
+export declare function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable;
+//# sourceMappingURL=convert-table.d.ts.map

package/dist/lib/convert-table.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"convert-table.d.ts","sourceRoot":"","sources":["../../src/lib/convert-table.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAClF,OAAO,KAAK,EAAC,KAAK,IAAI,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAE5D;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,gBAAgB,GAAG,UAAU,CAKvF;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa,CAkB5E;AAED;;;GAGG;AACH,wBAAgB,+BAA+B,CAAC,aAAa,EAAE,aAAa,GAAG,cAAc,CAmB5F"}

package/dist/lib/parse-arrow-in-batches.d.ts

@@ -1,4 +1,5 @@
+import { ArrowTableBatch } from '@loaders.gl/schema';
 /**
  */
-export declare function parseArrowInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>): AsyncIterable<any>;
+export declare function parseArrowInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>): AsyncIterable<ArrowTableBatch>;
 //# sourceMappingURL=parse-arrow-in-batches.d.ts.map

package/dist/lib/parse-arrow-in-batches.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-in-batches.ts"],"names":[],"mappings":"AAIA;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,GAAG,CAAC,CA6BpB"}
+{"version":3,"file":"parse-arrow-in-batches.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-in-batches.ts"],"names":[],"mappings":"AACA,OAAO,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AAInD;GACG;AACH,wBAAgB,mBAAmB,CACjC,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,GAChE,aAAa,CAAC,eAAe,CAAC,CAsChC"}

package/dist/lib/parse-arrow-sync.d.ts

@@ -1,3 +1,4 @@
+import type { ArrowTable, ColumnarTable, ObjectRowTable } from '@loaders.gl/schema';
 import type { ArrowLoaderOptions } from '../arrow-loader';
-export default function parseArrowSync(arrayBuffer: any, options?: ArrowLoaderOptions): {};
+export default function parseArrowSync(arrayBuffer: any, options?: ArrowLoaderOptions): ArrowTable | ColumnarTable | ObjectRowTable;
 //# sourceMappingURL=parse-arrow-sync.d.ts.map

package/dist/lib/parse-arrow-sync.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-sync.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,iBAAiB,CAAC;AAIxD,MAAM,CAAC,OAAO,UAAU,cAAc,CAAC,WAAW,KAAA,EAAE,OAAO,CAAC,EAAE,kBAAkB,MAyB/E"}
+{"version":3,"file":"parse-arrow-sync.d.ts","sourceRoot":"","sources":["../../src/lib/parse-arrow-sync.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,UAAU,EAAE,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAGlF,OAAO,KAAK,EAAC,kBAAkB,EAAC,MAAM,iBAAiB,CAAC;AAIxD,MAAM,CAAC,OAAO,UAAU,cAAc,CACpC,WAAW,KAAA,EACX,OAAO,CAAC,EAAE,kBAAkB,GAC3B,UAAU,GAAG,aAAa,GAAG,cAAc,CAoB7C"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/arrow",
-  "version": "4.0.0-alpha.23",
+  "version": "4.0.0-alpha.24",
   "description": "Simple columnar table loader for the Apache Arrow format",
   "license": "MIT",
   "publishConfig": {
@@ -39,9 +39,9 @@
     "build-worker2": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --platform=browser --external:{stream}"
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "4.0.0-alpha.23",
-    "@loaders.gl/schema": "4.0.0-alpha.23",
-    "apache-arrow": "^9.0.0"
+    "@loaders.gl/loader-utils": "4.0.0-alpha.24",
+    "@loaders.gl/schema": "4.0.0-alpha.24",
+    "apache-arrow": "^13.0.0"
   },
-  "gitHead": "e212f2a0c0e342f7cb65ce84fa2ff39f64b7d94b"
+  "gitHead": "97a8990595c132fb14e3445a8768d9f4cb98ff05"
 }

package/src/index.ts

@@ -1,11 +1,12 @@
 import type {LoaderWithParser} from '@loaders.gl/loader-utils';
 import type {ArrowLoaderOptions} from './arrow-loader';
+import {ArrowTable, ArrowTableBatch, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
+import {TableBatchBuilder} from '@loaders.gl/schema';
 import {ArrowLoader as ArrowWorkerLoader} from './arrow-loader';
 import parseSync from './lib/parse-arrow-sync';
 import {parseArrowInBatches} from './lib/parse-arrow-in-batches';
 
-import {TableBatchBuilder} from '@loaders.gl/schema';
-import ArrowTableBatchAggregator from './lib/arrow-table-batch';
+import {ArrowTableBatchAggregator} from './lib/arrow-table-batch';
 
 // Make the ArrowBatch type available
 TableBatchBuilder.ArrowBatch = ArrowTableBatchAggregator;
@@ -23,7 +24,11 @@ export type {ArrowLoaderOptions};
 export {ArrowWorkerLoader};
 
 /** ArrowJS table loader */
-export const ArrowLoader: LoaderWithParser = {
+export const ArrowLoader: LoaderWithParser<
+  ArrowTable | ColumnarTable | ObjectRowTable,
+  ArrowTableBatch,
+  ArrowLoaderOptions
+> = {
   ...ArrowWorkerLoader,
   parse: async (arraybuffer: ArrayBuffer, options?: ArrowLoaderOptions) =>
     parseSync(arraybuffer, options),

package/src/lib/arrow-table-batch.ts

@@ -11,7 +11,7 @@ import {
 } from 'apache-arrow';
 import {ColumnarTableBatchAggregator} from '@loaders.gl/schema';
 
-export default class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
+export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
   arrowSchema: Schema | null;
 
   constructor(schema, options) {

package/src/lib/convert-table.ts

@@ -0,0 +1,65 @@
+// loaders.gl, MIT license
+import type {ColumnarTable, ObjectRowTable, ArrowTable} from '@loaders.gl/schema';
+import type {Table as ApacheArrowTable} from 'apache-arrow';
+
+/**
+ * Wrap an apache arrow table in a loaders.gl table wrapper.
+ * From this additional conversions are available.
+ * @param arrowTable
+ * @returns
+ */
+export function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable {
+  return {
+    shape: 'arrow-table',
+    data: arrowTable
+  };
+}
+
+/**
+ * Convert an Apache Arrow table to a ColumnarTable
+ * @note Currently does not convert schema
+ */
+export function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {
+  // TODO - avoid calling `getColumn` on columns we are not interested in?
+  // Add options object?
+
+  const arrowTable = table.data;
+  const columnarTable = {};
+
+  for (const field of arrowTable.schema.fields) {
+    // This (is intended to) coalesce all record batches into a single typed array
+    const arrowColumn = arrowTable.getChild(field.name);
+    const values = arrowColumn?.toArray();
+    columnarTable[field.name] = values;
+  }
+
+  return {
+    shape: 'columnar-table',
+    data: columnarTable
+  };
+}
+
+/**
+ *
+ * @note - should be part of schema module
+ */
+export function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {
+  const tableKeys = Object.keys(columnarTable);
+  const tableRowsCount = columnarTable[tableKeys[0]].length;
+
+  const rowFormatTable: {}[] = [];
+
+  for (let index = 0; index < tableRowsCount; index++) {
+    const tableItem = {};
+    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
+      const fieldName = tableKeys[keyIndex];
+      tableItem[fieldName] = columnarTable[fieldName][index];
+    }
+    rowFormatTable.push(tableItem);
+  }
+
+  return {
+    shape: 'object-row-table',
+    data: rowFormatTable
+  };
+}

package/src/lib/parse-arrow-in-batches.ts

@@ -1,4 +1,5 @@
 // TODO - this import defeats the sophisticated typescript checking in ArrowJS
+import {ArrowTableBatch} from '@loaders.gl/schema';
 import {RecordBatchReader} from 'apache-arrow';
 // import {isIterable} from '@loaders.gl/core';
 
@@ -6,7 +7,7 @@ import {RecordBatchReader} from 'apache-arrow';
  */
 export function parseArrowInBatches(
   asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
-): AsyncIterable<any> {
+): AsyncIterable<ArrowTableBatch> {
   // Creates the appropriate RecordBatchReader subclasses from the input
   // This will also close the underlying source in case of early termination or errors
 
@@ -25,25 +26,33 @@ export function parseArrowInBatches(
   }
   */
 
-  async function* makeArrowAsyncIterator() {
+  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {
+    // @ts-ignore
     const readers = RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
-      for await (const batch of reader) {
-        yield processBatch(batch);
+      for await (const recordBatch of reader) {
+        const arrowTabledBatch: ArrowTableBatch = {
+          shape: 'arrow-table',
+          batchType: 'data',
+          data: recordBatch,
+          length: recordBatch.data.length
+        };
+        // processBatch(recordBatch);
+        yield arrowTabledBatch;
       }
       break; // only processing one stream of batches
     }
   }
-  return makeArrowAsyncIterator();
-}
 
-function processBatch(batch) {
-  const values = {
-    metadata: batch.schema.metadata,
-    length: batch.length
-  };
-  batch.schema.fields.forEach(({name}, index) => {
-    values[name] = batch.getChildAt(index).toArray();
-  });
-  return values;
+  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;
 }
+
+// function processBatch(batch: RecordBatch): ArrowTableBatch {
+//   const values = {};
+//   batch.schema.fields.forEach(({name}, index) => {
+//     values[name] = batch.getChildAt(index)?.toArray();
+//   });
+//   return {
+
+//   };
+// }

package/src/lib/parse-arrow-sync.ts

@@ -1,46 +1,31 @@
-import type {ArrowLoaderOptions} from '../arrow-loader';
+import type {ArrowTable, ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
+import {convertTable} from '@loaders.gl/schema';
 import {tableFromIPC} from 'apache-arrow';
+import type {ArrowLoaderOptions} from '../arrow-loader';
+import {convertApacheArrowToArrowTable, convertArrowToColumnarTable} from './convert-table';
 
 // Parses arrow to a columnar table
-export default function parseArrowSync(arrayBuffer, options?: ArrowLoaderOptions) {
-  const arrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
-
-  // Extract columns
-
-  // TODO - avoid calling `getColumn` on columns we are not interested in?
-  // Add options object?
-  const columnarTable = {};
+export default function parseArrowSync(
+  arrayBuffer,
+  options?: ArrowLoaderOptions
+): ArrowTable | ColumnarTable | ObjectRowTable {
+  const apacheArrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
+  const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);
 
-  for (const field of arrowTable.schema.fields) {
-    // This (is intended to) coalesce all record batches into a single typed array
-    const arrowColumn = arrowTable.getChild(field.name);
-    const values = arrowColumn?.toArray();
-    columnarTable[field.name] = values;
-  }
-
-  switch (options?.arrow?.shape) {
+  const shape = options?.arrow?.shape || 'arrow-table';
+  switch (shape) {
     case 'arrow-table':
       return arrowTable;
-    case 'object-row-table':
-      return convertColumnarToRowFormatTable(columnarTable);
+
     case 'columnar-table':
-    default:
-      return columnarTable;
-  }
-}
+      return convertArrowToColumnarTable(arrowTable);
 
-function convertColumnarToRowFormatTable(columnarTable) {
-  const tableKeys = Object.keys(columnarTable);
-  const tableRowsCount = columnarTable[tableKeys[0]].length;
-  const rowFormatTable: {}[] = [];
+    case 'object-row-table':
+      const columnarTable = convertArrowToColumnarTable(arrowTable);
+      return convertTable(columnarTable, 'object-row-table');
 
-  for (let index = 0; index < tableRowsCount; index++) {
-    const tableItem = {};
-    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {
-      const fieldName = tableKeys[keyIndex];
-      tableItem[fieldName] = columnarTable[fieldName][index];
-    }
-    rowFormatTable.push(tableItem);
+    default:
+      // TODO
+      throw new Error(shape);
   }
-  return rowFormatTable;
 }

package/dist/arrow-loader.js

@@ -1,28 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ArrowLoader = void 0;
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-/** ArrowJS table loader */
-exports.ArrowLoader = {
-    name: 'Apache Arrow',
-    id: 'arrow',
-    module: 'arrow',
-    version: VERSION,
-    // worker: true,
-    category: 'table',
-    extensions: ['arrow', 'feather'],
-    mimeTypes: [
-        'application/vnd.apache.arrow.file',
-        'application/vnd.apache.arrow.stream',
-        'application/octet-stream'
-    ],
-    binary: true,
-    tests: ['ARROW'],
-    options: {
-        arrow: {
-            shape: 'columnar-table'
-        }
-    }
-};

package/dist/arrow-writer.js

@@ -1,25 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ArrowWriter = void 0;
-const encode_arrow_1 = require("./lib/encode-arrow");
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-/** Apache Arrow writer */
-exports.ArrowWriter = {
-    name: 'Apache Arrow',
-    id: 'arrow',
-    module: 'arrow',
-    version: VERSION,
-    extensions: ['arrow', 'feather'],
-    mimeTypes: [
-        'application/vnd.apache.arrow.file',
-        'application/vnd.apache.arrow.stream',
-        'application/octet-stream'
-    ],
-    encodeSync(data, options) {
-        return (0, encode_arrow_1.encodeArrowSync)(data);
-    },
-    binary: true,
-    options: {}
-};

package/dist/bundle.js

@@ -1,5 +0,0 @@
-"use strict";
-// @ts-nocheck
-const moduleExports = require('./index');
-globalThis.loaders = globalThis.loaders || {};
-module.exports = Object.assign(globalThis.loaders, moduleExports);

package/dist/index.js

@@ -1,27 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.ArrowLoader = exports.ArrowWorkerLoader = exports.ArrowWriter = exports.VECTOR_TYPES = void 0;
-const arrow_loader_1 = require("./arrow-loader");
-Object.defineProperty(exports, "ArrowWorkerLoader", { enumerable: true, get: function () { return arrow_loader_1.ArrowLoader; } });
-const parse_arrow_sync_1 = __importDefault(require("./lib/parse-arrow-sync"));
-const parse_arrow_in_batches_1 = require("./lib/parse-arrow-in-batches");
-const schema_1 = require("@loaders.gl/schema");
-const arrow_table_batch_1 = __importDefault(require("./lib/arrow-table-batch"));
-// Make the ArrowBatch type available
-schema_1.TableBatchBuilder.ArrowBatch = arrow_table_batch_1.default;
-// Types
-var types_1 = require("./types");
-Object.defineProperty(exports, "VECTOR_TYPES", { enumerable: true, get: function () { return types_1.VECTOR_TYPES; } });
-// Arrow writer
-var arrow_writer_1 = require("./arrow-writer");
-Object.defineProperty(exports, "ArrowWriter", { enumerable: true, get: function () { return arrow_writer_1.ArrowWriter; } });
-/** ArrowJS table loader */
-exports.ArrowLoader = {
-    ...arrow_loader_1.ArrowLoader,
-    parse: async (arraybuffer, options) => (0, parse_arrow_sync_1.default)(arraybuffer, options),
-    parseSync: parse_arrow_sync_1.default,
-    parseInBatches: parse_arrow_in_batches_1.parseArrowInBatches
-};

package/dist/lib/arrow-table-batch.js

@@ -1,65 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const apache_arrow_1 = require("apache-arrow");
-const schema_1 = require("@loaders.gl/schema");
-class ArrowTableBatchAggregator extends schema_1.ColumnarTableBatchAggregator {
-    constructor(schema, options) {
-        super(schema, options);
-        this.arrowSchema = null;
-    }
-    getBatch() {
-        const batch = super.getBatch();
-        if (batch) {
-            // Get the arrow schema
-            this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);
-            // Get arrow format vectors
-            const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);
-            // Create the record batch
-            const recordBatch = new apache_arrow_1.RecordBatch(this.arrowSchema, (0, apache_arrow_1.makeData)({
-                type: new apache_arrow_1.Struct(this.arrowSchema.fields),
-                children: arrowVectors.map(({ data }) => data[0])
-            }));
-            return {
-                shape: 'arrow-table',
-                batchType: 'data',
-                data: recordBatch,
-                length: batch.length
-            };
-        }
-        return null;
-    }
-}
-exports.default = ArrowTableBatchAggregator;
-// Convert from a simple loaders.gl schema to an Arrow schema
-function getArrowSchema(schema) {
-    const arrowFields = [];
-    for (const key in schema) {
-        const field = schema[key];
-        if (field.type === Float32Array) {
-            // TODO - just store the original field as metadata?
-            const metadata = new Map(); // field;
-            // arrow: new Field(name, nullable, metadata)
-            const arrowField = new apache_arrow_1.Field(field.name, new apache_arrow_1.Float32(), field.nullable, metadata);
-            arrowFields.push(arrowField);
-        }
-    }
-    if (arrowFields.length === 0) {
-        throw new Error('No arrow convertible fields');
-    }
-    return new apache_arrow_1.Schema(arrowFields);
-}
-// Convert from simple loaders.gl arrays to arrow vectors
-function getArrowVectors(arrowSchema, data) {
-    const arrowVectors = [];
-    for (const field of arrowSchema.fields) {
-        const vector = data[field.name];
-        if (vector instanceof Float32Array) {
-            const arrowVector = (0, apache_arrow_1.makeVector)(vector);
-            arrowVectors.push(arrowVector);
-        }
-    }
-    if (arrowSchema.fields.length !== arrowVectors.length) {
-        throw new Error('Some columns not arrow convertible');
-    }
-    return arrowVectors;
-}