@loaders.gl/arrow 4.0.0-alpha.24 → 4.0.0-alpha.25

package/dist/arrow-worker.js

@@ -561,8 +561,9 @@
     switch (table.shape) {
       case "array-row-table":
       case "object-row-table":
-      case "geojson-row-table":
         return table.data.length;
+      case "geojson-table":
+        return table.features.length;
       case "arrow-table":
         return table.data.numRows;
       case "columnar-table":
@@ -580,8 +581,9 @@
         const columnIndex = getTableColumnIndex(table, columnName);
         return table.data[rowIndex][columnIndex];
       case "object-row-table":
-      case "geojson-row-table":
         return table.data[rowIndex][columnName];
+      case "geojson-table":
+        return table.features[rowIndex][columnName];
       case "columnar-table":
         const column = table.data[columnName];
         return column[rowIndex];
@@ -604,7 +606,6 @@
       case "object-row-table":
         return copy ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
       case "array-row-table":
-      case "geojson-row-table":
         if (table.schema) {
           const objectRow2 = target || {};
           for (let i = 0; i < table.schema.fields.length; i++) {
@@ -613,6 +614,15 @@
           return objectRow2;
         }
         throw new Error("no schema");
+      case "geojson-table":
+        if (table.schema) {
+          const objectRow2 = target || {};
+          for (let i = 0; i < table.schema.fields.length; i++) {
+            objectRow2[table.schema.fields[i].name] = table.features[rowIndex][i];
+          }
+          return objectRow2;
+        }
+        throw new Error("no schema");
       case "columnar-table":
         if (table.schema) {
           const objectRow2 = target || {};
@@ -644,7 +654,6 @@
       case "array-row-table":
         return copy ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
       case "object-row-table":
-      case "geojson-row-table":
         if (table.schema) {
           const arrayRow2 = target || [];
           for (let i = 0; i < table.schema.fields.length; i++) {
@@ -653,6 +662,15 @@
           return arrayRow2;
         }
         return Object.values(table.data[rowIndex]);
+      case "geojson-table":
+        if (table.schema) {
+          const arrayRow2 = target || [];
+          for (let i = 0; i < table.schema.fields.length; i++) {
+            arrayRow2[i] = table.features[rowIndex][table.schema.fields[i].name];
+          }
+          return arrayRow2;
+        }
+        return Object.values(table.features[rowIndex]);
       case "columnar-table":
         if (table.schema) {
           const arrayRow2 = target || [];
@@ -11479,7 +11497,7 @@ return true;`);
   RecordBatchStreamWriter["throughDOM"] = recordBatchWriterThroughDOMStream;
 
   // src/arrow-loader.ts
-  var VERSION = true ? "4.0.0-alpha.24" : "latest";
+  var VERSION = true ? "4.0.0-alpha.25" : "latest";
   var ArrowLoader = {
     name: "Apache Arrow",
     id: "arrow",
@@ -11549,7 +11567,7 @@ return true;`);
           const arrowTabledBatch = {
             shape: "arrow-table",
             batchType: "data",
-            data: recordBatch,
+            data: new Table([recordBatch]),
             length: recordBatch.data.length
           };
           yield arrowTabledBatch;
@@ -11578,7 +11596,7 @@ return true;`);
         return {
           shape: "arrow-table",
           batchType: "data",
-          data: recordBatch,
+          data: new Table([recordBatch]),
           length: batch.length
         };
       }

package/dist/dist.min.js

@@ -406,8 +406,9 @@
     switch (table.shape) {
       case "array-row-table":
       case "object-row-table":
-      case "geojson-row-table":
         return table.data.length;
+      case "geojson-table":
+        return table.features.length;
       case "arrow-table":
         return table.data.numRows;
       case "columnar-table":
@@ -425,8 +426,9 @@
         const columnIndex = getTableColumnIndex(table, columnName);
         return table.data[rowIndex][columnIndex];
       case "object-row-table":
-      case "geojson-row-table":
         return table.data[rowIndex][columnName];
+      case "geojson-table":
+        return table.features[rowIndex][columnName];
       case "columnar-table":
         const column = table.data[columnName];
         return column[rowIndex];
@@ -449,7 +451,6 @@
       case "object-row-table":
         return copy ? Object.fromEntries(Object.entries(table.data[rowIndex])) : table.data[rowIndex];
       case "array-row-table":
-      case "geojson-row-table":
         if (table.schema) {
           const objectRow2 = target || {};
           for (let i = 0; i < table.schema.fields.length; i++) {
@@ -458,6 +459,15 @@
           return objectRow2;
         }
         throw new Error("no schema");
+      case "geojson-table":
+        if (table.schema) {
+          const objectRow2 = target || {};
+          for (let i = 0; i < table.schema.fields.length; i++) {
+            objectRow2[table.schema.fields[i].name] = table.features[rowIndex][i];
+          }
+          return objectRow2;
+        }
+        throw new Error("no schema");
       case "columnar-table":
         if (table.schema) {
           const objectRow2 = target || {};
@@ -489,7 +499,6 @@
       case "array-row-table":
         return copy ? Array.from(table.data[rowIndex]) : table.data[rowIndex];
       case "object-row-table":
-      case "geojson-row-table":
         if (table.schema) {
           const arrayRow2 = target || [];
           for (let i = 0; i < table.schema.fields.length; i++) {
@@ -498,6 +507,15 @@
           return arrayRow2;
         }
         return Object.values(table.data[rowIndex]);
+      case "geojson-table":
+        if (table.schema) {
+          const arrayRow2 = target || [];
+          for (let i = 0; i < table.schema.fields.length; i++) {
+            arrayRow2[i] = table.features[rowIndex][table.schema.fields[i].name];
+          }
+          return arrayRow2;
+        }
+        return Object.values(table.features[rowIndex]);
       case "columnar-table":
         if (table.schema) {
           const arrayRow2 = target || [];
@@ -12487,7 +12505,7 @@ return true;`);
           const arrowTabledBatch = {
             shape: "arrow-table",
             batchType: "data",
-            data: recordBatch,
+            data: new Table([recordBatch]),
             length: recordBatch.data.length
           };
           yield arrowTabledBatch;
@@ -12555,7 +12573,7 @@ return true;`);
             return {
               shape: "arrow-table",
               batchType: "data",
-              data: recordBatch,
+              data: new Table([recordBatch]),
               length: batch.length
             };
           }

package/dist/es5/arrow-loader.js

@@ -4,7 +4,7 @@ Object.defineProperty(exports, "__esModule", {
   value: true
 });
 exports.ArrowLoader = void 0;
-var VERSION = typeof "4.0.0-alpha.24" !== 'undefined' ? "4.0.0-alpha.24" : 'latest';
+var VERSION = typeof "4.0.0-alpha.25" !== 'undefined' ? "4.0.0-alpha.25" : 'latest';
 var ArrowLoader = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/es5/arrow-writer.js

@@ -5,7 +5,7 @@ Object.defineProperty(exports, "__esModule", {
 });
 exports.ArrowWriter = void 0;
 var _encodeArrow = require("./lib/encode-arrow");
-var VERSION = typeof "4.0.0-alpha.24" !== 'undefined' ? "4.0.0-alpha.24" : 'latest';
+var VERSION = typeof "4.0.0-alpha.25" !== 'undefined' ? "4.0.0-alpha.25" : 'latest';
 var ArrowWriter = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/es5/lib/arrow-table-batch.js

@@ -48,7 +48,7 @@ var ArrowTableBatchAggregator = function (_ColumnarTableBatchAg) {
         return {
           shape: 'arrow-table',
           batchType: 'data',
-          data: recordBatch,
+          data: new _apacheArrow.Table([recordBatch]),
           length: batch.length
         };
       }

package/dist/es5/lib/arrow-table-batch.js.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-table-batch.js","names":["_apacheArrow","require","_schema","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","_createSuper","Derived","hasNativeReflectConstruct","_isNativeReflectConstruct","_createSuperInternal","Super","_getPrototypeOf2","default","result","NewTarget","Reflect","construct","arguments","apply","_possibleConstructorReturn2","sham","Proxy","Boolean","valueOf","ArrowTableBatchAggregator","_ColumnarTableBatchAg","_inherits2","_super","schema","options","_this","_classCallCheck2","_defineProperty2","_assertThisInitialized2","arrowSchema","_createClass2","key","getBatch","batch","_get2","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","RecordBatch","makeData","type","Struct","fields","children","map","_ref","shape","batchType","ColumnarTableBatchAggregator","exports","arrowFields","field","Float32Array","metadata","Map","arrowField","Field","Float32","nullable","push","Error","Schema","_iterator","_step","vector","arrowVector","makeVector"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {\n  Schema,\n  Field,\n  RecordBatch,\n  Struct,\n  makeVector,\n  makeData,\n  Vector,\n  Float32\n} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n\n      // Create the record batch\n      const recordBatch = new RecordBatch(\n        this.arrowSchema,\n        makeData({\n          type: new Struct(this.arrowSchema.fields),\n          children: arrowVectors.map(({data}) => data[0])\n        })\n      );\n\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: recordBatch,\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema): Schema {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      // TODO - just store the original field as metadata?\n      const metadata = new Map(); // field;\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data): Vector[] {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = makeVector(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"mappings":";;;;;;;;;;;;;;;AACA,IAAAA,YAAA,GAAAC,OAAA;AAUA,IAAAC,OAAA,GAAAD,OAAA;AAAgE,SAAAE,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAAA,SAAAC,aAAAC,OAAA,QAAAC,yBAAA,GAAAC,yBAAA,oBAAAC,qBAAA,QAAAC,KAAA,OAAAC,gBAAA,CAAAC,OAAA,EAAAN,OAAA,GAAAO,MAAA,MAAAN,yBAAA,QAAAO,SAAA,OAAAH,gBAAA,CAAAC,OAAA,QAAAd,WAAA,EAAAe,MAAA,GAAAE,OAAA,CAAAC,SAAA,CAAAN,KAAA,EAAAO,SAAA,EAAAH,SAAA,YAAAD,MAAA,GAAAH,KAAA,CAAAQ,KAAA,OAAAD,SAAA,gBAAAE,2BAAA,CAAAP,OAAA,QAAAC,MAAA;AAAA,SAAAL,0BAAA,eAAAO,OAAA,qBAAAA,OAAA,CAAAC,SAAA,oBAAAD,OAAA,CAAAC,SAAA,CAAAI,IAAA,2BAAAC,KAAA,oCAAAC,OAAA,CAAA3B,SAAA,CAAA4B,OAAA,CAAApC,IAAA,CAAA4B,OAAA,CAAAC,SAAA,CAAAM,OAAA,8CAAA1C,CAAA;AAAA,IAEnD4C,yBAAyB,aAAAC,qBAAA;EAAA,IAAAC,UAAA,CAAAd,OAAA,EAAAY,yBAAA,EAAAC,qBAAA;EAAA,IAAAE,MAAA,GAAAtB,YAAA,CAAAmB,yBAAA;EAGpC,SAAAA,0BAAYI,MAAM,EAAEC,OAAO,EAAE;IAAA,IAAAC,KAAA;IAAA,IAAAC,gBAAA,CAAAnB,OAAA,QAAAY,yBAAA;IAC3BM,KAAA,GAAAH,MAAA,CAAAxC,IAAA,OAAMyC,MAAM,EAAEC,OAAO;IAAE,IAAAG,gBAAA,CAAApB,OAAA,MAAAqB,uBAAA,CAAArB,OAAA,EAAAkB,KAAA;IACvBA,KAAA,CAAKI,WAAW,GAAG,IAAI;IAAC,OAAAJ,KAAA;EAC1B;EAAC,IAAAK,aAAA,CAAAvB,OAAA,EAAAY,yBAAA;IAAAY,GAAA;IAAAzD,KAAA,EAED,SAAA0D,SAAA,EAAmC;MACjC,IAAMC,KAAK,OAAAC,KAAA,CAAA3B,OAAA,MAAAD,gBAAA,CAAAC,OAAA,EAAAY,yBAAA,CAAA7B,SAAA,qBAAAR,IAAA,MAAmB;MAC9B,IAAImD,KAAK,EAAE;QAET,IAAI,CAACJ,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIM,cAAc,CAACF,KAAK,CAACV,MAAM,CAAC;QAGnE,IAAMa,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACR,WAAW,EAAEI,KAAK,CAACK,IAAI,CAAC;QAGlE,IAAMC,WAAW,GAAG,IAAIC,wBAAW,CACjC,IAAI,CAACX,WAAW,EAChB,IAAAY,qBAAQ,EAAC;UACPC,IAAI,EAAE,IAAIC,mBAAM,CAAC,IAAI,CAACd,WAAW,CAACe,MAAM,CAAC;UACzCC,QAAQ,EAAET,YAAY,CAACU,GAAG,CAAC,UAAAC,IAAA;YAAA,IAAET,IAAI,GAAAS,IAAA,CAAJT,IAAI;YAAA,OAAMA,IAAI,CAAC,CAAC,CAAC;UAAA;QAChD,CAAC,CACH,CAAC;QAED,OAAO;UACLU,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBX,IAAI,EAAEC,WAAW;UACjBvE,MAAM,EAAEiE,KAAK,CAACjE;QAChB,CAAC;MACH;MAEA,OAAO,IAAI;IACb;EAAC;EAAA,OAAAmD,yBAAA;AAAA,EAnC4C+B,oCAA4B;AAAAC,OAAA,CAAAhC,yBAAA,GAAAA,yBAAA;AAuC3E,SAASgB,cAAcA,CAACZ,MAAM,EAAU;EACtC,IAAM6B,WAAoB,GAAG,EAAE;EAC/B,KAAK,IAAMrB,GAAG,IAAIR,MAAM,EAAE;IACxB,IAAM8B,KAAK,GAAG9B,MAAM,CAACQ,GAAG,CAAC;IACzB,IAAIsB,KAAK,CAACX,IAAI,KAAKY,YAAY,EAAE;MAE/B,IAAMC,QAAQ,GAAG,IAAIC,GAAG,CAAC,CAAC;MAE1B,IAAMC,UAAU,GAAG,IAAIC,kBAAK,CAACL,KAAK,CAAC3D,IAAI,EAAE,IAAIiE,oBAAO,CAAC,CAAC,EAAEN,KAAK,CAACO,QAAQ,EAAEL,QAAQ,CAAC;MACjFH,WAAW,CAACS,IAAI,CAACJ,UAAU,CAAC;IAC9B;EACF;EACA,IAAIL,WAAW,CAACpF,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAI8F,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIC,mBAAM,CAACX,WAAW,CAAC;AAChC;AAGA,SAASf,eAAeA,CAACR,WAAW,EAAES,IAAI,EAAY;EACpD,IAAMF,YAAmB,GAAG,EAAE;EAAC,IAAA4B,SAAA,GAAAzG,0BAAA,CACXsE,WAAW,CAACe,MAAM;IAAAqB,KAAA;EAAA;IAAtC,KAAAD,SAAA,CAAA7F,CAAA,MAAA8F,KAAA,GAAAD,SAAA,CAAA5F,CAAA,IAAAC,IAAA,GAAwC;MAAA,IAA7BgF,KAAK,GAAAY,KAAA,CAAA3F,KAAA;MACd,IAAM4F,MAAM,GAAG5B,IAAI,CAACe,KAAK,CAAC3D,IAAI,CAAC;MAC/B,IAAIwE,MAAM,YAAYZ,YAAY,EAAE;QAClC,IAAMa,WAAW,GAAG,IAAAC,uBAAU,EAACF,MAAM,CAAC;QACtC9B,YAAY,CAACyB,IAAI,CAACM,WAAW,CAAC;MAChC;IACF;EAAC,SAAAtF,GAAA;IAAAmF,SAAA,CAAAzF,CAAA,CAAAM,GAAA;EAAA;IAAAmF,SAAA,CAAAvF,CAAA;EAAA;EACD,IAAIoD,WAAW,CAACe,MAAM,CAAC5E,MAAM,KAAKoE,YAAY,CAACpE,MAAM,EAAE;IACrD,MAAM,IAAI8F,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAO1B,YAAY;AACrB"}
+{"version":3,"file":"arrow-table-batch.js","names":["_apacheArrow","require","_schema","_createForOfIteratorHelper","o","allowArrayLike","it","Symbol","iterator","Array","isArray","_unsupportedIterableToArray","length","i","F","s","n","done","value","e","_e","f","TypeError","normalCompletion","didErr","err","call","step","next","_e2","return","minLen","_arrayLikeToArray","Object","prototype","toString","slice","constructor","name","from","test","arr","len","arr2","_createSuper","Derived","hasNativeReflectConstruct","_isNativeReflectConstruct","_createSuperInternal","Super","_getPrototypeOf2","default","result","NewTarget","Reflect","construct","arguments","apply","_possibleConstructorReturn2","sham","Proxy","Boolean","valueOf","ArrowTableBatchAggregator","_ColumnarTableBatchAg","_inherits2","_super","schema","options","_this","_classCallCheck2","_defineProperty2","_assertThisInitialized2","arrowSchema","_createClass2","key","getBatch","batch","_get2","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","RecordBatch","makeData","type","Struct","fields","children","map","_ref","shape","batchType","ApacheArrowTable","ColumnarTableBatchAggregator","exports","arrowFields","field","Float32Array","metadata","Map","arrowField","Field","Float32","nullable","push","Error","Schema","_iterator","_step","vector","arrowVector","makeVector"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {\n  Table as ApacheArrowTable,\n  Schema,\n  Field,\n  RecordBatch,\n  Struct,\n  makeVector,\n  makeData,\n  Vector,\n  Float32\n} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n\n      // Create the record batch\n      const recordBatch = new RecordBatch(\n        this.arrowSchema,\n        makeData({\n          type: new Struct(this.arrowSchema.fields),\n          children: arrowVectors.map(({data}) => data[0])\n        })\n      );\n\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: new ApacheArrowTable([recordBatch]),\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema): Schema {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      // TODO - just store the original field as metadata?\n      const metadata = new Map(); // field;\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data): Vector[] {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = makeVector(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"mappings":";;;;;;;;;;;;;;;AACA,IAAAA,YAAA,GAAAC,OAAA;AAWA,IAAAC,OAAA,GAAAD,OAAA;AAAgE,SAAAE,2BAAAC,CAAA,EAAAC,cAAA,QAAAC,EAAA,UAAAC,MAAA,oBAAAH,CAAA,CAAAG,MAAA,CAAAC,QAAA,KAAAJ,CAAA,qBAAAE,EAAA,QAAAG,KAAA,CAAAC,OAAA,CAAAN,CAAA,MAAAE,EAAA,GAAAK,2BAAA,CAAAP,CAAA,MAAAC,cAAA,IAAAD,CAAA,WAAAA,CAAA,CAAAQ,MAAA,qBAAAN,EAAA,EAAAF,CAAA,GAAAE,EAAA,MAAAO,CAAA,UAAAC,CAAA,YAAAA,EAAA,eAAAC,CAAA,EAAAD,CAAA,EAAAE,CAAA,WAAAA,EAAA,QAAAH,CAAA,IAAAT,CAAA,CAAAQ,MAAA,WAAAK,IAAA,mBAAAA,IAAA,SAAAC,KAAA,EAAAd,CAAA,CAAAS,CAAA,UAAAM,CAAA,WAAAA,EAAAC,EAAA,UAAAA,EAAA,KAAAC,CAAA,EAAAP,CAAA,gBAAAQ,SAAA,iJAAAC,gBAAA,SAAAC,MAAA,UAAAC,GAAA,WAAAV,CAAA,WAAAA,EAAA,IAAAT,EAAA,GAAAA,EAAA,CAAAoB,IAAA,CAAAtB,CAAA,MAAAY,CAAA,WAAAA,EAAA,QAAAW,IAAA,GAAArB,EAAA,CAAAsB,IAAA,IAAAL,gBAAA,GAAAI,IAAA,CAAAV,IAAA,SAAAU,IAAA,KAAAR,CAAA,WAAAA,EAAAU,GAAA,IAAAL,MAAA,SAAAC,GAAA,GAAAI,GAAA,KAAAR,CAAA,WAAAA,EAAA,eAAAE,gBAAA,IAAAjB,EAAA,CAAAwB,MAAA,UAAAxB,EAAA,CAAAwB,MAAA,oBAAAN,MAAA,QAAAC,GAAA;AAAA,SAAAd,4BAAAP,CAAA,EAAA2B,MAAA,SAAA3B,CAAA,qBAAAA,CAAA,sBAAA4B,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA,OAAAf,CAAA,GAAAiB,MAAA,CAAAC,SAAA,CAAAC,QAAA,CAAAT,IAAA,CAAAtB,CAAA,EAAAgC,KAAA,aAAApB,CAAA,iBAAAZ,CAAA,CAAAiC,WAAA,EAAArB,CAAA,GAAAZ,CAAA,CAAAiC,WAAA,CAAAC,IAAA,MAAAtB,CAAA,cAAAA,CAAA,mBAAAP,KAAA,CAAA8B,IAAA,CAAAnC,CAAA,OAAAY,CAAA,+DAAAwB,IAAA,CAAAxB,CAAA,UAAAgB,iBAAA,CAAA5B,CAAA,EAAA2B,MAAA;AAAA,SAAAC,kBAAAS,GAAA,EAAAC,GAAA,QAAAA,GAAA,YAAAA,GAAA,GAAAD,GAAA,CAAA7B,MAAA,EAAA8B,GAAA,GAAAD,GAAA,CAAA7B,MAAA,WAAAC,CAAA,MAAA8B,IAAA,OAAAlC,KAAA,CAAAiC,GAAA,GAAA7B,CAAA,GAAA6B,GAAA,EAAA7B,CAAA,IAAA8B,IAAA,CAAA9B,CAAA,IAAA4B,GAAA,CAAA5B,CAAA,UAAA8B,IAAA;AAAA,SAAAC,aAAAC,OAAA,QAAAC,yBAAA,GAAAC,yBAAA,oBAAAC,qBAAA,QAAAC,KAAA,OAAAC,gBAAA,CAAAC,OAAA,EAAAN,OAAA,GAAAO,MAAA,MAAAN,yBAAA,QAAAO,SAAA,OAAAH,gBAAA,CAAAC,OAAA,QAAAd,WAAA,EAAAe,MAAA,GAAAE,OAAA,CAAAC,SAAA,CAAAN,KAAA,EAAAO,SAAA,EAAAH,SAAA,YAAAD,MAAA,GAAAH,KAAA,CAAAQ,KAAA,OAAAD,SAAA,gBAAAE,2BAAA,CAAAP,OAAA,QAAAC,MAAA;AAAA,SAAAL,0BAAA,eAAAO,OAAA,qBAAAA,OAAA,CAAAC,SAAA,oBAAAD,OAAA,CAAAC,SAAA,CAAAI,IAAA,2BAAAC,KAAA,oCAAAC,OAAA,CAAA3B,SAAA,CAAA4B,OAAA,CAAApC,IAAA,CAAA4B,OAAA,CAAAC,SAAA,CAAAM,OAAA,8CAAA1C,CAAA;AAAA,IAEnD4C,yBAAyB,aAAAC,qBAAA;EAAA,IAAAC,UAAA,CAAAd,OAAA,EAAAY,yBAAA,EAAAC,qBAAA;EAAA,IAAAE,MAAA,GAAAtB,YAAA,CAAAmB,yBAAA;EAGpC,SAAAA,0BAAYI,MAAM,EAAEC,OAAO,EAAE;IAAA,IAAAC,KAAA;IAAA,IAAAC,gBAAA,CAAAnB,OAAA,QAAAY,yBAAA;IAC3BM,KAAA,GAAAH,MAAA,CAAAxC,IAAA,OAAMyC,MAAM,EAAEC,OAAO;IAAE,IAAAG,gBAAA,CAAApB,OAAA,MAAAqB,uBAAA,CAAArB,OAAA,EAAAkB,KAAA;IACvBA,KAAA,CAAKI,WAAW,GAAG,IAAI;IAAC,OAAAJ,KAAA;EAC1B;EAAC,IAAAK,aAAA,CAAAvB,OAAA,EAAAY,yBAAA;IAAAY,GAAA;IAAAzD,KAAA,EAED,SAAA0D,SAAA,EAAmC;MACjC,IAAMC,KAAK,OAAAC,KAAA,CAAA3B,OAAA,MAAAD,gBAAA,CAAAC,OAAA,EAAAY,yBAAA,CAAA7B,SAAA,qBAAAR,IAAA,MAAmB;MAC9B,IAAImD,KAAK,EAAE;QAET,IAAI,CAACJ,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIM,cAAc,CAACF,KAAK,CAACV,MAAM,CAAC;QAGnE,IAAMa,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACR,WAAW,EAAEI,KAAK,CAACK,IAAI,CAAC;QAGlE,IAAMC,WAAW,GAAG,IAAIC,wBAAW,CACjC,IAAI,CAACX,WAAW,EAChB,IAAAY,qBAAQ,EAAC;UACPC,IAAI,EAAE,IAAIC,mBAAM,CAAC,IAAI,CAACd,WAAW,CAACe,MAAM,CAAC;UACzCC,QAAQ,EAAET,YAAY,CAACU,GAAG,CAAC,UAAAC,IAAA;YAAA,IAAET,IAAI,GAAAS,IAAA,CAAJT,IAAI;YAAA,OAAMA,IAAI,CAAC,CAAC,CAAC;UAAA;QAChD,CAAC,CACH,CAAC;QAED,OAAO;UACLU,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBX,IAAI,EAAE,IAAIY,kBAAgB,CAAC,CAACX,WAAW,CAAC,CAAC;UACzCvE,MAAM,EAAEiE,KAAK,CAACjE;QAChB,CAAC;MACH;MAEA,OAAO,IAAI;IACb;EAAC;EAAA,OAAAmD,yBAAA;AAAA,EAnC4CgC,oCAA4B;AAAAC,OAAA,CAAAjC,yBAAA,GAAAA,yBAAA;AAuC3E,SAASgB,cAAcA,CAACZ,MAAM,EAAU;EACtC,IAAM8B,WAAoB,GAAG,EAAE;EAC/B,KAAK,IAAMtB,GAAG,IAAIR,MAAM,EAAE;IACxB,IAAM+B,KAAK,GAAG/B,MAAM,CAACQ,GAAG,CAAC;IACzB,IAAIuB,KAAK,CAACZ,IAAI,KAAKa,YAAY,EAAE;MAE/B,IAAMC,QAAQ,GAAG,IAAIC,GAAG,CAAC,CAAC;MAE1B,IAAMC,UAAU,GAAG,IAAIC,kBAAK,CAACL,KAAK,CAAC5D,IAAI,EAAE,IAAIkE,oBAAO,CAAC,CAAC,EAAEN,KAAK,CAACO,QAAQ,EAAEL,QAAQ,CAAC;MACjFH,WAAW,CAACS,IAAI,CAACJ,UAAU,CAAC;IAC9B;EACF;EACA,IAAIL,WAAW,CAACrF,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAI+F,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIC,mBAAM,CAACX,WAAW,CAAC;AAChC;AAGA,SAAShB,eAAeA,CAACR,WAAW,EAAES,IAAI,EAAY;EACpD,IAAMF,YAAmB,GAAG,EAAE;EAAC,IAAA6B,SAAA,GAAA1G,0BAAA,CACXsE,WAAW,CAACe,MAAM;IAAAsB,KAAA;EAAA;IAAtC,KAAAD,SAAA,CAAA9F,CAAA,MAAA+F,KAAA,GAAAD,SAAA,CAAA7F,CAAA,IAAAC,IAAA,GAAwC;MAAA,IAA7BiF,KAAK,GAAAY,KAAA,CAAA5F,KAAA;MACd,IAAM6F,MAAM,GAAG7B,IAAI,CAACgB,KAAK,CAAC5D,IAAI,CAAC;MAC/B,IAAIyE,MAAM,YAAYZ,YAAY,EAAE;QAClC,IAAMa,WAAW,GAAG,IAAAC,uBAAU,EAACF,MAAM,CAAC;QACtC/B,YAAY,CAAC0B,IAAI,CAACM,WAAW,CAAC;MAChC;IACF;EAAC,SAAAvF,GAAA;IAAAoF,SAAA,CAAA1F,CAAA,CAAAM,GAAA;EAAA;IAAAoF,SAAA,CAAAxF,CAAA;EAAA;EACD,IAAIoD,WAAW,CAACe,MAAM,CAAC5E,MAAM,KAAKoE,YAAY,CAACpE,MAAM,EAAE;IACrD,MAAM,IAAI+F,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAO3B,YAAY;AACrB"}

package/dist/es5/lib/parse-arrow-in-batches.js

@@ -51,7 +51,7 @@ function parseArrowInBatches(asyncIterator) {
             arrowTabledBatch = {
               shape: 'arrow-table',
               batchType: 'data',
-              data: recordBatch,
+              data: new _apacheArrow.Table([recordBatch]),
               length: recordBatch.data.length
             };
             _context.next = 20;

package/dist/es5/lib/parse-arrow-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-in-batches.js","names":["_apacheArrow","require","_asyncIterator","iterable","method","async","sync","retry","Symbol","asyncIterator","iterator","call","AsyncFromSyncIterator","TypeError","s","AsyncFromSyncIteratorContinuation","r","Object","Promise","reject","done","resolve","value","then","n","next","prototype","apply","arguments","return","_return","ret","throw","_throw","thr","parseArrowInBatches","makeArrowAsyncIterator","_makeArrowAsyncIterator","_wrapAsyncGenerator2","default","_regenerator","mark","_callee","readers","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","reader","_iteratorAbruptCompletion2","_didIteratorError2","_iteratorError2","_iterator2","_step2","recordBatch","arrowTabledBatch","wrap","_callee$","_context","prev","RecordBatchReader","readAll","_awaitAsyncGenerator2","sent","shape","batchType","data","length","t0","finish","abrupt","t1","stop"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {ArrowTableBatch} from '@loaders.gl/schema';\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: recordBatch,\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n\n//   };\n// }\n"],"mappings":";;;;;;;;;;AAEA,IAAAA,YAAA,GAAAC,OAAA;AAA+C,SAAAC,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAC,MAAA,KAAAH,KAAA,GAAAG,MAAA,CAAAC,aAAA,EAAAH,IAAA,GAAAE,MAAA,CAAAE,QAAA,GAAAH,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAAO,IAAA,CAAAR,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAM,qBAAA,CAAAR,MAAA,CAAAO,IAAA,CAAAR,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAO,SAAA;AAAA,SAAAD,sBAAAE,CAAA,aAAAC,kCAAAC,CAAA,QAAAC,MAAA,CAAAD,CAAA,MAAAA,CAAA,SAAAE,OAAA,CAAAC,MAAA,KAAAN,SAAA,CAAAG,CAAA,+BAAAI,IAAA,GAAAJ,CAAA,CAAAI,IAAA,SAAAF,OAAA,CAAAG,OAAA,CAAAL,CAAA,CAAAM,KAAA,EAAAC,IAAA,WAAAD,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAF,IAAA,EAAAA,IAAA,iBAAAR,qBAAA,YAAAA,sBAAAE,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAU,CAAA,GAAAV,CAAA,CAAAW,IAAA,KAAAb,qBAAA,CAAAc,SAAA,KAAAZ,CAAA,QAAAU,CAAA,QAAAC,IAAA,WAAAA,KAAA,WAAAV,iCAAA,MAAAS,CAAA,CAAAG,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAC,MAAA,WAAAC,QAAAR,KAAA,QAAAS,GAAA,QAAAjB,CAAA,CAAAe,MAAA,oBAAAE,GAAA,GAAAb,OAAA,CAAAG,OAAA,GAAAC,KAAA,EAAAA,KAAA,EAAAF,IAAA,UAAAL,iCAAA,CAAAgB,GAAA,CAAAJ,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAI,KAAA,WAAAC,OAAAX,KAAA,QAAAY,GAAA,QAAApB,CAAA,CAAAe,MAAA,oBAAAK,GAAA,GAAAhB,OAAA,CAAAC,MAAA,CAAAG,KAAA,IAAAP,iCAAA,CAAAmB,GAAA,CAAAP,KAAA,MAAAb,CAAA,EAAAc,SAAA,aAAAhB,qBAAA,CAAAE,CAAA;AAKxC,SAASqB,mBAAmBA,CACjC1B,aAAiE,EACjC;EAAA,SAmBhB2B,sBAAsBA,CAAA;IAAA,OAAAC,uBAAA,CAAAV,KAAA,OAAAC,SAAA;EAAA;EAAA,SAAAS,wBAAA;IAAAA,uBAAA,OAAAC,oBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAtC,SAAAC,QAAA;MAAA,IAAAC,OAAA,EAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,MAAA,EAAAC,0BAAA,EAAAC,kBAAA,EAAAC,eAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,WAAA,EAAAC,gBAAA;MAAA,OAAAhB,YAAA,CAAAD,OAAA,CAAAkB,IAAA,UAAAC,SAAAC,QAAA;QAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAlC,IAAA;UAAA;YAEQkB,OAAO,GAAGkB,8BAAiB,CAACC,OAAO,CAACrD,aAAa,CAAC;YAAAmC,yBAAA;YAAAC,iBAAA;YAAAc,QAAA,CAAAC,IAAA;YAAAb,SAAA,GAAA7C,cAAA,CAC7ByC,OAAO;UAAA;YAAAgB,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAQ,SAAA,CAAAtB,IAAA;UAAA;YAAA,MAAAmB,yBAAA,KAAAI,KAAA,GAAAW,QAAA,CAAAK,IAAA,EAAA5C,IAAA;cAAAuC,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAjBwB,MAAM,GAAAD,KAAA,CAAA1B,KAAA;YAAA4B,0BAAA;YAAAC,kBAAA;YAAAQ,QAAA,CAAAC,IAAA;YAAAP,UAAA,GAAAnD,cAAA,CACW+C,MAAM;UAAA;YAAAU,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAc,UAAA,CAAA5B,IAAA;UAAA;YAAA,MAAAyB,0BAAA,KAAAI,MAAA,GAAAK,QAAA,CAAAK,IAAA,EAAA5C,IAAA;cAAAuC,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAArB8B,WAAW,GAAAD,MAAA,CAAAhC,KAAA;YACpBkC,gBAAiC,GAAG;cACxCS,KAAK,EAAE,aAAa;cACpBC,SAAS,EAAE,MAAM;cACjBC,IAAI,EAAEZ,WAAW;cACjBa,MAAM,EAAEb,WAAW,CAACY,IAAI,CAACC;YAC3B,CAAC;YAAAT,QAAA,CAAAlC,IAAA;YAED,OAAM+B,gBAAgB;UAAA;YAAAN,0BAAA;YAAAS,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAU,EAAA,GAAAV,QAAA;YAAAR,kBAAA;YAAAC,eAAA,GAAAO,QAAA,CAAAU,EAAA;UAAA;YAAAV,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAC,IAAA;YAAA,MAAAV,0BAAA,IAAAG,UAAA,CAAAxB,MAAA;cAAA8B,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAc,UAAA,CAAAxB,MAAA;UAAA;YAAA8B,QAAA,CAAAC,IAAA;YAAA,KAAAT,kBAAA;cAAAQ,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAA,MAAA2B,eAAA;UAAA;YAAA,OAAAO,QAAA,CAAAW,MAAA;UAAA;YAAA,OAAAX,QAAA,CAAAW,MAAA;UAAA;YAAA,OAAAX,QAAA,CAAAY,MAAA;UAAA;YAAA3B,yBAAA;YAAAe,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAa,EAAA,GAAAb,QAAA;YAAAd,iBAAA;YAAAC,cAAA,GAAAa,QAAA,CAAAa,EAAA;UAAA;YAAAb,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAC,IAAA;YAAA,MAAAhB,yBAAA,IAAAG,SAAA,CAAAlB,MAAA;cAAA8B,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAQ,SAAA,CAAAlB,MAAA;UAAA;YAAA8B,QAAA,CAAAC,IAAA;YAAA,KAAAf,iBAAA;cAAAc,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAA,MAAAqB,cAAA;UAAA;YAAA,OAAAa,QAAA,CAAAW,MAAA;UAAA;YAAA,OAAAX,QAAA,CAAAW,MAAA;UAAA;UAAA;YAAA,OAAAX,QAAA,CAAAc,IAAA;QAAA;MAAA,GAAA/B,OAAA;IAAA,CAI3B;IAAA,OAAAL,uBAAA,CAAAV,KAAA,OAAAC,SAAA;EAAA;EAED,OAAOQ,sBAAsB,CAAC,CAAC;AACjC"}
+{"version":3,"file":"parse-arrow-in-batches.js","names":["_apacheArrow","require","_asyncIterator","iterable","method","async","sync","retry","Symbol","asyncIterator","iterator","call","AsyncFromSyncIterator","TypeError","s","AsyncFromSyncIteratorContinuation","r","Object","Promise","reject","done","resolve","value","then","n","next","prototype","apply","arguments","return","_return","ret","throw","_throw","thr","parseArrowInBatches","makeArrowAsyncIterator","_makeArrowAsyncIterator","_wrapAsyncGenerator2","default","_regenerator","mark","_callee","readers","_iteratorAbruptCompletion","_didIteratorError","_iteratorError","_iterator","_step","reader","_iteratorAbruptCompletion2","_didIteratorError2","_iteratorError2","_iterator2","_step2","recordBatch","arrowTabledBatch","wrap","_callee$","_context","prev","RecordBatchReader","readAll","_awaitAsyncGenerator2","sent","shape","batchType","data","ApacheArrowTable","length","t0","finish","abrupt","t1","stop"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {ArrowTableBatch} from '@loaders.gl/schema';\nimport {RecordBatchReader, Table as ApacheArrowTable} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: new ApacheArrowTable([recordBatch]),\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n\n//   };\n// }\n"],"mappings":";;;;;;;;;;AAEA,IAAAA,YAAA,GAAAC,OAAA;AAA0E,SAAAC,eAAAC,QAAA,QAAAC,MAAA,EAAAC,KAAA,EAAAC,IAAA,EAAAC,KAAA,iCAAAC,MAAA,KAAAH,KAAA,GAAAG,MAAA,CAAAC,aAAA,EAAAH,IAAA,GAAAE,MAAA,CAAAE,QAAA,GAAAH,KAAA,WAAAF,KAAA,aAAAD,MAAA,GAAAD,QAAA,CAAAE,KAAA,WAAAD,MAAA,CAAAO,IAAA,CAAAR,QAAA,OAAAG,IAAA,aAAAF,MAAA,GAAAD,QAAA,CAAAG,IAAA,eAAAM,qBAAA,CAAAR,MAAA,CAAAO,IAAA,CAAAR,QAAA,IAAAE,KAAA,sBAAAC,IAAA,6BAAAO,SAAA;AAAA,SAAAD,sBAAAE,CAAA,aAAAC,kCAAAC,CAAA,QAAAC,MAAA,CAAAD,CAAA,MAAAA,CAAA,SAAAE,OAAA,CAAAC,MAAA,KAAAN,SAAA,CAAAG,CAAA,+BAAAI,IAAA,GAAAJ,CAAA,CAAAI,IAAA,SAAAF,OAAA,CAAAG,OAAA,CAAAL,CAAA,CAAAM,KAAA,EAAAC,IAAA,WAAAD,KAAA,aAAAA,KAAA,EAAAA,KAAA,EAAAF,IAAA,EAAAA,IAAA,iBAAAR,qBAAA,YAAAA,sBAAAE,CAAA,SAAAA,CAAA,GAAAA,CAAA,OAAAU,CAAA,GAAAV,CAAA,CAAAW,IAAA,KAAAb,qBAAA,CAAAc,SAAA,KAAAZ,CAAA,QAAAU,CAAA,QAAAC,IAAA,WAAAA,KAAA,WAAAV,iCAAA,MAAAS,CAAA,CAAAG,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAC,MAAA,WAAAC,QAAAR,KAAA,QAAAS,GAAA,QAAAjB,CAAA,CAAAe,MAAA,oBAAAE,GAAA,GAAAb,OAAA,CAAAG,OAAA,GAAAC,KAAA,EAAAA,KAAA,EAAAF,IAAA,UAAAL,iCAAA,CAAAgB,GAAA,CAAAJ,KAAA,MAAAb,CAAA,EAAAc,SAAA,OAAAI,KAAA,WAAAC,OAAAX,KAAA,QAAAY,GAAA,QAAApB,CAAA,CAAAe,MAAA,oBAAAK,GAAA,GAAAhB,OAAA,CAAAC,MAAA,CAAAG,KAAA,IAAAP,iCAAA,CAAAmB,GAAA,CAAAP,KAAA,MAAAb,CAAA,EAAAc,SAAA,aAAAhB,qBAAA,CAAAE,CAAA;AAKnE,SAASqB,mBAAmBA,CACjC1B,aAAiE,EACjC;EAAA,SAmBhB2B,sBAAsBA,CAAA;IAAA,OAAAC,uBAAA,CAAAV,KAAA,OAAAC,SAAA;EAAA;EAAA,SAAAS,wBAAA;IAAAA,uBAAA,OAAAC,oBAAA,CAAAC,OAAA,EAAAC,YAAA,CAAAD,OAAA,CAAAE,IAAA,CAAtC,SAAAC,QAAA;MAAA,IAAAC,OAAA,EAAAC,yBAAA,EAAAC,iBAAA,EAAAC,cAAA,EAAAC,SAAA,EAAAC,KAAA,EAAAC,MAAA,EAAAC,0BAAA,EAAAC,kBAAA,EAAAC,eAAA,EAAAC,UAAA,EAAAC,MAAA,EAAAC,WAAA,EAAAC,gBAAA;MAAA,OAAAhB,YAAA,CAAAD,OAAA,CAAAkB,IAAA,UAAAC,SAAAC,QAAA;QAAA,kBAAAA,QAAA,CAAAC,IAAA,GAAAD,QAAA,CAAAlC,IAAA;UAAA;YAEQkB,OAAO,GAAGkB,8BAAiB,CAACC,OAAO,CAACrD,aAAa,CAAC;YAAAmC,yBAAA;YAAAC,iBAAA;YAAAc,QAAA,CAAAC,IAAA;YAAAb,SAAA,GAAA7C,cAAA,CAC7ByC,OAAO;UAAA;YAAAgB,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAQ,SAAA,CAAAtB,IAAA;UAAA;YAAA,MAAAmB,yBAAA,KAAAI,KAAA,GAAAW,QAAA,CAAAK,IAAA,EAAA5C,IAAA;cAAAuC,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAjBwB,MAAM,GAAAD,KAAA,CAAA1B,KAAA;YAAA4B,0BAAA;YAAAC,kBAAA;YAAAQ,QAAA,CAAAC,IAAA;YAAAP,UAAA,GAAAnD,cAAA,CACW+C,MAAM;UAAA;YAAAU,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAc,UAAA,CAAA5B,IAAA;UAAA;YAAA,MAAAyB,0BAAA,KAAAI,MAAA,GAAAK,QAAA,CAAAK,IAAA,EAAA5C,IAAA;cAAAuC,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAArB8B,WAAW,GAAAD,MAAA,CAAAhC,KAAA;YACpBkC,gBAAiC,GAAG;cACxCS,KAAK,EAAE,aAAa;cACpBC,SAAS,EAAE,MAAM;cACjBC,IAAI,EAAE,IAAIC,kBAAgB,CAAC,CAACb,WAAW,CAAC,CAAC;cACzCc,MAAM,EAAEd,WAAW,CAACY,IAAI,CAACE;YAC3B,CAAC;YAAAV,QAAA,CAAAlC,IAAA;YAED,OAAM+B,gBAAgB;UAAA;YAAAN,0BAAA;YAAAS,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAW,EAAA,GAAAX,QAAA;YAAAR,kBAAA;YAAAC,eAAA,GAAAO,QAAA,CAAAW,EAAA;UAAA;YAAAX,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAC,IAAA;YAAA,MAAAV,0BAAA,IAAAG,UAAA,CAAAxB,MAAA;cAAA8B,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAc,UAAA,CAAAxB,MAAA;UAAA;YAAA8B,QAAA,CAAAC,IAAA;YAAA,KAAAT,kBAAA;cAAAQ,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAA,MAAA2B,eAAA;UAAA;YAAA,OAAAO,QAAA,CAAAY,MAAA;UAAA;YAAA,OAAAZ,QAAA,CAAAY,MAAA;UAAA;YAAA,OAAAZ,QAAA,CAAAa,MAAA;UAAA;YAAA5B,yBAAA;YAAAe,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA;UAAA;YAAAkC,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAc,EAAA,GAAAd,QAAA;YAAAd,iBAAA;YAAAC,cAAA,GAAAa,QAAA,CAAAc,EAAA;UAAA;YAAAd,QAAA,CAAAC,IAAA;YAAAD,QAAA,CAAAC,IAAA;YAAA,MAAAhB,yBAAA,IAAAG,SAAA,CAAAlB,MAAA;cAAA8B,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAAkC,QAAA,CAAAlC,IAAA;YAAA,WAAAsC,qBAAA,CAAAxB,OAAA,EAAAQ,SAAA,CAAAlB,MAAA;UAAA;YAAA8B,QAAA,CAAAC,IAAA;YAAA,KAAAf,iBAAA;cAAAc,QAAA,CAAAlC,IAAA;cAAA;YAAA;YAAA,MAAAqB,cAAA;UAAA;YAAA,OAAAa,QAAA,CAAAY,MAAA;UAAA;YAAA,OAAAZ,QAAA,CAAAY,MAAA;UAAA;UAAA;YAAA,OAAAZ,QAAA,CAAAe,IAAA;QAAA;MAAA,GAAAhC,OAAA;IAAA,CAI3B;IAAA,OAAAL,uBAAA,CAAAV,KAAA,OAAAC,SAAA;EAAA;EAED,OAAOQ,sBAAsB,CAAC,CAAC;AACjC"}

package/dist/esm/arrow-loader.js

@@ -1,4 +1,4 @@
-const VERSION = typeof "4.0.0-alpha.24" !== 'undefined' ? "4.0.0-alpha.24" : 'latest';
+const VERSION = typeof "4.0.0-alpha.25" !== 'undefined' ? "4.0.0-alpha.25" : 'latest';
 export const ArrowLoader = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/esm/arrow-writer.js

@@ -1,5 +1,5 @@
 import { encodeArrowSync } from './lib/encode-arrow';
-const VERSION = typeof "4.0.0-alpha.24" !== 'undefined' ? "4.0.0-alpha.24" : 'latest';
+const VERSION = typeof "4.0.0-alpha.25" !== 'undefined' ? "4.0.0-alpha.25" : 'latest';
 export const ArrowWriter = {
   name: 'Apache Arrow',
   id: 'arrow',

package/dist/esm/lib/arrow-table-batch.js

@@ -1,5 +1,5 @@
 import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-import { Schema, Field, RecordBatch, Struct, makeVector, makeData, Float32 } from 'apache-arrow';
+import { Table as ApacheArrowTable, Schema, Field, RecordBatch, Struct, makeVector, makeData, Float32 } from 'apache-arrow';
 import { ColumnarTableBatchAggregator } from '@loaders.gl/schema';
 export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
   constructor(schema, options) {
@@ -24,7 +24,7 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
       return {
         shape: 'arrow-table',
         batchType: 'data',
-        data: recordBatch,
+        data: new ApacheArrowTable([recordBatch]),
         length: batch.length
       };
     }

package/dist/esm/lib/arrow-table-batch.js.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-table-batch.js","names":["Schema","Field","RecordBatch","Struct","makeVector","makeData","Float32","ColumnarTableBatchAggregator","ArrowTableBatchAggregator","constructor","schema","options","_defineProperty","arrowSchema","getBatch","batch","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","type","fields","children","map","_ref","shape","batchType","length","arrowFields","key","field","Float32Array","metadata","Map","arrowField","name","nullable","push","Error","vector","arrowVector"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {\n  Schema,\n  Field,\n  RecordBatch,\n  Struct,\n  makeVector,\n  makeData,\n  Vector,\n  Float32\n} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n\n      // Create the record batch\n      const recordBatch = new RecordBatch(\n        this.arrowSchema,\n        makeData({\n          type: new Struct(this.arrowSchema.fields),\n          children: arrowVectors.map(({data}) => data[0])\n        })\n      );\n\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: recordBatch,\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema): Schema {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      // TODO - just store the original field as metadata?\n      const metadata = new Map(); // field;\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data): Vector[] {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = makeVector(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"mappings":";AACA,SACEA,MAAM,EACNC,KAAK,EACLC,WAAW,EACXC,MAAM,EACNC,UAAU,EACVC,QAAQ,EAERC,OAAO,QACF,cAAc;AACrB,SAAQC,4BAA4B,QAAO,oBAAoB;AAE/D,OAAO,MAAMC,yBAAyB,SAASD,4BAA4B,CAAC;EAG1EE,WAAWA,CAACC,MAAM,EAAEC,OAAO,EAAE;IAC3B,KAAK,CAACD,MAAM,EAAEC,OAAO,CAAC;IAACC,eAAA;IACvB,IAAI,CAACC,WAAW,GAAG,IAAI;EACzB;EAEAC,QAAQA,CAAA,EAA2B;IACjC,MAAMC,KAAK,GAAG,KAAK,CAACD,QAAQ,CAAC,CAAC;IAC9B,IAAIC,KAAK,EAAE;MAET,IAAI,CAACF,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIG,cAAc,CAACD,KAAK,CAACL,MAAM,CAAC;MAGnE,MAAMO,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACL,WAAW,EAAEE,KAAK,CAACI,IAAI,CAAC;MAGlE,MAAMC,WAAW,GAAG,IAAIlB,WAAW,CACjC,IAAI,CAACW,WAAW,EAChBR,QAAQ,CAAC;QACPgB,IAAI,EAAE,IAAIlB,MAAM,CAAC,IAAI,CAACU,WAAW,CAACS,MAAM,CAAC;QACzCC,QAAQ,EAAEN,YAAY,CAACO,GAAG,CAACC,IAAA;UAAA,IAAC;YAACN;UAAI,CAAC,GAAAM,IAAA;UAAA,OAAKN,IAAI,CAAC,CAAC,CAAC;QAAA;MAChD,CAAC,CACH,CAAC;MAED,OAAO;QACLO,KAAK,EAAE,aAAa;QACpBC,SAAS,EAAE,MAAM;QACjBR,IAAI,EAAEC,WAAW;QACjBQ,MAAM,EAAEb,KAAK,CAACa;MAChB,CAAC;IACH;IAEA,OAAO,IAAI;EACb;AACF;AAGA,SAASZ,cAAcA,CAACN,MAAM,EAAU;EACtC,MAAMmB,WAAoB,GAAG,EAAE;EAC/B,KAAK,MAAMC,GAAG,IAAIpB,MAAM,EAAE;IACxB,MAAMqB,KAAK,GAAGrB,MAAM,CAACoB,GAAG,CAAC;IACzB,IAAIC,KAAK,CAACV,IAAI,KAAKW,YAAY,EAAE;MAE/B,MAAMC,QAAQ,GAAG,IAAIC,GAAG,CAAC,CAAC;MAE1B,MAAMC,UAAU,GAAG,IAAIlC,KAAK,CAAC8B,KAAK,CAACK,IAAI,EAAE,IAAI9B,OAAO,CAAC,CAAC,EAAEyB,KAAK,CAACM,QAAQ,EAAEJ,QAAQ,CAAC;MACjFJ,WAAW,CAACS,IAAI,CAACH,UAAU,CAAC;IAC9B;EACF;EACA,IAAIN,WAAW,CAACD,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAIW,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIvC,MAAM,CAAC6B,WAAW,CAAC;AAChC;AAGA,SAASX,eAAeA,CAACL,WAAW,EAAEM,IAAI,EAAY;EACpD,MAAMF,YAAmB,GAAG,EAAE;EAC9B,KAAK,MAAMc,KAAK,IAAIlB,WAAW,CAACS,MAAM,EAAE;IACtC,MAAMkB,MAAM,GAAGrB,IAAI,CAACY,KAAK,CAACK,IAAI,CAAC;IAC/B,IAAII,MAAM,YAAYR,YAAY,EAAE;MAClC,MAAMS,WAAW,GAAGrC,UAAU,CAACoC,MAAM,CAAC;MACtCvB,YAAY,CAACqB,IAAI,CAACG,WAAW,CAAC;IAChC;EACF;EACA,IAAI5B,WAAW,CAACS,MAAM,CAACM,MAAM,KAAKX,YAAY,CAACW,MAAM,EAAE;IACrD,MAAM,IAAIW,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAOtB,YAAY;AACrB"}
+{"version":3,"file":"arrow-table-batch.js","names":["Table","ApacheArrowTable","Schema","Field","RecordBatch","Struct","makeVector","makeData","Float32","ColumnarTableBatchAggregator","ArrowTableBatchAggregator","constructor","schema","options","_defineProperty","arrowSchema","getBatch","batch","getArrowSchema","arrowVectors","getArrowVectors","data","recordBatch","type","fields","children","map","_ref","shape","batchType","length","arrowFields","key","field","Float32Array","metadata","Map","arrowField","name","nullable","push","Error","vector","arrowVector"],"sources":["../../../src/lib/arrow-table-batch.ts"],"sourcesContent":["import type {ArrowTableBatch} from '@loaders.gl/schema';\nimport {\n  Table as ApacheArrowTable,\n  Schema,\n  Field,\n  RecordBatch,\n  Struct,\n  makeVector,\n  makeData,\n  Vector,\n  Float32\n} from 'apache-arrow';\nimport {ColumnarTableBatchAggregator} from '@loaders.gl/schema';\n\nexport class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {\n  arrowSchema: Schema | null;\n\n  constructor(schema, options) {\n    super(schema, options);\n    this.arrowSchema = null;\n  }\n\n  getBatch(): ArrowTableBatch | null {\n    const batch = super.getBatch();\n    if (batch) {\n      // Get the arrow schema\n      this.arrowSchema = this.arrowSchema || getArrowSchema(batch.schema);\n\n      // Get arrow format vectors\n      const arrowVectors = getArrowVectors(this.arrowSchema, batch.data);\n\n      // Create the record batch\n      const recordBatch = new RecordBatch(\n        this.arrowSchema,\n        makeData({\n          type: new Struct(this.arrowSchema.fields),\n          children: arrowVectors.map(({data}) => data[0])\n        })\n      );\n\n      return {\n        shape: 'arrow-table',\n        batchType: 'data',\n        data: new ApacheArrowTable([recordBatch]),\n        length: batch.length\n      };\n    }\n\n    return null;\n  }\n}\n\n// Convert from a simple loaders.gl schema to an Arrow schema\nfunction getArrowSchema(schema): Schema {\n  const arrowFields: Field[] = [];\n  for (const key in schema) {\n    const field = schema[key];\n    if (field.type === Float32Array) {\n      // TODO - just store the original field as metadata?\n      const metadata = new Map(); // field;\n      // arrow: new Field(name, nullable, metadata)\n      const arrowField = new Field(field.name, new Float32(), field.nullable, metadata);\n      arrowFields.push(arrowField);\n    }\n  }\n  if (arrowFields.length === 0) {\n    throw new Error('No arrow convertible fields');\n  }\n\n  return new Schema(arrowFields);\n}\n\n// Convert from simple loaders.gl arrays to arrow vectors\nfunction getArrowVectors(arrowSchema, data): Vector[] {\n  const arrowVectors: any[] = [];\n  for (const field of arrowSchema.fields) {\n    const vector = data[field.name];\n    if (vector instanceof Float32Array) {\n      const arrowVector = makeVector(vector);\n      arrowVectors.push(arrowVector);\n    }\n  }\n  if (arrowSchema.fields.length !== arrowVectors.length) {\n    throw new Error('Some columns not arrow convertible');\n  }\n  return arrowVectors;\n}\n"],"mappings":";AACA,SACEA,KAAK,IAAIC,gBAAgB,EACzBC,MAAM,EACNC,KAAK,EACLC,WAAW,EACXC,MAAM,EACNC,UAAU,EACVC,QAAQ,EAERC,OAAO,QACF,cAAc;AACrB,SAAQC,4BAA4B,QAAO,oBAAoB;AAE/D,OAAO,MAAMC,yBAAyB,SAASD,4BAA4B,CAAC;EAG1EE,WAAWA,CAACC,MAAM,EAAEC,OAAO,EAAE;IAC3B,KAAK,CAACD,MAAM,EAAEC,OAAO,CAAC;IAACC,eAAA;IACvB,IAAI,CAACC,WAAW,GAAG,IAAI;EACzB;EAEAC,QAAQA,CAAA,EAA2B;IACjC,MAAMC,KAAK,GAAG,KAAK,CAACD,QAAQ,CAAC,CAAC;IAC9B,IAAIC,KAAK,EAAE;MAET,IAAI,CAACF,WAAW,GAAG,IAAI,CAACA,WAAW,IAAIG,cAAc,CAACD,KAAK,CAACL,MAAM,CAAC;MAGnE,MAAMO,YAAY,GAAGC,eAAe,CAAC,IAAI,CAACL,WAAW,EAAEE,KAAK,CAACI,IAAI,CAAC;MAGlE,MAAMC,WAAW,GAAG,IAAIlB,WAAW,CACjC,IAAI,CAACW,WAAW,EAChBR,QAAQ,CAAC;QACPgB,IAAI,EAAE,IAAIlB,MAAM,CAAC,IAAI,CAACU,WAAW,CAACS,MAAM,CAAC;QACzCC,QAAQ,EAAEN,YAAY,CAACO,GAAG,CAACC,IAAA;UAAA,IAAC;YAACN;UAAI,CAAC,GAAAM,IAAA;UAAA,OAAKN,IAAI,CAAC,CAAC,CAAC;QAAA;MAChD,CAAC,CACH,CAAC;MAED,OAAO;QACLO,KAAK,EAAE,aAAa;QACpBC,SAAS,EAAE,MAAM;QACjBR,IAAI,EAAE,IAAIpB,gBAAgB,CAAC,CAACqB,WAAW,CAAC,CAAC;QACzCQ,MAAM,EAAEb,KAAK,CAACa;MAChB,CAAC;IACH;IAEA,OAAO,IAAI;EACb;AACF;AAGA,SAASZ,cAAcA,CAACN,MAAM,EAAU;EACtC,MAAMmB,WAAoB,GAAG,EAAE;EAC/B,KAAK,MAAMC,GAAG,IAAIpB,MAAM,EAAE;IACxB,MAAMqB,KAAK,GAAGrB,MAAM,CAACoB,GAAG,CAAC;IACzB,IAAIC,KAAK,CAACV,IAAI,KAAKW,YAAY,EAAE;MAE/B,MAAMC,QAAQ,GAAG,IAAIC,GAAG,CAAC,CAAC;MAE1B,MAAMC,UAAU,GAAG,IAAIlC,KAAK,CAAC8B,KAAK,CAACK,IAAI,EAAE,IAAI9B,OAAO,CAAC,CAAC,EAAEyB,KAAK,CAACM,QAAQ,EAAEJ,QAAQ,CAAC;MACjFJ,WAAW,CAACS,IAAI,CAACH,UAAU,CAAC;IAC9B;EACF;EACA,IAAIN,WAAW,CAACD,MAAM,KAAK,CAAC,EAAE;IAC5B,MAAM,IAAIW,KAAK,CAAC,6BAA6B,CAAC;EAChD;EAEA,OAAO,IAAIvC,MAAM,CAAC6B,WAAW,CAAC;AAChC;AAGA,SAASX,eAAeA,CAACL,WAAW,EAAEM,IAAI,EAAY;EACpD,MAAMF,YAAmB,GAAG,EAAE;EAC9B,KAAK,MAAMc,KAAK,IAAIlB,WAAW,CAACS,MAAM,EAAE;IACtC,MAAMkB,MAAM,GAAGrB,IAAI,CAACY,KAAK,CAACK,IAAI,CAAC;IAC/B,IAAII,MAAM,YAAYR,YAAY,EAAE;MAClC,MAAMS,WAAW,GAAGrC,UAAU,CAACoC,MAAM,CAAC;MACtCvB,YAAY,CAACqB,IAAI,CAACG,WAAW,CAAC;IAChC;EACF;EACA,IAAI5B,WAAW,CAACS,MAAM,CAACM,MAAM,KAAKX,YAAY,CAACW,MAAM,EAAE;IACrD,MAAM,IAAIW,KAAK,CAAC,oCAAoC,CAAC;EACvD;EACA,OAAOtB,YAAY;AACrB"}

package/dist/esm/lib/parse-arrow-in-batches.js

@@ -1,4 +1,4 @@
-import { RecordBatchReader } from 'apache-arrow';
+import { RecordBatchReader, Table as ApacheArrowTable } from 'apache-arrow';
 export function parseArrowInBatches(asyncIterator) {
   async function* makeArrowAsyncIterator() {
     const readers = RecordBatchReader.readAll(asyncIterator);
@@ -7,7 +7,7 @@ export function parseArrowInBatches(asyncIterator) {
         const arrowTabledBatch = {
           shape: 'arrow-table',
           batchType: 'data',
-          data: recordBatch,
+          data: new ApacheArrowTable([recordBatch]),
           length: recordBatch.data.length
         };
         yield arrowTabledBatch;

package/dist/esm/lib/parse-arrow-in-batches.js.map

@@ -1 +1 @@
-{"version":3,"file":"parse-arrow-in-batches.js","names":["RecordBatchReader","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","readAll","reader","recordBatch","arrowTabledBatch","shape","batchType","data","length"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {ArrowTableBatch} from '@loaders.gl/schema';\nimport {RecordBatchReader} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: recordBatch,\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n\n//   };\n// }\n"],"mappings":"AAEA,SAAQA,iBAAiB,QAAO,cAAc;AAK9C,OAAO,SAASC,mBAAmBA,CACjCC,aAAiE,EACjC;EAmBhC,gBAAgBC,sBAAsBA,CAAA,EAAmC;IAEvE,MAAMC,OAAO,GAAGJ,iBAAiB,CAACK,OAAO,CAACH,aAAa,CAAC;IACxD,WAAW,MAAMI,MAAM,IAAIF,OAAO,EAAE;MAClC,WAAW,MAAMG,WAAW,IAAID,MAAM,EAAE;QACtC,MAAME,gBAAiC,GAAG;UACxCC,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBC,IAAI,EAAEJ,WAAW;UACjBK,MAAM,EAAEL,WAAW,CAACI,IAAI,CAACC;QAC3B,CAAC;QAED,MAAMJ,gBAAgB;MACxB;MACA;IACF;EACF;EAEA,OAAOL,sBAAsB,CAAC,CAAC;AACjC"}
+{"version":3,"file":"parse-arrow-in-batches.js","names":["RecordBatchReader","Table","ApacheArrowTable","parseArrowInBatches","asyncIterator","makeArrowAsyncIterator","readers","readAll","reader","recordBatch","arrowTabledBatch","shape","batchType","data","length"],"sources":["../../../src/lib/parse-arrow-in-batches.ts"],"sourcesContent":["// TODO - this import defeats the sophisticated typescript checking in ArrowJS\nimport {ArrowTableBatch} from '@loaders.gl/schema';\nimport {RecordBatchReader, Table as ApacheArrowTable} from 'apache-arrow';\n// import {isIterable} from '@loaders.gl/core';\n\n/**\n */\nexport function parseArrowInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>\n): AsyncIterable<ArrowTableBatch> {\n  // Creates the appropriate RecordBatchReader subclasses from the input\n  // This will also close the underlying source in case of early termination or errors\n\n  // As an optimization, return a non-async iterator\n  /*\n  if (isIterable(readers)) {\n    function* makeArrowIterator() {\n      for (const reader of readers) {\n        for (const batch of reader) {\n          yield processBatch(batch, reader);\n        }\n        break; // only processing one stream of batches\n      }\n    }\n    const arrowIterator = makeArrowIterator();\n  }\n  */\n\n  async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {\n    // @ts-ignore\n    const readers = RecordBatchReader.readAll(asyncIterator);\n    for await (const reader of readers) {\n      for await (const recordBatch of reader) {\n        const arrowTabledBatch: ArrowTableBatch = {\n          shape: 'arrow-table',\n          batchType: 'data',\n          data: new ApacheArrowTable([recordBatch]),\n          length: recordBatch.data.length\n        };\n        // processBatch(recordBatch);\n        yield arrowTabledBatch;\n      }\n      break; // only processing one stream of batches\n    }\n  }\n\n  return makeArrowAsyncIterator() as any; // as AsyncIterator<ArrowTableBatch>;\n}\n\n// function processBatch(batch: RecordBatch): ArrowTableBatch {\n//   const values = {};\n//   batch.schema.fields.forEach(({name}, index) => {\n//     values[name] = batch.getChildAt(index)?.toArray();\n//   });\n//   return {\n\n//   };\n// }\n"],"mappings":"AAEA,SAAQA,iBAAiB,EAAEC,KAAK,IAAIC,gBAAgB,QAAO,cAAc;AAKzE,OAAO,SAASC,mBAAmBA,CACjCC,aAAiE,EACjC;EAmBhC,gBAAgBC,sBAAsBA,CAAA,EAAmC;IAEvE,MAAMC,OAAO,GAAGN,iBAAiB,CAACO,OAAO,CAACH,aAAa,CAAC;IACxD,WAAW,MAAMI,MAAM,IAAIF,OAAO,EAAE;MAClC,WAAW,MAAMG,WAAW,IAAID,MAAM,EAAE;QACtC,MAAME,gBAAiC,GAAG;UACxCC,KAAK,EAAE,aAAa;UACpBC,SAAS,EAAE,MAAM;UACjBC,IAAI,EAAE,IAAIX,gBAAgB,CAAC,CAACO,WAAW,CAAC,CAAC;UACzCK,MAAM,EAAEL,WAAW,CAACI,IAAI,CAACC;QAC3B,CAAC;QAED,MAAMJ,gBAAgB;MACxB;MACA;IACF;EACF;EAEA,OAAOL,sBAAsB,CAAC,CAAC;AACjC"}

package/dist/lib/arrow-table-batch.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"arrow-table-batch.d.ts","sourceRoot":"","sources":["../../src/lib/arrow-table-batch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AACxD,OAAO,EACL,MAAM,EAQP,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,4BAA4B,EAAC,MAAM,oBAAoB,CAAC;AAEhE,qBAAa,yBAA0B,SAAQ,4BAA4B;IACzE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEf,MAAM,KAAA,EAAE,OAAO,KAAA;IAK3B,QAAQ,IAAI,eAAe,GAAG,IAAI;CA4BnC"}
+{"version":3,"file":"arrow-table-batch.d.ts","sourceRoot":"","sources":["../../src/lib/arrow-table-batch.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,eAAe,EAAC,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAEL,MAAM,EAQP,MAAM,cAAc,CAAC;AACtB,OAAO,EAAC,4BAA4B,EAAC,MAAM,oBAAoB,CAAC;AAEhE,qBAAa,yBAA0B,SAAQ,4BAA4B;IACzE,WAAW,EAAE,MAAM,GAAG,IAAI,CAAC;gBAEf,MAAM,KAAA,EAAE,OAAO,KAAA;IAK3B,QAAQ,IAAI,eAAe,GAAG,IAAI;CA4BnC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/arrow",
-  "version": "4.0.0-alpha.24",
+  "version": "4.0.0-alpha.25",
   "description": "Simple columnar table loader for the Apache Arrow format",
   "license": "MIT",
   "publishConfig": {
@@ -39,9 +39,9 @@
     "build-worker2": "esbuild src/workers/arrow-worker.ts --bundle --outfile=dist/arrow-worker.js --platform=browser --external:{stream}"
   },
   "dependencies": {
-    "@loaders.gl/loader-utils": "4.0.0-alpha.24",
-    "@loaders.gl/schema": "4.0.0-alpha.24",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.25",
+    "@loaders.gl/schema": "4.0.0-alpha.25",
     "apache-arrow": "^13.0.0"
   },
-  "gitHead": "97a8990595c132fb14e3445a8768d9f4cb98ff05"
+  "gitHead": "40135f391b869388dbbcd615bbe51178d0c370be"
 }

package/src/lib/arrow-table-batch.ts

@@ -1,5 +1,6 @@
 import type {ArrowTableBatch} from '@loaders.gl/schema';
 import {
+  Table as ApacheArrowTable,
   Schema,
   Field,
   RecordBatch,
@@ -40,7 +41,7 @@ export class ArrowTableBatchAggregator extends ColumnarTableBatchAggregator {
       return {
         shape: 'arrow-table',
         batchType: 'data',
-        data: recordBatch,
+        data: new ApacheArrowTable([recordBatch]),
         length: batch.length
       };
     }

package/src/lib/parse-arrow-in-batches.ts

@@ -1,6 +1,6 @@
 // TODO - this import defeats the sophisticated typescript checking in ArrowJS
 import {ArrowTableBatch} from '@loaders.gl/schema';
-import {RecordBatchReader} from 'apache-arrow';
+import {RecordBatchReader, Table as ApacheArrowTable} from 'apache-arrow';
 // import {isIterable} from '@loaders.gl/core';
 
 /**
@@ -34,7 +34,7 @@ export function parseArrowInBatches(
         const arrowTabledBatch: ArrowTableBatch = {
           shape: 'arrow-table',
           batchType: 'data',
-          data: recordBatch,
+          data: new ApacheArrowTable([recordBatch]),
           length: recordBatch.data.length
         };
         // processBatch(recordBatch);