@loaders.gl/arrow 4.0.0 → 4.0.2

package/src/lib/parse-arrow-in-batches.ts

@@ -1,6 +1,6 @@
 // TODO - this import defeats the sophisticated typescript checking in ArrowJS
 import type {ArrowTableBatch} from './arrow-table';
-import {RecordBatchReader, Table as ApacheArrowTable} from 'apache-arrow';
+import * as arrow from 'apache-arrow';
 // import {isIterable} from '@loaders.gl/core';
 
 /**
@@ -8,7 +8,7 @@ import {RecordBatchReader, Table as ApacheArrowTable} from 'apache-arrow';
 export function parseArrowInBatches(
   asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>
 ): AsyncIterable<ArrowTableBatch> {
-  // Creates the appropriate RecordBatchReader subclasses from the input
+  // Creates the appropriate arrow.RecordBatchReader subclasses from the input
   // This will also close the underlying source in case of early termination or errors
 
   // As an optimization, return a non-async iterator
@@ -28,13 +28,13 @@ export function parseArrowInBatches(
 
   async function* makeArrowAsyncIterator(): AsyncIterator<ArrowTableBatch> {
     // @ts-ignore
-    const readers = RecordBatchReader.readAll(asyncIterator);
+    const readers = arrow.RecordBatchReader.readAll(asyncIterator);
     for await (const reader of readers) {
       for await (const recordBatch of reader) {
         const arrowTabledBatch: ArrowTableBatch = {
           shape: 'arrow-table',
           batchType: 'data',
-          data: new ApacheArrowTable([recordBatch]),
+          data: new arrow.Table([recordBatch]),
           length: recordBatch.data.length
         };
         // processBatch(recordBatch);

package/src/lib/parse-arrow-sync.ts

@@ -1,16 +1,19 @@
 import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
 import type {ArrowTable} from './arrow-table';
 import {convertTable} from '@loaders.gl/schema';
-import {tableFromIPC} from 'apache-arrow';
+import * as arrow from 'apache-arrow';
 import type {ArrowLoaderOptions} from '../arrow-loader';
-import {convertApacheArrowToArrowTable, convertArrowToColumnarTable} from './convert-table';
+import {
+  convertApacheArrowToArrowTable,
+  convertArrowToColumnarTable
+} from '../tables/convert-arrow-to-table';
 
 // Parses arrow to a columnar table
 export default function parseArrowSync(
   arrayBuffer,
   options?: ArrowLoaderOptions
 ): ArrowTable | ColumnarTable | ObjectRowTable {
-  const apacheArrowTable = tableFromIPC([new Uint8Array(arrayBuffer)]);
+  const apacheArrowTable = arrow.tableFromIPC([new Uint8Array(arrayBuffer)]);
   const arrowTable = convertApacheArrowToArrowTable(apacheArrowTable);
 
   const shape = options?.arrow?.shape || 'arrow-table';

package/src/schema/arrow-type-utils.ts

@@ -2,45 +2,27 @@
 // Copyright (c) vis.gl contributors
 
 import type {TypedArray} from '@loaders.gl/schema';
-import {
-  DataType,
-  Float32,
-  Float64,
-  Int16,
-  Int32,
-  Int8,
-  Uint16,
-  Uint32,
-  Uint8
-  // Int8Vector,
-  // Uint8Vector,
-  // Int16Vector,
-  // Uint16Vector,
-  // Int32Vector,
-  // Uint32Vector,
-  // Float32Vector,
-  // Float64Vector
-} from 'apache-arrow';
-// import {AbstractVector} from 'apache-arrow/vector';
+import * as arrow from 'apache-arrow';
 
-export function getArrowType(array: TypedArray): DataType {
+/** Return an Apache Arrow Type instance that corresponds to the type of the elements in the supplied Typed Array */
+export function getArrowType(array: TypedArray): arrow.DataType {
   switch (array.constructor) {
     case Int8Array:
-      return new Int8();
+      return new arrow.Int8();
     case Uint8Array:
-      return new Uint8();
+      return new arrow.Uint8();
     case Int16Array:
-      return new Int16();
+      return new arrow.Int16();
     case Uint16Array:
-      return new Uint16();
+      return new arrow.Uint16();
     case Int32Array:
-      return new Int32();
+      return new arrow.Int32();
     case Uint32Array:
-      return new Uint32();
+      return new arrow.Uint32();
     case Float32Array:
-      return new Float32();
+      return new arrow.Float32();
     case Float64Array:
-      return new Float64();
+      return new arrow.Float64();
     default:
       throw new Error('array type not supported');
   }

package/src/schema/convert-arrow-schema.ts

@@ -0,0 +1,265 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
+import type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';
+import * as arrow from 'apache-arrow';
+
+/** Convert Apache Arrow Schema (class instance) to a serialized Schema (plain data) */
+export function serializeArrowSchema(arrowSchema: arrow.Schema): Schema {
+  return {
+    fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),
+    metadata: serializeArrowMetadata(arrowSchema.metadata)
+  };
+}
+
+/** Convert a serialized Schema (plain data) to an Apache Arrow Schema (class instance) */
+export function deserializeArrowSchema(schema: Schema): arrow.Schema {
+  return new arrow.Schema(
+    schema.fields.map((field) => deserializeArrowField(field)),
+    deserializeArrowMetadata(schema.metadata)
+  );
+}
+
+/** Convert Apache Arrow Schema metadata (Map<string, string>) to serialized metadata (Record<string, string> */
+export function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {
+  return Object.fromEntries(arrowMetadata);
+}
+
+/** Convert serialized metadata (Record<string, string> to Apache Arrow Schema metadata (Map<string, string>) to */
+export function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {
+  return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();
+}
+
+/** Convert Apache Arrow Field (class instance) to serialized Field (plain data) */
+export function serializeArrowField(field: arrow.Field): Field {
+  return {
+    name: field.name,
+    type: serializeArrowType(field.type),
+    nullable: field.nullable,
+    metadata: serializeArrowMetadata(field.metadata)
+  };
+}
+
+/** Convert a serialized Field (plain data) to an Apache Arrow Field (class instance)*/
+export function deserializeArrowField(field: Field): arrow.Field {
+  return new arrow.Field(
+    field.name,
+    deserializeArrowType(field.type),
+    field.nullable,
+    deserializeArrowMetadata(field.metadata)
+  );
+}
+
+/** Converts a serializable loaders.gl data type to hydrated arrow data type */
+// eslint-disable-next-line complexity
+export function serializeArrowType(arrowType: arrow.DataType): DataType {
+  switch (arrowType.constructor) {
+    case arrow.Null:
+      return 'null';
+    case arrow.Binary:
+      return 'binary';
+    case arrow.Bool:
+      return 'bool';
+    case arrow.Int:
+      const intType = arrowType as arrow.Int;
+      return `${intType.isSigned ? 'u' : ''}int${intType.bitWidth}`;
+    case arrow.Int8:
+      return 'int8';
+    case arrow.Int16:
+      return 'int16';
+    case arrow.Int32:
+      return 'int32';
+    case arrow.Int64:
+      return 'int64';
+    case arrow.Uint8:
+      return 'uint8';
+    case arrow.Uint16:
+      return 'uint16';
+    case arrow.Uint32:
+      return 'uint32';
+    case arrow.Uint64:
+      return 'uint64';
+    case arrow.Float:
+      const precision = (arrowType as arrow.Float).precision;
+      // return `float(precision + 1) * 16`;
+      switch (precision) {
+        case arrow.Precision.HALF:
+          return 'float16';
+        case arrow.Precision.SINGLE:
+          return 'float32';
+        case arrow.Precision.DOUBLE:
+          return 'float64';
+        default:
+          return 'float16';
+      }
+    case arrow.Float16:
+      return 'float16';
+    case arrow.Float32:
+      return 'float32';
+    case arrow.Float64:
+      return 'float64';
+    case arrow.Utf8:
+      return 'utf8';
+    case Date:
+      const dateUnit = (arrowType as arrow.Date_).unit;
+      return dateUnit === arrow.DateUnit.DAY ? 'date-day' : 'date-millisecond';
+    case arrow.DateDay:
+      return 'date-day';
+    case arrow.DateMillisecond:
+      return 'date-millisecond';
+    case arrow.Time:
+      const timeUnit = (arrowType as arrow.Time).unit;
+      switch (timeUnit) {
+        case arrow.TimeUnit.SECOND:
+          return 'time-second';
+        case arrow.TimeUnit.MILLISECOND:
+          return 'time-millisecond';
+        case arrow.TimeUnit.MICROSECOND:
+          return 'time-microsecond';
+        case arrow.TimeUnit.NANOSECOND:
+          return 'time-nanosecond';
+        default:
+          return 'time-second';
+      }
+    case arrow.TimeMillisecond:
+      return 'time-millisecond';
+    case arrow.TimeSecond:
+      return 'time-second';
+    case arrow.TimeMicrosecond:
+      return 'time-microsecond';
+    case arrow.TimeNanosecond:
+      return 'time-nanosecond';
+    case arrow.Timestamp:
+      const timeStampUnit = (arrowType as arrow.Timestamp).unit;
+      switch (timeStampUnit) {
+        case arrow.TimeUnit.SECOND:
+          return 'timestamp-second';
+        case arrow.TimeUnit.MILLISECOND:
+          return 'timestamp-millisecond';
+        case arrow.TimeUnit.MICROSECOND:
+          return 'timestamp-microsecond';
+        case arrow.TimeUnit.NANOSECOND:
+          return 'timestamp-nanosecond';
+        default:
+          return 'timestamp-second';
+      }
+    case arrow.TimestampSecond:
+      return 'timestamp-second';
+    case arrow.TimestampMillisecond:
+      return 'timestamp-millisecond';
+    case arrow.TimestampMicrosecond:
+      return 'timestamp-microsecond';
+    case arrow.TimestampNanosecond:
+      return 'timestamp-nanosecond';
+    case arrow.Interval:
+      const intervalUnit = (arrowType as arrow.Interval).unit;
+      switch (intervalUnit) {
+        case arrow.IntervalUnit.DAY_TIME:
+          return 'interval-daytime';
+        case arrow.IntervalUnit.YEAR_MONTH:
+          return 'interval-yearmonth';
+        default:
+          return 'interval-daytime';
+      }
+    case arrow.IntervalDayTime:
+      return 'interval-daytime';
+    case arrow.IntervalYearMonth:
+      return 'interval-yearmonth';
+    case arrow.List:
+      const listType = arrowType as arrow.List;
+      const listField = listType.valueField;
+      return {
+        type: 'list',
+        children: [serializeArrowField(listField)]
+      };
+    case arrow.FixedSizeList:
+      return {
+        type: 'fixed-size-list',
+        listSize: (arrowType as arrow.FixedSizeList).listSize,
+        children: [serializeArrowField((arrowType as arrow.FixedSizeList).children[0])]
+      };
+    // case arrow.Struct:
+    //   return {type: 'struct', children: (arrowType as arrow.Struct).children};
+    default:
+      throw new Error('array type not supported');
+  }
+}
+
+/** Converts a serializable loaders.gl data type to hydrated arrow data type */
+// eslint-disable-next-line complexity
+export function deserializeArrowType(dataType: DataType): arrow.DataType {
+  if (typeof dataType === 'object') {
+    switch (dataType.type) {
+      case 'list':
+        const field = deserializeArrowField(dataType.children[0]);
+        return new arrow.List(field);
+      case 'fixed-size-list':
+        const child = deserializeArrowField(dataType.children[0]);
+        return new arrow.FixedSizeList(dataType.listSize, child);
+      case 'struct':
+        const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));
+        return new arrow.Struct(children);
+      default:
+        throw new Error('array type not supported');
+    }
+  }
+
+  switch (dataType) {
+    case 'null':
+      return new arrow.Null();
+    case 'binary':
+      return new arrow.Binary();
+    case 'bool':
+      return new arrow.Bool();
+    case 'int8':
+      return new arrow.Int8();
+    case 'int16':
+      return new arrow.Int16();
+    case 'int32':
+      return new arrow.Int32();
+    case 'int64':
+      return new arrow.Int64();
+    case 'uint8':
+      return new arrow.Uint8();
+    case 'uint16':
+      return new arrow.Uint16();
+    case 'uint32':
+      return new arrow.Uint32();
+    case 'uint64':
+      return new arrow.Uint64();
+    case 'float16':
+      return new arrow.Float16();
+    case 'float32':
+      return new arrow.Float32();
+    case 'float64':
+      return new arrow.Float64();
+    case 'utf8':
+      return new arrow.Utf8();
+    case 'date-day':
+      return new arrow.DateDay();
+    case 'date-millisecond':
+      return new arrow.DateMillisecond();
+    case 'time-second':
+      return new arrow.TimeSecond();
+    case 'time-millisecond':
+      return new arrow.TimeMillisecond();
+    case 'time-microsecond':
+      return new arrow.TimeMicrosecond();
+    case 'time-nanosecond':
+      return new arrow.TimeNanosecond();
+    case 'timestamp-second':
+      return new arrow.TimestampSecond();
+    case 'timestamp-millisecond':
+      return new arrow.TimestampMillisecond();
+    case 'timestamp-microsecond':
+      return new arrow.TimestampMicrosecond();
+    case 'timestamp-nanosecond':
+      return new arrow.TimestampNanosecond();
+    case 'interval-daytime':
+      return new arrow.IntervalDayTime();
+    case 'interval-yearmonth':
+      return new arrow.IntervalYearMonth();
+    default:
+      throw new Error('array type not supported');
+  }
+}

package/src/{lib/convert-table.ts → tables/convert-arrow-to-table.ts}

@@ -1,8 +1,9 @@
 // loaders.gl, MIT license
 // Copyright (c) vis.gl contributors
+
 import type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';
-import type {Table as ApacheArrowTable} from 'apache-arrow';
-import type {ArrowTable} from './arrow-table';
+import type * as arrow from 'apache-arrow';
+import type {ArrowTable} from '../lib/arrow-table';
 
 /**
  * Wrap an apache arrow table in a loaders.gl table wrapper.
@@ -10,7 +11,7 @@ import type {ArrowTable} from './arrow-table';
  * @param arrowTable
  * @returns
  */
-export function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable {
+export function convertApacheArrowToArrowTable(arrowTable: arrow.Table): ArrowTable {
   return {
     shape: 'arrow-table',
     data: arrowTable

package/src/types.ts

@@ -1,3 +1,6 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
 type TypedIntArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array;
 
 type TypedFloatArray = Float32Array | Float64Array;

package/src/workers/arrow-worker.ts

@@ -1,3 +1,6 @@
+// loaders.gl, MIT license
+// Copyright (c) vis.gl contributors
+
 import {createLoaderWorker} from '@loaders.gl/loader-utils';
 import {ArrowLoader} from '../index';
 

package/dist/lib/convert-table.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-table.d.ts","sourceRoot":"","sources":["../../src/lib/convert-table.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AACtE,OAAO,KAAK,EAAC,KAAK,IAAI,gBAAgB,EAAC,MAAM,cAAc,CAAC;AAC5D,OAAO,KAAK,EAAC,UAAU,EAAC,MAAM,eAAe,CAAC;AAE9C;;;;;GAKG;AACH,wBAAgB,8BAA8B,CAAC,UAAU,EAAE,gBAAgB,GAAG,UAAU,CAKvF;AAED;;;GAGG;AACH,wBAAgB,2BAA2B,CAAC,KAAK,EAAE,UAAU,GAAG,aAAa,CAkB5E;AAED;;;GAGG;AACH,wBAAgB,+BAA+B,CAAC,aAAa,EAAE,aAAa,GAAG,cAAc,CAmB5F"}

package/dist/lib/convert-table.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-table.js","names":["convertApacheArrowToArrowTable","arrowTable","shape","data","convertArrowToColumnarTable","table","columnarTable","field","schema","fields","arrowColumn","getChild","name","values","toArray","convertColumnarToRowFormatTable","tableKeys","Object","keys","tableRowsCount","length","rowFormatTable","index","tableItem","keyIndex","fieldName","push"],"sources":["../../src/lib/convert-table.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\nimport type {ColumnarTable, ObjectRowTable} from '@loaders.gl/schema';\nimport type {Table as ApacheArrowTable} from 'apache-arrow';\nimport type {ArrowTable} from './arrow-table';\n\n/**\n * Wrap an apache arrow table in a loaders.gl table wrapper.\n * From this additional conversions are available.\n * @param arrowTable\n * @returns\n */\nexport function convertApacheArrowToArrowTable(arrowTable: ApacheArrowTable): ArrowTable {\n  return {\n    shape: 'arrow-table',\n    data: arrowTable\n  };\n}\n\n/**\n * Convert an Apache Arrow table to a ColumnarTable\n * @note Currently does not convert schema\n */\nexport function convertArrowToColumnarTable(table: ArrowTable): ColumnarTable {\n  // TODO - avoid calling `getColumn` on columns we are not interested in?\n  // Add options object?\n\n  const arrowTable = table.data;\n  const columnarTable = {};\n\n  for (const field of arrowTable.schema.fields) {\n    // This (is intended to) coalesce all record batches into a single typed array\n    const arrowColumn = arrowTable.getChild(field.name);\n    const values = arrowColumn?.toArray();\n    columnarTable[field.name] = values;\n  }\n\n  return {\n    shape: 'columnar-table',\n    data: columnarTable\n  };\n}\n\n/**\n *\n * @note - should be part of schema module\n */\nexport function convertColumnarToRowFormatTable(columnarTable: ColumnarTable): ObjectRowTable {\n  const tableKeys = Object.keys(columnarTable);\n  const tableRowsCount = columnarTable[tableKeys[0]].length;\n\n  const rowFormatTable: {}[] = [];\n\n  for (let index = 0; index < tableRowsCount; index++) {\n    const tableItem = {};\n    for (let keyIndex = 0; keyIndex < tableKeys.length; keyIndex++) {\n      const fieldName = tableKeys[keyIndex];\n      tableItem[fieldName] = columnarTable[fieldName][index];\n    }\n    rowFormatTable.push(tableItem);\n  }\n\n  return {\n    shape: 'object-row-table',\n    data: rowFormatTable\n  };\n}\n"],"mappings":"AAYA,OAAO,SAASA,8BAA8BA,CAACC,UAA4B,EAAc;EACvF,OAAO;IACLC,KAAK,EAAE,aAAa;IACpBC,IAAI,EAAEF;EACR,CAAC;AACH;AAMA,OAAO,SAASG,2BAA2BA,CAACC,KAAiB,EAAiB;EAI5E,MAAMJ,UAAU,GAAGI,KAAK,CAACF,IAAI;EAC7B,MAAMG,aAAa,GAAG,CAAC,CAAC;EAExB,KAAK,MAAMC,KAAK,IAAIN,UAAU,CAACO,MAAM,CAACC,MAAM,EAAE;IAE5C,MAAMC,WAAW,GAAGT,UAAU,CAACU,QAAQ,CAACJ,KAAK,CAACK,IAAI,CAAC;IACnD,MAAMC,MAAM,GAAGH,WAAW,aAAXA,WAAW,uBAAXA,WAAW,CAAEI,OAAO,CAAC,CAAC;IACrCR,aAAa,CAACC,KAAK,CAACK,IAAI,CAAC,GAAGC,MAAM;EACpC;EAEA,OAAO;IACLX,KAAK,EAAE,gBAAgB;IACvBC,IAAI,EAAEG;EACR,CAAC;AACH;AAMA,OAAO,SAASS,+BAA+BA,CAACT,aAA4B,EAAkB;EAC5F,MAAMU,SAAS,GAAGC,MAAM,CAACC,IAAI,CAACZ,aAAa,CAAC;EAC5C,MAAMa,cAAc,GAAGb,aAAa,CAACU,SAAS,CAAC,CAAC,CAAC,CAAC,CAACI,MAAM;EAEzD,MAAMC,cAAoB,GAAG,EAAE;EAE/B,KAAK,IAAIC,KAAK,GAAG,CAAC,EAAEA,KAAK,GAAGH,cAAc,EAAEG,KAAK,EAAE,EAAE;IACnD,MAAMC,SAAS,GAAG,CAAC,CAAC;IACpB,KAAK,IAAIC,QAAQ,GAAG,CAAC,EAAEA,QAAQ,GAAGR,SAAS,CAACI,MAAM,EAAEI,QAAQ,EAAE,EAAE;MAC9D,MAAMC,SAAS,GAAGT,SAAS,CAACQ,QAAQ,CAAC;MACrCD,SAAS,CAACE,SAAS,CAAC,GAAGnB,aAAa,CAACmB,SAAS,CAAC,CAACH,KAAK,CAAC;IACxD;IACAD,cAAc,CAACK,IAAI,CAACH,SAAS,CAAC;EAChC;EAEA,OAAO;IACLrB,KAAK,EAAE,kBAAkB;IACzBC,IAAI,EAAEkB;EACR,CAAC;AACH"}

package/dist/schema/convert-schema-arrow.d.ts

@@ -1,13 +0,0 @@
-import type { DataType, Field, Schema, SchemaMetadata } from '@loaders.gl/schema';
-import { Field as ArrowField, Schema as ArrowSchema, DataType as ArrowDataType } from 'apache-arrow';
-export declare function serializeArrowSchema(arrowSchema: ArrowSchema): Schema;
-export declare function deserializeArrowSchema(schema: Schema): ArrowSchema;
-export declare function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata;
-export declare function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string>;
-export declare function serializeArrowField(field: ArrowField): Field;
-export declare function deserializeArrowField(field: Field): ArrowField;
-/** Converts a serializable loaders.gl data type to hydrated arrow data type */
-export declare function serializeArrowType(arrowType: ArrowDataType): DataType;
-/** Converts a serializable loaders.gl data type to hydrated arrow data type */
-export declare function deserializeArrowType(dataType: DataType): ArrowDataType;
-//# sourceMappingURL=convert-schema-arrow.d.ts.map

package/dist/schema/convert-schema-arrow.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-schema-arrow.d.ts","sourceRoot":"","sources":["../../src/schema/convert-schema-arrow.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAChF,OAAO,EACL,KAAK,IAAI,UAAU,EACnB,MAAM,IAAI,WAAW,EACrB,QAAQ,IAAI,aAAa,EAkC1B,MAAM,cAAc,CAAC;AAEtB,wBAAgB,oBAAoB,CAAC,WAAW,EAAE,WAAW,GAAG,MAAM,CAKrE;AAED,wBAAgB,sBAAsB,CAAC,MAAM,EAAE,MAAM,GAAG,WAAW,CAKlE;AAED,wBAAgB,sBAAsB,CAAC,aAAa,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,GAAG,cAAc,CAEzF;AAED,wBAAgB,wBAAwB,CAAC,QAAQ,CAAC,EAAE,cAAc,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAEvF;AAED,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,UAAU,GAAG,KAAK,CAO5D;AAED,wBAAgB,qBAAqB,CAAC,KAAK,EAAE,KAAK,GAAG,UAAU,CAO9D;AAED,+EAA+E;AAE/E,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,aAAa,GAAG,QAAQ,CAqErE;AAED,+EAA+E;AAE/E,wBAAgB,oBAAoB,CAAC,QAAQ,EAAE,QAAQ,GAAG,aAAa,CA0EtE"}

package/dist/schema/convert-schema-arrow.js

@@ -1,158 +0,0 @@
-import { Field as ArrowField, Schema as ArrowSchema, Null, Binary, Bool, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64, Float16, Float32, Float64, Utf8, DateDay, DateMillisecond, TimeMillisecond, TimeSecond, TimestampSecond, TimestampMillisecond, TimestampMicrosecond, TimestampNanosecond, IntervalDayTime, IntervalYearMonth, FixedSizeList, Struct } from 'apache-arrow';
-export function serializeArrowSchema(arrowSchema) {
-  return {
-    fields: arrowSchema.fields.map(arrowField => serializeArrowField(arrowField)),
-    metadata: serializeArrowMetadata(arrowSchema.metadata)
-  };
-}
-export function deserializeArrowSchema(schema) {
-  return new ArrowSchema(schema.fields.map(field => deserializeArrowField(field)), deserializeArrowMetadata(schema.metadata));
-}
-export function serializeArrowMetadata(arrowMetadata) {
-  return Object.fromEntries(arrowMetadata);
-}
-export function deserializeArrowMetadata(metadata) {
-  return metadata ? new Map(Object.entries(metadata)) : new Map();
-}
-export function serializeArrowField(field) {
-  return {
-    name: field.name,
-    type: serializeArrowType(field.type),
-    nullable: field.nullable,
-    metadata: serializeArrowMetadata(field.metadata)
-  };
-}
-export function deserializeArrowField(field) {
-  return new ArrowField(field.name, deserializeArrowType(field.type), field.nullable, deserializeArrowMetadata(field.metadata));
-}
-export function serializeArrowType(arrowType) {
-  switch (arrowType.constructor) {
-    case Null:
-      return 'null';
-    case Binary:
-      return 'binary';
-    case Bool:
-      return 'bool';
-    case Int8:
-      return 'int8';
-    case Int16:
-      return 'int16';
-    case Int32:
-      return 'int32';
-    case Int64:
-      return 'int64';
-    case Uint8:
-      return 'uint8';
-    case Uint16:
-      return 'uint16';
-    case Uint32:
-      return 'uint32';
-    case Uint64:
-      return 'uint64';
-    case Float16:
-      return 'float16';
-    case Float32:
-      return 'float32';
-    case Float64:
-      return 'float64';
-    case Utf8:
-      return 'utf8';
-    case DateDay:
-      return 'date-day';
-    case DateMillisecond:
-      return 'date-millisecond';
-    case TimeMillisecond:
-      return 'time-millisecond';
-    case TimeSecond:
-      return 'time-second';
-    case TimestampSecond:
-      return 'timestamp-second';
-    case TimestampMillisecond:
-      return 'timestamp-millisecond';
-    case TimestampMicrosecond:
-      return 'timestamp-microsecond';
-    case TimestampNanosecond:
-      return 'timestamp-nanosecond';
-    case IntervalDayTime:
-      return 'interval-daytime';
-    case IntervalYearMonth:
-      return 'interval-yearmonth';
-    case FixedSizeList:
-      return {
-        type: 'fixed-size-list',
-        listSize: arrowType.listSize,
-        children: [serializeArrowField(arrowType.children[0])]
-      };
-    default:
-      throw new Error('array type not supported');
-  }
-}
-export function deserializeArrowType(dataType) {
-  if (typeof dataType === 'object') {
-    switch (dataType.type) {
-      case 'fixed-size-list':
-        const child = deserializeArrowField(dataType.children[0]);
-        return new FixedSizeList(dataType.listSize, child);
-      case 'struct':
-        const children = dataType.children.map(arrowField => deserializeArrowField(arrowField));
-        return new Struct(children);
-      default:
-        throw new Error('array type not supported');
-    }
-  }
-  switch (dataType) {
-    case 'null':
-      return new Null();
-    case 'binary':
-      return new Binary();
-    case 'bool':
-      return new Bool();
-    case 'int8':
-      return new Int8();
-    case 'int16':
-      return new Int16();
-    case 'int32':
-      return new Int32();
-    case 'int64':
-      return new Int64();
-    case 'uint8':
-      return new Uint8();
-    case 'uint16':
-      return new Uint16();
-    case 'uint32':
-      return new Uint32();
-    case 'uint64':
-      return new Uint64();
-    case 'float16':
-      return new Float16();
-    case 'float32':
-      return new Float32();
-    case 'float64':
-      return new Float64();
-    case 'utf8':
-      return new Utf8();
-    case 'date-day':
-      return new DateDay();
-    case 'date-millisecond':
-      return new DateMillisecond();
-    case 'time-millisecond':
-      return new TimeMillisecond();
-    case 'time-second':
-      return new TimeSecond();
-    case 'timestamp-second':
-      return new TimestampSecond();
-    case 'timestamp-millisecond':
-      return new TimestampMillisecond();
-    case 'timestamp-microsecond':
-      return new TimestampMicrosecond();
-    case 'timestamp-nanosecond':
-      return new TimestampNanosecond();
-    case 'interval-daytime':
-      return new IntervalDayTime();
-    case 'interval-yearmonth':
-      return new IntervalYearMonth();
-    default:
-      throw new Error('array type not supported');
-  }
-}
-//# sourceMappingURL=convert-schema-arrow.js.map

package/dist/schema/convert-schema-arrow.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"convert-schema-arrow.js","names":["Field","ArrowField","Schema","ArrowSchema","Null","Binary","Bool","Int8","Int16","Int32","Int64","Uint8","Uint16","Uint32","Uint64","Float16","Float32","Float64","Utf8","DateDay","DateMillisecond","TimeMillisecond","TimeSecond","TimestampSecond","TimestampMillisecond","TimestampMicrosecond","TimestampNanosecond","IntervalDayTime","IntervalYearMonth","FixedSizeList","Struct","serializeArrowSchema","arrowSchema","fields","map","arrowField","serializeArrowField","metadata","serializeArrowMetadata","deserializeArrowSchema","schema","field","deserializeArrowField","deserializeArrowMetadata","arrowMetadata","Object","fromEntries","Map","entries","name","type","serializeArrowType","nullable","deserializeArrowType","arrowType","constructor","listSize","children","Error","dataType","child"],"sources":["../../src/schema/convert-schema-arrow.ts"],"sourcesContent":["// loaders.gl, MIT license\n// Copyright (c) vis.gl contributors\n\nimport type {DataType, Field, Schema, SchemaMetadata} from '@loaders.gl/schema';\nimport {\n  Field as ArrowField,\n  Schema as ArrowSchema,\n  DataType as ArrowDataType,\n  Null,\n  Binary,\n  Bool,\n  // Int,\n  Int8,\n  Int16,\n  Int32,\n  Int64,\n  Uint8,\n  Uint16,\n  Uint32,\n  Uint64,\n  // Float,\n  Float16,\n  Float32,\n  Float64,\n  Utf8,\n  // Date,\n  DateDay,\n  DateMillisecond,\n  // Time,\n  TimeMillisecond,\n  TimeSecond,\n  // Timestamp,\n  TimestampSecond,\n  TimestampMillisecond,\n  TimestampMicrosecond,\n  TimestampNanosecond,\n  // Interval,\n  IntervalDayTime,\n  IntervalYearMonth,\n  FixedSizeList,\n  Struct\n} from 'apache-arrow';\n\nexport function serializeArrowSchema(arrowSchema: ArrowSchema): Schema {\n  return {\n    fields: arrowSchema.fields.map((arrowField) => serializeArrowField(arrowField)),\n    metadata: serializeArrowMetadata(arrowSchema.metadata)\n  };\n}\n\nexport function deserializeArrowSchema(schema: Schema): ArrowSchema {\n  return new ArrowSchema(\n    schema.fields.map((field) => deserializeArrowField(field)),\n    deserializeArrowMetadata(schema.metadata)\n  );\n}\n\nexport function serializeArrowMetadata(arrowMetadata: Map<string, string>): SchemaMetadata {\n  return Object.fromEntries(arrowMetadata);\n}\n\nexport function deserializeArrowMetadata(metadata?: SchemaMetadata): Map<string, string> {\n  return metadata ? new Map(Object.entries(metadata)) : new Map<string, string>();\n}\n\nexport function serializeArrowField(field: ArrowField): Field {\n  return {\n    name: field.name,\n    type: serializeArrowType(field.type),\n    nullable: field.nullable,\n    metadata: serializeArrowMetadata(field.metadata)\n  };\n}\n\nexport function deserializeArrowField(field: Field): ArrowField {\n  return new ArrowField(\n    field.name,\n    deserializeArrowType(field.type),\n    field.nullable,\n    deserializeArrowMetadata(field.metadata)\n  );\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function serializeArrowType(arrowType: ArrowDataType): DataType {\n  switch (arrowType.constructor) {\n    case Null:\n      return 'null';\n    case Binary:\n      return 'binary';\n    case Bool:\n      return 'bool';\n    // case Int: return 'int';\n    case Int8:\n      return 'int8';\n    case Int16:\n      return 'int16';\n    case Int32:\n      return 'int32';\n    case Int64:\n      return 'int64';\n    case Uint8:\n      return 'uint8';\n    case Uint16:\n      return 'uint16';\n    case Uint32:\n      return 'uint32';\n    case Uint64:\n      return 'uint64';\n    // case Float: return 'float';\n    case Float16:\n      return 'float16';\n    case Float32:\n      return 'float32';\n    case Float64:\n      return 'float64';\n    case Utf8:\n      return 'utf8';\n    // case Date: return 'date';\n    case DateDay:\n      return 'date-day';\n    case DateMillisecond:\n      return 'date-millisecond';\n    // case Time: return 'time';\n    case TimeMillisecond:\n      return 'time-millisecond';\n    case TimeSecond:\n      return 'time-second';\n    // case Timestamp: return 'timestamp';\n    case TimestampSecond:\n      return 'timestamp-second';\n    case TimestampMillisecond:\n      return 'timestamp-millisecond';\n    case TimestampMicrosecond:\n      return 'timestamp-microsecond';\n    case TimestampNanosecond:\n      return 'timestamp-nanosecond';\n    // case Interval: return 'interval';\n    case IntervalDayTime:\n      return 'interval-daytime';\n    case IntervalYearMonth:\n      return 'interval-yearmonth';\n    case FixedSizeList:\n      return {\n        type: 'fixed-size-list',\n        listSize: (arrowType as FixedSizeList).listSize,\n        children: [serializeArrowField((arrowType as FixedSizeList).children[0])]\n      };\n    // case Struct:\n    //   return {type: 'struct', children: (arrowType as Struct).children};\n    default:\n      throw new Error('array type not supported');\n  }\n}\n\n/** Converts a serializable loaders.gl data type to hydrated arrow data type */\n// eslint-disable-next-line complexity\nexport function deserializeArrowType(dataType: DataType): ArrowDataType {\n  if (typeof dataType === 'object') {\n    switch (dataType.type) {\n      case 'fixed-size-list':\n        const child = deserializeArrowField(dataType.children[0]);\n        return new FixedSizeList(dataType.listSize, child);\n      case 'struct':\n        const children = dataType.children.map((arrowField) => deserializeArrowField(arrowField));\n        return new Struct(children);\n      default:\n        throw new Error('array type not supported');\n    }\n  }\n\n  switch (dataType) {\n    case 'null':\n      return new Null();\n    case 'binary':\n      return new Binary();\n    case 'bool':\n      return new Bool();\n    // case 'int': return new Int();\n    case 'int8':\n      return new Int8();\n    case 'int16':\n      return new Int16();\n    case 'int32':\n      return new Int32();\n    case 'int64':\n      return new Int64();\n    case 'uint8':\n      return new Uint8();\n    case 'uint16':\n      return new Uint16();\n    case 'uint32':\n      return new Uint32();\n    case 'uint64':\n      return new Uint64();\n    // case 'float': return new Float();\n    case 'float16':\n      return new Float16();\n    case 'float32':\n      return new Float32();\n    case 'float64':\n      return new Float64();\n    case 'utf8':\n      return new Utf8();\n    // case 'date': return new Date();\n    case 'date-day':\n      return new DateDay();\n    case 'date-millisecond':\n      return new DateMillisecond();\n    // case 'time': return new Time();\n    case 'time-millisecond':\n      return new TimeMillisecond();\n    case 'time-second':\n      return new TimeSecond();\n    // case 'timestamp': return new Timestamp();\n    case 'timestamp-second':\n      return new TimestampSecond();\n    case 'timestamp-millisecond':\n      return new TimestampMillisecond();\n    case 'timestamp-microsecond':\n      return new TimestampMicrosecond();\n    case 'timestamp-nanosecond':\n      return new TimestampNanosecond();\n    // case 'interval': return new Interval();\n    case 'interval-daytime':\n      return new IntervalDayTime();\n    case 'interval-yearmonth':\n      return new IntervalYearMonth();\n    default:\n      throw new Error('array type not supported');\n  }\n}\n"],"mappings":"AAIA,SACEA,KAAK,IAAIC,UAAU,EACnBC,MAAM,IAAIC,WAAW,EAErBC,IAAI,EACJC,MAAM,EACNC,IAAI,EAEJC,IAAI,EACJC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,KAAK,EACLC,MAAM,EACNC,MAAM,EACNC,MAAM,EAENC,OAAO,EACPC,OAAO,EACPC,OAAO,EACPC,IAAI,EAEJC,OAAO,EACPC,eAAe,EAEfC,eAAe,EACfC,UAAU,EAEVC,eAAe,EACfC,oBAAoB,EACpBC,oBAAoB,EACpBC,mBAAmB,EAEnBC,eAAe,EACfC,iBAAiB,EACjBC,aAAa,EACbC,MAAM,QACD,cAAc;AAErB,OAAO,SAASC,oBAAoBA,CAACC,WAAwB,EAAU;EACrE,OAAO;IACLC,MAAM,EAAED,WAAW,CAACC,MAAM,CAACC,GAAG,CAAEC,UAAU,IAAKC,mBAAmB,CAACD,UAAU,CAAC,CAAC;IAC/EE,QAAQ,EAAEC,sBAAsB,CAACN,WAAW,CAACK,QAAQ;EACvD,CAAC;AACH;AAEA,OAAO,SAASE,sBAAsBA,CAACC,MAAc,EAAe;EAClE,OAAO,IAAIrC,WAAW,CACpBqC,MAAM,CAACP,MAAM,CAACC,GAAG,CAAEO,KAAK,IAAKC,qBAAqB,CAACD,KAAK,CAAC,CAAC,EAC1DE,wBAAwB,CAACH,MAAM,CAACH,QAAQ,CAC1C,CAAC;AACH;AAEA,OAAO,SAASC,sBAAsBA,CAACM,aAAkC,EAAkB;EACzF,OAAOC,MAAM,CAACC,WAAW,CAACF,aAAa,CAAC;AAC1C;AAEA,OAAO,SAASD,wBAAwBA,CAACN,QAAyB,EAAuB;EACvF,OAAOA,QAAQ,GAAG,IAAIU,GAAG,CAACF,MAAM,CAACG,OAAO,CAACX,QAAQ,CAAC,CAAC,GAAG,IAAIU,GAAG,CAAiB,CAAC;AACjF;AAEA,OAAO,SAASX,mBAAmBA,CAACK,KAAiB,EAAS;EAC5D,OAAO;IACLQ,IAAI,EAAER,KAAK,CAACQ,IAAI;IAChBC,IAAI,EAAEC,kBAAkB,CAACV,KAAK,CAACS,IAAI,CAAC;IACpCE,QAAQ,EAAEX,KAAK,CAACW,QAAQ;IACxBf,QAAQ,EAAEC,sBAAsB,CAACG,KAAK,CAACJ,QAAQ;EACjD,CAAC;AACH;AAEA,OAAO,SAASK,qBAAqBA,CAACD,KAAY,EAAc;EAC9D,OAAO,IAAIxC,UAAU,CACnBwC,KAAK,CAACQ,IAAI,EACVI,oBAAoB,CAACZ,KAAK,CAACS,IAAI,CAAC,EAChCT,KAAK,CAACW,QAAQ,EACdT,wBAAwB,CAACF,KAAK,CAACJ,QAAQ,CACzC,CAAC;AACH;AAIA,OAAO,SAASc,kBAAkBA,CAACG,SAAwB,EAAY;EACrE,QAAQA,SAAS,CAACC,WAAW;IAC3B,KAAKnD,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,IAAI;MACP,OAAO,MAAM;IACf,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,KAAK;MACR,OAAO,OAAO;IAChB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IACjB,KAAKC,MAAM;MACT,OAAO,QAAQ;IAEjB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,OAAO;MACV,OAAO,SAAS;IAClB,KAAKC,IAAI;MACP,OAAO,MAAM;IAEf,KAAKC,OAAO;MACV,OAAO,UAAU;IACnB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAE3B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,UAAU;MACb,OAAO,aAAa;IAEtB,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,oBAAoB;MACvB,OAAO,uBAAuB;IAChC,KAAKC,mBAAmB;MACtB,OAAO,sBAAsB;IAE/B,KAAKC,eAAe;MAClB,OAAO,kBAAkB;IAC3B,KAAKC,iBAAiB;MACpB,OAAO,oBAAoB;IAC7B,KAAKC,aAAa;MAChB,OAAO;QACLqB,IAAI,EAAE,iBAAiB;QACvBM,QAAQ,EAAGF,SAAS,CAAmBE,QAAQ;QAC/CC,QAAQ,EAAE,CAACrB,mBAAmB,CAAEkB,SAAS,CAAmBG,QAAQ,CAAC,CAAC,CAAC,CAAC;MAC1E,CAAC;IAGH;MACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF;AAIA,OAAO,SAASL,oBAAoBA,CAACM,QAAkB,EAAiB;EACtE,IAAI,OAAOA,QAAQ,KAAK,QAAQ,EAAE;IAChC,QAAQA,QAAQ,CAACT,IAAI;MACnB,KAAK,iBAAiB;QACpB,MAAMU,KAAK,GAAGlB,qBAAqB,CAACiB,QAAQ,CAACF,QAAQ,CAAC,CAAC,CAAC,CAAC;QACzD,OAAO,IAAI5B,aAAa,CAAC8B,QAAQ,CAACH,QAAQ,EAAEI,KAAK,CAAC;MACpD,KAAK,QAAQ;QACX,MAAMH,QAAQ,GAAGE,QAAQ,CAACF,QAAQ,CAACvB,GAAG,CAAEC,UAAU,IAAKO,qBAAqB,CAACP,UAAU,CAAC,CAAC;QACzF,OAAO,IAAIL,MAAM,CAAC2B,QAAQ,CAAC;MAC7B;QACE,MAAM,IAAIC,KAAK,CAAC,0BAA0B,CAAC;IAC/C;EACF;EAEA,QAAQC,QAAQ;IACd,KAAK,MAAM;MACT,OAAO,IAAIvD,IAAI,CAAC,CAAC;IACnB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IACnB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,OAAO;MACV,OAAO,IAAIC,KAAK,CAAC,CAAC;IACpB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IACrB,KAAK,QAAQ;MACX,OAAO,IAAIC,MAAM,CAAC,CAAC;IAErB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,SAAS;MACZ,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,MAAM;MACT,OAAO,IAAIC,IAAI,CAAC,CAAC;IAEnB,KAAK,UAAU;MACb,OAAO,IAAIC,OAAO,CAAC,CAAC;IACtB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAE9B,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,aAAa;MAChB,OAAO,IAAIC,UAAU,CAAC,CAAC;IAEzB,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,uBAAuB;MAC1B,OAAO,IAAIC,oBAAoB,CAAC,CAAC;IACnC,KAAK,sBAAsB;MACzB,OAAO,IAAIC,mBAAmB,CAAC,CAAC;IAElC,KAAK,kBAAkB;MACrB,OAAO,IAAIC,eAAe,CAAC,CAAC;IAC9B,KAAK,oBAAoB;MACvB,OAAO,IAAIC,iBAAiB,CAAC,CAAC;IAChC;MACE,MAAM,IAAI8B,KAAK,CAAC,0BAA0B,CAAC;EAC/C;AACF"}