@loaders.gl/schema 3.1.0-alpha.4 → 3.1.0-beta.3

package/dist/lib/utils/async-queue.d.ts

@@ -0,0 +1,18 @@
+export default class AsyncQueue<T> {
+    private _values;
+    private _settlers;
+    private _closed;
+    constructor();
+    close(): void;
+    [Symbol.asyncIterator](): AsyncIterator<T>;
+    enqueue(value: T | Error): void;
+    /**
+     * @returns a Promise for an IteratorResult
+     */
+    next(): Promise<any>;
+}
+/**
+ * @returns a Promise for an Array with the elements in `asyncIterable`
+ */
+export declare function takeAsync(asyncIterable: AsyncIterable<any>, count?: number): Promise<any[]>;
+//# sourceMappingURL=async-queue.d.ts.map

package/dist/lib/utils/async-queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"async-queue.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/async-queue.ts"],"names":[],"mappings":"AAcA,MAAM,CAAC,OAAO,OAAO,UAAU,CAAC,CAAC;IAC/B,OAAO,CAAC,OAAO,CAAwB;IACvC,OAAO,CAAC,SAAS,CAA8E;IAC/F,OAAO,CAAC,OAAO,CAAU;;IAazB,KAAK,IAAI,IAAI;IAOb,CAAC,MAAM,CAAC,aAAa,CAAC,IAAI,aAAa,CAAC,CAAC,CAAC;IAI1C,OAAO,CAAC,KAAK,EAAE,CAAC,GAAG,KAAK,GAAG,IAAI;IAoB/B;;OAEG;IACH,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC;CAoBrB;AAED;;GAEG;AACH,wBAAsB,SAAS,CAC7B,aAAa,EAAE,aAAa,CAAC,GAAG,CAAC,EACjC,KAAK,SAAW,GACf,OAAO,CAAC,GAAG,EAAE,CAAC,CAWhB"}

package/dist/lib/utils/async-queue.js

@@ -0,0 +1,92 @@
+"use strict";
+// From https://github.com/rauschma/async-iter-demo/tree/master/src under MIT license
+// http://2ality.com/2016/10/asynchronous-iteration.html
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.takeAsync = void 0;
+class ArrayQueue extends Array {
+    enqueue(value) {
+        // Add at the end
+        return this.push(value);
+    }
+    dequeue() {
+        // Remove first element
+        return this.shift();
+    }
+}
+class AsyncQueue {
+    constructor() {
+        // enqueues > dequeues
+        this._values = new ArrayQueue();
+        // dequeues > enqueues
+        this._settlers = new ArrayQueue();
+        this._closed = false;
+    }
+    close() {
+        while (this._settlers.length > 0) {
+            this._settlers.dequeue().resolve({ done: true });
+        }
+        this._closed = true;
+    }
+    [Symbol.asyncIterator]() {
+        return this;
+    }
+    enqueue(value) {
+        if (this._closed) {
+            throw new Error('Closed');
+        }
+        if (this._settlers.length > 0) {
+            if (this._values.length > 0) {
+                throw new Error('Illegal internal state');
+            }
+            const settler = this._settlers.dequeue();
+            if (value instanceof Error) {
+                settler.reject(value);
+            }
+            else {
+                settler.resolve({ value });
+            }
+        }
+        else {
+            this._values.enqueue(value);
+        }
+    }
+    /**
+     * @returns a Promise for an IteratorResult
+     */
+    next() {
+        if (this._values.length > 0) {
+            const value = this._values.dequeue();
+            if (value instanceof Error) {
+                return Promise.reject(value);
+            }
+            return Promise.resolve({ value });
+        }
+        if (this._closed) {
+            if (this._settlers.length > 0) {
+                throw new Error('Illegal internal state');
+            }
+            return Promise.resolve({ done: true });
+        }
+        // Wait for new values to be enqueued
+        return new Promise((resolve, reject) => {
+            this._settlers.enqueue({ resolve, reject });
+        });
+    }
+}
+exports.default = AsyncQueue;
+/**
+ * @returns a Promise for an Array with the elements in `asyncIterable`
+ */
+async function takeAsync(asyncIterable, count = Infinity) {
+    const result = [];
+    const iterator = asyncIterable[Symbol.asyncIterator]();
+    while (result.length < count) {
+        const { value, done } = await iterator.next();
+        if (done) {
+            break;
+        }
+        result.push(value);
+    }
+    return result;
+}
+exports.takeAsync = takeAsync;

package/dist/lib/utils/row-utils.d.ts

@@ -0,0 +1,9 @@
+/** Convert an object row to an array row */
+export declare function convertToObjectRow(arrayRow: any[], headers: string[]): {
+    [columnName: string]: any;
+};
+/** Convert an object row to an array row */
+export declare function convertToArrayRow(objectRow: {
+    [columnName: string]: any;
+}, headers: string[]): any[];
+//# sourceMappingURL=row-utils.d.ts.map

package/dist/lib/utils/row-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"row-utils.d.ts","sourceRoot":"","sources":["../../../src/lib/utils/row-utils.ts"],"names":[],"mappings":"AAAA,4CAA4C;AAC5C,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,GAAG,EAAE,EACf,OAAO,EAAE,MAAM,EAAE,GAChB;IAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,CAY7B;AAED,4CAA4C;AAC5C,wBAAgB,iBAAiB,CAC/B,SAAS,EAAE;IAAC,CAAC,UAAU,EAAE,MAAM,GAAG,GAAG,CAAA;CAAC,EACtC,OAAO,EAAE,MAAM,EAAE,GAChB,GAAG,EAAE,CAYP"}

package/dist/lib/utils/row-utils.js

@@ -0,0 +1,33 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.convertToArrayRow = exports.convertToObjectRow = void 0;
+/** Convert an object row to an array row */
+function convertToObjectRow(arrayRow, headers) {
+    if (!arrayRow) {
+        throw new Error('null row');
+    }
+    if (!headers) {
+        throw new Error('no headers');
+    }
+    const objectRow = {};
+    for (let i = 0; i < headers.length; i++) {
+        objectRow[headers[i]] = arrayRow[i];
+    }
+    return objectRow;
+}
+exports.convertToObjectRow = convertToObjectRow;
+/** Convert an object row to an array row */
+function convertToArrayRow(objectRow, headers) {
+    if (!objectRow) {
+        throw new Error('null row');
+    }
+    if (!headers) {
+        throw new Error('no headers');
+    }
+    const arrayRow = new Array(headers.length);
+    for (let i = 0; i < headers.length; i++) {
+        arrayRow[i] = objectRow[headers[i]];
+    }
+    return arrayRow;
+}
+exports.convertToArrayRow = convertToArrayRow;

package/dist/types.d.ts

@@ -0,0 +1,8 @@
+/** Any typed array */
+export declare type TypedArray = Int8Array | Uint8Array | Int16Array | Uint16Array | Int32Array | Uint32Array | Uint8ClampedArray | Float32Array | Float64Array;
+export declare type BigTypedArray = TypedArray | BigInt64Array | BigUint64Array;
+/** Any numeric array: typed array or `number[]` */
+export declare type NumberArray = number[] | TypedArray;
+/** Any array: typed array or js array (`any[]`) */
+export declare type AnyArray = any[] | TypedArray;
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,sBAAsB;AACtB,oBAAY,UAAU,GAClB,SAAS,GACT,UAAU,GACV,UAAU,GACV,WAAW,GACX,UAAU,GACV,WAAW,GACX,iBAAiB,GACjB,YAAY,GACZ,YAAY,CAAC;AAEjB,oBAAY,aAAa,GAAG,UAAU,GAAG,aAAa,GAAG,cAAc,CAAC;AAExE,mDAAmD;AACnD,oBAAY,WAAW,GAAG,MAAM,EAAE,GAAG,UAAU,CAAC;AAEhD,mDAAmD;AACnD,oBAAY,QAAQ,GAAG,GAAG,EAAE,GAAG,UAAU,CAAC"}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/schema",
-  "version": "3.1.0-alpha.4",
+  "version": "3.1.0-beta.3",
   "description": "Table format APIs for JSON, CSV, etc...",
   "license": "MIT",
   "publishConfig": {
@@ -18,7 +18,7 @@
     "point cloud",
     "PLY"
   ],
-  "types": "src/index.ts",
+  "types": "dist/index.d.ts",
   "main": "dist/es5/index.js",
   "module": "dist/esm/index.js",
   "sideEffects": false,
@@ -29,12 +29,10 @@
   ],
   "scripts": {
     "pre-build": "npm run build-bundle",
-    "build-bundle": "webpack --display=minimal --config ../../scripts/webpack/bundle.js"
+    "build-bundle": "esbuild src/bundle.ts --bundle --outfile=dist/bundle.js"
   },
   "dependencies": {
-    "@types/geojson": "^7946.0.7",
-    "apache-arrow": "^4.0.0",
-    "d3-dsv": "^1.2.0"
+    "@types/geojson": "^7946.0.7"
   },
-  "gitHead": "e309784af37ef9f3640c7733c7851124c72e1fa3"
+  "gitHead": "3537c382b3ea4e092b24b6f94c3edee72076039c"
 }

package/src/category/common.ts

@@ -1,4 +1,5 @@
-import type {RecordBatch} from 'apache-arrow';
+// import type {RecordBatch} from 'apache-arrow';
+type RecordBatch = any;
 
 export type Field = any;
 

package/src/category/mesh/convert-mesh.ts

@@ -1,6 +1,6 @@
 import type {Mesh} from './mesh-types';
 import type {ColumnarTable, ArrowTable} from '../table/table-types';
-import {convertMeshToArrowTable} from './mesh-to-arrow-table';
+// import {convertMeshToArrowTable} from './mesh-to-arrow-table';
 
 type TargetShape = 'mesh' | 'columnar-table' | 'arrow-table';
 
@@ -17,11 +17,11 @@ export function convertMesh(
       return mesh;
     case 'columnar-table':
       return convertMeshToColumnarTable(mesh);
-    case 'arrow-table':
-      return {
-        shape: 'arrow-table',
-        data: convertMeshToArrowTable(mesh)
-      };
+    // case 'arrow-table':
+    //   return {
+    //     shape: 'arrow-table',
+    //     data: convertMeshToArrowTable(mesh)
+    //   };
     default:
       throw new Error(`Unsupported shape ${options?.shape}`);
   }

package/src/category/mesh/deduce-mesh-schema.ts

@@ -1,5 +1,5 @@
 import {MeshAttribute, MeshAttributes} from './mesh-types';
-import {Schema, Field, FixedSizeList} from '../../lib/schema';
+import {Schema, Field, FixedSizeList} from '../../lib/schema/schema';
 import {getArrowTypeFromTypedArray} from '../../lib/arrow/arrow-like-type-utils';
 
 /**

package/src/category/mesh/mesh-to-arrow-table.ts

@@ -1,3 +1,4 @@
+/*
 import {
   Table,
   Schema,
@@ -6,7 +7,7 @@ import {
   Field,
   Data,
   FixedSizeListVector
-} from 'apache-arrow/Arrow.dom';
+} from 'apache-arrow';
 import {AbstractVector} from 'apache-arrow/vector';
 import {getArrowType, getArrowVector} from '../../lib/arrow/arrow-type-utils';
 import type {Mesh} from './mesh-types';
@@ -18,7 +19,7 @@ import {makeMeshAttributeMetadata} from './deduce-mesh-schema';
  * @param metadata
  * @param batchSize
  * @returns
- */
+ *
 export function convertMeshToArrowTable(mesh: Mesh, batchSize?: number): Table {
   const vectors: AbstractVector[] = [];
   const fields: Field[] = [];
@@ -39,3 +40,4 @@ export function convertMeshToArrowTable(mesh: Mesh, batchSize?: number): Table {
   const table = new Table(schema, recordBatch);
   return table;
 }
+*/

package/src/category/mesh/mesh-types.ts

@@ -1,4 +1,4 @@
-import type {Schema} from '../../lib/schema';
+import type {Schema} from '../../lib/schema/schema';
 import type {TypedArray} from '../../types';
 import type {ColumnarTable, ArrowTable} from '../table/table-types';
 

package/src/category/table/deduce-table-schema.ts

@@ -14,7 +14,7 @@ import {
   // Utf8,
   // TimestampMillisecond,
   // Null
-} from '../../lib/schema';
+} from '../../lib/schema/schema';
 
 // const TYPED_ARRAY_TO_TYPE = {
 //   Int8Array: new Int8(),

package/src/category/table/table-types.ts

@@ -1,7 +1,10 @@
-import type {Schema} from '../../lib/schema';
-import type {Table as ApacheArrowTable, RecordBatch} from 'apache-arrow/Arrow.dom';
+import type {Schema} from '../../lib/schema/schema';
 import type {AnyArray} from '../../types';
 import type {Batch} from '../common';
+// TODO Arrow dependency causes issues
+// import type {Table as ApacheArrowTable, RecordBatch} from 'apache-arrow/Arrow.dom';
+type ApacheArrowTable = any;
+type RecordBatch = any;
 
 /** A general table */
 export interface Table {

package/src/index.ts

@@ -112,7 +112,7 @@ export {
   IntervalYearMonth,
   FixedSizeList,
   Struct
-} from './lib/schema';
+} from './lib/schema/schema';
 
 // EXPERIMENTAL APIs
 

package/src/lib/arrow/arrow-like-type-utils.ts

@@ -1,5 +1,15 @@
 import type {TypedArray} from '../../types';
-import {DataType, Float32, Float64, Int16, Int32, Int8, Uint16, Uint32, Uint8} from '../schema';
+import {
+  DataType,
+  Float32,
+  Float64,
+  Int16,
+  Int32,
+  Int8,
+  Uint16,
+  Uint32,
+  Uint8
+} from '../schema/schema';
 
 export function getArrowTypeFromTypedArray(array: TypedArray): DataType {
   switch (array.constructor) {

package/src/lib/arrow/arrow-type-utils.ts

@@ -1,3 +1,4 @@
+/*
 import type {TypedArray} from '../../types';
 import {
   DataType,
@@ -65,3 +66,4 @@ export function getArrowVector(array: TypedArray): AbstractVector {
       throw new Error('array type not supported');
   }
 }
+*/

package/src/lib/arrow/get-type-info.ts

@@ -1,4 +1,4 @@
-import {Type} from '../schema';
+import {Type} from '../schema/schema';
 import {AnyArray} from '../../types';
 
 /**

package/src/lib/batches/base-table-batch-aggregator.ts

@@ -1,4 +1,4 @@
-import type {Schema} from '../schema';
+import type {Schema} from '../schema/schema';
 import type {TableBatch} from '../../category/table/table-types';
 import {TableBatchAggregator, TableBatchOptions} from './table-batch-aggregator';
 

package/src/lib/batches/columnar-table-batch-aggregator.ts

@@ -1,4 +1,4 @@
-import type {Schema} from '../schema';
+import type {Schema} from '../schema/schema';
 import type {ColumnarTableBatch, ArrowTableBatch} from '../../category/table/table-types';
 import {TableBatchAggregator} from './table-batch-aggregator';
 

package/src/lib/batches/row-table-batch-aggregator.ts

@@ -1,4 +1,4 @@
-import type {Schema} from '../schema';
+import type {Schema} from '../schema/schema';
 import type {TableBatch} from '../../category/table/table-types';
 // import type {ArrayRowTableBatch, ObjectRowTableBatch} from '../../category/table';
 import {convertToArrayRow, convertToObjectRow} from '../utils/row-utils';

package/src/lib/batches/table-batch-aggregator.ts

@@ -1,4 +1,4 @@
-import type {Schema} from '../schema';
+import type {Schema} from '../schema/schema';
 import type {TableBatch} from '../../category/table/table-types';
 
 export interface TableBatchOptions {

package/src/lib/batches/table-batch-builder.ts

@@ -1,4 +1,4 @@
-import type {Schema} from '../schema';
+import type {Schema} from '../schema/schema';
 import type {TableBatch} from '../../category/table/table-types';
 import type {TableBatchAggregator, TableBatchConstructor} from './table-batch-aggregator';
 import BaseTableBatchAggregator from './base-table-batch-aggregator';
@@ -114,7 +114,7 @@ export default class TableBatchBuilder {
 
   // INTERNAL
 
-  _estimateRowMB(row) {
+  _estimateRowMB(row: any[] | object): number {
     return Array.isArray(row) ? row.length * 8 : Object.keys(row).length * 8;
   }
 

package/src/lib/schema/impl/schema.ts

@@ -82,8 +82,8 @@ export default class Schema {
 }
 
 // Warn if any duplicated field names
-function checkNames(fields) {
-  const usedNames = {};
+function checkNames(fields: Field[]): void {
+  const usedNames: Record<string, boolean> = {};
   for (const field of fields) {
     if (usedNames[field.name]) {
       // eslint-disable-next-line

package/src/lib/schema/impl/type.ts

@@ -333,10 +333,10 @@ const TimeUnit = {
 };
 
 export class Time extends DataType {
-  readonly unit: number;
+  readonly unit: any;
   readonly bitWidth: number;
 
-  constructor(unit, bitWidth) {
+  constructor(unit: any, bitWidth: number) {
     super();
     this.unit = unit;
     this.bitWidth = bitWidth;

package/src/lib/schema-utils/deduce-column-type.ts

@@ -33,7 +33,9 @@
 //   TYPED_ARRAY_TO_TYPE.BigUint64Array = new Uint64();
 // }
 
-export function deduceTypeFromColumn(value) {
+export function deduceTypeFromColumn(
+  value: unknown
+): StringConstructor | DateConstructor | Float32ArrayConstructor | null {
   if (value instanceof Date) {
     return Date;
   } else if (value instanceof Number) {
@@ -44,7 +46,9 @@ export function deduceTypeFromColumn(value) {
   return null;
 }
 
-export function deduceTypeFromValue(value) {
+export function deduceTypeFromValue(
+  value: unknown
+): StringConstructor | DateConstructor | Float32ArrayConstructor | null {
   if (value instanceof Date) {
     return Date;
   } else if (value instanceof Number) {

package/src/lib/utils/async-queue.ts

@@ -2,7 +2,7 @@
 // http://2ality.com/2016/10/asynchronous-iteration.html
 
 class ArrayQueue<T> extends Array<T> {
-  enqueue(value) {
+  enqueue(value: T) {
     // Add at the end
     return this.push(value);
   }
@@ -13,15 +13,18 @@ class ArrayQueue<T> extends Array<T> {
 }
 
 export default class AsyncQueue<T> {
-  private _values: ArrayQueue<T>;
-  private _settlers: ArrayQueue<{resolve; reject}>;
+  private _values: ArrayQueue<T | Error>;
+  private _settlers: ArrayQueue<{resolve: (value: any) => void; reject: (reason?: any) => void}>;
   private _closed: boolean;
 
   constructor() {
     // enqueues > dequeues
     this._values = new ArrayQueue<T>();
     // dequeues > enqueues
-    this._settlers = new ArrayQueue<{resolve; reject}>();
+    this._settlers = new ArrayQueue<{
+      resolve: (value: any) => void;
+      reject: (reason?: any) => void;
+    }>();
     this._closed = false;
   }