@loaders.gl/json 3.1.0-alpha.3 → 3.1.0-beta.2

package/dist/lib/jsonpath/jsonpath.d.ts

@@ -0,0 +1,32 @@
+/**
+ * A parser for a minimal subset of the jsonpath standard
+ * Full JSON path parsers for JS exist but are quite large (bundle size)
+ *
+ * Supports
+ *
+ *   `$.component.component.component`
+ */
+export default class JSONPath {
+    path: string[];
+    constructor(path?: JSONPath | string[] | string | null);
+    clone(): JSONPath;
+    toString(): string;
+    push(name: string): void;
+    pop(): string | undefined;
+    set(name: string): void;
+    equals(other: JSONPath): boolean;
+    /**
+     * Sets the value pointed at by path
+     * TODO - handle root path
+     * @param object
+     * @param value
+     */
+    setFieldAtPath(object: any, value: any): void;
+    /**
+     * Gets the value pointed at by path
+     * TODO - handle root path
+     * @param object
+     */
+    getFieldAtPath(object: any): any;
+}
+//# sourceMappingURL=jsonpath.d.ts.map

package/dist/lib/jsonpath/jsonpath.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"jsonpath.d.ts","sourceRoot":"","sources":["../../../src/lib/jsonpath/jsonpath.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,MAAM,CAAC,OAAO,OAAO,QAAQ;IAC3B,IAAI,EAAE,MAAM,EAAE,CAAC;gBAEH,IAAI,GAAE,QAAQ,GAAG,MAAM,EAAE,GAAG,MAAM,GAAG,IAAW;IAuB5D,KAAK,IAAI,QAAQ;IAIjB,QAAQ,IAAI,MAAM;IAIlB,IAAI,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIxB,GAAG;IAIH,GAAG,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI;IAIvB,MAAM,CAAC,KAAK,EAAE,QAAQ,GAAG,OAAO;IAchC;;;;;OAKG;IACH,cAAc,CAAC,MAAM,KAAA,EAAE,KAAK,KAAA;IAW5B;;;;OAIG;IACH,cAAc,CAAC,MAAM,KAAA;CAUtB"}

package/dist/lib/jsonpath/jsonpath.js

@@ -1,86 +1,89 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-export default class JSONPath {
-  constructor(path = null) {
-    _defineProperty(this, "path", void 0);
-
-    this.path = ['$'];
-
-    if (path instanceof JSONPath) {
-      this.path = [...path.path];
-      return;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+/**
+ * A parser for a minimal subset of the jsonpath standard
+ * Full JSON path parsers for JS exist but are quite large (bundle size)
+ *
+ * Supports
+ *
+ *   `$.component.component.component`
+ */
+class JSONPath {
+    constructor(path = null) {
+        this.path = ['$'];
+        if (path instanceof JSONPath) {
+            // @ts-ignore
+            this.path = [...path.path];
+            return;
+        }
+        if (Array.isArray(path)) {
+            this.path.push(...path);
+            return;
+        }
+        // Parse a string as a JSONPath
+        if (typeof path === 'string') {
+            this.path = path.split('.');
+            if (this.path[0] !== '$') {
+                throw new Error('JSONPaths must start with $');
+            }
+        }
     }
-
-    if (Array.isArray(path)) {
-      this.path.push(...path);
-      return;
+    clone() {
+        return new JSONPath(this);
     }
-
-    if (typeof path === 'string') {
-      this.path = path.split('.');
-
-      if (this.path[0] !== '$') {
-        throw new Error('JSONPaths must start with $');
-      }
+    toString() {
+        return this.path.join('.');
     }
-  }
-
-  clone() {
-    return new JSONPath(this);
-  }
-
-  toString() {
-    return this.path.join('.');
-  }
-
-  push(name) {
-    this.path.push(name);
-  }
-
-  pop() {
-    return this.path.pop();
-  }
-
-  set(name) {
-    this.path[this.path.length - 1] = name;
-  }
-
-  equals(other) {
-    if (!this || !other || this.path.length !== other.path.length) {
-      return false;
+    push(name) {
+        this.path.push(name);
     }
-
-    for (let i = 0; i < this.path.length; ++i) {
-      if (this.path[i] !== other.path[i]) {
-        return false;
-      }
+    pop() {
+        return this.path.pop();
     }
-
-    return true;
-  }
-
-  setFieldAtPath(object, value) {
-    const path = [...this.path];
-    path.shift();
-    const field = path.pop();
-
-    for (const component of path) {
-      object = object[component];
+    set(name) {
+        this.path[this.path.length - 1] = name;
     }
-
-    object[field] = value;
-  }
-
-  getFieldAtPath(object) {
-    const path = [...this.path];
-    path.shift();
-    const field = path.pop();
-
-    for (const component of path) {
-      object = object[component];
+    equals(other) {
+        if (!this || !other || this.path.length !== other.path.length) {
+            return false;
+        }
+        for (let i = 0; i < this.path.length; ++i) {
+            if (this.path[i] !== other.path[i]) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Sets the value pointed at by path
+     * TODO - handle root path
+     * @param object
+     * @param value
+     */
+    setFieldAtPath(object, value) {
+        const path = [...this.path];
+        path.shift();
+        const field = path.pop();
+        for (const component of path) {
+            object = object[component];
+        }
+        // @ts-ignore
+        object[field] = value;
+    }
+    /**
+     * Gets the value pointed at by path
+     * TODO - handle root path
+     * @param object
+     */
+    getFieldAtPath(object) {
+        const path = [...this.path];
+        path.shift();
+        const field = path.pop();
+        for (const component of path) {
+            object = object[component];
+        }
+        // @ts-ignore
+        return object[field];
     }
-
-    return object[field];
-  }
-
 }
-//# sourceMappingURL=jsonpath.js.map
+exports.default = JSONPath;

package/dist/lib/parse-json-in-batches.d.ts

@@ -0,0 +1,4 @@
+import type { Batch } from '@loaders.gl/schema';
+import type { JSONLoaderOptions } from '../json-loader';
+export default function parseJSONInBatches(binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options: JSONLoaderOptions): AsyncIterable<Batch>;
+//# sourceMappingURL=parse-json-in-batches.d.ts.map

package/dist/lib/parse-json-in-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-json-in-batches.d.ts","sourceRoot":"","sources":["../../src/lib/parse-json-in-batches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,gBAAgB,CAAC;AAOtD,wBAA+B,kBAAkB,CAC/C,mBAAmB,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACvE,OAAO,EAAE,iBAAiB,GACzB,aAAa,CAAC,KAAK,CAAC,CA+EtB"}

package/dist/lib/parse-json-in-batches.js

@@ -1,87 +1,80 @@
-import { TableBatchBuilder } from '@loaders.gl/schema';
-import { makeTextDecoderIterator } from '@loaders.gl/loader-utils';
-import StreamingJSONParser from './parser/streaming-json-parser';
-export default async function* parseJSONInBatches(binaryAsyncIterator, options) {
-  var _options$json;
-
-  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);
-  const {
-    metadata
-  } = options;
-  const {
-    jsonpaths
-  } = options.json || {};
-  let isFirstChunk = true;
-  const schema = null;
-  const shape = (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'row-table';
-  const tableBatchBuilder = new TableBatchBuilder(schema, { ...options,
-    shape
-  });
-  const parser = new StreamingJSONParser({
-    jsonpaths
-  });
-
-  for await (const chunk of asyncIterator) {
-    const rows = parser.write(chunk);
-    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
-
-    if (rows.length > 0 && isFirstChunk) {
-      if (metadata) {
-        const initialBatch = {
-          shape,
-          batchType: 'partial-result',
-          data: [],
-          length: 0,
-          bytesUsed: 0,
-          container: parser.getPartialResult(),
-          jsonpath
-        };
-        yield initialBatch;
-      }
-
-      isFirstChunk = false;
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const schema_1 = require("@loaders.gl/schema");
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const streaming_json_parser_1 = __importDefault(require("./parser/streaming-json-parser"));
+// TODO - support batch size 0 = no batching/single batch?
+// eslint-disable-next-line max-statements, complexity
+async function* parseJSONInBatches(binaryAsyncIterator, options) {
+    const asyncIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
+    const { metadata } = options;
+    const { jsonpaths } = options.json || {};
+    let isFirstChunk = true;
+    // TODO fix Schema deduction
+    const schema = null; // new Schema([]);
+    const shape = options?.json?.shape || 'row-table';
+    // @ts-ignore
+    const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
+        ...options,
+        shape
+    });
+    const parser = new streaming_json_parser_1.default({ jsonpaths });
+    for await (const chunk of asyncIterator) {
+        const rows = parser.write(chunk);
+        const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
+        if (rows.length > 0 && isFirstChunk) {
+            if (metadata) {
+                const initialBatch = {
+                    // Common fields
+                    shape,
+                    batchType: 'partial-result',
+                    data: [],
+                    length: 0,
+                    bytesUsed: 0,
+                    // JSON additions
+                    container: parser.getPartialResult(),
+                    jsonpath
+                };
+                yield initialBatch;
+            }
+            isFirstChunk = false;
+            // schema = deduceSchema(rows);
+        }
+        // Add the row
+        for (const row of rows) {
+            tableBatchBuilder.addRow(row);
+            // If a batch has been completed, emit it
+            const batch = tableBatchBuilder.getFullBatch({ jsonpath });
+            if (batch) {
+                yield batch;
+            }
+        }
+        tableBatchBuilder.chunkComplete(chunk);
+        const batch = tableBatchBuilder.getFullBatch({ jsonpath });
+        if (batch) {
+            yield batch;
+        }
     }
-
-    for (const row of rows) {
-      tableBatchBuilder.addRow(row);
-      const batch = tableBatchBuilder.getFullBatch({
-        jsonpath
-      });
-
-      if (batch) {
+    // yield final batch
+    const jsonpath = parser.getStreamingJsonPathAsString();
+    const batch = tableBatchBuilder.getFinalBatch({ jsonpath });
+    if (batch) {
         yield batch;
-      }
     }
-
-    tableBatchBuilder.chunkComplete(chunk);
-    const batch = tableBatchBuilder.getFullBatch({
-      jsonpath
-    });
-
-    if (batch) {
-      yield batch;
+    if (metadata) {
+        const finalBatch = {
+            shape,
+            batchType: 'final-result',
+            container: parser.getPartialResult(),
+            jsonpath: parser.getStreamingJsonPathAsString(),
+            data: [],
+            length: 0
+            // schema: null
+        };
+        yield finalBatch;
     }
-  }
-
-  const jsonpath = parser.getStreamingJsonPathAsString();
-  const batch = tableBatchBuilder.getFinalBatch({
-    jsonpath
-  });
-
-  if (batch) {
-    yield batch;
-  }
-
-  if (metadata) {
-    const finalBatch = {
-      shape,
-      batchType: 'final-result',
-      container: parser.getPartialResult(),
-      jsonpath: parser.getStreamingJsonPathAsString(),
-      data: [],
-      length: 0
-    };
-    yield finalBatch;
-  }
 }
-//# sourceMappingURL=parse-json-in-batches.js.map
+exports.default = parseJSONInBatches;

package/dist/lib/parse-json.d.ts

@@ -0,0 +1,3 @@
+import type { JSONLoaderOptions } from '../json-loader';
+export default function parseJSONSync(jsonText: string, options: JSONLoaderOptions): any;
+//# sourceMappingURL=parse-json.d.ts.map

package/dist/lib/parse-json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-json.d.ts","sourceRoot":"","sources":["../../src/lib/parse-json.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,gBAAgB,CAAC;AAEtD,MAAM,CAAC,OAAO,UAAU,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,OAUjF"}

package/dist/lib/parse-json.js

@@ -1,34 +1,29 @@
-export default function parseJSONSync(jsonText, options) {
-  try {
-    var _options$json;
-
-    const json = JSON.parse(jsonText);
-
-    if ((_options$json = options.json) !== null && _options$json !== void 0 && _options$json.table) {
-      return getFirstArray(json) || json;
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+function parseJSONSync(jsonText, options) {
+    try {
+        const json = JSON.parse(jsonText);
+        if (options.json?.table) {
+            return getFirstArray(json) || json;
+        }
+        return json;
+    }
+    catch (error) {
+        throw new Error('JSONLoader: failed to parse JSON');
     }
-
-    return json;
-  } catch (error) {
-    throw new Error('JSONLoader: failed to parse JSON');
-  }
 }
-
+exports.default = parseJSONSync;
 function getFirstArray(json) {
-  if (Array.isArray(json)) {
-    return json;
-  }
-
-  if (json && typeof json === 'object') {
-    for (const value of Object.values(json)) {
-      const array = getFirstArray(value);
-
-      if (array) {
-        return array;
-      }
+    if (Array.isArray(json)) {
+        return json;
+    }
+    if (json && typeof json === 'object') {
+        for (const value of Object.values(json)) {
+            const array = getFirstArray(value);
+            if (array) {
+                return array;
+            }
+        }
     }
-  }
-
-  return null;
+    return null;
 }
-//# sourceMappingURL=parse-json.js.map

package/dist/lib/parse-ndjson-in-batches.d.ts

@@ -0,0 +1,4 @@
+import type { Batch } from '@loaders.gl/schema';
+import { LoaderOptions } from '@loaders.gl/loader-utils';
+export default function parseNDJSONInBatches(binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: LoaderOptions): AsyncIterable<Batch>;
+//# sourceMappingURL=parse-ndjson-in-batches.d.ts.map

package/dist/lib/parse-ndjson-in-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-ndjson-in-batches.d.ts","sourceRoot":"","sources":["../../src/lib/parse-ndjson-in-batches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EACL,aAAa,EAId,MAAM,0BAA0B,CAAC;AAElC,wBAA+B,oBAAoB,CACjD,mBAAmB,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACvE,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,KAAK,CAAC,CA+BtB"}

package/dist/lib/parse-ndjson-in-batches.js

@@ -1,37 +1,35 @@
-import { TableBatchBuilder } from '@loaders.gl/schema';
-import { makeLineIterator, makeNumberedLineIterator, makeTextDecoderIterator } from '@loaders.gl/loader-utils';
-export default async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
-  const textIterator = makeTextDecoderIterator(binaryAsyncIterator);
-  const lineIterator = makeLineIterator(textIterator);
-  const numberedLineIterator = makeNumberedLineIterator(lineIterator);
-  const schema = null;
-  const shape = 'row-table';
-  const tableBatchBuilder = new TableBatchBuilder(schema, { ...options,
-    shape
-  });
-
-  for await (const {
-    counter,
-    line
-  } of numberedLineIterator) {
-    try {
-      const row = JSON.parse(line);
-      tableBatchBuilder.addRow(row);
-      tableBatchBuilder.chunkComplete(line);
-      const batch = tableBatchBuilder.getFullBatch();
-
-      if (batch) {
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const schema_1 = require("@loaders.gl/schema");
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
+    const textIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
+    const lineIterator = (0, loader_utils_1.makeLineIterator)(textIterator);
+    const numberedLineIterator = (0, loader_utils_1.makeNumberedLineIterator)(lineIterator);
+    const schema = null;
+    const shape = 'row-table';
+    // @ts-ignore
+    const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
+        ...options,
+        shape
+    });
+    for await (const { counter, line } of numberedLineIterator) {
+        try {
+            const row = JSON.parse(line);
+            tableBatchBuilder.addRow(row);
+            tableBatchBuilder.chunkComplete(line);
+            const batch = tableBatchBuilder.getFullBatch();
+            if (batch) {
+                yield batch;
+            }
+        }
+        catch (error) {
+            throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
+        }
+    }
+    const batch = tableBatchBuilder.getFinalBatch();
+    if (batch) {
         yield batch;
-      }
-    } catch (error) {
-      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
     }
-  }
-
-  const batch = tableBatchBuilder.getFinalBatch();
-
-  if (batch) {
-    yield batch;
-  }
 }
-//# sourceMappingURL=parse-ndjson-in-batches.js.map
+exports.default = parseNDJSONInBatches;

package/dist/lib/parse-ndjson.d.ts

@@ -0,0 +1,2 @@
+export default function parseNDJSONSync(ndjsonText: string): any[];
+//# sourceMappingURL=parse-ndjson.d.ts.map

package/dist/lib/parse-ndjson.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-ndjson.d.ts","sourceRoot":"","sources":["../../src/lib/parse-ndjson.ts"],"names":[],"mappings":"AAAA,MAAM,CAAC,OAAO,UAAU,eAAe,CAAC,UAAU,EAAE,MAAM,SASzD"}

package/dist/lib/parse-ndjson.js

@@ -1,11 +1,14 @@
-export default function parseNDJSONSync(ndjsonText) {
-  const lines = ndjsonText.trim().split('\n');
-  return lines.map((line, counter) => {
-    try {
-      return JSON.parse(line);
-    } catch (error) {
-      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
-    }
-  });
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+function parseNDJSONSync(ndjsonText) {
+    const lines = ndjsonText.trim().split('\n');
+    return lines.map((line, counter) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch (error) {
+            throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
+        }
+    });
 }
-//# sourceMappingURL=parse-ndjson.js.map
+exports.default = parseNDJSONSync;

package/dist/lib/parser/json-parser.d.ts

@@ -0,0 +1,22 @@
+import ClarinetParser, { ClarinetParserOptions } from '../clarinet/clarinet';
+import JSONPath from '../jsonpath/jsonpath';
+export default class JSONParser {
+    readonly parser: ClarinetParser;
+    result: undefined;
+    previousStates: never[];
+    currentState: Readonly<{
+        container: never[];
+        key: null;
+    }>;
+    jsonpath: JSONPath;
+    constructor(options: ClarinetParserOptions);
+    reset(): void;
+    write(chunk: any): void;
+    close(): void;
+    _pushOrSet(value: any): void;
+    _openArray(newContainer?: never[]): void;
+    _closeArray(): void;
+    _openObject(newContainer?: {}): void;
+    _closeObject(): void;
+}
+//# sourceMappingURL=json-parser.d.ts.map

package/dist/lib/parser/json-parser.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-parser.d.ts","sourceRoot":"","sources":["../../../src/lib/parser/json-parser.ts"],"names":[],"mappings":"AAEA,OAAO,cAAc,EAAE,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAC3E,OAAO,QAAQ,MAAM,sBAAsB,CAAC;AAI5C,MAAM,CAAC,OAAO,OAAO,UAAU;IAC7B,QAAQ,CAAC,MAAM,EAAE,cAAc,CAAC;IAChC,MAAM,YAAa;IACnB,cAAc,UAAM;IACpB,YAAY;;;OAA6C;IACzD,QAAQ,EAAE,QAAQ,CAAkB;gBAExB,OAAO,EAAE,qBAAqB;IAiD1C,KAAK,IAAI,IAAI;IAOb,KAAK,CAAC,KAAK,KAAA,GAAG,IAAI;IAIlB,KAAK,IAAI,IAAI;IAMb,UAAU,CAAC,KAAK,KAAA,GAAG,IAAI;IAUvB,UAAU,CAAC,YAAY,UAAK,GAAG,IAAI;IAOnC,WAAW,IAAI,IAAI;IAKnB,WAAW,CAAC,YAAY,KAAK,GAAG,IAAI;IAOpC,YAAY,IAAI,IAAI;CAIrB"}