@loaders.gl/json 4.2.0-alpha.3 → 4.2.0-alpha.5

package/dist/lib/json-parser/streaming-json-parser.js

@@ -1,71 +1,94 @@
 import { default as JSONParser } from "./json-parser.js";
 import JSONPath from "../jsonpath/jsonpath.js";
+/**
+ * The `StreamingJSONParser` looks for the first array in the JSON structure.
+ * and emits an array of chunks
+ */
 export default class StreamingJSONParser extends JSONParser {
-  constructor() {
-    let options = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : {};
-    super({
-      onopenarray: () => {
-        if (!this.streamingArray) {
-          if (this._matchJSONPath()) {
-            this.streamingJsonPath = this.getJsonPath().clone();
-            this.streamingArray = [];
-            this._openArray(this.streamingArray);
-            return;
-          }
-        }
-        this._openArray();
-      },
-      onopenobject: name => {
-        if (!this.topLevelObject) {
-          this.topLevelObject = {};
-          this._openObject(this.topLevelObject);
-        } else {
-          this._openObject({});
-        }
-        if (typeof name !== 'undefined') {
-          this.parser.emit('onkey', name);
+    constructor(options = {}) {
+        super({
+            onopenarray: () => {
+                if (!this.streamingArray) {
+                    if (this._matchJSONPath()) {
+                        // @ts-ignore
+                        this.streamingJsonPath = this.getJsonPath().clone();
+                        this.streamingArray = [];
+                        this._openArray(this.streamingArray);
+                        return;
+                    }
+                }
+                this._openArray();
+            },
+            // Redefine onopenarray to inject value for top-level object
+            onopenobject: (name) => {
+                if (!this.topLevelObject) {
+                    this.topLevelObject = {};
+                    this._openObject(this.topLevelObject);
+                }
+                else {
+                    this._openObject({});
+                }
+                if (typeof name !== 'undefined') {
+                    this.parser.emit('onkey', name);
+                }
+            }
+        });
+        this.streamingJsonPath = null;
+        this.streamingArray = null;
+        this.topLevelObject = null;
+        const jsonpaths = options.jsonpaths || [];
+        this.jsonPaths = jsonpaths.map((jsonpath) => new JSONPath(jsonpath));
+    }
+    /**
+     * write REDEFINITION
+     * - super.write() chunk to parser
+     * - get the contents (so far) of "topmost-level" array as batch of rows
+     * - clear top-level array
+     * - return the batch of rows\
+     */
+    write(chunk) {
+        super.write(chunk);
+        let array = [];
+        if (this.streamingArray) {
+            array = [...this.streamingArray];
+            this.streamingArray.length = 0;
         }
-      }
-    });
-    this.jsonPaths = void 0;
-    this.streamingJsonPath = null;
-    this.streamingArray = null;
-    this.topLevelObject = null;
-    const jsonpaths = options.jsonpaths || [];
-    this.jsonPaths = jsonpaths.map(jsonpath => new JSONPath(jsonpath));
-  }
-  write(chunk) {
-    super.write(chunk);
-    let array = [];
-    if (this.streamingArray) {
-      array = [...this.streamingArray];
-      this.streamingArray.length = 0;
+        return array;
+    }
+    /**
+     * Returns a partially formed result object
+     * Useful for returning the "wrapper" object when array is not top level
+     * e.g. GeoJSON
+     */
+    getPartialResult() {
+        return this.topLevelObject;
     }
-    return array;
-  }
-  getPartialResult() {
-    return this.topLevelObject;
-  }
-  getStreamingJsonPath() {
-    return this.streamingJsonPath;
-  }
-  getStreamingJsonPathAsString() {
-    return this.streamingJsonPath && this.streamingJsonPath.toString();
-  }
-  getJsonPath() {
-    return this.jsonpath;
-  }
-  _matchJSONPath() {
-    const currentPath = this.getJsonPath();
-    if (this.jsonPaths.length === 0) {
-      return true;
+    getStreamingJsonPath() {
+        return this.streamingJsonPath;
     }
-    for (const jsonPath of this.jsonPaths) {
-      if (jsonPath.equals(currentPath)) {
-        return true;
-      }
+    getStreamingJsonPathAsString() {
+        return this.streamingJsonPath && this.streamingJsonPath.toString();
+    }
+    getJsonPath() {
+        return this.jsonpath;
+    }
+    // PRIVATE METHODS
+    /**
+     * Checks is this.getJsonPath matches the jsonpaths provided in options
+     */
+    _matchJSONPath() {
+        const currentPath = this.getJsonPath();
+        // console.debug(`Testing JSONPath`, currentPath);
+        // Backwards compatibility, match any array
+        // TODO implement using wildcard once that is supported
+        if (this.jsonPaths.length === 0) {
+            return true;
+        }
+        for (const jsonPath of this.jsonPaths) {
+            if (jsonPath.equals(currentPath)) {
+                return true;
+            }
+        }
+        return false;
     }
-    return false;
-  }
 }
-//# sourceMappingURL=streaming-json-parser.js.map

package/dist/lib/jsonpath/jsonpath.js

@@ -1,66 +1,86 @@
+/**
+ * A parser for a minimal subset of the jsonpath standard
+ * Full JSON path parsers for JS exist but are quite large (bundle size)
+ *
+ * Supports
+ *
+ *   `$.component.component.component`
+ */
 export default class JSONPath {
-  constructor() {
-    let path = arguments.length > 0 && arguments[0] !== undefined ? arguments[0] : null;
-    this.path = void 0;
-    this.path = ['$'];
-    if (path instanceof JSONPath) {
-      this.path = [...path.path];
-      return;
+    constructor(path = null) {
+        this.path = ['$'];
+        if (path instanceof JSONPath) {
+            // @ts-ignore
+            this.path = [...path.path];
+            return;
+        }
+        if (Array.isArray(path)) {
+            this.path.push(...path);
+            return;
+        }
+        // Parse a string as a JSONPath
+        if (typeof path === 'string') {
+            this.path = path.split('.');
+            if (this.path[0] !== '$') {
+                throw new Error('JSONPaths must start with $');
+            }
+        }
     }
-    if (Array.isArray(path)) {
-      this.path.push(...path);
-      return;
+    clone() {
+        return new JSONPath(this);
     }
-    if (typeof path === 'string') {
-      this.path = path.split('.');
-      if (this.path[0] !== '$') {
-        throw new Error('JSONPaths must start with $');
-      }
+    toString() {
+        return this.path.join('.');
     }
-  }
-  clone() {
-    return new JSONPath(this);
-  }
-  toString() {
-    return this.path.join('.');
-  }
-  push(name) {
-    this.path.push(name);
-  }
-  pop() {
-    return this.path.pop();
-  }
-  set(name) {
-    this.path[this.path.length - 1] = name;
-  }
-  equals(other) {
-    if (!this || !other || this.path.length !== other.path.length) {
-      return false;
+    push(name) {
+        this.path.push(name);
     }
-    for (let i = 0; i < this.path.length; ++i) {
-      if (this.path[i] !== other.path[i]) {
-        return false;
-      }
+    pop() {
+        return this.path.pop();
     }
-    return true;
-  }
-  setFieldAtPath(object, value) {
-    const path = [...this.path];
-    path.shift();
-    const field = path.pop();
-    for (const component of path) {
-      object = object[component];
+    set(name) {
+        this.path[this.path.length - 1] = name;
     }
-    object[field] = value;
-  }
-  getFieldAtPath(object) {
-    const path = [...this.path];
-    path.shift();
-    const field = path.pop();
-    for (const component of path) {
-      object = object[component];
+    equals(other) {
+        if (!this || !other || this.path.length !== other.path.length) {
+            return false;
+        }
+        for (let i = 0; i < this.path.length; ++i) {
+            if (this.path[i] !== other.path[i]) {
+                return false;
+            }
+        }
+        return true;
+    }
+    /**
+     * Sets the value pointed at by path
+     * TODO - handle root path
+     * @param object
+     * @param value
+     */
+    setFieldAtPath(object, value) {
+        const path = [...this.path];
+        path.shift();
+        const field = path.pop();
+        for (const component of path) {
+            object = object[component];
+        }
+        // @ts-ignore
+        object[field] = value;
+    }
+    /**
+     * Gets the value pointed at by path
+     * TODO - handle root path
+     * @param object
+     */
+    getFieldAtPath(object) {
+        const path = [...this.path];
+        path.shift();
+        const field = path.pop();
+        for (const component of path) {
+            object = object[component];
+        }
+        // @ts-ignore
+        return object[field];
     }
-    return object[field];
-  }
 }
-//# sourceMappingURL=jsonpath.js.map

package/dist/lib/parsers/parse-json-in-batches.d.ts

@@ -1,5 +1,5 @@
 import type { TableBatch } from '@loaders.gl/schema';
-import type { JSONLoaderOptions, MetadataBatch, JSONBatch } from '../../json-loader';
+import type { JSONLoaderOptions, MetadataBatch, JSONBatch } from "../../json-loader.js";
 export declare function parseJSONInBatches(binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options: JSONLoaderOptions): AsyncIterable<TableBatch | MetadataBatch | JSONBatch>;
 export declare function rebuildJsonObject(batch: any, data: any): any;
 //# sourceMappingURL=parse-json-in-batches.d.ts.map

package/dist/lib/parsers/parse-json-in-batches.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-json-in-batches.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-json-in-batches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAS,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAC3D,OAAO,KAAK,EAAC,iBAAiB,EAAE,aAAa,EAAE,SAAS,EAAC,MAAM,mBAAmB,CAAC;AAQnF,wBAAuB,kBAAkB,CACvC,mBAAmB,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACvE,OAAO,EAAE,iBAAiB,GACzB,aAAa,CAAC,UAAU,GAAG,aAAa,GAAG,SAAS,CAAC,CA2EvD;AAED,wBAAgB,iBAAiB,CAAC,KAAK,KAAA,EAAE,IAAI,KAAA,OAmB5C"}
+{"version":3,"file":"parse-json-in-batches.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-json-in-batches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAS,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAC3D,OAAO,KAAK,EAAC,iBAAiB,EAAE,aAAa,EAAE,SAAS,EAAC,6BAA0B;AAQnF,wBAAuB,kBAAkB,CACvC,mBAAmB,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACvE,OAAO,EAAE,iBAAiB,GACzB,aAAa,CAAC,UAAU,GAAG,aAAa,GAAG,SAAS,CAAC,CA2EvD;AAED,wBAAgB,iBAAiB,CAAC,KAAK,KAAA,EAAE,IAAI,KAAA,OAmB5C"}

package/dist/lib/parsers/parse-json-in-batches.js

@@ -2,86 +2,87 @@ import { TableBatchBuilder } from '@loaders.gl/schema';
 import { assert, makeTextDecoderIterator } from '@loaders.gl/loader-utils';
 import StreamingJSONParser from "../json-parser/streaming-json-parser.js";
 import JSONPath from "../jsonpath/jsonpath.js";
+// TODO - support batch size 0 = no batching/single batch?
+// eslint-disable-next-line max-statements, complexity
 export async function* parseJSONInBatches(binaryAsyncIterator, options) {
-  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);
-  const {
-    metadata
-  } = options;
-  const {
-    jsonpaths
-  } = options.json || {};
-  let isFirstChunk = true;
-  const schema = null;
-  const tableBatchBuilder = new TableBatchBuilder(schema, options);
-  const parser = new StreamingJSONParser({
-    jsonpaths
-  });
-  for await (const chunk of asyncIterator) {
-    const rows = parser.write(chunk);
-    const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
-    if (rows.length > 0 && isFirstChunk) {
-      if (metadata) {
-        var _options$json;
-        const initialBatch = {
-          shape: (options === null || options === void 0 ? void 0 : (_options$json = options.json) === null || _options$json === void 0 ? void 0 : _options$json.shape) || 'array-row-table',
-          batchType: 'partial-result',
-          data: [],
-          length: 0,
-          bytesUsed: 0,
-          container: parser.getPartialResult(),
-          jsonpath
-        };
-        yield initialBatch;
-      }
-      isFirstChunk = false;
+    const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);
+    const { metadata } = options;
+    const { jsonpaths } = options.json || {};
+    let isFirstChunk = true;
+    // @ts-expect-error TODO fix Schema deduction
+    const schema = null;
+    const tableBatchBuilder = new TableBatchBuilder(schema, options);
+    const parser = new StreamingJSONParser({ jsonpaths });
+    for await (const chunk of asyncIterator) {
+        const rows = parser.write(chunk);
+        const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
+        if (rows.length > 0 && isFirstChunk) {
+            if (metadata) {
+                const initialBatch = {
+                    // Common fields
+                    shape: options?.json?.shape || 'array-row-table',
+                    batchType: 'partial-result',
+                    data: [],
+                    length: 0,
+                    bytesUsed: 0,
+                    // JSON additions
+                    container: parser.getPartialResult(),
+                    jsonpath
+                };
+                yield initialBatch;
+            }
+            isFirstChunk = false;
+            // schema = deduceSchema(rows);
+        }
+        // Add the row
+        for (const row of rows) {
+            tableBatchBuilder.addRow(row);
+            // If a batch has been completed, emit it
+            const batch = tableBatchBuilder.getFullBatch({ jsonpath });
+            if (batch) {
+                yield batch;
+            }
+        }
+        tableBatchBuilder.chunkComplete(chunk);
+        const batch = tableBatchBuilder.getFullBatch({ jsonpath });
+        if (batch) {
+            yield batch;
+        }
     }
-    for (const row of rows) {
-      tableBatchBuilder.addRow(row);
-      const batch = tableBatchBuilder.getFullBatch({
-        jsonpath
-      });
-      if (batch) {
+    // yield final batch
+    const jsonpath = parser.getStreamingJsonPathAsString();
+    const batch = tableBatchBuilder.getFinalBatch({ jsonpath });
+    if (batch) {
         yield batch;
-      }
     }
-    tableBatchBuilder.chunkComplete(chunk);
-    const batch = tableBatchBuilder.getFullBatch({
-      jsonpath
-    });
-    if (batch) {
-      yield batch;
+    if (metadata) {
+        const finalBatch = {
+            shape: 'json',
+            batchType: 'final-result',
+            container: parser.getPartialResult(),
+            jsonpath: parser.getStreamingJsonPathAsString(),
+            /** Data Just to avoid crashing? */
+            data: [],
+            length: 0
+            // schema: null
+        };
+        yield finalBatch;
     }
-  }
-  const jsonpath = parser.getStreamingJsonPathAsString();
-  const batch = tableBatchBuilder.getFinalBatch({
-    jsonpath
-  });
-  if (batch) {
-    yield batch;
-  }
-  if (metadata) {
-    const finalBatch = {
-      shape: 'json',
-      batchType: 'final-result',
-      container: parser.getPartialResult(),
-      jsonpath: parser.getStreamingJsonPathAsString(),
-      data: [],
-      length: 0
-    };
-    yield finalBatch;
-  }
 }
 export function rebuildJsonObject(batch, data) {
-  assert(batch.batchType === 'final-result');
-  if (batch.jsonpath === '$') {
-    return data;
-  }
-  if (batch.jsonpath && batch.jsonpath.length > 1) {
-    const topLevelObject = batch.container;
-    const streamingPath = new JSONPath(batch.jsonpath);
-    streamingPath.setFieldAtPath(topLevelObject, data);
-    return topLevelObject;
-  }
-  return batch.container;
+    // Last batch will have this special type and will provide all the root object of the parsed file
+    assert(batch.batchType === 'final-result');
+    // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects
+    if (batch.jsonpath === '$') {
+        return data;
+    }
+    // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object
+    if (batch.jsonpath && batch.jsonpath.length > 1) {
+        const topLevelObject = batch.container;
+        const streamingPath = new JSONPath(batch.jsonpath);
+        streamingPath.setFieldAtPath(topLevelObject, data);
+        return topLevelObject;
+    }
+    // No jsonpath, in this case nothing was streamed.
+    return batch.container;
 }
-//# sourceMappingURL=parse-json-in-batches.js.map

package/dist/lib/parsers/parse-json.d.ts

@@ -1,4 +1,4 @@
 import type { RowTable } from '@loaders.gl/schema';
-import type { JSONLoaderOptions } from '../../json-loader';
+import type { JSONLoaderOptions } from "../../json-loader.js";
 export declare function parseJSONSync(jsonText: string, options: JSONLoaderOptions): RowTable;
 //# sourceMappingURL=parse-json.d.ts.map

package/dist/lib/parsers/parse-json.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"parse-json.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-json.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAEzD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,QAAQ,CAWpF"}
+{"version":3,"file":"parse-json.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-json.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,6BAA0B;AAEzD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,QAAQ,CAWpF"}

package/dist/lib/parsers/parse-json.js

@@ -1,29 +1,28 @@
 import { makeTableFromData } from '@loaders.gl/schema';
 export function parseJSONSync(jsonText, options) {
-  try {
-    var _options$json;
-    const json = JSON.parse(jsonText);
-    if ((_options$json = options.json) !== null && _options$json !== void 0 && _options$json.table) {
-      const data = getFirstArray(json) || json;
-      return makeTableFromData(data);
+    try {
+        const json = JSON.parse(jsonText);
+        if (options.json?.table) {
+            const data = getFirstArray(json) || json;
+            return makeTableFromData(data);
+        }
+        return json;
+    }
+    catch (error) {
+        throw new Error('JSONLoader: failed to parse JSON');
     }
-    return json;
-  } catch (error) {
-    throw new Error('JSONLoader: failed to parse JSON');
-  }
 }
 function getFirstArray(json) {
-  if (Array.isArray(json)) {
-    return json;
-  }
-  if (json && typeof json === 'object') {
-    for (const value of Object.values(json)) {
-      const array = getFirstArray(value);
-      if (array) {
-        return array;
-      }
+    if (Array.isArray(json)) {
+        return json;
+    }
+    if (json && typeof json === 'object') {
+        for (const value of Object.values(json)) {
+            const array = getFirstArray(value);
+            if (array) {
+                return array;
+            }
+        }
     }
-  }
-  return null;
+    return null;
 }
-//# sourceMappingURL=parse-json.js.map

package/dist/lib/parsers/parse-ndjson-in-batches.js

@@ -1,34 +1,32 @@
 import { TableBatchBuilder } from '@loaders.gl/schema';
 import { makeLineIterator, makeNumberedLineIterator, makeTextDecoderIterator } from '@loaders.gl/loader-utils';
 export async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
-  const textIterator = makeTextDecoderIterator(binaryAsyncIterator);
-  const lineIterator = makeLineIterator(textIterator);
-  const numberedLineIterator = makeNumberedLineIterator(lineIterator);
-  const schema = null;
-  const shape = 'row-table';
-  const tableBatchBuilder = new TableBatchBuilder(schema, {
-    ...options,
-    shape
-  });
-  for await (const {
-    counter,
-    line
-  } of numberedLineIterator) {
-    try {
-      const row = JSON.parse(line);
-      tableBatchBuilder.addRow(row);
-      tableBatchBuilder.chunkComplete(line);
-      const batch = tableBatchBuilder.getFullBatch();
-      if (batch) {
+    const textIterator = makeTextDecoderIterator(binaryAsyncIterator);
+    const lineIterator = makeLineIterator(textIterator);
+    const numberedLineIterator = makeNumberedLineIterator(lineIterator);
+    const schema = null;
+    const shape = 'row-table';
+    // @ts-ignore
+    const tableBatchBuilder = new TableBatchBuilder(schema, {
+        ...options,
+        shape
+    });
+    for await (const { counter, line } of numberedLineIterator) {
+        try {
+            const row = JSON.parse(line);
+            tableBatchBuilder.addRow(row);
+            tableBatchBuilder.chunkComplete(line);
+            const batch = tableBatchBuilder.getFullBatch();
+            if (batch) {
+                yield batch;
+            }
+        }
+        catch (error) {
+            throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
+        }
+    }
+    const batch = tableBatchBuilder.getFinalBatch();
+    if (batch) {
         yield batch;
-      }
-    } catch (error) {
-      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
     }
-  }
-  const batch = tableBatchBuilder.getFinalBatch();
-  if (batch) {
-    yield batch;
-  }
 }
-//# sourceMappingURL=parse-ndjson-in-batches.js.map

package/dist/lib/parsers/parse-ndjson.js

@@ -1,13 +1,13 @@
 import { makeTableFromData } from '@loaders.gl/schema';
 export function parseNDJSONSync(ndjsonText) {
-  const lines = ndjsonText.trim().split('\n');
-  const parsedLines = lines.map((line, counter) => {
-    try {
-      return JSON.parse(line);
-    } catch (error) {
-      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
-    }
-  });
-  return makeTableFromData(parsedLines);
+    const lines = ndjsonText.trim().split('\n');
+    const parsedLines = lines.map((line, counter) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch (error) {
+            throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
+        }
+    });
+    return makeTableFromData(parsedLines);
 }
-//# sourceMappingURL=parse-ndjson.js.map