@loaders.gl/json 3.4.11 → 3.4.13

package/dist/lib/parse-json-in-batches.js

@@ -1,100 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.rebuildJsonObject = void 0;
-const schema_1 = require("@loaders.gl/schema");
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const streaming_json_parser_1 = __importDefault(require("./parser/streaming-json-parser"));
-const jsonpath_1 = __importDefault(require("./jsonpath/jsonpath"));
-// TODO - support batch size 0 = no batching/single batch?
-// eslint-disable-next-line max-statements, complexity
-async function* parseJSONInBatches(binaryAsyncIterator, options) {
-    const asyncIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
-    const { metadata } = options;
-    const { jsonpaths } = options.json || {};
-    let isFirstChunk = true;
-    // TODO fix Schema deduction
-    const schema = null; // new Schema([]);
-    const shape = options?.json?.shape || 'row-table';
-    // @ts-ignore
-    const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
-        ...options,
-        shape
-    });
-    const parser = new streaming_json_parser_1.default({ jsonpaths });
-    for await (const chunk of asyncIterator) {
-        const rows = parser.write(chunk);
-        const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
-        if (rows.length > 0 && isFirstChunk) {
-            if (metadata) {
-                const initialBatch = {
-                    // Common fields
-                    shape,
-                    batchType: 'partial-result',
-                    data: [],
-                    length: 0,
-                    bytesUsed: 0,
-                    // JSON additions
-                    container: parser.getPartialResult(),
-                    jsonpath
-                };
-                yield initialBatch;
-            }
-            isFirstChunk = false;
-            // schema = deduceSchema(rows);
-        }
-        // Add the row
-        for (const row of rows) {
-            tableBatchBuilder.addRow(row);
-            // If a batch has been completed, emit it
-            const batch = tableBatchBuilder.getFullBatch({ jsonpath });
-            if (batch) {
-                yield batch;
-            }
-        }
-        tableBatchBuilder.chunkComplete(chunk);
-        const batch = tableBatchBuilder.getFullBatch({ jsonpath });
-        if (batch) {
-            yield batch;
-        }
-    }
-    // yield final batch
-    const jsonpath = parser.getStreamingJsonPathAsString();
-    const batch = tableBatchBuilder.getFinalBatch({ jsonpath });
-    if (batch) {
-        yield batch;
-    }
-    if (metadata) {
-        const finalBatch = {
-            shape,
-            batchType: 'final-result',
-            container: parser.getPartialResult(),
-            jsonpath: parser.getStreamingJsonPathAsString(),
-            data: [],
-            length: 0
-            // schema: null
-        };
-        yield finalBatch;
-    }
-}
-exports.default = parseJSONInBatches;
-function rebuildJsonObject(batch, data) {
-    // Last batch will have this special type and will provide all the root object of the parsed file
-    (0, loader_utils_1.assert)(batch.batchType === 'final-result');
-    // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects
-    if (batch.jsonpath === '$') {
-        return data;
-    }
-    // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object
-    if (batch.jsonpath && batch.jsonpath.length > 1) {
-        const topLevelObject = batch.container;
-        const streamingPath = new jsonpath_1.default(batch.jsonpath);
-        streamingPath.setFieldAtPath(topLevelObject, data);
-        return topLevelObject;
-    }
-    // No jsonpath, in this case nothing was streamed.
-    return batch.container;
-}
-exports.rebuildJsonObject = rebuildJsonObject;

package/dist/lib/parse-json.js

@@ -1,29 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-function parseJSONSync(jsonText, options) {
-    try {
-        const json = JSON.parse(jsonText);
-        if (options.json?.table) {
-            return getFirstArray(json) || json;
-        }
-        return json;
-    }
-    catch (error) {
-        throw new Error('JSONLoader: failed to parse JSON');
-    }
-}
-exports.default = parseJSONSync;
-function getFirstArray(json) {
-    if (Array.isArray(json)) {
-        return json;
-    }
-    if (json && typeof json === 'object') {
-        for (const value of Object.values(json)) {
-            const array = getFirstArray(value);
-            if (array) {
-                return array;
-            }
-        }
-    }
-    return null;
-}

package/dist/lib/parse-ndjson-in-batches.js

@@ -1,35 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const schema_1 = require("@loaders.gl/schema");
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
-    const textIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
-    const lineIterator = (0, loader_utils_1.makeLineIterator)(textIterator);
-    const numberedLineIterator = (0, loader_utils_1.makeNumberedLineIterator)(lineIterator);
-    const schema = null;
-    const shape = 'row-table';
-    // @ts-ignore
-    const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
-        ...options,
-        shape
-    });
-    for await (const { counter, line } of numberedLineIterator) {
-        try {
-            const row = JSON.parse(line);
-            tableBatchBuilder.addRow(row);
-            tableBatchBuilder.chunkComplete(line);
-            const batch = tableBatchBuilder.getFullBatch();
-            if (batch) {
-                yield batch;
-            }
-        }
-        catch (error) {
-            throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
-        }
-    }
-    const batch = tableBatchBuilder.getFinalBatch();
-    if (batch) {
-        yield batch;
-    }
-}
-exports.default = parseNDJSONInBatches;

package/dist/lib/parse-ndjson.js

@@ -1,14 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-function parseNDJSONSync(ndjsonText) {
-    const lines = ndjsonText.trim().split('\n');
-    return lines.map((line, counter) => {
-        try {
-            return JSON.parse(line);
-        }
-        catch (error) {
-            throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
-        }
-    });
-}
-exports.default = parseNDJSONSync;

package/dist/lib/parser/json-parser.js

@@ -1,98 +0,0 @@
-"use strict";
-// @ts-nocheck
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const clarinet_1 = __importDefault(require("../clarinet/clarinet"));
-const jsonpath_1 = __importDefault(require("../jsonpath/jsonpath"));
-// JSONParser builds a JSON object using the events emitted by the Clarinet parser
-class JSONParser {
-    constructor(options) {
-        this.result = undefined;
-        this.previousStates = [];
-        this.currentState = Object.freeze({ container: [], key: null });
-        this.jsonpath = new jsonpath_1.default();
-        this.reset();
-        this.parser = new clarinet_1.default({
-            onready: () => {
-                this.jsonpath = new jsonpath_1.default();
-                this.previousStates.length = 0;
-                this.currentState.container.length = 0;
-            },
-            onopenobject: (name) => {
-                this._openObject({});
-                if (typeof name !== 'undefined') {
-                    this.parser.emit('onkey', name);
-                }
-            },
-            onkey: (name) => {
-                this.jsonpath.set(name);
-                this.currentState.key = name;
-            },
-            oncloseobject: () => {
-                this._closeObject();
-            },
-            onopenarray: () => {
-                this._openArray();
-            },
-            onclosearray: () => {
-                this._closeArray();
-            },
-            onvalue: (value) => {
-                this._pushOrSet(value);
-            },
-            onerror: (error) => {
-                throw error;
-            },
-            onend: () => {
-                this.result = this.currentState.container.pop();
-            },
-            ...options
-        });
-    }
-    reset() {
-        this.result = undefined;
-        this.previousStates = [];
-        this.currentState = Object.freeze({ container: [], key: null });
-        this.jsonpath = new jsonpath_1.default();
-    }
-    write(chunk) {
-        this.parser.write(chunk);
-    }
-    close() {
-        this.parser.close();
-    }
-    // PRIVATE METHODS
-    _pushOrSet(value) {
-        const { container, key } = this.currentState;
-        if (key !== null) {
-            container[key] = value;
-            this.currentState.key = null;
-        }
-        else {
-            container.push(value);
-        }
-    }
-    _openArray(newContainer = []) {
-        this.jsonpath.push(null);
-        this._pushOrSet(newContainer);
-        this.previousStates.push(this.currentState);
-        this.currentState = { container: newContainer, isArray: true, key: null };
-    }
-    _closeArray() {
-        this.jsonpath.pop();
-        this.currentState = this.previousStates.pop();
-    }
-    _openObject(newContainer = {}) {
-        this.jsonpath.push(null);
-        this._pushOrSet(newContainer);
-        this.previousStates.push(this.currentState);
-        this.currentState = { container: newContainer, isArray: false, key: null };
-    }
-    _closeObject() {
-        this.jsonpath.pop();
-        this.currentState = this.previousStates.pop();
-    }
-}
-exports.default = JSONParser;

package/dist/lib/parser/streaming-json-parser.js

@@ -1,100 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-const json_parser_1 = __importDefault(require("./json-parser"));
-const jsonpath_1 = __importDefault(require("../jsonpath/jsonpath"));
-/**
- * The `StreamingJSONParser` looks for the first array in the JSON structure.
- * and emits an array of chunks
- */
-class StreamingJSONParser extends json_parser_1.default {
-    constructor(options = {}) {
-        super({
-            onopenarray: () => {
-                if (!this.streamingArray) {
-                    if (this._matchJSONPath()) {
-                        // @ts-ignore
-                        this.streamingJsonPath = this.getJsonPath().clone();
-                        this.streamingArray = [];
-                        this._openArray(this.streamingArray);
-                        return;
-                    }
-                }
-                this._openArray();
-            },
-            // Redefine onopenarray to inject value for top-level object
-            onopenobject: (name) => {
-                if (!this.topLevelObject) {
-                    this.topLevelObject = {};
-                    this._openObject(this.topLevelObject);
-                }
-                else {
-                    this._openObject({});
-                }
-                if (typeof name !== 'undefined') {
-                    this.parser.emit('onkey', name);
-                }
-            }
-        });
-        this.streamingJsonPath = null;
-        this.streamingArray = null;
-        this.topLevelObject = null;
-        const jsonpaths = options.jsonpaths || [];
-        this.jsonPaths = jsonpaths.map((jsonpath) => new jsonpath_1.default(jsonpath));
-    }
-    /**
-     * write REDEFINITION
-     * - super.write() chunk to parser
-     * - get the contents (so far) of "topmost-level" array as batch of rows
-     * - clear top-level array
-     * - return the batch of rows\
-     */
-    write(chunk) {
-        super.write(chunk);
-        let array = [];
-        if (this.streamingArray) {
-            array = [...this.streamingArray];
-            this.streamingArray.length = 0;
-        }
-        return array;
-    }
-    /**
-     * Returns a partially formed result object
-     * Useful for returning the "wrapper" object when array is not top level
-     * e.g. GeoJSON
-     */
-    getPartialResult() {
-        return this.topLevelObject;
-    }
-    getStreamingJsonPath() {
-        return this.streamingJsonPath;
-    }
-    getStreamingJsonPathAsString() {
-        return this.streamingJsonPath && this.streamingJsonPath.toString();
-    }
-    getJsonPath() {
-        return this.jsonpath;
-    }
-    // PRIVATE METHODS
-    /**
-     * Checks is this.getJsonPath matches the jsonpaths provided in options
-     */
-    _matchJSONPath() {
-        const currentPath = this.getJsonPath();
-        // console.debug(`Testing JSONPath`, currentPath);
-        // Backwards compatibility, match any array
-        // TODO implement using wildcard once that is supported
-        if (this.jsonPaths.length === 0) {
-            return true;
-        }
-        for (const jsonPath of this.jsonPaths) {
-            if (jsonPath.equals(currentPath)) {
-                return true;
-            }
-        }
-        return false;
-    }
-}
-exports.default = StreamingJSONParser;

package/dist/ndgeoson-loader.js

@@ -1,40 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports._typecheckNDJSONLoader = exports.NDJSONLoader = void 0;
-const parse_ndjson_1 = __importDefault(require("./lib/parse-ndjson"));
-const parse_ndjson_in_batches_1 = __importDefault(require("./lib/parse-ndjson-in-batches"));
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-const DEFAULT_NDGEOJSON_LOADER_OPTIONS = {
-    geojson: {
-        shape: 'object-row-table'
-    },
-    gis: {
-        format: 'geojson'
-    }
-};
-exports.NDJSONLoader = {
-    name: 'NDJSON',
-    id: 'ndjson',
-    module: 'json',
-    version: VERSION,
-    extensions: ['ndjson', 'ndgeojson'],
-    mimeTypes: [
-        'application/geo+x-ndjson',
-        'application/geo+x-ldjson',
-        'application/jsonlines',
-        'application/geo+json-seq',
-        'application/x-ndjson'
-    ],
-    category: 'table',
-    text: true,
-    parse: async (arrayBuffer) => (0, parse_ndjson_1.default)(new TextDecoder().decode(arrayBuffer)),
-    parseTextSync: parse_ndjson_1.default,
-    parseInBatches: parse_ndjson_in_batches_1.default,
-    options: DEFAULT_NDGEOJSON_LOADER_OPTIONS
-};
-exports._typecheckNDJSONLoader = exports.NDJSONLoader;

package/dist/ndjson-loader.js

@@ -1,29 +0,0 @@
-"use strict";
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.NDJSONLoader = void 0;
-const parse_ndjson_1 = __importDefault(require("./lib/parse-ndjson"));
-const parse_ndjson_in_batches_1 = __importDefault(require("./lib/parse-ndjson-in-batches"));
-// __VERSION__ is injected by babel-plugin-version-inline
-// @ts-ignore TS2304: Cannot find name '__VERSION__'.
-const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
-exports.NDJSONLoader = {
-    name: 'NDJSON',
-    id: 'ndjson',
-    module: 'json',
-    version: VERSION,
-    extensions: ['ndjson', 'jsonl'],
-    mimeTypes: [
-        'application/x-ndjson',
-        'application/jsonlines',
-        'application/json-seq'
-    ],
-    category: 'table',
-    text: true,
-    parse: async (arrayBuffer) => (0, parse_ndjson_1.default)(new TextDecoder().decode(arrayBuffer)),
-    parseTextSync: parse_ndjson_1.default,
-    parseInBatches: parse_ndjson_in_batches_1.default,
-    options: {}
-};

package/dist/workers/geojson-worker.js

@@ -1,5 +0,0 @@
-"use strict";
-Object.defineProperty(exports, "__esModule", { value: true });
-const loader_utils_1 = require("@loaders.gl/loader-utils");
-const geojson_loader_1 = require("../geojson-loader");
-(0, loader_utils_1.createLoaderWorker)(geojson_loader_1.GeoJSONLoader);