@loaders.gl/json 4.0.0-alpha.5 → 4.0.0-alpha.7

package/dist/lib/parsers/parse-json-in-batches.js

@@ -0,0 +1,100 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.rebuildJsonObject = exports.parseJSONInBatches = void 0;
+const schema_1 = require("@loaders.gl/schema");
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const streaming_json_parser_1 = __importDefault(require("../json-parser/streaming-json-parser"));
+const jsonpath_1 = __importDefault(require("../jsonpath/jsonpath"));
+// TODO - support batch size 0 = no batching/single batch?
+// eslint-disable-next-line max-statements, complexity
+async function* parseJSONInBatches(binaryAsyncIterator, options) {
+    const asyncIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
+    const { metadata } = options;
+    const { jsonpaths } = options.json || {};
+    let isFirstChunk = true;
+    // TODO fix Schema deduction
+    const schema = null; // new Schema([]);
+    const shape = options?.json?.shape || 'row-table';
+    // @ts-ignore
+    const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
+        ...options,
+        shape
+    });
+    const parser = new streaming_json_parser_1.default({ jsonpaths });
+    for await (const chunk of asyncIterator) {
+        const rows = parser.write(chunk);
+        const jsonpath = rows.length > 0 && parser.getStreamingJsonPathAsString();
+        if (rows.length > 0 && isFirstChunk) {
+            if (metadata) {
+                const initialBatch = {
+                    // Common fields
+                    shape,
+                    batchType: 'partial-result',
+                    data: [],
+                    length: 0,
+                    bytesUsed: 0,
+                    // JSON additions
+                    container: parser.getPartialResult(),
+                    jsonpath
+                };
+                yield initialBatch;
+            }
+            isFirstChunk = false;
+            // schema = deduceSchema(rows);
+        }
+        // Add the row
+        for (const row of rows) {
+            tableBatchBuilder.addRow(row);
+            // If a batch has been completed, emit it
+            const batch = tableBatchBuilder.getFullBatch({ jsonpath });
+            if (batch) {
+                yield batch;
+            }
+        }
+        tableBatchBuilder.chunkComplete(chunk);
+        const batch = tableBatchBuilder.getFullBatch({ jsonpath });
+        if (batch) {
+            yield batch;
+        }
+    }
+    // yield final batch
+    const jsonpath = parser.getStreamingJsonPathAsString();
+    const batch = tableBatchBuilder.getFinalBatch({ jsonpath });
+    if (batch) {
+        yield batch;
+    }
+    if (metadata) {
+        const finalBatch = {
+            shape,
+            batchType: 'final-result',
+            container: parser.getPartialResult(),
+            jsonpath: parser.getStreamingJsonPathAsString(),
+            data: [],
+            length: 0
+            // schema: null
+        };
+        yield finalBatch;
+    }
+}
+exports.parseJSONInBatches = parseJSONInBatches;
+function rebuildJsonObject(batch, data) {
+    // Last batch will have this special type and will provide all the root object of the parsed file
+    (0, loader_utils_1.assert)(batch.batchType === 'final-result');
+    // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects
+    if (batch.jsonpath === '$') {
+        return data;
+    }
+    // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object
+    if (batch.jsonpath && batch.jsonpath.length > 1) {
+        const topLevelObject = batch.container;
+        const streamingPath = new jsonpath_1.default(batch.jsonpath);
+        streamingPath.setFieldAtPath(topLevelObject, data);
+        return topLevelObject;
+    }
+    // No jsonpath, in this case nothing was streamed.
+    return batch.container;
+}
+exports.rebuildJsonObject = rebuildJsonObject;

package/dist/lib/parsers/parse-json.d.ts

@@ -0,0 +1,4 @@
+import type { RowTable } from '@loaders.gl/schema';
+import type { JSONLoaderOptions } from '../../json-loader';
+export declare function parseJSONSync(jsonText: string, options: JSONLoaderOptions): RowTable;
+//# sourceMappingURL=parse-json.d.ts.map

package/dist/lib/parsers/parse-json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-json.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-json.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAC,QAAQ,EAAC,MAAM,oBAAoB,CAAC;AAEjD,OAAO,KAAK,EAAC,iBAAiB,EAAC,MAAM,mBAAmB,CAAC;AAEzD,wBAAgB,aAAa,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,iBAAiB,GAAG,QAAQ,CAWpF"}

package/dist/lib/parsers/parse-json.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseJSONSync = void 0;
+const schema_1 = require("@loaders.gl/schema");
+function parseJSONSync(jsonText, options) {
+    try {
+        const json = JSON.parse(jsonText);
+        if (options.json?.table) {
+            const data = getFirstArray(json) || json;
+            return (0, schema_1.makeTableFromData)(data);
+        }
+        return json;
+    }
+    catch (error) {
+        throw new Error('JSONLoader: failed to parse JSON');
+    }
+}
+exports.parseJSONSync = parseJSONSync;
+function getFirstArray(json) {
+    if (Array.isArray(json)) {
+        return json;
+    }
+    if (json && typeof json === 'object') {
+        for (const value of Object.values(json)) {
+            const array = getFirstArray(value);
+            if (array) {
+                return array;
+            }
+        }
+    }
+    return null;
+}

package/dist/lib/{parse-ndjson-in-batches.d.ts → parsers/parse-ndjson-in-batches.d.ts}

@@ -1,4 +1,4 @@
 import type { Batch } from '@loaders.gl/schema';
 import { LoaderOptions } from '@loaders.gl/loader-utils';
-export default function parseNDJSONInBatches(binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: LoaderOptions): AsyncIterable<Batch>;
+export declare function parseNDJSONInBatches(binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: LoaderOptions): AsyncIterable<Batch>;
 //# sourceMappingURL=parse-ndjson-in-batches.d.ts.map

package/dist/lib/parsers/parse-ndjson-in-batches.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-ndjson-in-batches.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-ndjson-in-batches.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAE9C,OAAO,EACL,aAAa,EAId,MAAM,0BAA0B,CAAC;AAElC,wBAAuB,oBAAoB,CACzC,mBAAmB,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACvE,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,KAAK,CAAC,CA+BtB"}

package/dist/lib/parsers/parse-ndjson-in-batches.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseNDJSONInBatches = void 0;
+const schema_1 = require("@loaders.gl/schema");
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+async function* parseNDJSONInBatches(binaryAsyncIterator, options) {
+    const textIterator = (0, loader_utils_1.makeTextDecoderIterator)(binaryAsyncIterator);
+    const lineIterator = (0, loader_utils_1.makeLineIterator)(textIterator);
+    const numberedLineIterator = (0, loader_utils_1.makeNumberedLineIterator)(lineIterator);
+    const schema = null;
+    const shape = 'row-table';
+    // @ts-ignore
+    const tableBatchBuilder = new schema_1.TableBatchBuilder(schema, {
+        ...options,
+        shape
+    });
+    for await (const { counter, line } of numberedLineIterator) {
+        try {
+            const row = JSON.parse(line);
+            tableBatchBuilder.addRow(row);
+            tableBatchBuilder.chunkComplete(line);
+            const batch = tableBatchBuilder.getFullBatch();
+            if (batch) {
+                yield batch;
+            }
+        }
+        catch (error) {
+            throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter}`);
+        }
+    }
+    const batch = tableBatchBuilder.getFinalBatch();
+    if (batch) {
+        yield batch;
+    }
+}
+exports.parseNDJSONInBatches = parseNDJSONInBatches;

package/dist/lib/parsers/parse-ndjson.d.ts

@@ -0,0 +1,3 @@
+import type { ArrayRowTable, ObjectRowTable } from '@loaders.gl/schema';
+export declare function parseNDJSONSync(ndjsonText: string): ArrayRowTable | ObjectRowTable;
+//# sourceMappingURL=parse-ndjson.d.ts.map

package/dist/lib/parsers/parse-ndjson.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"parse-ndjson.d.ts","sourceRoot":"","sources":["../../../src/lib/parsers/parse-ndjson.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,oBAAoB,CAAC;AAGtE,wBAAgB,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,aAAa,GAAG,cAAc,CAWlF"}

package/dist/lib/parsers/parse-ndjson.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseNDJSONSync = void 0;
+const schema_1 = require("@loaders.gl/schema");
+function parseNDJSONSync(ndjsonText) {
+    const lines = ndjsonText.trim().split('\n');
+    const parsedLines = lines.map((line, counter) => {
+        try {
+            return JSON.parse(line);
+        }
+        catch (error) {
+            throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
+        }
+    });
+    return (0, schema_1.makeTableFromData)(parsedLines);
+}
+exports.parseNDJSONSync = parseNDJSONSync;

package/dist/ndgeoson-loader.d.ts

@@ -0,0 +1,34 @@
+import type { LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
+import { parseNDJSONSync } from './lib/parsers/parse-ndjson';
+import { parseNDJSONInBatches } from './lib/parsers/parse-ndjson-in-batches';
+export type NDGeoJSONLoaderOptions = LoaderOptions & {
+    geojson?: {
+        shape?: 'object-row-table';
+    };
+    gis?: {
+        format: 'geojson';
+    };
+};
+export declare const NDJSONLoader: {
+    name: string;
+    id: string;
+    module: string;
+    version: any;
+    extensions: string[];
+    mimeTypes: string[];
+    category: string;
+    text: boolean;
+    parse: (arrayBuffer: ArrayBuffer) => Promise<import("@loaders.gl/schema").ArrayRowTable | import("@loaders.gl/schema").ObjectRowTable>;
+    parseTextSync: typeof parseNDJSONSync;
+    parseInBatches: typeof parseNDJSONInBatches;
+    options: {
+        geojson: {
+            shape: string;
+        };
+        gis: {
+            format: string;
+        };
+    };
+};
+export declare const _typecheckNDJSONLoader: LoaderWithParser;
+//# sourceMappingURL=ndgeoson-loader.d.ts.map

package/dist/ndgeoson-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"ndgeoson-loader.d.ts","sourceRoot":"","sources":["../src/ndgeoson-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAC9E,OAAO,EAAC,eAAe,EAAC,MAAM,4BAA4B,CAAC;AAC3D,OAAO,EAAC,oBAAoB,EAAC,MAAM,uCAAuC,CAAC;AAM3E,MAAM,MAAM,sBAAsB,GAAG,aAAa,GAAG;IACnD,OAAO,CAAC,EAAE;QACR,KAAK,CAAC,EAAE,kBAAkB,CAAC;KAC5B,CAAC;IACF,GAAG,CAAC,EAAE;QACJ,MAAM,EAAE,SAAS,CAAC;KACnB,CAAC;CACH,CAAC;AAWF,eAAO,MAAM,YAAY;;;;;;;;;yBAeI,WAAW;;;;;;;;;;;CAIvC,CAAC;AAEF,eAAO,MAAM,sBAAsB,EAAE,gBAA+B,CAAC"}

package/dist/ndgeoson-loader.js

@@ -0,0 +1,37 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports._typecheckNDJSONLoader = exports.NDJSONLoader = void 0;
+const parse_ndjson_1 = require("./lib/parsers/parse-ndjson");
+const parse_ndjson_in_batches_1 = require("./lib/parsers/parse-ndjson-in-batches");
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+const DEFAULT_NDGEOJSON_LOADER_OPTIONS = {
+    geojson: {
+        shape: 'object-row-table'
+    },
+    gis: {
+        format: 'geojson'
+    }
+};
+exports.NDJSONLoader = {
+    name: 'NDJSON',
+    id: 'ndjson',
+    module: 'json',
+    version: VERSION,
+    extensions: ['ndjson', 'ndgeojson'],
+    mimeTypes: [
+        'application/geo+x-ndjson',
+        'application/geo+x-ldjson',
+        'application/jsonlines',
+        'application/geo+json-seq',
+        'application/x-ndjson'
+    ],
+    category: 'table',
+    text: true,
+    parse: async (arrayBuffer) => (0, parse_ndjson_1.parseNDJSONSync)(new TextDecoder().decode(arrayBuffer)),
+    parseTextSync: parse_ndjson_1.parseNDJSONSync,
+    parseInBatches: parse_ndjson_in_batches_1.parseNDJSONInBatches,
+    options: DEFAULT_NDGEOJSON_LOADER_OPTIONS
+};
+exports._typecheckNDJSONLoader = exports.NDJSONLoader;

package/dist/ndjson-loader.d.ts

@@ -1,22 +1,4 @@
-import type { Batch } from '@loaders.gl/schema';
-import type { LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
-export declare const NDJSONLoader: {
-    name: string;
-    id: string;
-    module: string;
-    version: any;
-    extensions: string[];
-    mimeTypes: string[];
-    category: string;
-    text: boolean;
-    parse: typeof parse;
-    parseTextSync: typeof parseTextSync;
-    parseInBatches: typeof parseInBatches;
-    options: {};
-};
-declare function parse(arrayBuffer: ArrayBuffer): Promise<any[]>;
-declare function parseTextSync(text: string): any[];
-declare function parseInBatches(asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>, options?: LoaderOptions): AsyncIterable<Batch>;
-export declare const _typecheckNDJSONLoader: LoaderWithParser;
-export {};
+import { LoaderWithParser, LoaderOptions } from '@loaders.gl/loader-utils';
+import { ObjectRowTable, ArrayRowTable, TableBatch } from '@loaders.gl/schema';
+export declare const NDJSONLoader: LoaderWithParser<ObjectRowTable | ArrayRowTable, TableBatch, LoaderOptions>;
 //# sourceMappingURL=ndjson-loader.d.ts.map

package/dist/ndjson-loader.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ndjson-loader.d.ts","sourceRoot":"","sources":["../src/ndjson-loader.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAC,KAAK,EAAC,MAAM,oBAAoB,CAAC;AAC9C,OAAO,KAAK,EAAC,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AAQ9E,eAAO,MAAM,YAAY;;;;;;;;;;;;;CAaxB,CAAC;AAEF,iBAAe,KAAK,CAAC,WAAW,EAAE,WAAW,kBAE5C;AAED,iBAAS,aAAa,CAAC,IAAI,EAAE,MAAM,SAElC;AAED,iBAAS,cAAc,CACrB,aAAa,EAAE,aAAa,CAAC,WAAW,CAAC,GAAG,QAAQ,CAAC,WAAW,CAAC,EACjE,OAAO,CAAC,EAAE,aAAa,GACtB,aAAa,CAAC,KAAK,CAAC,CAEtB;AAED,eAAO,MAAM,sBAAsB,EAAE,gBAA+B,CAAC"}
+{"version":3,"file":"ndjson-loader.d.ts","sourceRoot":"","sources":["../src/ndjson-loader.ts"],"names":[],"mappings":"AAEA,OAAO,EAAC,gBAAgB,EAAE,aAAa,EAAC,MAAM,0BAA0B,CAAC;AACzE,OAAO,EAAC,cAAc,EAAE,aAAa,EAAE,UAAU,EAAC,MAAM,oBAAoB,CAAC;AAQ7E,eAAO,MAAM,YAAY,EAAE,gBAAgB,CACzC,cAAc,GAAG,aAAa,EAC9B,UAAU,EACV,aAAa,CAkBd,CAAC"}

package/dist/ndjson-loader.js

@@ -1,32 +1,27 @@
-import parseNDJSONSync from './lib/parse-ndjson';
-import parseNDJSONInBatches from './lib/parse-ndjson-in-batches';
-const VERSION = typeof "4.0.0-alpha.5" !== 'undefined' ? "4.0.0-alpha.5" : 'latest';
-export const NDJSONLoader = {
-  name: 'NDJSON',
-  id: 'ndjson',
-  module: 'json',
-  version: VERSION,
-  extensions: ['ndjson'],
-  mimeTypes: ['application/x-ndjson'],
-  category: 'table',
-  text: true,
-  parse,
-  parseTextSync,
-  parseInBatches,
-  options: {}
+"use strict";
+// loaders.gl, MIT license
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NDJSONLoader = void 0;
+const parse_ndjson_1 = require("./lib/parsers/parse-ndjson");
+const parse_ndjson_in_batches_1 = require("./lib/parsers/parse-ndjson-in-batches");
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+exports.NDJSONLoader = {
+    name: 'NDJSON',
+    id: 'ndjson',
+    module: 'json',
+    version: VERSION,
+    extensions: ['ndjson', 'jsonl'],
+    mimeTypes: [
+        'application/x-ndjson',
+        'application/jsonlines',
+        'application/json-seq'
+    ],
+    category: 'table',
+    text: true,
+    parse: async (arrayBuffer) => (0, parse_ndjson_1.parseNDJSONSync)(new TextDecoder().decode(arrayBuffer)),
+    parseTextSync: parse_ndjson_1.parseNDJSONSync,
+    parseInBatches: parse_ndjson_in_batches_1.parseNDJSONInBatches,
+    options: {}
 };
-
-async function parse(arrayBuffer) {
-  return parseTextSync(new TextDecoder().decode(arrayBuffer));
-}
-
-function parseTextSync(text) {
-  return parseNDJSONSync(text);
-}
-
-function parseInBatches(asyncIterator, options) {
-  return parseNDJSONInBatches(asyncIterator, options);
-}
-
-export const _typecheckNDJSONLoader = NDJSONLoader;
-//# sourceMappingURL=ndjson-loader.js.map

package/dist/workers/geojson-worker.js

@@ -1,4 +1,5 @@
-import { createLoaderWorker } from '@loaders.gl/loader-utils';
-import { GeoJSONLoader } from '../geojson-loader';
-createLoaderWorker(GeoJSONLoader);
-//# sourceMappingURL=geojson-worker.js.map
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const loader_utils_1 = require("@loaders.gl/loader-utils");
+const geojson_loader_1 = require("../geojson-loader");
+(0, loader_utils_1.createLoaderWorker)(geojson_loader_1.GeoJSONLoader);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@loaders.gl/json",
-  "version": "4.0.0-alpha.5",
+  "version": "4.0.0-alpha.7",
   "description": "Framework-independent loader for JSON and streaming JSON formats",
   "license": "MIT",
   "publishConfig": {
@@ -21,8 +21,8 @@
     "JSON async iterator"
   ],
   "types": "dist/index.d.ts",
-  "main": "dist/index.js",
-  "module": "dist/index.js",
+  "main": "dist/es5/index.js",
+  "module": "dist/esm/index.js",
   "sideEffects": false,
   "files": [
     "src",
@@ -35,9 +35,9 @@
     "build-worker": "esbuild src/workers/geojson-worker.ts --bundle --outfile=dist/geojson-worker.js --define:__VERSION__=\\\"$npm_package_version\\\""
   },
   "dependencies": {
-    "@loaders.gl/gis": "4.0.0-alpha.5",
-    "@loaders.gl/loader-utils": "4.0.0-alpha.5",
-    "@loaders.gl/schema": "4.0.0-alpha.5"
+    "@loaders.gl/gis": "4.0.0-alpha.7",
+    "@loaders.gl/loader-utils": "4.0.0-alpha.7",
+    "@loaders.gl/schema": "4.0.0-alpha.7"
   },
-  "gitHead": "7a71a54bdf1ddf985cc3af3db90b82e7fa97d025"
+  "gitHead": "afb59c4d8e5d8ebb9c28f111cb0c96c5527d0ffd"
 }

package/src/geojson-loader.ts

@@ -1,8 +1,11 @@
+// loaders.gl, MIT license
+
 import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils';
 import type {JSONLoaderOptions} from './json-loader';
 import {geojsonToBinary} from '@loaders.gl/gis';
-import parseJSONSync from './lib/parse-json';
-import parseJSONInBatches from './lib/parse-json-in-batches';
+import {parseJSONSync} from './lib/parsers/parse-json';
+import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
+import {GeoJSONRowTable} from '@loaders.gl/schema';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
@@ -13,7 +16,7 @@ export type GeoJSONLoaderOptions = JSONLoaderOptions & {
     shape?: 'object-row-table';
   };
   gis?: {
-    format: 'geojson';
+    format?: 'geojson' | 'binary';
   };
 };
 
@@ -61,12 +64,13 @@ function parseTextSync(text, options) {
   options = {...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options};
   options.json = {...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson};
   options.gis = options.gis || {};
-  const json = parseJSONSync(text, options);
+  const table = parseJSONSync(text, options) as GeoJSONRowTable;
+  table.shape = 'geojson-row-table';
   switch (options.gis.format) {
     case 'binary':
-      return geojsonToBinary(json);
+      return geojsonToBinary(table.data);
     default:
-      return json;
+      return table;
   }
 }
 

package/src/geojson-writer.ts

@@ -0,0 +1,27 @@
+// loaders.gl, MIT license
+// Copyright Foursquare, Inc 20222
+
+import type {Writer} from '@loaders.gl/loader-utils';
+import type {Table, TableBatch} from '@loaders.gl/schema';
+import type {GeoJSONWriterOptions} from './lib/encoders/geojson-encoder';
+import {encodeTableAsGeojsonInBatches} from './lib/encoders/geojson-encoder';
+
+export type {GeoJSONWriterOptions};
+
+export const GeoJSONWriter: Writer<Table, TableBatch, GeoJSONWriterOptions> = {
+  id: 'geojson',
+  version: 'latest',
+  module: 'geojson',
+  name: 'GeoJSON',
+  extensions: ['geojson'],
+  mimeTypes: ['application/geo+json'],
+  options: {
+    geojson: {
+      featureArray: false,
+      geometryColumn: null
+    }
+  },
+  text: true,
+  encodeInBatches: (tableIterator: AsyncIterable<TableBatch>, options) =>
+    encodeTableAsGeojsonInBatches(tableIterator, options)
+};

package/src/index.ts

@@ -1,7 +1,12 @@
+// loaders.gl, MIT license
+
 export type {JSONLoaderOptions} from './json-loader';
 export {JSONLoader} from './json-loader';
 export {NDJSONLoader} from './ndjson-loader';
 
+export type {JSONWriterOptions} from './json-writer';
+export {JSONWriter} from './json-writer';
+
 // EXPERIMENTAL EXPORTS - WARNING: MAY BE REMOVED WIHTOUT NOTICE IN FUTURE RELEASES
 export type {GeoJSONLoaderOptions as _GeoJSONLoaderOptions} from './geojson-loader';
 export {
@@ -9,7 +14,10 @@ export {
   GeoJSONWorkerLoader as _GeoJSONWorkerLoader
 } from './geojson-loader';
 
+export type {GeoJSONWriterOptions as _GeoJSONWriterOptions} from './geojson-writer';
+export {GeoJSONWriter as _GeoJSONWriter} from './geojson-writer';
+
 export {default as _JSONPath} from './lib/jsonpath/jsonpath';
 export {default as _ClarinetParser} from './lib/clarinet/clarinet';
 
-export {rebuildJsonObject as _rebuildJsonObject} from './lib/parse-json-in-batches';
+export {rebuildJsonObject as _rebuildJsonObject} from './lib/parsers/parse-json-in-batches';

package/src/json-loader.ts

@@ -1,25 +1,29 @@
-import type {Batch} from '@loaders.gl/schema';
+// loaders.gl, MIT license
+
+import type {Table, TableBatch} from '@loaders.gl/schema';
 import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
-import parseJSONSync from './lib/parse-json';
-import parseJSONInBatches from './lib/parse-json-in-batches';
+import {parseJSONSync} from './lib/parsers/parse-json';
+import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
+type ParseJSONOptions = {
+  shape?: 'row-table';
+  table?: boolean;
+  jsonpaths?: string[];
+};
+
 /**
  * @param table -
  * @param jsonpaths -
  */
 export type JSONLoaderOptions = LoaderOptions & {
-  json?: {
-    shape?: 'row-table';
-    table?: false;
-    jsonpaths?: string[];
-  };
+  json?: ParseJSONOptions;
 };
 
-const DEFAULT_JSON_LOADER_OPTIONS = {
+const DEFAULT_JSON_LOADER_OPTIONS: {json: Required<ParseJSONOptions>} = {
   json: {
     shape: 'row-table',
     table: false,
@@ -28,26 +32,13 @@ const DEFAULT_JSON_LOADER_OPTIONS = {
   }
 };
 
-export const JSONLoader: LoaderWithParser = {
+export const JSONLoader: LoaderWithParser<Table, TableBatch, JSONLoaderOptions> = {
   name: 'JSON',
   id: 'json',
   module: 'json',
   version: VERSION,
   extensions: ['json', 'geojson'],
   mimeTypes: ['application/json'],
-  // TODO - support various line based JSON formats
-  /*
-  extensions: {
-    json: null,
-    jsonl: {stream: true},
-    ndjson: {stream: true}
-  },
-  mimeTypes: {
-    'application/json': null,
-    'application/json-seq': {stream: true},
-    'application/x-ndjson': {stream: true}
-  },
-  */
   category: 'table',
   text: true,
   parse,
@@ -68,7 +59,7 @@ function parseTextSync(text: string, options?: JSONLoaderOptions) {
 function parseInBatches(
   asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
   options?: JSONLoaderOptions
-): AsyncIterable<Batch> {
+): AsyncIterable<TableBatch> {
   const jsonOptions = {...options, json: {...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json}};
   return parseJSONInBatches(asyncIterator, jsonOptions as JSONLoaderOptions);
 }

package/src/json-writer.ts

@@ -0,0 +1,24 @@
+// loaders.gl, MIT license
+// Copyright 2022 Foursquare Labs, Inc.
+
+/* global TextEncoder */
+import type {Writer} from '@loaders.gl/loader-utils';
+import type {Table, TableBatch} from '@loaders.gl/schema';
+import type {JSONWriterOptions} from './lib/encoders/json-encoder';
+import {encodeTableAsJSON} from './lib/encoders/json-encoder';
+
+export type {JSONWriterOptions};
+
+export const JSONWriter: Writer<Table, TableBatch, JSONWriterOptions> = {
+  id: 'json',
+  version: 'latest',
+  module: 'json',
+  name: 'JSON',
+  extensions: ['json'],
+  mimeTypes: ['application/json'],
+  options: {},
+  text: true,
+  encode: async (table: Table, options: JSONWriterOptions) =>
+    new TextEncoder().encode(encodeTableAsJSON(table, options)).buffer,
+  encodeText: (table: Table, options: JSONWriterOptions) => encodeTableAsJSON(table, options)
+};