@loaders.gl/json 4.0.0-alpha.4 → 4.0.0-alpha.6

package/src/geojson-writer.ts

@@ -0,0 +1,27 @@
+// loaders.gl, MIT license
+// Copyright Foursquare, Inc 20222
+
+import type {Writer} from '@loaders.gl/loader-utils';
+import type {Table, TableBatch} from '@loaders.gl/schema';
+import type {GeoJSONWriterOptions} from './lib/encoders/geojson-encoder';
+import {encodeTableAsGeojsonInBatches} from './lib/encoders/geojson-encoder';
+
+export type {GeoJSONWriterOptions};
+
+export const GeoJSONWriter: Writer<Table, TableBatch, GeoJSONWriterOptions> = {
+  id: 'geojson',
+  version: 'latest',
+  module: 'geojson',
+  name: 'GeoJSON',
+  extensions: ['geojson'],
+  mimeTypes: ['application/geo+json'],
+  options: {
+    geojson: {
+      featureArray: false,
+      geometryColumn: null
+    }
+  },
+  text: true,
+  encodeInBatches: (tableIterator: AsyncIterable<TableBatch>, options) =>
+    encodeTableAsGeojsonInBatches(tableIterator, options)
+};

package/src/index.ts

@@ -1,7 +1,12 @@
+// loaders.gl, MIT license
+
 export type {JSONLoaderOptions} from './json-loader';
 export {JSONLoader} from './json-loader';
 export {NDJSONLoader} from './ndjson-loader';
 
+export type {JSONWriterOptions} from './json-writer';
+export {JSONWriter} from './json-writer';
+
 // EXPERIMENTAL EXPORTS - WARNING: MAY BE REMOVED WIHTOUT NOTICE IN FUTURE RELEASES
 export type {GeoJSONLoaderOptions as _GeoJSONLoaderOptions} from './geojson-loader';
 export {
@@ -9,5 +14,10 @@ export {
   GeoJSONWorkerLoader as _GeoJSONWorkerLoader
 } from './geojson-loader';
 
+export type {GeoJSONWriterOptions as _GeoJSONWriterOptions} from './geojson-writer';
+export {GeoJSONWriter as _GeoJSONWriter} from './geojson-writer';
+
 export {default as _JSONPath} from './lib/jsonpath/jsonpath';
 export {default as _ClarinetParser} from './lib/clarinet/clarinet';
+
+export {rebuildJsonObject as _rebuildJsonObject} from './lib/parsers/parse-json-in-batches';

package/src/json-loader.ts

@@ -1,25 +1,29 @@
-import type {Batch} from '@loaders.gl/schema';
+// loaders.gl, MIT license
+
+import type {Table, TableBatch} from '@loaders.gl/schema';
 import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
-import parseJSONSync from './lib/parse-json';
-import parseJSONInBatches from './lib/parse-json-in-batches';
+import {parseJSONSync} from './lib/parsers/parse-json';
+import {parseJSONInBatches} from './lib/parsers/parse-json-in-batches';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
+type ParseJSONOptions = {
+  shape?: 'row-table';
+  table?: boolean;
+  jsonpaths?: string[];
+};
+
 /**
  * @param table -
  * @param jsonpaths -
  */
 export type JSONLoaderOptions = LoaderOptions & {
-  json?: {
-    shape?: 'row-table';
-    table?: false;
-    jsonpaths?: string[];
-  };
+  json?: ParseJSONOptions;
 };
 
-const DEFAULT_JSON_LOADER_OPTIONS = {
+const DEFAULT_JSON_LOADER_OPTIONS: {json: Required<ParseJSONOptions>} = {
   json: {
     shape: 'row-table',
     table: false,
@@ -28,26 +32,13 @@ const DEFAULT_JSON_LOADER_OPTIONS = {
   }
 };
 
-export const JSONLoader: LoaderWithParser = {
+export const JSONLoader: LoaderWithParser<Table, TableBatch, JSONLoaderOptions> = {
   name: 'JSON',
   id: 'json',
   module: 'json',
   version: VERSION,
   extensions: ['json', 'geojson'],
   mimeTypes: ['application/json'],
-  // TODO - support various line based JSON formats
-  /*
-  extensions: {
-    json: null,
-    jsonl: {stream: true},
-    ndjson: {stream: true}
-  },
-  mimeTypes: {
-    'application/json': null,
-    'application/json-seq': {stream: true},
-    'application/x-ndjson': {stream: true}
-  },
-  */
   category: 'table',
   text: true,
   parse,
@@ -68,7 +59,7 @@ function parseTextSync(text: string, options?: JSONLoaderOptions) {
 function parseInBatches(
   asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
   options?: JSONLoaderOptions
-): AsyncIterable<Batch> {
+): AsyncIterable<TableBatch> {
   const jsonOptions = {...options, json: {...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json}};
   return parseJSONInBatches(asyncIterator, jsonOptions as JSONLoaderOptions);
 }

package/src/json-writer.ts

@@ -0,0 +1,24 @@
+// loaders.gl, MIT license
+// Copyright 2022 Foursquare Labs, Inc.
+
+/* global TextEncoder */
+import type {Writer} from '@loaders.gl/loader-utils';
+import type {Table, TableBatch} from '@loaders.gl/schema';
+import type {JSONWriterOptions} from './lib/encoders/json-encoder';
+import {encodeTableAsJSON} from './lib/encoders/json-encoder';
+
+export type {JSONWriterOptions};
+
+export const JSONWriter: Writer<Table, TableBatch, JSONWriterOptions> = {
+  id: 'json',
+  version: 'latest',
+  module: 'json',
+  name: 'JSON',
+  extensions: ['json'],
+  mimeTypes: ['application/json'],
+  options: {},
+  text: true,
+  encode: async (table: Table, options: JSONWriterOptions) =>
+    new TextEncoder().encode(encodeTableAsJSON(table, options)).buffer,
+  encodeText: (table: Table, options: JSONWriterOptions) => encodeTableAsJSON(table, options)
+};

package/src/lib/encoders/encode-utils.ts

@@ -0,0 +1,54 @@
+// loaders.gl, MIT license
+// Copyright 2022 Foursquare Labs, Inc.
+
+import {Table, getTableLength, getTableNumCols, getTableRowAsArray} from '@loaders.gl/schema';
+
+type Row = {[key: string]: unknown};
+
+/**
+ * Attempts to identify which column contains geometry
+ * Currently just returns name (key) of first object-valued column
+ * @todo look for hints in schema metadata
+ * @todo look for WKB
+ */
+export function detectGeometryColumnIndex(table: Table): number {
+  // TODO - look for hints in schema metadata
+
+  // look for a column named geometry
+  const geometryIndex = table.schema?.fields.findIndex((field) => field.name === 'geometry') ?? -1;
+  if (geometryIndex > -1) {
+    return geometryIndex;
+  }
+
+  // look at the data
+  // TODO - this drags in the indices
+  if (getTableLength(table) > 0) {
+    const row = getTableRowAsArray(table, 0);
+    for (let columnIndex = 0; columnIndex < getTableNumCols(table); columnIndex++) {
+      const value = row?.[columnIndex];
+      if (value && typeof value === 'object') {
+        return columnIndex;
+      }
+    }
+  }
+
+  throw new Error('Failed to detect geometry column');
+}
+
+/**
+ * Return a row as a property (key/value) object, excluding selected columns
+ */
+export function getRowPropertyObject(
+  table: Table,
+  row: Row,
+  excludeColumnIndices: number[] = []
+): {[columnName: string]: unknown} {
+  const properties = {};
+  for (let columnIndex = 0; columnIndex < getTableNumCols(table); ++columnIndex) {
+    const columnName = table.schema?.fields[columnIndex].name;
+    if (columnName && !excludeColumnIndices.includes(columnIndex)) {
+      properties[columnName] = row[columnName];
+    }
+  }
+  return properties;
+}

package/src/lib/encoders/geojson-encoder.ts

@@ -0,0 +1,139 @@
+// loaders.gl, MIT license
+// Copyright 2022 Foursquare Labs, Inc.
+
+import {Feature, getTableLength} from '@loaders.gl/schema';
+import {Table, TableBatch, getTableRowAsObject} from '@loaders.gl/schema';
+import {detectGeometryColumnIndex, getRowPropertyObject} from './encode-utils';
+import {Utf8ArrayBufferEncoder} from './utf8-encoder';
+
+type Row = {[key: string]: unknown};
+
+export type GeoJSONWriterOptions = {
+  geojson?: {
+    featureArray?: boolean;
+    geometryColumn?: number | null;
+  };
+  chunkSize?: number;
+};
+
+/**
+ * Encode a table as GeoJSON
+ */
+// eslint-disable-next-line max-statements
+export async function* encodeTableAsGeojsonInBatches(
+  batchIterator: AsyncIterable<TableBatch>, // | Iterable<TableBatch>,
+  inputOpts: GeoJSONWriterOptions = {}
+): AsyncIterable<ArrayBuffer> {
+  const options: Required<GeoJSONWriterOptions> = {geojson: {}, chunkSize: 10000, ...inputOpts};
+
+  const utf8Encoder = new Utf8ArrayBufferEncoder(options.chunkSize);
+
+  if (!options.geojson.featureArray) {
+    utf8Encoder.push('{\n', '"type": "FeatureCollection",\n', '"features":\n');
+  }
+  utf8Encoder.push('['); // Note no newline
+
+  let geometryColumn = options.geojson.geometryColumn;
+
+  let isFirstLine = true;
+
+  for await (const batch of batchIterator) {
+    const {table, start, end = getTableLength(batch.table) - start} = batch;
+
+    // Deduce geometry column if not already done
+    if (!geometryColumn) {
+      geometryColumn = geometryColumn || detectGeometryColumnIndex(table);
+    }
+
+    for (let rowIndex = start; rowIndex < end; ++rowIndex) {
+      // Add a comma except on final feature
+      if (!isFirstLine) {
+        utf8Encoder.push(',');
+      }
+      utf8Encoder.push('\n');
+      isFirstLine = false;
+
+      encodeRow(table, rowIndex, geometryColumn, utf8Encoder);
+
+      // eslint-disable-next-line max-depth
+      if (utf8Encoder.isFull()) {
+        yield utf8Encoder.getArrayBufferBatch();
+      }
+    }
+    const arrayBufferBatch = utf8Encoder.getArrayBufferBatch();
+    if (arrayBufferBatch.byteLength > 0) {
+      yield arrayBufferBatch;
+    }
+  }
+
+  utf8Encoder.push('\n');
+
+  // Add completing rows and emit final batch
+  utf8Encoder.push(']\n');
+  if (!options.geojson.featureArray) {
+    utf8Encoder.push('}');
+  }
+
+  // Note: Since we pushed a few final lines, the last batch will always exist, no need to check first
+  yield utf8Encoder.getArrayBufferBatch();
+}
+
+// Helpers
+
+/**
+ * Encode a row. Currently this ignores properties in the geometry column.
+ */
+function encodeRow(
+  table: Table,
+  rowIndex: number,
+  geometryColumnIndex: number,
+  utf8Encoder: Utf8ArrayBufferEncoder
+): void {
+  const row = getTableRowAsObject(table, rowIndex);
+  if (!row) return;
+  const featureWithProperties = getFeatureFromRow(table, row, geometryColumnIndex);
+  const featureString = JSON.stringify(featureWithProperties);
+  utf8Encoder.push(featureString);
+}
+
+/**
+ * Encode a row as a Feature. Currently this ignores properties objects in the geometry column.
+ */
+function getFeatureFromRow(table: Table, row: Row, geometryColumnIndex: number): Feature {
+  // Extract non-feature/geometry properties
+  const properties = getRowPropertyObject(table, row, [geometryColumnIndex]);
+
+  // Extract geometry feature
+  const columnName = table.schema?.fields[geometryColumnIndex].name;
+  let featureOrGeometry =
+    columnName && (row[columnName] as {[key: string]: unknown} | string | null | undefined);
+
+  // GeoJSON support null geometries
+  if (!featureOrGeometry) {
+    // @ts-ignore Feature type does not support null geometries
+    return {type: 'Feature', geometry: null, properties};
+  }
+
+  // Support string geometries?
+  // TODO: This assumes GeoJSON strings, which may not be the correct format
+  // (could be WKT, encoded WKB...)
+  if (typeof featureOrGeometry === 'string') {
+    try {
+      featureOrGeometry = JSON.parse(featureOrGeometry);
+    } catch (err) {
+      throw new Error('Invalid string geometry');
+    }
+  }
+
+  if (typeof featureOrGeometry !== 'object' || typeof featureOrGeometry?.type !== 'string') {
+    throw new Error('invalid geometry column value');
+  }
+
+  if (featureOrGeometry?.type === 'Feature') {
+    // @ts-ignore Feature type does not support null geometries
+    return {...featureOrGeometry, properties};
+  }
+
+  // @ts-ignore Feature type does not support null geometries
+  return {type: 'Feature', geometry: featureOrGeometry, properties};
+}

package/src/lib/encoders/json-encoder.ts

@@ -0,0 +1,30 @@
+// loaders.gl, MIT license
+// Copyright 2022 Foursquare Labs, Inc.
+
+import {Table, makeRowIterator} from '@loaders.gl/schema';
+
+type RowArray = unknown[];
+type RowObject = {[key: string]: unknown};
+type TableJSON = RowArray[] | RowObject[];
+
+export type JSONWriterOptions = {
+  shape?: 'object-row-table' | 'array-row-table';
+  wrapper?: (table: TableJSON) => unknown;
+};
+
+/**
+ * Encode a table as a JSON string
+ */
+export function encodeTableAsJSON(table: Table, options: JSONWriterOptions = {}): string {
+  const shape = options.shape || 'object-row-table';
+
+  const strings: string[] = [];
+  const rowIterator = makeRowIterator(table, shape);
+  for (const row of rowIterator) {
+    // Round elements etc
+    // processRow(wrappedRow, table.schema);
+    // const wrappedRow = options.wrapper ? options.wrapper(row) : row;
+    strings.push(JSON.stringify(row));
+  }
+  return `[${strings.join(',')}]`;
+}

package/src/lib/encoders/utf8-encoder.ts

@@ -0,0 +1,35 @@
+// loaders.gl, MIT License
+
+/* global TextEncoder */
+export class Utf8ArrayBufferEncoder {
+  private readonly chunkSize: number;
+  private strings: string[] = [];
+  private totalLength = 0;
+  private textEncoder: TextEncoder = new TextEncoder();
+
+  constructor(chunkSize: number) {
+    this.chunkSize = chunkSize;
+  }
+
+  push(...strings: string[]): void {
+    for (const string of strings) {
+      this.strings.push(string);
+      this.totalLength += string.length;
+    }
+  }
+
+  isFull(): boolean {
+    return this.totalLength >= this.chunkSize;
+  }
+
+  getArrayBufferBatch(): ArrayBufferLike {
+    return this.textEncoder.encode(this.getStringBatch()).buffer;
+  }
+
+  getStringBatch(): string {
+    const stringChunk = this.strings.join('');
+    this.strings = [];
+    this.totalLength = 0;
+    return stringChunk;
+  }
+}

package/src/lib/{parse-json-in-batches.ts → parsers/parse-json-in-batches.ts}

@@ -1,15 +1,16 @@
-import type {Batch} from '@loaders.gl/schema';
-import type {JSONLoaderOptions} from '../json-loader';
+import type {TableBatch} from '@loaders.gl/schema';
+import type {JSONLoaderOptions} from '../../json-loader';
 import {TableBatchBuilder} from '@loaders.gl/schema';
-import {makeTextDecoderIterator} from '@loaders.gl/loader-utils';
-import StreamingJSONParser from './parser/streaming-json-parser';
+import {assert, makeTextDecoderIterator} from '@loaders.gl/loader-utils';
+import StreamingJSONParser from '../json-parser/streaming-json-parser';
+import JSONPath from '../jsonpath/jsonpath';
 
 // TODO - support batch size 0 = no batching/single batch?
 // eslint-disable-next-line max-statements, complexity
-export default async function* parseJSONInBatches(
+export async function* parseJSONInBatches(
   binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
   options: JSONLoaderOptions
-): AsyncIterable<Batch> {
+): AsyncIterable<TableBatch> {
   const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);
 
   const {metadata} = options;
@@ -35,7 +36,7 @@ export default async function* parseJSONInBatches(
 
     if (rows.length > 0 && isFirstChunk) {
       if (metadata) {
-        const initialBatch: Batch = {
+        const initialBatch: TableBatch = {
           // Common fields
           shape,
           batchType: 'partial-result',
@@ -77,7 +78,7 @@ export default async function* parseJSONInBatches(
   }
 
   if (metadata) {
-    const finalBatch: Batch = {
+    const finalBatch: TableBatch = {
       shape,
       batchType: 'final-result',
       container: parser.getPartialResult(),
@@ -89,3 +90,24 @@ export default async function* parseJSONInBatches(
     yield finalBatch;
   }
 }
+
+export function rebuildJsonObject(batch, data) {
+  // Last batch will have this special type and will provide all the root object of the parsed file
+  assert(batch.batchType === 'final-result');
+
+  // The streamed JSON data is a top level array (jsonpath = '$'), just return the array of row objects
+  if (batch.jsonpath === '$') {
+    return data;
+  }
+
+  // (jsonpath !== '$') The streamed data is not a top level array, so stitch it back in to the top-level object
+  if (batch.jsonpath && batch.jsonpath.length > 1) {
+    const topLevelObject = batch.container;
+    const streamingPath = new JSONPath(batch.jsonpath);
+    streamingPath.setFieldAtPath(topLevelObject, data);
+    return topLevelObject;
+  }
+
+  // No jsonpath, in this case nothing was streamed.
+  return batch.container;
+}

package/src/lib/{parse-json.ts → parsers/parse-json.ts}

@@ -1,10 +1,14 @@
-import type {JSONLoaderOptions} from '../json-loader';
+// loaders.gl, MIT license
+import type {RowTable} from '@loaders.gl/schema';
+import {makeTableFromData} from '@loaders.gl/schema';
+import type {JSONLoaderOptions} from '../../json-loader';
 
-export default function parseJSONSync(jsonText: string, options: JSONLoaderOptions) {
+export function parseJSONSync(jsonText: string, options: JSONLoaderOptions): RowTable {
   try {
     const json = JSON.parse(jsonText);
     if (options.json?.table) {
-      return getFirstArray(json) || json;
+      const data = getFirstArray(json) || json;
+      return makeTableFromData(data);
     }
     return json;
   } catch (error) {

package/src/lib/{parse-ndjson-in-batches.ts → parsers/parse-ndjson-in-batches.ts}

@@ -7,7 +7,7 @@ import {
   makeTextDecoderIterator
 } from '@loaders.gl/loader-utils';
 
-export default async function* parseNDJSONInBatches(
+export async function* parseNDJSONInBatches(
   binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
   options?: LoaderOptions
 ): AsyncIterable<Batch> {

package/src/lib/parsers/parse-ndjson.ts

@@ -0,0 +1,15 @@
+import type {ArrayRowTable, ObjectRowTable} from '@loaders.gl/schema';
+import {makeTableFromData} from '@loaders.gl/schema';
+
+export function parseNDJSONSync(ndjsonText: string): ArrayRowTable | ObjectRowTable {
+  const lines = ndjsonText.trim().split('\n');
+  const parsedLines = lines.map((line, counter) => {
+    try {
+      return JSON.parse(line);
+    } catch (error) {
+      throw new Error(`NDJSONLoader: failed to parse JSON on line ${counter + 1}`);
+    }
+  });
+
+  return makeTableFromData(parsedLines);
+}

package/src/ndgeoson-loader.ts

@@ -0,0 +1,48 @@
+import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import {parseNDJSONSync} from './lib/parsers/parse-ndjson';
+import {parseNDJSONInBatches} from './lib/parsers/parse-ndjson-in-batches';
+
+// __VERSION__ is injected by babel-plugin-version-inline
+// @ts-ignore TS2304: Cannot find name '__VERSION__'.
+const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
+
+export type NDGeoJSONLoaderOptions = LoaderOptions & {
+  geojson?: {
+    shape?: 'object-row-table';
+  };
+  gis?: {
+    format: 'geojson';
+  };
+};
+
+const DEFAULT_NDGEOJSON_LOADER_OPTIONS = {
+  geojson: {
+    shape: 'object-row-table'
+  },
+  gis: {
+    format: 'geojson'
+  }
+};
+
+export const NDJSONLoader = {
+  name: 'NDJSON',
+  id: 'ndjson',
+  module: 'json',
+  version: VERSION,
+  extensions: ['ndjson', 'ndgeojson'],
+  mimeTypes: [
+    'application/geo+x-ndjson',
+    'application/geo+x-ldjson',
+    'application/jsonlines', // https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#cm-batch
+    'application/geo+json-seq',
+    'application/x-ndjson'
+  ],
+  category: 'table',
+  text: true,
+  parse: async (arrayBuffer: ArrayBuffer) => parseNDJSONSync(new TextDecoder().decode(arrayBuffer)),
+  parseTextSync: parseNDJSONSync,
+  parseInBatches: parseNDJSONInBatches,
+  options: DEFAULT_NDGEOJSON_LOADER_OPTIONS
+};
+
+export const _typecheckNDJSONLoader: LoaderWithParser = NDJSONLoader;

package/src/ndjson-loader.ts

@@ -1,40 +1,33 @@
-import type {Batch} from '@loaders.gl/schema';
-import type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
-import parseNDJSONSync from './lib/parse-ndjson';
-import parseNDJSONInBatches from './lib/parse-ndjson-in-batches';
+// loaders.gl, MIT license
+
+import {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';
+import {ObjectRowTable, ArrayRowTable, TableBatch} from '@loaders.gl/schema';
+import {parseNDJSONSync} from './lib/parsers/parse-ndjson';
+import {parseNDJSONInBatches} from './lib/parsers/parse-ndjson-in-batches';
 
 // __VERSION__ is injected by babel-plugin-version-inline
 // @ts-ignore TS2304: Cannot find name '__VERSION__'.
 const VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';
 
-export const NDJSONLoader = {
+export const NDJSONLoader: LoaderWithParser<
+  ObjectRowTable | ArrayRowTable,
+  TableBatch,
+  LoaderOptions
+> = {
   name: 'NDJSON',
   id: 'ndjson',
   module: 'json',
   version: VERSION,
-  extensions: ['ndjson'],
-  mimeTypes: ['application/x-ndjson'],
+  extensions: ['ndjson', 'jsonl'],
+  mimeTypes: [
+    'application/x-ndjson',
+    'application/jsonlines', // https://docs.aws.amazon.com/sagemaker/latest/dg/cdf-inference.html#cm-batch
+    'application/json-seq'
+  ],
   category: 'table',
   text: true,
-  parse,
-  parseTextSync,
-  parseInBatches,
+  parse: async (arrayBuffer: ArrayBuffer) => parseNDJSONSync(new TextDecoder().decode(arrayBuffer)),
+  parseTextSync: parseNDJSONSync,
+  parseInBatches: parseNDJSONInBatches,
   options: {}
 };
-
-async function parse(arrayBuffer: ArrayBuffer) {
-  return parseTextSync(new TextDecoder().decode(arrayBuffer));
-}
-
-function parseTextSync(text: string) {
-  return parseNDJSONSync(text);
-}
-
-function parseInBatches(
-  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
-  options?: LoaderOptions
-): AsyncIterable<Batch> {
-  return parseNDJSONInBatches(asyncIterator, options);
-}
-
-export const _typecheckNDJSONLoader: LoaderWithParser = NDJSONLoader;

package/dist/bundle.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/bundle.ts"],"names":["moduleExports","require","globalThis","loaders","module","exports","Object","assign"],"mappings":"AACA,MAAMA,aAAa,GAAGC,OAAO,CAAC,SAAD,CAA7B;;AACAC,UAAU,CAACC,OAAX,GAAqBD,UAAU,CAACC,OAAX,IAAsB,EAA3C;AACAC,MAAM,CAACC,OAAP,GAAiBC,MAAM,CAACC,MAAP,CAAcL,UAAU,CAACC,OAAzB,EAAkCH,aAAlC,CAAjB","sourcesContent":["// @ts-nocheck\nconst moduleExports = require('./index');\nglobalThis.loaders = globalThis.loaders || {};\nmodule.exports = Object.assign(globalThis.loaders, moduleExports);\n"],"file":"bundle.js"}

package/dist/geojson-loader.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/geojson-loader.ts"],"names":["geojsonToBinary","parseJSONSync","parseJSONInBatches","VERSION","DEFAULT_GEOJSON_LOADER_OPTIONS","geojson","shape","json","jsonpaths","gis","format","GeoJSONWorkerLoader","name","id","module","version","worker","extensions","mimeTypes","category","text","options","GeoJSONLoader","parse","parseTextSync","parseInBatches","arrayBuffer","TextDecoder","decode","asyncIterator","geojsonIterator","makeBinaryGeometryIterator","batch","data"],"mappings":"AAEA,SAAQA,eAAR,QAA8B,iBAA9B;AACA,OAAOC,aAAP,MAA0B,kBAA1B;AACA,OAAOC,kBAAP,MAA+B,6BAA/B;AAIA,MAAMC,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAWA,MAAMC,8BAA8B,GAAG;AACrCC,EAAAA,OAAO,EAAE;AACPC,IAAAA,KAAK,EAAE;AADA,GAD4B;AAIrCC,EAAAA,IAAI,EAAE;AACJC,IAAAA,SAAS,EAAE,CAAC,GAAD,EAAM,YAAN;AADP,GAJ+B;AAOrCC,EAAAA,GAAG,EAAE;AACHC,IAAAA,MAAM,EAAE;AADL;AAPgC,CAAvC;AAeA,OAAO,MAAMC,mBAA2B,GAAG;AACzCC,EAAAA,IAAI,EAAE,SADmC;AAEzCC,EAAAA,EAAE,EAAE,SAFqC;AAGzCC,EAAAA,MAAM,EAAE,SAHiC;AAIzCC,EAAAA,OAAO,EAAEZ,OAJgC;AAKzCa,EAAAA,MAAM,EAAE,IALiC;AAMzCC,EAAAA,UAAU,EAAE,CAAC,SAAD,CAN6B;AAOzCC,EAAAA,SAAS,EAAE,CAAC,sBAAD,CAP8B;AAQzCC,EAAAA,QAAQ,EAAE,UAR+B;AASzCC,EAAAA,IAAI,EAAE,IATmC;AAUzCC,EAAAA,OAAO,EAAEjB;AAVgC,CAApC;AAaP,OAAO,MAAMkB,aAA+B,GAAG,EAC7C,GAAGX,mBAD0C;AAE7CY,EAAAA,KAF6C;AAG7CC,EAAAA,aAH6C;AAI7CC,EAAAA;AAJ6C,CAAxC;;AAOP,eAAeF,KAAf,CAAqBG,WAArB,EAAkCL,OAAlC,EAA2C;AACzC,SAAOG,aAAa,CAAC,IAAIG,WAAJ,GAAkBC,MAAlB,CAAyBF,WAAzB,CAAD,EAAwCL,OAAxC,CAApB;AACD;;AAED,SAASG,aAAT,CAAuBJ,IAAvB,EAA6BC,OAA7B,EAAsC;AAEpCA,EAAAA,OAAO,GAAG,EAAC,GAAGjB,8BAAJ;AAAoC,OAAGiB;AAAvC,GAAV;AACAA,EAAAA,OAAO,CAACd,IAAR,GAAe,EAAC,GAAGH,8BAA8B,CAACC,OAAnC;AAA4C,OAAGgB,OAAO,CAAChB;AAAvD,GAAf;AACAgB,EAAAA,OAAO,CAACZ,GAAR,GAAcY,OAAO,CAACZ,GAAR,IAAe,EAA7B;AACA,QAAMF,IAAI,GAAGN,aAAa,CAACmB,IAAD,EAAOC,OAAP,CAA1B;;AACA,UAAQA,OAAO,CAACZ,GAAR,CAAYC,MAApB;AACE,SAAK,QAAL;AACE,aAAOV,eAAe,CAACO,IAAD,CAAtB;;AACF;AACE,aAAOA,IAAP;AAJJ;AAMD;;AAED,SAASkB,cAAT,CAAwBI,aAAxB,EAAuCR,OAAvC,EAAoE;AAElEA,EAAAA,OAAO,GAAG,EAAC,GAAGjB,8BAAJ;AAAoC,OAAGiB;AAAvC,GAAV;AACAA,EAAAA,OAAO,CAACd,IAAR,GAAe,EAAC,GAAGH,8BAA8B,CAACC,OAAnC;AAA4C,OAAGgB,OAAO,CAAChB;AAAvD,GAAf;AAEA,QAAMyB,eAAe,GAAG5B,kBAAkB,CAAC2B,aAAD,EAAgBR,OAAhB,CAA1C;;AAEA,UAAQA,OAAO,CAACZ,GAAR,CAAYC,MAApB;AACE,SAAK,QAAL;AACE,aAAOqB,0BAA0B,CAACD,eAAD,CAAjC;;AACF;AACE,aAAOA,eAAP;AAJJ;AAMD;;AAED,gBAAgBC,0BAAhB,CAA2CD,eAA3C,EAA4D;AAC1D,aAAW,MAAME,KAAjB,IAA0BF,eAA1B,EAA2C;AACzCE,IAAAA,KAAK,CAACC,IAAN,GAAajC,eAAe,CAACgC,KAAK,CAACC,IAAP,CAA5B;AACA,UAAMD,KAAN;AACD;AACF","sourcesContent":["import type {Loader, LoaderWithParser} from '@loaders.gl/loader-utils';\nimport type {JSONLoaderOptions} from './json-loader';\nimport {geojsonToBinary} from '@loaders.gl/gis';\nimport parseJSONSync from './lib/parse-json';\nimport parseJSONInBatches from './lib/parse-json-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\nexport type GeoJSONLoaderOptions = JSONLoaderOptions & {\n  geojson?: {\n    shape?: 'object-row-table';\n  };\n  gis?: {\n    format: 'geojson';\n  };\n};\n\nconst DEFAULT_GEOJSON_LOADER_OPTIONS = {\n  geojson: {\n    shape: 'object-row-table'\n  },\n  json: {\n    jsonpaths: ['$', '$.features']\n  },\n  gis: {\n    format: 'geojson'\n  }\n};\n\n/**\n * GeoJSON loader\n */\nexport const GeoJSONWorkerLoader: Loader = {\n  name: 'GeoJSON',\n  id: 'geojson',\n  module: 'geojson',\n  version: VERSION,\n  worker: true,\n  extensions: ['geojson'],\n  mimeTypes: ['application/geo+json'],\n  category: 'geometry',\n  text: true,\n  options: DEFAULT_GEOJSON_LOADER_OPTIONS\n};\n\nexport const GeoJSONLoader: LoaderWithParser = {\n  ...GeoJSONWorkerLoader,\n  parse,\n  parseTextSync,\n  parseInBatches\n};\n\nasync function parse(arrayBuffer, options) {\n  return parseTextSync(new TextDecoder().decode(arrayBuffer), options);\n}\n\nfunction parseTextSync(text, options) {\n  // Apps can call the parse method directly, we so apply default options here\n  options = {...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options};\n  options.json = {...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson};\n  options.gis = options.gis || {};\n  const json = parseJSONSync(text, options);\n  switch (options.gis.format) {\n    case 'binary':\n      return geojsonToBinary(json);\n    default:\n      return json;\n  }\n}\n\nfunction parseInBatches(asyncIterator, options): AsyncIterable<any> {\n  // Apps can call the parse method directly, we so apply default options here\n  options = {...DEFAULT_GEOJSON_LOADER_OPTIONS, ...options};\n  options.json = {...DEFAULT_GEOJSON_LOADER_OPTIONS.geojson, ...options.geojson};\n\n  const geojsonIterator = parseJSONInBatches(asyncIterator, options);\n\n  switch (options.gis.format) {\n    case 'binary':\n      return makeBinaryGeometryIterator(geojsonIterator);\n    default:\n      return geojsonIterator;\n  }\n}\n\nasync function* makeBinaryGeometryIterator(geojsonIterator) {\n  for await (const batch of geojsonIterator) {\n    batch.data = geojsonToBinary(batch.data);\n    yield batch;\n  }\n}\n"],"file":"geojson-loader.js"}

package/dist/index.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/index.ts"],"names":["JSONLoader","NDJSONLoader","GeoJSONLoader","_GeoJSONLoader","GeoJSONWorkerLoader","_GeoJSONWorkerLoader","default","_JSONPath","_ClarinetParser"],"mappings":"AACA,SAAQA,UAAR,QAAyB,eAAzB;AACA,SAAQC,YAAR,QAA2B,iBAA3B;AAIA,SACEC,aAAa,IAAIC,cADnB,EAEEC,mBAAmB,IAAIC,oBAFzB,QAGO,kBAHP;AAKA,SAAQC,OAAO,IAAIC,SAAnB,QAAmC,yBAAnC;AACA,SAAQD,OAAO,IAAIE,eAAnB,QAAyC,yBAAzC","sourcesContent":["export type {JSONLoaderOptions} from './json-loader';\nexport {JSONLoader} from './json-loader';\nexport {NDJSONLoader} from './ndjson-loader';\n\n// EXPERIMENTAL EXPORTS - WARNING: MAY BE REMOVED WIHTOUT NOTICE IN FUTURE RELEASES\nexport type {GeoJSONLoaderOptions as _GeoJSONLoaderOptions} from './geojson-loader';\nexport {\n  GeoJSONLoader as _GeoJSONLoader,\n  GeoJSONWorkerLoader as _GeoJSONWorkerLoader\n} from './geojson-loader';\n\nexport {default as _JSONPath} from './lib/jsonpath/jsonpath';\nexport {default as _ClarinetParser} from './lib/clarinet/clarinet';\n"],"file":"index.js"}

package/dist/json-loader.js.map

@@ -1 +0,0 @@
-{"version":3,"sources":["../src/json-loader.ts"],"names":["parseJSONSync","parseJSONInBatches","VERSION","DEFAULT_JSON_LOADER_OPTIONS","json","shape","table","jsonpaths","JSONLoader","name","id","module","version","extensions","mimeTypes","category","text","parse","parseTextSync","parseInBatches","options","arrayBuffer","TextDecoder","decode","jsonOptions","asyncIterator"],"mappings":"AAEA,OAAOA,aAAP,MAA0B,kBAA1B;AACA,OAAOC,kBAAP,MAA+B,6BAA/B;AAIA,MAAMC,OAAO,GAAG,2BAAuB,WAAvB,qBAAmD,QAAnE;AAcA,MAAMC,2BAA2B,GAAG;AAClCC,EAAAA,IAAI,EAAE;AACJC,IAAAA,KAAK,EAAE,WADH;AAEJC,IAAAA,KAAK,EAAE,KAFH;AAGJC,IAAAA,SAAS,EAAE;AAHP;AAD4B,CAApC;AASA,OAAO,MAAMC,UAA4B,GAAG;AAC1CC,EAAAA,IAAI,EAAE,MADoC;AAE1CC,EAAAA,EAAE,EAAE,MAFsC;AAG1CC,EAAAA,MAAM,EAAE,MAHkC;AAI1CC,EAAAA,OAAO,EAAEV,OAJiC;AAK1CW,EAAAA,UAAU,EAAE,CAAC,MAAD,EAAS,SAAT,CAL8B;AAM1CC,EAAAA,SAAS,EAAE,CAAC,kBAAD,CAN+B;AAoB1CC,EAAAA,QAAQ,EAAE,OApBgC;AAqB1CC,EAAAA,IAAI,EAAE,IArBoC;AAsB1CC,EAAAA,KAtB0C;AAuB1CC,EAAAA,aAvB0C;AAwB1CC,EAAAA,cAxB0C;AAyB1CC,EAAAA,OAAO,EAAEjB;AAzBiC,CAArC;;AA4BP,eAAec,KAAf,CAAqBI,WAArB,EAA+CD,OAA/C,EAA4E;AAC1E,SAAOF,aAAa,CAAC,IAAII,WAAJ,GAAkBC,MAAlB,CAAyBF,WAAzB,CAAD,EAAwCD,OAAxC,CAApB;AACD;;AAED,SAASF,aAAT,CAAuBF,IAAvB,EAAqCI,OAArC,EAAkE;AAChE,QAAMI,WAAW,GAAG,EAAC,GAAGJ,OAAJ;AAAahB,IAAAA,IAAI,EAAE,EAAC,GAAGD,2BAA2B,CAACC,IAAhC;AAAsC,UAAGgB,OAAH,aAAGA,OAAH,uBAAGA,OAAO,CAAEhB,IAAZ;AAAtC;AAAnB,GAApB;AACA,SAAOJ,aAAa,CAACgB,IAAD,EAAOQ,WAAP,CAApB;AACD;;AAED,SAASL,cAAT,CACEM,aADF,EAEEL,OAFF,EAGwB;AACtB,QAAMI,WAAW,GAAG,EAAC,GAAGJ,OAAJ;AAAahB,IAAAA,IAAI,EAAE,EAAC,GAAGD,2BAA2B,CAACC,IAAhC;AAAsC,UAAGgB,OAAH,aAAGA,OAAH,uBAAGA,OAAO,CAAEhB,IAAZ;AAAtC;AAAnB,GAApB;AACA,SAAOH,kBAAkB,CAACwB,aAAD,EAAgBD,WAAhB,CAAzB;AACD","sourcesContent":["import type {Batch} from '@loaders.gl/schema';\nimport type {LoaderWithParser, LoaderOptions} from '@loaders.gl/loader-utils';\nimport parseJSONSync from './lib/parse-json';\nimport parseJSONInBatches from './lib/parse-json-in-batches';\n\n// __VERSION__ is injected by babel-plugin-version-inline\n// @ts-ignore TS2304: Cannot find name '__VERSION__'.\nconst VERSION = typeof __VERSION__ !== 'undefined' ? __VERSION__ : 'latest';\n\n/**\n * @param table -\n * @param jsonpaths -\n */\nexport type JSONLoaderOptions = LoaderOptions & {\n  json?: {\n    shape?: 'row-table';\n    table?: false;\n    jsonpaths?: string[];\n  };\n};\n\nconst DEFAULT_JSON_LOADER_OPTIONS = {\n  json: {\n    shape: 'row-table',\n    table: false,\n    jsonpaths: []\n    // batchSize: 'auto'\n  }\n};\n\nexport const JSONLoader: LoaderWithParser = {\n  name: 'JSON',\n  id: 'json',\n  module: 'json',\n  version: VERSION,\n  extensions: ['json', 'geojson'],\n  mimeTypes: ['application/json'],\n  // TODO - support various line based JSON formats\n  /*\n  extensions: {\n    json: null,\n    jsonl: {stream: true},\n    ndjson: {stream: true}\n  },\n  mimeTypes: {\n    'application/json': null,\n    'application/json-seq': {stream: true},\n    'application/x-ndjson': {stream: true}\n  },\n  */\n  category: 'table',\n  text: true,\n  parse,\n  parseTextSync,\n  parseInBatches,\n  options: DEFAULT_JSON_LOADER_OPTIONS\n};\n\nasync function parse(arrayBuffer: ArrayBuffer, options?: JSONLoaderOptions) {\n  return parseTextSync(new TextDecoder().decode(arrayBuffer), options);\n}\n\nfunction parseTextSync(text: string, options?: JSONLoaderOptions) {\n  const jsonOptions = {...options, json: {...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json}};\n  return parseJSONSync(text, jsonOptions as JSONLoaderOptions);\n}\n\nfunction parseInBatches(\n  asyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,\n  options?: JSONLoaderOptions\n): AsyncIterable<Batch> {\n  const jsonOptions = {...options, json: {...DEFAULT_JSON_LOADER_OPTIONS.json, ...options?.json}};\n  return parseJSONInBatches(asyncIterator, jsonOptions as JSONLoaderOptions);\n}\n"],"file":"json-loader.js"}