@loaders.gl/json 4.4.0-alpha.1 → 4.4.0-alpha.9

package/src/lib/jsonpath/jsonpath.ts

@@ -2,6 +2,8 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
+/* eslint-disable no-continue */
+
 /**
  * A parser for a minimal subset of the jsonpath standard
  * Full JSON path parsers for JS exist but are quite large (bundle size)
@@ -14,26 +16,7 @@ export default class JSONPath {
   path: string[];
 
   constructor(path: JSONPath | string[] | string | null = null) {
-    this.path = ['$'];
-
-    if (path instanceof JSONPath) {
-      // @ts-ignore
-      this.path = [...path.path];
-      return;
-    }
-
-    if (Array.isArray(path)) {
-      this.path.push(...path);
-      return;
-    }
-
-    // Parse a string as a JSONPath
-    if (typeof path === 'string') {
-      this.path = path.split('.');
-      if (this.path[0] !== '$') {
-        throw new Error('JSONPaths must start with $');
-      }
-    }
+    this.path = parseJsonPath(path);
   }
 
   clone(): JSONPath {
@@ -41,7 +24,7 @@ export default class JSONPath {
   }
 
   toString(): string {
-    return this.path.join('.');
+    return formatJsonPath(this.path);
   }
 
   push(name: string): void {
@@ -103,3 +86,267 @@ export default class JSONPath {
     return object[field];
   }
 }
+
+type BracketSegment =
+  | {type: 'property'; value: string; nextIndex: number}
+  | {type: 'array-selector'; nextIndex: number};
+
+function parseJsonPath(path: JSONPath | string[] | string | null): string[] {
+  if (path instanceof JSONPath) {
+    return [...path.path];
+  }
+
+  if (Array.isArray(path)) {
+    return ['$'].concat(path);
+  }
+
+  if (typeof path === 'string') {
+    return parseJsonPathString(path);
+  }
+
+  return ['$'];
+}
+
+// eslint-disable-next-line complexity, max-statements
+function parseJsonPathString(pathString: string): string[] {
+  const trimmedPath = pathString.trim();
+  if (!trimmedPath.startsWith('$')) {
+    throw new Error('JSONPath must start with $');
+  }
+
+  const segments: string[] = ['$'];
+  let index = 1;
+  let arrayElementSelectorEncountered = false;
+
+  while (index < trimmedPath.length) {
+    const character = trimmedPath[index];
+    if (character === '.') {
+      if (arrayElementSelectorEncountered) {
+        throw new Error('JSONPath cannot select fields after array element selectors');
+      }
+      index += 1;
+      if (trimmedPath[index] === '.') {
+        throw new Error('JSONPath descendant selectors (..) are not supported');
+      }
+      const {value, nextIndex, isWildcard} = parseDotSegment(trimmedPath, index);
+      if (isWildcard) {
+        if (nextIndex < trimmedPath.length) {
+          throw new Error('JSONPath wildcard selectors must terminate the path');
+        }
+        arrayElementSelectorEncountered = true;
+        index = nextIndex;
+        continue;
+      }
+      segments.push(value);
+      index = nextIndex;
+      continue;
+    }
+
+    if (character === '[') {
+      const parsedSegment = parseBracketSegment(trimmedPath, index);
+      if (parsedSegment.type === 'property') {
+        if (arrayElementSelectorEncountered) {
+          throw new Error('JSONPath cannot select fields after array element selectors');
+        }
+        segments.push(parsedSegment.value);
+      } else {
+        arrayElementSelectorEncountered = true;
+      }
+      index = parsedSegment.nextIndex;
+      continue;
+    }
+
+    if (character === '@') {
+      throw new Error('JSONPath current node selector (@) is not supported');
+    }
+
+    if (character.trim() === '') {
+      index += 1;
+      continue;
+    }
+
+    throw new Error(`Unexpected character "${character}" in JSONPath`);
+  }
+
+  return segments;
+}
+
+function parseDotSegment(
+  pathString: string,
+  startIndex: number
+): {
+  value: string;
+  nextIndex: number;
+  isWildcard: boolean;
+} {
+  if (startIndex >= pathString.length) {
+    throw new Error('JSONPath cannot end with a period');
+  }
+
+  if (pathString[startIndex] === '*') {
+    return {value: '*', nextIndex: startIndex + 1, isWildcard: true};
+  }
+
+  const firstCharacter = pathString[startIndex];
+  if (firstCharacter === '@') {
+    throw new Error('JSONPath current node selector (@) is not supported');
+  }
+  if (!isIdentifierStartCharacter(firstCharacter)) {
+    throw new Error('JSONPath property names after period must start with a letter, $ or _');
+  }
+
+  let endIndex = startIndex + 1;
+  while (endIndex < pathString.length && isIdentifierCharacter(pathString[endIndex])) {
+    endIndex++;
+  }
+
+  if (endIndex === startIndex) {
+    throw new Error('JSONPath is missing a property name after period');
+  }
+
+  return {
+    value: pathString.slice(startIndex, endIndex),
+    nextIndex: endIndex,
+    isWildcard: false
+  };
+}
+
+function parseBracketSegment(pathString: string, startIndex: number): BracketSegment {
+  const contentStartIndex = startIndex + 1;
+  if (contentStartIndex >= pathString.length) {
+    throw new Error('JSONPath has unterminated bracket');
+  }
+
+  const firstCharacter = pathString[contentStartIndex];
+  if (firstCharacter === "'" || firstCharacter === '"') {
+    const {value, nextIndex} = parseBracketProperty(pathString, contentStartIndex);
+    return {type: 'property', value, nextIndex};
+  }
+
+  const closingBracketIndex = pathString.indexOf(']', contentStartIndex);
+  if (closingBracketIndex === -1) {
+    throw new Error('JSONPath has unterminated bracket');
+  }
+
+  const content = pathString.slice(contentStartIndex, closingBracketIndex).trim();
+  const unsupportedSelectorMessage = getUnsupportedBracketSelectorMessage(content);
+  if (unsupportedSelectorMessage) {
+    throw new Error(unsupportedSelectorMessage);
+  }
+  if (content === '*') {
+    return {type: 'array-selector', nextIndex: closingBracketIndex + 1};
+  }
+
+  if (/^\d+$/.test(content)) {
+    throw new Error('JSONPath array index selectors are not supported');
+  }
+
+  if (/^\d*\s*:\s*\d*(\s*:\s*\d*)?$/.test(content)) {
+    return {type: 'array-selector', nextIndex: closingBracketIndex + 1};
+  }
+
+  throw new Error(`Unsupported bracket selector "[${content}]" in JSONPath`);
+}
+
+function getUnsupportedBracketSelectorMessage(content: string): string | null {
+  if (!content.length) {
+    return 'JSONPath bracket selectors cannot be empty';
+  }
+  if (content.startsWith('(')) {
+    return 'JSONPath script selectors are not supported';
+  }
+  if (content.startsWith('?')) {
+    return 'JSONPath filter selectors are not supported';
+  }
+  if (content.includes(',')) {
+    return 'JSONPath union selectors are not supported';
+  }
+  if (content.startsWith('@') || content.includes('@.')) {
+    return 'JSONPath current node selector (@) is not supported';
+  }
+  return null;
+}
+
+// eslint-disable-next-line complexity, max-statements
+function parseBracketProperty(
+  pathString: string,
+  startIndex: number
+): {
+  value: string;
+  nextIndex: number;
+} {
+  const quoteCharacter = pathString[startIndex];
+  let index = startIndex + 1;
+  let value = '';
+  let terminated = false;
+
+  while (index < pathString.length) {
+    const character = pathString[index];
+    if (character === '\\') {
+      index += 1;
+      if (index >= pathString.length) {
+        break;
+      }
+      value += pathString[index];
+      index += 1;
+      continue;
+    }
+
+    if (character === quoteCharacter) {
+      terminated = true;
+      index += 1;
+      break;
+    }
+
+    value += character;
+    index += 1;
+  }
+
+  if (!terminated) {
+    throw new Error('JSONPath string in bracket property selector is unterminated');
+  }
+
+  while (index < pathString.length && pathString[index].trim() === '') {
+    index += 1;
+  }
+
+  if (pathString[index] !== ']') {
+    throw new Error('JSONPath property selectors must end with ]');
+  }
+
+  if (!value.length) {
+    throw new Error('JSONPath property selectors cannot be empty');
+  }
+
+  return {value, nextIndex: index + 1};
+}
+
+function isIdentifierCharacter(character: string): boolean {
+  return /[a-zA-Z0-9$_]/.test(character);
+}
+
+function isIdentifierStartCharacter(character: string): boolean {
+  return /[a-zA-Z_$]/.test(character);
+}
+
+function isIdentifierSegment(segment: string): boolean {
+  return /^[a-zA-Z_$][a-zA-Z0-9_$]*$/.test(segment);
+}
+
+function formatJsonPath(path: string[]): string {
+  return path
+    .map((segment, index) => {
+      if (index === 0) {
+        return segment;
+      }
+      if (segment === '*') {
+        return '.*';
+      }
+      if (isIdentifierSegment(segment)) {
+        return `.${segment}`;
+      }
+      const escapedSegment = segment.replace(/\\/g, '\\\\').replace(/'/g, "\\'");
+      return `['${escapedSegment}']`;
+    })
+    .join('');
+}

package/src/lib/parsers/parse-json-in-batches.ts

@@ -6,26 +6,28 @@ import type {Schema, TableBatch} from '@loaders.gl/schema';
 import type {JSONLoaderOptions, MetadataBatch, JSONBatch} from '../../json-loader';
 
 import {TableBatchBuilder} from '@loaders.gl/schema-utils';
-import {assert, makeTextDecoderIterator} from '@loaders.gl/loader-utils';
+import {assert, makeTextDecoderIterator, toArrayBufferIterator} from '@loaders.gl/loader-utils';
 import StreamingJSONParser from '../json-parser/streaming-json-parser';
 import JSONPath from '../jsonpath/jsonpath';
 
 // TODO - support batch size 0 = no batching/single batch?
 // eslint-disable-next-line max-statements, complexity
 export async function* parseJSONInBatches(
-  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  binaryAsyncIterator:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>,
   options: JSONLoaderOptions
 ): AsyncIterable<TableBatch | MetadataBatch | JSONBatch> {
-  const asyncIterator = makeTextDecoderIterator(binaryAsyncIterator);
+  const asyncIterator = makeTextDecoderIterator(toArrayBufferIterator(binaryAsyncIterator));
 
-  const {metadata} = options;
+  const metadata = Boolean(options?.core?.metadata || (options as any)?.metadata);
   const {jsonpaths} = options.json || {};
 
   let isFirstChunk: boolean = true;
 
   // @ts-expect-error TODO fix Schema deduction
   const schema: Schema = null;
-  const tableBatchBuilder = new TableBatchBuilder(schema, options);
+  const tableBatchBuilder = new TableBatchBuilder(schema, options?.core);
 
   const parser = new StreamingJSONParser({jsonpaths});
 

package/src/lib/parsers/parse-ndjson-in-batches.ts

@@ -8,14 +8,17 @@ import {
   LoaderOptions,
   makeLineIterator,
   makeNumberedLineIterator,
-  makeTextDecoderIterator
+  makeTextDecoderIterator,
+  toArrayBufferIterator
 } from '@loaders.gl/loader-utils';
 
 export async function* parseNDJSONInBatches(
-  binaryAsyncIterator: AsyncIterable<ArrayBuffer> | Iterable<ArrayBuffer>,
+  binaryAsyncIterator:
+    | AsyncIterable<ArrayBufferLike | ArrayBufferView>
+    | Iterable<ArrayBufferLike | ArrayBufferView>,
   options?: LoaderOptions
 ): AsyncIterable<TableBatch> {
-  const textIterator = makeTextDecoderIterator(binaryAsyncIterator);
+  const textIterator = makeTextDecoderIterator(toArrayBufferIterator(binaryAsyncIterator));
   const lineIterator = makeLineIterator(textIterator);
   const numberedLineIterator = makeNumberedLineIterator(lineIterator);
 
@@ -23,7 +26,7 @@ export async function* parseNDJSONInBatches(
   const shape = 'row-table';
   // @ts-ignore
   const tableBatchBuilder = new TableBatchBuilder(schema, {
-    ...options,
+    ...(options?.core || options),
     shape
   });