@loaders.gl/json 3.1.0-alpha.3 → 4.0.0-alpha.4

package/src/lib/clarinet/clarinet.ts

@@ -0,0 +1,539 @@
+// loaders.gl, MIT license
+// This is a fork of the clarinet library, originally BSD license (see LICENSE file)
+// loaders.gl changes:
+// - typescript port
+
+export type ClarinetEvent =
+  | 'onvalue'
+  | 'onstring'
+  | 'onkey'
+  | 'onopenobject'
+  | 'oncloseobject'
+  | 'onopenarray'
+  | 'onclosearray'
+  | 'onerror'
+  | 'onend'
+  | 'onready';
+
+// Removes the MAX_BUFFER_LENGTH, originally set to 64 * 1024
+const MAX_BUFFER_LENGTH = Number.MAX_SAFE_INTEGER;
+// const DEBUG = false;
+
+enum STATE {
+  BEGIN = 0,
+  VALUE, // general stuff
+  OPEN_OBJECT, // {
+  CLOSE_OBJECT, // }
+  OPEN_ARRAY, // [
+  CLOSE_ARRAY, // ]
+  TEXT_ESCAPE, // \ stuff
+  STRING, // ""
+  BACKSLASH,
+  END, // No more stack
+  OPEN_KEY, // , "a"
+  CLOSE_KEY, // :
+  TRUE, // r
+  TRUE2, // u
+  TRUE3, // e
+  FALSE, // a
+  FALSE2, // l
+  FALSE3, // s
+  FALSE4, // e
+  NULL, // u
+  NULL2, // l
+  NULL3, // l
+  NUMBER_DECIMAL_POINT, // .
+  NUMBER_DIGIT // [0-9]
+}
+
+const Char = {
+  tab: 0x09, // \t
+  lineFeed: 0x0a, // \n
+  carriageReturn: 0x0d, // \r
+  space: 0x20, // " "
+
+  doubleQuote: 0x22, // "
+  plus: 0x2b, // +
+  comma: 0x2c, // ,
+  minus: 0x2d, // -
+  period: 0x2e, // .
+
+  _0: 0x30, // 0
+  _9: 0x39, // 9
+
+  colon: 0x3a, // :
+
+  E: 0x45, // E
+
+  openBracket: 0x5b, // [
+  backslash: 0x5c, // \
+  closeBracket: 0x5d, // ]
+
+  a: 0x61, // a
+  b: 0x62, // b
+  e: 0x65, // e
+  f: 0x66, // f
+  l: 0x6c, // l
+  n: 0x6e, // n
+  r: 0x72, // r
+  s: 0x73, // s
+  t: 0x74, // t
+  u: 0x75, // u
+
+  openBrace: 0x7b, // {
+  closeBrace: 0x7d // }
+};
+
+const stringTokenPattern = /[\\"\n]/g;
+
+type ParserEvent = (parser: ClarinetParser, event: string, data?: any) => void;
+
+export type ClarinetParserOptions = {
+  onready?: ParserEvent;
+  onopenobject?: ParserEvent;
+  onkey?: ParserEvent;
+  oncloseobject?: ParserEvent;
+  onopenarray?: ParserEvent;
+  onclosearray?: ParserEvent;
+  onvalue?: ParserEvent;
+  onerror?: ParserEvent;
+  onend?: ParserEvent;
+  onchunkparsed?: ParserEvent;
+};
+
+const DEFAULT_OPTIONS: Required<ClarinetParserOptions> = {
+  onready: () => {},
+  onopenobject: () => {},
+  onkey: () => {},
+  oncloseobject: () => {},
+  onopenarray: () => {},
+  onclosearray: () => {},
+  onvalue: () => {},
+  onerror: () => {},
+  onend: () => {},
+  onchunkparsed: () => {}
+};
+export default class ClarinetParser {
+  protected options: Required<ClarinetParserOptions> = DEFAULT_OPTIONS;
+
+  bufferCheckPosition = MAX_BUFFER_LENGTH;
+  q = '';
+  c = '';
+  p = '';
+  closed = false;
+  closedRoot = false;
+  sawRoot = false;
+  // tag = null;
+  error: Error | null = null;
+  state = STATE.BEGIN;
+  stack: STATE[] = [];
+  // mostly just for error reporting
+  position: number = 0;
+  column: number = 0;
+  line: number = 1;
+  slashed: boolean = false;
+  unicodeI: number = 0;
+  unicodeS: string | null = null;
+  depth: number = 0;
+
+  textNode;
+  numberNode;
+
+  constructor(options: ClarinetParserOptions = {}) {
+    this.options = {...DEFAULT_OPTIONS, ...options};
+    this.textNode = undefined;
+    this.numberNode = '';
+    this.emit('onready');
+  }
+
+  end() {
+    if (this.state !== STATE.VALUE || this.depth !== 0) this._error('Unexpected end');
+
+    this._closeValue();
+    this.c = '';
+    this.closed = true;
+    this.emit('onend');
+    return this;
+  }
+
+  resume() {
+    this.error = null;
+    return this;
+  }
+
+  close() {
+    return this.write(null);
+  }
+
+  // protected
+
+  emit(event: string, data?: any): void {
+    // if (DEBUG) console.log('-- emit', event, data);
+    this.options[event]?.(data, this);
+  }
+
+  emitNode(event: string, data?: any): void {
+    this._closeValue();
+    this.emit(event, data);
+  }
+
+  /* eslint-disable no-continue */
+  // eslint-disable-next-line complexity, max-statements
+  write(chunk) {
+    if (this.error) {
+      throw this.error;
+    }
+    if (this.closed) {
+      return this._error('Cannot write after close. Assign an onready handler.');
+    }
+    if (chunk === null) {
+      return this.end();
+    }
+    let i = 0;
+    let c = chunk.charCodeAt(0);
+    let p = this.p;
+    // if (DEBUG) console.log(`write -> [${  chunk  }]`);
+    while (c) {
+      p = c;
+      this.c = c = chunk.charCodeAt(i++);
+      // if chunk doesnt have next, like streaming char by char
+      // this way we need to check if previous is really previous
+      // if not we need to reset to what the this says is the previous
+      // from buffer
+      if (p !== c) {
+        this.p = p;
+      } else {
+        p = this.p;
+      }
+
+      if (!c) break;
+
+      // if (DEBUG) console.log(i, c, STATE[this.state]);
+      this.position++;
+      if (c === Char.lineFeed) {
+        this.line++;
+        this.column = 0;
+      } else this.column++;
+
+      switch (this.state) {
+        case STATE.BEGIN:
+          if (c === Char.openBrace) this.state = STATE.OPEN_OBJECT;
+          else if (c === Char.openBracket) this.state = STATE.OPEN_ARRAY;
+          else if (!isWhitespace(c)) {
+            this._error('Non-whitespace before {[.');
+          }
+          continue;
+
+        case STATE.OPEN_KEY:
+        case STATE.OPEN_OBJECT:
+          if (isWhitespace(c)) continue;
+          if (this.state === STATE.OPEN_KEY) this.stack.push(STATE.CLOSE_KEY);
+          else if (c === Char.closeBrace) {
+            this.emit('onopenobject');
+            this.depth++;
+            this.emit('oncloseobject');
+            this.depth--;
+            this.state = this.stack.pop() || STATE.VALUE;
+            continue;
+          } else this.stack.push(STATE.CLOSE_OBJECT);
+          if (c === Char.doubleQuote) this.state = STATE.STRING;
+          else this._error('Malformed object key should start with "');
+          continue;
+
+        case STATE.CLOSE_KEY:
+        case STATE.CLOSE_OBJECT:
+          if (isWhitespace(c)) continue;
+          // let event = this.state === STATE.CLOSE_KEY ? 'key' : 'object';
+          if (c === Char.colon) {
+            if (this.state === STATE.CLOSE_OBJECT) {
+              this.stack.push(STATE.CLOSE_OBJECT);
+              this._closeValue('onopenobject');
+              this.depth++;
+            } else this._closeValue('onkey');
+            this.state = STATE.VALUE;
+          } else if (c === Char.closeBrace) {
+            this.emitNode('oncloseobject');
+            this.depth--;
+            this.state = this.stack.pop() || STATE.VALUE;
+          } else if (c === Char.comma) {
+            if (this.state === STATE.CLOSE_OBJECT) this.stack.push(STATE.CLOSE_OBJECT);
+            this._closeValue();
+            this.state = STATE.OPEN_KEY;
+          } else this._error('Bad object');
+          continue;
+
+        case STATE.OPEN_ARRAY: // after an array there always a value
+        case STATE.VALUE:
+          if (isWhitespace(c)) continue;
+          if (this.state === STATE.OPEN_ARRAY) {
+            this.emit('onopenarray');
+            this.depth++;
+            this.state = STATE.VALUE;
+            if (c === Char.closeBracket) {
+              this.emit('onclosearray');
+              this.depth--;
+              this.state = this.stack.pop() || STATE.VALUE;
+              continue;
+            } else {
+              this.stack.push(STATE.CLOSE_ARRAY);
+            }
+          }
+          if (c === Char.doubleQuote) this.state = STATE.STRING;
+          else if (c === Char.openBrace) this.state = STATE.OPEN_OBJECT;
+          else if (c === Char.openBracket) this.state = STATE.OPEN_ARRAY;
+          else if (c === Char.t) this.state = STATE.TRUE;
+          else if (c === Char.f) this.state = STATE.FALSE;
+          else if (c === Char.n) this.state = STATE.NULL;
+          else if (c === Char.minus) {
+            // keep and continue
+            this.numberNode += '-';
+          } else if (Char._0 <= c && c <= Char._9) {
+            this.numberNode += String.fromCharCode(c);
+            this.state = STATE.NUMBER_DIGIT;
+          } else this._error('Bad value');
+          continue;
+
+        case STATE.CLOSE_ARRAY:
+          if (c === Char.comma) {
+            this.stack.push(STATE.CLOSE_ARRAY);
+            this._closeValue('onvalue');
+            this.state = STATE.VALUE;
+          } else if (c === Char.closeBracket) {
+            this.emitNode('onclosearray');
+            this.depth--;
+            this.state = this.stack.pop() || STATE.VALUE;
+          } else if (isWhitespace(c)) continue;
+          else this._error('Bad array');
+          continue;
+
+        case STATE.STRING:
+          if (this.textNode === undefined) {
+            this.textNode = '';
+          }
+
+          // thanks thejh, this is an about 50% performance improvement.
+          let starti = i - 1;
+          let slashed = this.slashed;
+          let unicodeI = this.unicodeI;
+          // eslint-disable-next-line no-constant-condition, no-labels
+          STRING_BIGLOOP: while (true) {
+            // if (DEBUG) console.log(i, c, STATE[this.state], slashed);
+            // zero means "no unicode active". 1-4 mean "parse some more". end after 4.
+            while (unicodeI > 0) {
+              this.unicodeS += String.fromCharCode(c);
+              c = chunk.charCodeAt(i++);
+              this.position++;
+              if (unicodeI === 4) {
+                // TODO this might be slow? well, probably not used too often anyway
+                this.textNode += String.fromCharCode(parseInt(this.unicodeS as string, 16));
+                unicodeI = 0;
+                starti = i - 1;
+              } else {
+                unicodeI++;
+              }
+              // we can just break here: no stuff we skipped that still has to be sliced out or so
+              // eslint-disable-next-line no-labels
+              if (!c) break STRING_BIGLOOP;
+            }
+            if (c === Char.doubleQuote && !slashed) {
+              this.state = this.stack.pop() || STATE.VALUE;
+              this.textNode += chunk.substring(starti, i - 1);
+              this.position += i - 1 - starti;
+              break;
+            }
+            if (c === Char.backslash && !slashed) {
+              slashed = true;
+              this.textNode += chunk.substring(starti, i - 1);
+              this.position += i - 1 - starti;
+              c = chunk.charCodeAt(i++);
+              this.position++;
+              if (!c) break;
+            }
+            if (slashed) {
+              slashed = false;
+              if (c === Char.n) {
+                this.textNode += '\n';
+              } else if (c === Char.r) {
+                this.textNode += '\r';
+              } else if (c === Char.t) {
+                this.textNode += '\t';
+              } else if (c === Char.f) {
+                this.textNode += '\f';
+              } else if (c === Char.b) {
+                this.textNode += '\b';
+              } else if (c === Char.u) {
+                // \uxxxx. meh!
+                unicodeI = 1;
+                this.unicodeS = '';
+              } else {
+                this.textNode += String.fromCharCode(c);
+              }
+              c = chunk.charCodeAt(i++);
+              this.position++;
+              starti = i - 1;
+              if (!c) break;
+              else continue;
+            }
+
+            stringTokenPattern.lastIndex = i;
+            const reResult = stringTokenPattern.exec(chunk);
+            if (reResult === null) {
+              i = chunk.length + 1;
+              this.textNode += chunk.substring(starti, i - 1);
+              this.position += i - 1 - starti;
+              break;
+            }
+            i = reResult.index + 1;
+            c = chunk.charCodeAt(reResult.index);
+            if (!c) {
+              this.textNode += chunk.substring(starti, i - 1);
+              this.position += i - 1 - starti;
+              break;
+            }
+          }
+          this.slashed = slashed;
+          this.unicodeI = unicodeI;
+          continue;
+
+        case STATE.TRUE:
+          if (c === Char.r) this.state = STATE.TRUE2;
+          else this._error(`Invalid true started with t${c}`);
+          continue;
+
+        case STATE.TRUE2:
+          if (c === Char.u) this.state = STATE.TRUE3;
+          else this._error(`Invalid true started with tr${c}`);
+          continue;
+
+        case STATE.TRUE3:
+          if (c === Char.e) {
+            this.emit('onvalue', true);
+            this.state = this.stack.pop() || STATE.VALUE;
+          } else this._error(`Invalid true started with tru${c}`);
+          continue;
+
+        case STATE.FALSE:
+          if (c === Char.a) this.state = STATE.FALSE2;
+          else this._error(`Invalid false started with f${c}`);
+          continue;
+
+        case STATE.FALSE2:
+          if (c === Char.l) this.state = STATE.FALSE3;
+          else this._error(`Invalid false started with fa${c}`);
+          continue;
+
+        case STATE.FALSE3:
+          if (c === Char.s) this.state = STATE.FALSE4;
+          else this._error(`Invalid false started with fal${c}`);
+          continue;
+
+        case STATE.FALSE4:
+          if (c === Char.e) {
+            this.emit('onvalue', false);
+            this.state = this.stack.pop() || STATE.VALUE;
+          } else this._error(`Invalid false started with fals${c}`);
+          continue;
+
+        case STATE.NULL:
+          if (c === Char.u) this.state = STATE.NULL2;
+          else this._error(`Invalid null started with n${c}`);
+          continue;
+
+        case STATE.NULL2:
+          if (c === Char.l) this.state = STATE.NULL3;
+          else this._error(`Invalid null started with nu${c}`);
+          continue;
+
+        case STATE.NULL3:
+          if (c === Char.l) {
+            this.emit('onvalue', null);
+            this.state = this.stack.pop() || STATE.VALUE;
+          } else this._error(`Invalid null started with nul${c}`);
+          continue;
+
+        case STATE.NUMBER_DECIMAL_POINT:
+          if (c === Char.period) {
+            this.numberNode += '.';
+            this.state = STATE.NUMBER_DIGIT;
+          } else this._error('Leading zero not followed by .');
+          continue;
+
+        case STATE.NUMBER_DIGIT:
+          if (Char._0 <= c && c <= Char._9) this.numberNode += String.fromCharCode(c);
+          else if (c === Char.period) {
+            if (this.numberNode.indexOf('.') !== -1) this._error('Invalid number has two dots');
+            this.numberNode += '.';
+          } else if (c === Char.e || c === Char.E) {
+            if (this.numberNode.indexOf('e') !== -1 || this.numberNode.indexOf('E') !== -1)
+              this._error('Invalid number has two exponential');
+            this.numberNode += 'e';
+          } else if (c === Char.plus || c === Char.minus) {
+            // @ts-expect-error
+            if (!(p === Char.e || p === Char.E)) this._error('Invalid symbol in number');
+            this.numberNode += String.fromCharCode(c);
+          } else {
+            this._closeNumber();
+            i--; // go back one
+            this.state = this.stack.pop() || STATE.VALUE;
+          }
+          continue;
+
+        default:
+          this._error(`Unknown state: ${this.state}`);
+      }
+    }
+    if (this.position >= this.bufferCheckPosition) {
+      checkBufferLength(this);
+    }
+
+    this.emit('onchunkparsed');
+
+    return this;
+  }
+
+  _closeValue(event: string = 'onvalue'): void {
+    if (this.textNode !== undefined) {
+      this.emit(event, this.textNode);
+    }
+    this.textNode = undefined;
+  }
+
+  _closeNumber(): void {
+    if (this.numberNode) this.emit('onvalue', parseFloat(this.numberNode));
+    this.numberNode = '';
+  }
+
+  _error(message: string = ''): void {
+    this._closeValue();
+    message += `\nLine: ${this.line}\nColumn: ${this.column}\nChar: ${this.c}`;
+    const error = new Error(message);
+    this.error = error;
+    this.emit('onerror', error);
+  }
+}
+
+function isWhitespace(c): boolean {
+  return c === Char.carriageReturn || c === Char.lineFeed || c === Char.space || c === Char.tab;
+}
+
+function checkBufferLength(parser) {
+  const maxAllowed = Math.max(MAX_BUFFER_LENGTH, 10);
+  let maxActual = 0;
+
+  for (const buffer of ['textNode', 'numberNode']) {
+    const len = parser[buffer] === undefined ? 0 : parser[buffer].length;
+    if (len > maxAllowed) {
+      switch (buffer) {
+        case 'text':
+          // TODO - should this be closeValue?
+          // closeText(parser);
+          break;
+
+        default:
+          parser._error(`Max buffer length exceeded: ${buffer}`);
+      }
+    }
+    maxActual = Math.max(maxActual, len);
+  }
+  parser.bufferCheckPosition = MAX_BUFFER_LENGTH - maxActual + parser.position;
+}

package/src/lib/parser/json-parser.ts

@@ -1,17 +1,64 @@
 // @ts-nocheck
 
-import ClarinetParser from '../clarinet/clarinet';
+import ClarinetParser, {ClarinetParserOptions} from '../clarinet/clarinet';
 import JSONPath from '../jsonpath/jsonpath';
 
 // JSONParser builds a JSON object using the events emitted by the Clarinet parser
 
 export default class JSONParser {
-  jsonpath: JSONPath;
-  _parser?: ClarinetParser;
+  readonly parser: ClarinetParser;
+  result = undefined;
+  previousStates = [];
+  currentState = Object.freeze({container: [], key: null});
+  jsonpath: JSONPath = new JSONPath();
 
-  constructor() {
+  constructor(options: ClarinetParserOptions) {
     this.reset();
-    this._initializeParser();
+    this.parser = new ClarinetParser({
+      onready: () => {
+        this.jsonpath = new JSONPath();
+        this.previousStates.length = 0;
+        this.currentState.container.length = 0;
+      },
+
+      onopenobject: (name) => {
+        this._openObject({});
+        if (typeof name !== 'undefined') {
+          this.parser.emit('onkey', name);
+        }
+      },
+
+      onkey: (name) => {
+        this.jsonpath.set(name);
+        this.currentState.key = name;
+      },
+
+      oncloseobject: () => {
+        this._closeObject();
+      },
+
+      onopenarray: () => {
+        this._openArray();
+      },
+
+      onclosearray: () => {
+        this._closeArray();
+      },
+
+      onvalue: (value) => {
+        this._pushOrSet(value);
+      },
+
+      onerror: (error) => {
+        throw error;
+      },
+
+      onend: () => {
+        this.result = this.currentState.container.pop();
+      },
+
+      ...options
+    });
   }
 
   reset(): void {
@@ -64,54 +111,4 @@ export default class JSONParser {
     this.jsonpath.pop();
     this.currentState = this.previousStates.pop();
   }
-
-  _initializeParser(): void {
-    this._parser = new ClarinetParser({
-      onready: () => {
-        this.jsonpath = new JSONPath();
-        this.previousStates.length = 0;
-        this.currentState.container.length = 0;
-      },
-
-      onopenobject: (name) => {
-        this._openObject({});
-        if (typeof name !== 'undefined') {
-          this.parser.onkey(name);
-        }
-      },
-
-      onkey: (name) => {
-        this.jsonpath.set(name);
-        this.currentState.key = name;
-      },
-
-      oncloseobject: () => {
-        this._closeObject();
-      },
-
-      onopenarray: () => {
-        this._openArray();
-      },
-
-      onclosearray: () => {
-        this._closeArray();
-      },
-
-      onvalue: (value) => {
-        this._pushOrSet(value);
-      },
-
-      onerror: (error) => {
-        throw error;
-      },
-
-      onend: () => {
-        this.result = this.currentState.container.pop();
-      }
-    });
-  }
-
-  protected get parser(): ClarinetParser {
-    return this._parser as ClarinetParser;
-  }
 }