grammatik 0.1.1 → 0.1.2

package/src/lexer.d.ts

@@ -5,19 +5,60 @@
  * large win for punctuation-heavy grammars (JSON is ~60% punctuation by token count).
  *
  * @param {TokenDef[]} defs
- * @param {{ positions?: 'full' | 'offset' }} [options]
+ * @param {{ positions?: 'full' | 'offset', scanners?: boolean, fuse?: boolean, incremental?: boolean, recover?: boolean, errorToken?: string }} [options]
  *   `full` (default) records line and column on every token. `offset` records
  *   only `start`/`end`, which skips a scan of the entire input for newlines —
  *   worth taking when errors do not need to name a line, or when positions are
  *   resolved lazily from the offset afterwards.
- * @returns {{ tokenize: (text: string) => Token[], tokenNames: string[], positions: string }}
+ *
+ *   `incremental` (default false) is opt-in: when set, the returned lexer also
+ *   exposes `tokenizeIncremental` and `relex`, for editors that want to re-lex
+ *   only the region touched by a single edit instead of the whole document.
+ *   A lexer created without it is bit-for-bit the plain lexer — no overhead.
+ *
+ *   `scanners` (default true) runs patterns of a few common shapes as
+ *   table-driven loops instead of regex calls. The tokens are identical;
+ *   `false` forces every pattern through its regex.
+ *
+ *   `fuse` (default true) joins runs of three or more consecutive regex candidates
+ *   into one alternation, so a position where several tokens could start costs one
+ *   regex call rather than one per rejected token. Same tokens; `false` tries them
+ *   one at a time.
+ *
+ *   `recover` (default false, only meaningful with `incremental`) makes an
+ *   unmatched character produce a single-character token of type `errorToken`
+ *   (default `'Error'`) instead of throwing, so highlighting survives invalid
+ *   or half-typed input.
+ * @returns {{ tokenize: (text: string) => Token[], tokenizeIncremental?: (text: string) => IncrementalResult, relex?: (text: string, prev: IncrementalResult, edit: { start: number, oldEnd: number, newEnd: number }) => IncrementalResult, tokenNames: string[], positions: string, modeNames: string[] }}
  */
 export function createLexer(defs: TokenDef[], options?: {
     positions?: "full" | "offset";
+    scanners?: boolean;
+    fuse?: boolean;
+    incremental?: boolean;
+    recover?: boolean;
+    errorToken?: string;
 }): {
     tokenize: (text: string) => Token[];
+    tokenizeIncremental?: (text: string) => IncrementalResult;
+    relex?: (text: string, prev: IncrementalResult, edit: {
+        start: number;
+        oldEnd: number;
+        newEnd: number;
+    }) => IncrementalResult;
     tokenNames: string[];
     positions: string;
+    modeNames: string[];
+};
+/**
+ * Opaque state returned by `tokenizeIncremental`/`relex` — pass it back into
+ * `relex` unmodified; only `.tokens` is meant to be read directly.
+ */
+export type IncrementalResult = {
+    text: string;
+    tokens: Token[];
+    checkpoints: object[];
+    eof: object;
 };
 /**
  * A Chevrotain-style lexer. Token definitions are plain JSON, so a whole