@texuli/spec 0.5.0 → 0.6.0

package/dist/stream/tokenizer.js

@@ -0,0 +1,383 @@
+/**
+ * An incremental JSON tokenizer for untrusted streams (ADR-014). Text
+ * arrives in chunks split anywhere (inside a string, an escape, a number, a
+ * literal or a UTF-16 surrogate pair); the tokenizer reports each value as
+ * it closes. One pass over every code unit, no backtracking and no
+ * recursion, so the time is linear in the input and the memory is bounded
+ * by the byte budget and the depth cap. Framework-free: no DOM, no Node.
+ *
+ * It refuses, at a UTF-8 byte offset: malformed JSON, input over the byte
+ * budget, containers nested past the depth cap, duplicate keys and the
+ * prototype keys (`__proto__`, `constructor`, `prototype`). After an error
+ * it reports nothing more.
+ */
+/** Default byte budget: the catalog's default tree cap (256 KiB) plus 8 KiB of whitespace. */
+export const DEFAULT_STREAM_MAX_BYTES = 256 * 1024 + 8 * 1024;
+/** Default depth cap: the catalog's value-depth guard. */
+export const DEFAULT_STREAM_MAX_DEPTH = 128;
+/** Keys refused anywhere in a stream. */
+export const STREAM_PROTOTYPE_KEYS = Object.freeze([
+    '__proto__',
+    'constructor',
+    'prototype',
+]);
+const PROTOTYPE_KEYS = new Set(STREAM_PROTOTYPE_KEYS);
+const NUMBER = /^-?(?:0|[1-9]\d*)(?:\.\d+)?(?:[eE][+-]?\d+)?$/;
+const LITERALS = {
+    t: { text: 'true', value: true },
+    f: { text: 'false', value: false },
+    n: { text: 'null', value: null },
+};
+const ESCAPES = {
+    '"': '"',
+    '\\': '\\',
+    '/': '/',
+    b: '\b',
+    f: '\f',
+    n: '\n',
+    r: '\r',
+    t: '\t',
+};
+const isWhitespace = (code) => code === 0x20 || code === 0x0a || code === 0x0d || code === 0x09;
+export function createJsonTokenizer(options = {}) {
+    const maxBytes = options.maxBytes ?? DEFAULT_STREAM_MAX_BYTES;
+    const maxDepth = options.maxDepth ?? DEFAULT_STREAM_MAX_DEPTH;
+    const stack = [];
+    let expect = 'value';
+    let lexeme = 'none';
+    // The token being read, as chunks joined at its close (strings) or text.
+    let parts = [];
+    let tokenStart = 0;
+    let stringIsKey = false;
+    // Escape state inside a string: 0 none, 1 after `\`, 2–5 reading `\u` hex digits.
+    let escaping = 0;
+    let hex = 0;
+    let literal = null;
+    let bytes = 0;
+    // A high surrogate counted as 3 bytes; a low one right after it adds 1 (4 in all).
+    let pendingHigh = false;
+    let error = null;
+    let done = false;
+    let ended = false;
+    const fail = (code, offset, message) => {
+        error = { code, offset, message };
+    };
+    /** Bytes of one UTF-16 code unit, given the unit before it. */
+    const unitBytes = (code) => {
+        if (code < 0x80) {
+            pendingHigh = false;
+            return 1;
+        }
+        if (code < 0x800) {
+            pendingHigh = false;
+            return 2;
+        }
+        if (code >= 0xd800 && code <= 0xdbff) {
+            pendingHigh = true;
+            return 3;
+        }
+        if (code >= 0xdc00 && code <= 0xdfff && pendingHigh) {
+            pendingHigh = false;
+            return 1;
+        }
+        pendingHigh = false;
+        return 3;
+    };
+    /** A value closed: move the container state on. */
+    const closedValue = () => {
+        const top = stack[stack.length - 1];
+        expect = top ? 'comma-or-close' : 'after';
+    };
+    function emitScalar(events, value) {
+        events.push({ type: 'value', value, offset: tokenStart });
+        closedValue();
+    }
+    function finishNumber(events) {
+        const text = parts.join('');
+        parts = [];
+        lexeme = 'none';
+        if (!NUMBER.test(text)) {
+            fail('bad-json', tokenStart, `"${text.slice(0, 32)}" is not a JSON number.`);
+            return false;
+        }
+        emitScalar(events, Number(text));
+        return true;
+    }
+    function finishString(events) {
+        const text = parts.join('');
+        parts = [];
+        lexeme = 'none';
+        if (!stringIsKey) {
+            emitScalar(events, text);
+            return true;
+        }
+        if (PROTOTYPE_KEYS.has(text)) {
+            fail('prototype-key', tokenStart, `The key "${text}" is not allowed.`);
+            return false;
+        }
+        const frame = stack[stack.length - 1];
+        const keys = frame.keys;
+        if (keys.has(text)) {
+            fail('duplicate-key', tokenStart, `The key "${text.slice(0, 64)}" appears twice.`);
+            return false;
+        }
+        keys.add(text);
+        events.push({ type: 'key', key: text, offset: tokenStart });
+        expect = 'colon';
+        return true;
+    }
+    /** Start a value at `chunk[i]`; returns `false` on error. */
+    function startValue(events, char, offset) {
+        if (char === '{' || char === '[') {
+            if (stack.length >= maxDepth) {
+                fail('too-deep', offset, `Values nest deeper than ${maxDepth} levels.`);
+                return false;
+            }
+            const kind = char === '{' ? 'object' : 'array';
+            stack.push({ kind, keys: kind === 'object' ? new Set() : null });
+            events.push({ type: 'open', kind, offset });
+            expect = kind === 'object' ? 'key-or-close' : 'value-or-close';
+            return true;
+        }
+        tokenStart = offset;
+        if (char === '"') {
+            lexeme = 'string';
+            stringIsKey = false;
+            parts = [];
+            escaping = 0;
+            return true;
+        }
+        if (char === '-' || (char >= '0' && char <= '9')) {
+            lexeme = 'number';
+            parts = [char];
+            return true;
+        }
+        const found = LITERALS[char];
+        if (found) {
+            lexeme = 'literal';
+            literal = found;
+            parts = [char];
+            return true;
+        }
+        fail('bad-json', offset, `Unexpected "${char}" where a value should start.`);
+        return false;
+    }
+    function closeContainer(events, kind, offset) {
+        stack.pop();
+        events.push({ type: 'close', kind, offset });
+        closedValue();
+    }
+    function run(chunk, events) {
+        const length = chunk.length;
+        let i = 0;
+        while (i < length && error === null) {
+            const code = chunk.charCodeAt(i);
+            // ── Inside a string: copy runs of plain characters at once ──
+            if (lexeme === 'string') {
+                if (escaping === 0) {
+                    let j = i;
+                    while (j < length) {
+                        const c = chunk.charCodeAt(j);
+                        if (c === 0x22 || c === 0x5c || c < 0x20)
+                            break;
+                        bytes += unitBytes(c);
+                        j += 1;
+                    }
+                    if (bytes > maxBytes) {
+                        fail('too-large', maxBytes, `The stream is over its ${maxBytes}-byte budget.`);
+                        return;
+                    }
+                    if (j > i)
+                        parts.push(chunk.slice(i, j));
+                    i = j;
+                    if (i >= length)
+                        return;
+                    const c = chunk.charCodeAt(i);
+                    const at = bytes;
+                    bytes += unitBytes(c);
+                    i += 1;
+                    if (c === 0x22)
+                        finishString(events);
+                    else if (c === 0x5c)
+                        escaping = 1;
+                    else
+                        fail('bad-json', at, 'Control characters must be escaped inside strings.');
+                    continue;
+                }
+                const at = bytes;
+                bytes += unitBytes(code);
+                i += 1;
+                const char = chunk[i - 1];
+                if (escaping === 1) {
+                    if (char === 'u') {
+                        escaping = 2;
+                        hex = 0;
+                    }
+                    else if (Object.hasOwn(ESCAPES, char)) {
+                        parts.push(ESCAPES[char]);
+                        escaping = 0;
+                    }
+                    else {
+                        fail('bad-json', at, `"\\${char}" is not a JSON escape.`);
+                    }
+                    continue;
+                }
+                // Reading the four hex digits of `\u`.
+                if (!/^[0-9a-fA-F]$/.test(char)) {
+                    fail('bad-json', at, 'A "\\u" escape needs four hex digits.');
+                    continue;
+                }
+                hex = hex * 16 + Number.parseInt(char, 16);
+                escaping += 1;
+                if (escaping === 6) {
+                    // A pair of `\u` escapes forms a surrogate pair by concatenation, as JSON.parse does.
+                    parts.push(String.fromCharCode(hex));
+                    escaping = 0;
+                }
+                continue;
+            }
+            // ── Numbers and literals end at the first character that is not theirs ──
+            if (lexeme === 'number') {
+                const char = chunk[i];
+                if ((char >= '0' && char <= '9') ||
+                    char === '.' ||
+                    char === 'e' ||
+                    char === 'E' ||
+                    char === '+' ||
+                    char === '-') {
+                    bytes += 1;
+                    pendingHigh = false;
+                    if (bytes > maxBytes) {
+                        fail('too-large', maxBytes, `The stream is over its ${maxBytes}-byte budget.`);
+                        return;
+                    }
+                    parts.push(char);
+                    i += 1;
+                    continue;
+                }
+                if (!finishNumber(events))
+                    return;
+                // Fall through: this character is read as structure below.
+            }
+            else if (lexeme === 'literal') {
+                const current = literal;
+                const at = bytes;
+                const length0 = parts.length;
+                const char = chunk[i];
+                if (current.text[length0] !== char) {
+                    fail('bad-json', at, `Unexpected "${char}" in "${current.text}".`);
+                    return;
+                }
+                bytes += unitBytes(code);
+                parts.push(char);
+                i += 1;
+                if (parts.length === current.text.length) {
+                    parts = [];
+                    lexeme = 'none';
+                    literal = null;
+                    emitScalar(events, current.value);
+                }
+                continue;
+            }
+            // ── Structure ──
+            const at = bytes;
+            bytes += unitBytes(chunk.charCodeAt(i));
+            if (bytes > maxBytes) {
+                fail('too-large', maxBytes, `The stream is over its ${maxBytes}-byte budget.`);
+                return;
+            }
+            const char = chunk[i];
+            i += 1;
+            if (isWhitespace(chunk.charCodeAt(i - 1)))
+                continue;
+            switch (expect) {
+                case 'after':
+                    fail('bad-json', at, `Unexpected "${char}" after the end of the value.`);
+                    return;
+                case 'value':
+                    if (!startValue(events, char, at))
+                        return;
+                    break;
+                case 'value-or-close':
+                    if (char === ']')
+                        closeContainer(events, 'array', at);
+                    else if (!startValue(events, char, at))
+                        return;
+                    break;
+                case 'key':
+                case 'key-or-close':
+                    if (char === '}' && expect === 'key-or-close') {
+                        closeContainer(events, 'object', at);
+                    }
+                    else if (char === '"') {
+                        tokenStart = at;
+                        lexeme = 'string';
+                        stringIsKey = true;
+                        parts = [];
+                        escaping = 0;
+                    }
+                    else {
+                        fail('bad-json', at, `Expected a key in double quotes, got "${char}".`);
+                        return;
+                    }
+                    break;
+                case 'colon':
+                    if (char !== ':') {
+                        fail('bad-json', at, `Expected ":" after a key, got "${char}".`);
+                        return;
+                    }
+                    expect = 'value';
+                    break;
+                case 'comma-or-close': {
+                    const top = stack[stack.length - 1];
+                    if (char === ',')
+                        expect = top.kind === 'object' ? 'key' : 'value';
+                    else if (char === (top.kind === 'object' ? '}' : ']'))
+                        closeContainer(events, top.kind, at);
+                    else {
+                        fail('bad-json', at, `Expected "," or "${top.kind === 'object' ? '}' : ']'}", got "${char}".`);
+                        return;
+                    }
+                    break;
+                }
+            }
+        }
+    }
+    return {
+        write(chunk) {
+            const events = [];
+            if (error !== null || ended || typeof chunk !== 'string' || chunk.length === 0)
+                return events;
+            run(chunk, events);
+            return events;
+        },
+        end() {
+            const events = [];
+            if (error !== null || ended)
+                return events;
+            ended = true;
+            if (lexeme === 'number')
+                finishNumber(events);
+            if (error !== null)
+                return events;
+            if (lexeme !== 'none' || stack.length > 0 || expect !== 'after') {
+                fail('truncated', bytes, 'The stream ended before the value was complete.');
+                return events;
+            }
+            done = true;
+            return events;
+        },
+        get bytes() {
+            return bytes;
+        },
+        get depth() {
+            return stack.length;
+        },
+        get error() {
+            return error;
+        },
+        get done() {
+            return done;
+        },
+    };
+}

package/dist/version.d.ts

@@ -1,2 +1,2 @@
 /** The semantic version of the spec documents in this package. */
-export declare const SPEC_VERSION = "0.5.0";
+export declare const SPEC_VERSION = "0.6.0";

package/dist/version.js

@@ -1,2 +1,2 @@
 /** The semantic version of the spec documents in this package. */
-export const SPEC_VERSION = '0.5.0';
+export const SPEC_VERSION = '0.6.0';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@texuli/spec",
-  "version": "0.5.0",
+  "version": "0.6.0",
   "description": "The schema-first blueprint contract: JSON schemas, the catalog vocabulary, deterministic TypeScript and Kotlin projections, and the conformance harness.",
   "keywords": [
     "svelte",
@@ -42,6 +42,14 @@
     "./surface": {
       "types": "./dist/surface.d.ts",
       "default": "./dist/surface.js"
+    },
+    "./motion": {
+      "types": "./dist/motion.d.ts",
+      "default": "./dist/motion.js"
+    },
+    "./stream": {
+      "types": "./dist/stream/index.d.ts",
+      "default": "./dist/stream/index.js"
     }
   },
   "files": [