@kernlang/core 2.0.0 → 3.1.0

package/dist/parser.js

@@ -1,157 +1,461 @@
 let _parseWarnings = [];
-const MULTILINE_BLOCK_TYPES = new Set(['logic', 'handler', 'cleanup', 'body']);
-function parseLine(raw, lineNum) {
-    if (raw.trim() === '')
-        return null;
-    const indent = raw.search(/\S/);
-    let rest = raw.slice(indent);
-    const col = indent + 1;
-    // Extract type
-    const typeMatch = rest.match(/^([A-Za-z_][A-Za-z0-9_-]*)/);
-    if (!typeMatch)
-        return null;
-    const type = typeMatch[1];
-    rest = rest.slice(type.length);
-    const props = {};
-    const styles = {};
-    const pseudoStyles = {};
-    const themeRefs = [];
-    // Special: theme nodes have a bare name after the type: "theme bar {h:8}"
-    if (type === 'theme') {
-        rest = rest.replace(/^ +/, '');
-        const nameMatch = rest.match(/^([A-Za-z_][A-Za-z0-9_-]*)/);
-        if (nameMatch) {
-            props.name = nameMatch[1];
-            rest = rest.slice(nameMatch[0].length);
+// ── Tokenizer ────────────────────────────────────────────────────────────
+function isIdentStart(ch) {
+    return (ch >= 'A' && ch <= 'Z') || (ch >= 'a' && ch <= 'z') || ch === '_';
+}
+function isIdentChar(ch) {
+    return isIdentStart(ch) || (ch >= '0' && ch <= '9') || ch === '-';
+}
+function isDigit(ch) {
+    return ch >= '0' && ch <= '9';
+}
+/** Character-by-character tokenizer for a single KERN line (after indent stripped). */
+export function tokenizeLine(line) {
+    const tokens = [];
+    let i = 0;
+    while (i < line.length) {
+        const ch = line[i];
+        // Whitespace
+        if (ch === ' ' || ch === '\t') {
+            const start = i;
+            while (i < line.length && (line[i] === ' ' || line[i] === '\t'))
+                i++;
+            tokens.push({ kind: 'whitespace', value: line.slice(start, i), pos: start });
+            continue;
         }
-    }
-    // Special: import nodes support bare words for name and optional "default" flag
-    // Syntax: "import [default] <name> from=<path>"
-    if (type === 'import') {
-        rest = rest.replace(/^ +/, '');
-        // Check for "default" keyword
-        if (rest.startsWith('default')) {
-            const afterDefault = rest.slice(7);
-            if (afterDefault.length === 0 || afterDefault[0] === ' ') {
-                props.default = true;
-                rest = afterDefault.replace(/^ +/, '');
+        // Expression block {{ ... }}
+        if (ch === '{' && i + 1 < line.length && line[i + 1] === '{') {
+            const start = i;
+            i += 2;
+            let depth = 1;
+            while (i < line.length - 1 && depth > 0) {
+                if (line[i] === '{' && line[i + 1] === '{') {
+                    depth++;
+                    i += 2;
+                }
+                else if (line[i] === '}' && line[i + 1] === '}') {
+                    depth--;
+                    if (depth === 0)
+                        break;
+                    i += 2;
+                }
+                else
+                    i++;
             }
+            const inner = line.slice(start + 2, i).trim();
+            i += 2;
+            tokens.push({ kind: 'expr', value: inner, pos: start });
+            continue;
         }
-        // Capture the import name (bare word before from=)
-        const nameMatch = rest.match(/^([A-Za-z_][A-Za-z0-9_-]*)/);
-        if (nameMatch && !rest.startsWith('from=')) {
-            props.name = nameMatch[1];
-            rest = rest.slice(nameMatch[0].length);
-        }
-    }
-    // Parse the remainder: props, style blocks, theme refs
-    while (rest.length > 0) {
-        rest = rest.replace(/^ +/, '');
-        if (rest.length === 0)
-            break;
-        // Style block — find matching } respecting quotes
-        if (rest[0] === '{') {
-            let close = -1;
+        // Style block { ... } — find matching } respecting quotes
+        if (ch === '{') {
+            const start = i;
             let inQuote = false;
-            for (let j = 1; j < rest.length; j++) {
-                if (rest[j] === '"')
+            let j = i + 1;
+            while (j < line.length) {
+                if (line[j] === '"')
                     inQuote = !inQuote;
-                if (!inQuote && rest[j] === '}') {
-                    close = j;
+                if (!inQuote && line[j] === '}') {
+                    j++;
                     break;
                 }
+                j++;
             }
-            if (close === -1)
-                break;
-            const block = rest.slice(1, close);
-            parseStyleBlock(block, styles, pseudoStyles);
-            rest = rest.slice(close + 1);
+            tokens.push({ kind: 'style', value: line.slice(start + 1, j - 1), pos: start });
+            i = j;
             continue;
         }
-        // Theme ref
-        if (rest[0] === '$') {
-            const refMatch = rest.match(/^\$([A-Za-z_][A-Za-z0-9_-]*)/);
-            if (refMatch) {
-                themeRefs.push(refMatch[1]);
-                rest = rest.slice(refMatch[0].length);
-                continue;
+        // Quoted string "..."
+        if (ch === '"') {
+            const start = i;
+            i++;
+            while (i < line.length && line[i] !== '"')
+                i++;
+            const inner = line.slice(start + 1, i);
+            i++;
+            tokens.push({ kind: 'quoted', value: inner, pos: start });
+            continue;
+        }
+        // Theme ref $name
+        if (ch === '$' && i + 1 < line.length && isIdentStart(line[i + 1])) {
+            const start = i;
+            i++;
+            while (i < line.length && isIdentChar(line[i]))
+                i++;
+            tokens.push({ kind: 'themeRef', value: line.slice(start + 1, i), pos: start });
+            continue;
+        }
+        // Equals
+        if (ch === '=') {
+            tokens.push({ kind: 'equals', value: '=', pos: i });
+            i++;
+            continue;
+        }
+        // Comma
+        if (ch === ',') {
+            tokens.push({ kind: 'comma', value: ',', pos: i });
+            i++;
+            continue;
+        }
+        // Slash-prefixed path: /something
+        if (ch === '/') {
+            const start = i;
+            while (i < line.length && line[i] !== ' ' && line[i] !== '\t' && line[i] !== '{' && line[i] !== '$')
+                i++;
+            tokens.push({ kind: 'slash', value: line.slice(start, i), pos: start });
+            continue;
+        }
+        // Number (pure digits)
+        if (isDigit(ch)) {
+            const start = i;
+            while (i < line.length && isDigit(line[i]))
+                i++;
+            tokens.push({ kind: 'number', value: line.slice(start, i), pos: start });
+            continue;
+        }
+        // Identifier: [A-Za-z_][A-Za-z0-9_-]*
+        // Handles evolved: prefix (evolved:keyword → strips prefix, returns keyword)
+        if (isIdentStart(ch)) {
+            const start = i;
+            while (i < line.length && isIdentChar(line[i]))
+                i++;
+            if (line[i] === ':' && line.slice(start, i) === 'evolved' && i + 1 < line.length && isIdentStart(line[i + 1])) {
+                i++;
+                const nameStart = i;
+                while (i < line.length && isIdentChar(line[i]))
+                    i++;
+                tokens.push({ kind: 'identifier', value: line.slice(nameStart, i), pos: start });
+            }
+            else {
+                tokens.push({ kind: 'identifier', value: line.slice(start, i), pos: start });
             }
+            continue;
         }
-        // Prop: key={{ expression }}
-        const exprPropMatch = rest.match(/^([A-Za-z_][A-Za-z0-9_-]*)=\{\{/);
-        if (exprPropMatch) {
-            const key = exprPropMatch[1];
-            rest = rest.slice(exprPropMatch[0].length);
-            // Find matching }}
-            let depth = 1;
-            let j = 0;
-            for (; j < rest.length - 1; j++) {
-                if (rest[j] === '{' && rest[j + 1] === '{') {
-                    depth++;
-                    j++;
+        // Unknown character
+        tokens.push({ kind: 'unknown', value: ch, pos: i });
+        i++;
+    }
+    return tokens;
+}
+// ── Token stream ─────────────────────────────────────────────────────────
+// Opus: class-based cursor. Codex contribution: consumeAnyValue for evolved hints.
+class TokenStream {
+    tokens;
+    idx = 0;
+    constructor(tokens) { this.tokens = tokens; }
+    peek() { return this.tokens[this.idx]; }
+    next() { return this.tokens[this.idx++]; }
+    done() { return this.idx >= this.tokens.length; }
+    position() { return this.idx; }
+    setPosition(pos) { this.idx = pos; }
+    skipWS() {
+        while (this.idx < this.tokens.length && this.tokens[this.idx].kind === 'whitespace')
+            this.idx++;
+    }
+    /** Try to consume an identifier. Returns its value or null. */
+    tryIdent() {
+        if (this.idx < this.tokens.length && this.tokens[this.idx].kind === 'identifier') {
+            return this.tokens[this.idx++].value;
+        }
+        return null;
+    }
+    /** Try to consume a number token. Returns its value or null. */
+    tryNumber() {
+        if (this.idx < this.tokens.length && this.tokens[this.idx].kind === 'number') {
+            return this.tokens[this.idx++].value;
+        }
+        return null;
+    }
+    /** Check if the next non-WS token is an identifier followed by '='. */
+    isKeyValue() {
+        let j = this.idx;
+        while (j < this.tokens.length && this.tokens[j].kind === 'whitespace')
+            j++;
+        if (j >= this.tokens.length || this.tokens[j].kind !== 'identifier')
+            return false;
+        return j + 1 < this.tokens.length && this.tokens[j + 1].kind === 'equals';
+    }
+    /** Check if any remaining token contains '='. */
+    hasEquals() {
+        for (let j = this.idx; j < this.tokens.length; j++) {
+            if (this.tokens[j].kind === 'equals')
+                return true;
+        }
+        return false;
+    }
+    /** Check if there are more non-whitespace tokens. */
+    hasMore() {
+        let j = this.idx;
+        while (j < this.tokens.length && this.tokens[j].kind === 'whitespace')
+            j++;
+        return j < this.tokens.length;
+    }
+    /** Get remaining raw text from current position (for fallback / params). */
+    remainingRaw(line) {
+        if (this.idx >= this.tokens.length)
+            return '';
+        const startPos = this.tokens[this.idx].pos;
+        this.idx = this.tokens.length;
+        return line.slice(startPos);
+    }
+    /** Consume any single non-whitespace token as a value (for evolved positional args). */
+    consumeAnyValue() {
+        this.skipWS();
+        const tok = this.peek();
+        if (!tok || tok.kind === 'whitespace')
+            return undefined;
+        return this.next();
+    }
+}
+// ── Prop parsing (extracted from Codex's parsePropToken pattern) ──────────
+/** Map a value token to its JS representation. */
+function tokenValue(tok) {
+    if (tok.kind === 'expr')
+        return { __expr: true, code: tok.value };
+    if (tok.kind === 'quoted')
+        return tok.value;
+    return tok.value;
+}
+/** Try to parse a key=value prop from the stream. Returns true if consumed. */
+function parseProp(s, props) {
+    if (!s.isKeyValue())
+        return false;
+    s.skipWS();
+    const key = s.next().value; // identifier
+    s.next(); // =
+    const valTok = s.peek();
+    if (!valTok || valTok.kind === 'whitespace') {
+        props[key] = '';
+        return true;
+    }
+    // key={{expr}} or key="quoted"
+    if (valTok.kind === 'expr' || valTok.kind === 'quoted') {
+        props[key] = tokenValue(s.next());
+        return true;
+    }
+    // key=bareValue — collect tokens up to next WS/style/themeRef
+    let value = '';
+    while (!s.done()) {
+        const vt = s.peek();
+        if (vt.kind === 'whitespace' || vt.kind === 'style' || vt.kind === 'themeRef')
+            break;
+        value += vt.value;
+        s.next();
+    }
+    props[key] = value;
+    return true;
+}
+const MULTILINE_BLOCK_TYPES = new Set(['logic', 'handler', 'cleanup', 'body']);
+const _parserHints = new Map();
+/** Register parser hints for an evolved node type. */
+export function registerParserHints(keyword, hints) {
+    _parserHints.set(keyword, hints);
+    if (hints.multilineBlock) {
+        MULTILINE_BLOCK_TYPES.add(keyword);
+    }
+}
+/** Unregister parser hints (for rollback/testing). */
+export function unregisterParserHints(keyword) {
+    const hints = _parserHints.get(keyword);
+    if (hints?.multilineBlock) {
+        MULTILINE_BLOCK_TYPES.delete(keyword);
+    }
+    _parserHints.delete(keyword);
+}
+/** Clear all parser hints (for test isolation). */
+export function clearParserHints() {
+    for (const [keyword, hints] of _parserHints) {
+        if (hints.multilineBlock)
+            MULTILINE_BLOCK_TYPES.delete(keyword);
+    }
+    _parserHints.clear();
+}
+/** Consume a bare identifier into props if it's not a key=value pair. */
+function consumeBareIdent(s, props, propName) {
+    s.skipWS();
+    if (s.isKeyValue())
+        return;
+    const id = s.tryIdent();
+    if (id)
+        props[propName] = id;
+}
+const KEYWORD_HANDLERS = new Map([
+    ['theme', (s, props) => {
+            consumeBareIdent(s, props, 'name');
+        }],
+    ['import', (s, props) => {
+            s.skipWS();
+            const pos = s.position();
+            const id = s.tryIdent();
+            if (id === 'default') {
+                if (!s.done() && s.peek()?.kind !== 'equals') {
+                    props.default = true;
+                    s.skipWS();
                 }
-                else if (rest[j] === '}' && rest[j + 1] === '}') {
-                    depth--;
-                    j++;
-                    if (depth === 0)
-                        break;
+                else if (s.peek()?.kind === 'equals') {
+                    s.setPosition(pos);
+                    return;
+                }
+                else {
+                    props.default = true;
+                    return;
                 }
             }
-            const expr = rest.slice(0, j - 1).trim();
-            rest = rest.slice(j + 1);
-            props[key] = { __expr: true, code: expr };
-            continue;
-        }
-        // Prop: key=value or key="quoted value"
-        const propMatch = rest.match(/^([A-Za-z_][A-Za-z0-9_-]*)=/);
-        if (propMatch) {
-            const key = propMatch[1];
-            rest = rest.slice(propMatch[0].length);
-            let value;
-            if (rest[0] === '"') {
-                const endQuote = rest.indexOf('"', 1);
-                value = rest.slice(1, endQuote);
-                rest = rest.slice(endQuote + 1);
+            else if (id) {
+                s.setPosition(pos);
             }
-            else if (rest.startsWith('{{')) {
-                // Bare expression without key= prefix (e.g. value={{ x }})
-                rest = rest.slice(2);
-                let depth = 1;
-                let j = 0;
-                for (; j < rest.length - 1; j++) {
-                    if (rest[j] === '{' && rest[j + 1] === '{') {
-                        depth++;
-                        j++;
-                    }
-                    else if (rest[j] === '}' && rest[j + 1] === '}') {
-                        depth--;
-                        j++;
-                        if (depth === 0)
-                            break;
+            if (!s.isKeyValue()) {
+                s.skipWS();
+                const name = s.tryIdent();
+                if (name)
+                    props.name = name;
+            }
+        }],
+    ['route', (s, props) => {
+            s.skipWS();
+            const pos = s.position();
+            const verb = s.tryIdent();
+            if (verb && /^(GET|POST|PUT|DELETE|PATCH|HEAD|OPTIONS)$/i.test(verb)) {
+                props.method = verb.toLowerCase();
+                s.skipWS();
+                const tok = s.peek();
+                if (tok && tok.kind === 'slash') {
+                    props.path = tok.value;
+                    s.next();
+                }
+            }
+            else if (verb) {
+                s.setPosition(pos);
+            }
+        }],
+    ['params', (s, props, content) => {
+            s.skipWS();
+            const remaining = s.remainingRaw(content);
+            if (remaining.length > 0) {
+                const items = [];
+                const parts = remaining.split(',').map(p => p.trim()).filter(Boolean);
+                for (const part of parts) {
+                    const m = part.match(/^([A-Za-z_]\w*):([A-Za-z_]\w*(?:\[\])?)(?:\s*=\s*(.+))?$/);
+                    if (m) {
+                        const item = { name: m[1], type: m[2] };
+                        if (m[3] !== undefined)
+                            item.default = m[3].trim();
+                        items.push(item);
                     }
                 }
-                const expr = rest.slice(0, j - 1).trim();
-                rest = rest.slice(j + 1);
-                props[key] = { __expr: true, code: expr };
-                continue;
+                props.items = items;
             }
-            else {
-                const endMatch = rest.match(/^[^\s{$]+/);
-                value = endMatch ? endMatch[0] : '';
-                rest = rest.slice(value.length);
+        }],
+    ['auth', (s, props) => { consumeBareIdent(s, props, 'mode'); }],
+    ['validate', (s, props) => { consumeBareIdent(s, props, 'schema'); }],
+    ['error', (s, props) => {
+            s.skipWS();
+            const num = s.tryNumber();
+            if (num) {
+                props.status = parseInt(num, 10);
+                s.skipWS();
+                const tok = s.peek();
+                if (tok && tok.kind === 'quoted') {
+                    props.message = tok.value;
+                    s.next();
+                }
+            }
+        }],
+    ['derive', (s, props) => { consumeBareIdent(s, props, 'name'); }],
+    ['guard', (s, props) => { consumeBareIdent(s, props, 'name'); }],
+    ['effect', (s, props) => { consumeBareIdent(s, props, 'name'); }],
+    ['strategy', (s, props) => { consumeBareIdent(s, props, 'name'); }],
+    ['trigger', (s, props) => { consumeBareIdent(s, props, 'kind'); }],
+    ['respond', (s, props) => {
+            s.skipWS();
+            const num = s.tryNumber();
+            if (num)
+                props.status = parseInt(num, 10);
+        }],
+    ['middleware', (s, props, content) => {
+            s.skipWS();
+            if (!s.hasMore())
+                return;
+            if (s.hasEquals())
+                return;
+            const remaining = s.remainingRaw(content).trim();
+            if (remaining.length > 0) {
+                const names = remaining.split(',').map(n => n.trim()).filter(Boolean);
+                if (names.length > 1) {
+                    props.names = names;
+                }
+                else if (names.length === 1) {
+                    props.name = names[0];
+                }
+            }
+        }],
+]);
+// ── parseLine (token-based) ──────────────────────────────────────────────
+function parseLine(raw, lineNum) {
+    if (raw.trim() === '')
+        return null;
+    const indent = raw.search(/\S/);
+    const content = raw.slice(indent);
+    const col = indent + 1;
+    const tokens = tokenizeLine(content);
+    const s = new TokenStream(tokens);
+    // First token must be an identifier (the node type)
+    const typeToken = s.tryIdent();
+    if (!typeToken)
+        return null;
+    const type = typeToken;
+    const props = {};
+    const styles = {};
+    const pseudoStyles = {};
+    const themeRefs = [];
+    // ── Evolved node parser hints (v4) ──────────────────────────────────
+    const evolvedHints = _parserHints.get(type);
+    if (evolvedHints) {
+        if (evolvedHints.positionalArgs) {
+            for (const argName of evolvedHints.positionalArgs) {
+                const tok = s.consumeAnyValue();
+                if (tok)
+                    props[argName] = tok.value;
             }
-            props[key] = value;
+        }
+        if (evolvedHints.bareWord) {
+            s.skipWS();
+            if (!s.isKeyValue()) {
+                const id = s.tryIdent();
+                if (id)
+                    props[evolvedHints.bareWord] = id;
+            }
+        }
+    }
+    // ── Keyword-specific handling ──────────────────────────────────────
+    const handler = KEYWORD_HANDLERS.get(type);
+    if (handler)
+        handler(s, props, content);
+    // ── Generic prop/style/theme parsing ───────────────────────────────
+    while (!s.done()) {
+        s.skipWS();
+        if (s.done())
+            break;
+        const tok = s.peek();
+        // Style block
+        if (tok.kind === 'style') {
+            parseStyleBlock(tok.value, styles, pseudoStyles);
+            s.next();
             continue;
         }
-        // Unknown token — collect as warning, skip to next whitespace
-        const skipped = rest.match(/^\S+/);
-        if (skipped) {
-            const errCol = col + (raw.length - rest.length);
-            _parseWarnings.push(`Unexpected token "${skipped[0]}" at line ${lineNum}:${errCol}`);
-            rest = rest.slice(skipped[0].length);
+        // Theme ref
+        if (tok.kind === 'themeRef') {
+            themeRefs.push(tok.value);
+            s.next();
             continue;
         }
-        break;
+        // Key=value prop (extracted helper from Codex)
+        if (parseProp(s, props))
+            continue;
+        // Unknown token — skip with warning
+        const skipped = s.next();
+        const errCol = col + skipped.pos;
+        _parseWarnings.push(`Unexpected token "${skipped.value}" at line ${lineNum}:${errCol}`);
     }
     return {
         indent: indent / 2,
@@ -163,6 +467,7 @@ function parseLine(raw, lineNum) {
         loc: { line: lineNum, col },
     };
 }
+// ── Style block parsing (unchanged) ──────────────────────────────────────
 function splitStylePairs(block) {
     const pairs = [];
     let current = '';
@@ -212,7 +517,6 @@ function parseStyleBlock(block, styles, pseudoStyles) {
         if (quotedKeyMatch) {
             const key = quotedKeyMatch[1];
             let value = quotedKeyMatch[2].trim();
-            // Strip surrounding quotes from value if present
             if (value.startsWith('"') && value.endsWith('"')) {
                 value = value.slice(1, -1);
             }
@@ -224,7 +528,6 @@ function parseStyleBlock(block, styles, pseudoStyles) {
         if (colonIdx > 0) {
             const key = pair.slice(0, colonIdx).trim();
             let value = pair.slice(colonIdx + 1).trim();
-            // Strip surrounding quotes from value if present
             if (value.startsWith('"') && value.endsWith('"')) {
                 value = value.slice(1, -1);
             }
@@ -233,8 +536,6 @@ function parseStyleBlock(block, styles, pseudoStyles) {
     }
 }
 function expandMinified(source) {
-    // Detect minified S-expression format: node(child1,child2)
-    // Convert to indented format for the standard parser
     if (!source.includes('(') || source.split('\n').length > 2)
         return source;
     const result = [];
@@ -295,7 +596,6 @@ function expandMinified(source) {
 export function getParseWarnings() { return [..._parseWarnings]; }
 export function parse(source) {
     _parseWarnings = [];
-    // Handle minified S-expression format
     source = expandMinified(source);
     const lines = source.split('\n');
     const parsed = [];
@@ -307,7 +607,6 @@ export function parse(source) {
             const codeLines = [];
             const startLine = i + 1;
             const blockOpen = `${multilineType} <<<`;
-            // Check if inline close on same line
             const afterOpen = trimmed.slice(blockOpen.length);
             if (afterOpen.includes('>>>')) {
                 codeLines.push(afterOpen.split('>>>')[0]);
@@ -318,7 +617,6 @@ export function parse(source) {
                     codeLines.push(lines[i]);
                     i++;
                 }
-                // Capture text before >>> on closing line
                 if (i < lines.length) {
                     const closeLine = lines[i];
                     const closeIdx = closeLine.indexOf('>>>');
@@ -359,13 +657,11 @@ export function parse(source) {
             node.props.themeRefs = p.themeRefs;
         return node;
     }
-    // Build tree using indent levels
     const root = toNode(parsed[0]);
     const stack = [{ node: root, indent: parsed[0].indent }];
     for (let i = 1; i < parsed.length; i++) {
         const p = parsed[i];
         const node = toNode(p);
-        // Pop stack until we find a parent at a lower indent level
         while (stack.length > 1 && stack[stack.length - 1].indent >= p.indent) {
             stack.pop();
         }