@xdbml/parse 0.1.0-poc.1

package/dist/lexer.js

@@ -0,0 +1,549 @@
+/**
+ * xDBML lexer.
+ *
+ * Produces a stream of tokens with line/column positions. Keywords are NOT
+ * recognized as distinct token kinds at the lexer level. Identifiers carry
+ * their source value, and the parser interprets them as keywords by
+ * lowercased string comparison (per spec §3.8: keywords are
+ * case-insensitive, identifiers are case-sensitive — both end up as
+ * IDENTIFIER tokens here, with the parser making the keyword decision).
+ *
+ * Special multi-character punctuation handled here rather than in the parser:
+ *   - `<>` is lexed as MANY_TO_MANY (otherwise the parser would see `<` then `>`)
+ *   - `[*]` is lexed as ARRAY_WILDCARD only when followed-by/preceded-by a dot
+ *     in a path context; otherwise it's left as `LBRACKET`, `OP(*)`, `RBRACKET`
+ *     to avoid ambiguity with array settings. In practice the parser only
+ *     uses ARRAY_WILDCARD inside fieldPath, so we lex `[*]` greedily whenever
+ *     we see it and let the parser decide based on context.
+ *   - `'''...'''` lexes as a single STRING_LITERAL with the multiline flag.
+ */
+export const TokenKind = {
+    Identifier: 'Identifier',
+    QuotedIdentifier: 'QuotedIdentifier', // "double-quoted name"
+    StringLiteral: 'StringLiteral',
+    MultilineString: 'MultilineString',
+    NumberLiteral: 'NumberLiteral',
+    ExpressionLiteral: 'ExpressionLiteral', // `backtick-quoted`
+    LBrace: 'LBrace',
+    RBrace: 'RBrace',
+    LBracket: 'LBracket',
+    RBracket: 'RBracket',
+    LParen: 'LParen',
+    RParen: 'RParen',
+    Comma: 'Comma',
+    Colon: 'Colon',
+    Dot: 'Dot',
+    Tilde: 'Tilde',
+    Semicolon: 'Semicolon',
+    LAngle: 'LAngle', // <
+    RAngle: 'RAngle', // >
+    Minus: 'Minus', // -
+    ManyToMany: 'ManyToMany', // <>
+    ArrayWildcard: 'ArrayWildcard', // [*]
+    /**
+     * Standalone `*`. Used by v0.2 module-system import-all directives:
+     * `use * from './path'`. Note that `[*]` is a separate token
+     * (ArrayWildcard); this Star token is only produced when the asterisk
+     * appears outside that context.
+     */
+    Star: 'Star',
+    EOF: 'EOF',
+};
+export class LexError extends Error {
+    position;
+    constructor(message, position) {
+        super(`${message} (line ${position.line}, column ${position.column})`);
+        this.position = position;
+    }
+}
+export class Lexer {
+    text;
+    offset = 0;
+    line = 1;
+    column = 1;
+    constructor(text) {
+        this.text = text;
+    }
+    pos() {
+        return {
+            line: this.line,
+            column: this.column,
+            offset: this.offset,
+        };
+    }
+    peek(lookahead = 0) {
+        return this.text[this.offset + lookahead] ?? '';
+    }
+    advance() {
+        const c = this.text[this.offset];
+        this.offset += 1;
+        if (c === '\n') {
+            this.line += 1;
+            this.column = 1;
+        }
+        else {
+            this.column += 1;
+        }
+        return c;
+    }
+    matchSeq(seq) {
+        for (let i = 0; i < seq.length; i += 1) {
+            if (this.text[this.offset + i] !== seq[i]) {
+                return false;
+            }
+        }
+        return true;
+    }
+    isAtEnd() {
+        return this.offset >= this.text.length;
+    }
+    skipTrivia() {
+        while (!this.isAtEnd()) {
+            const c = this.peek();
+            if (c === ' ' || c === '\t' || c === '\r' || c === '\n') {
+                this.advance();
+                continue;
+            }
+            if (c === '/' && this.peek(1) === '/') {
+                // line comment
+                while (!this.isAtEnd() && this.peek() !== '\n') {
+                    this.advance();
+                }
+                continue;
+            }
+            if (c === '/' && this.peek(1) === '*') {
+                // block comment
+                this.advance();
+                this.advance();
+                while (!this.isAtEnd() && !(this.peek() === '*' && this.peek(1) === '/')) {
+                    this.advance();
+                }
+                if (!this.isAtEnd()) {
+                    this.advance(); // *
+                    this.advance(); // /
+                }
+                continue;
+            }
+            break;
+        }
+    }
+    isIdentStart(c) {
+        return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c === '_';
+    }
+    isIdentCont(c) {
+        return this.isIdentStart(c) || (c >= '0' && c <= '9');
+    }
+    isDigit(c) {
+        return c >= '0' && c <= '9';
+    }
+    lexIdentifier() {
+        const start = this.pos();
+        while (!this.isAtEnd() && this.isIdentCont(this.peek())) {
+            this.advance();
+        }
+        const end = this.pos();
+        const text = this.text.substring(start.offset, end.offset);
+        return {
+            kind: TokenKind.Identifier,
+            text,
+            start,
+            end,
+        };
+    }
+    lexNumber() {
+        const start = this.pos();
+        // optional leading minus is handled at parser level (it's an operator token)
+        while (!this.isAtEnd() && this.isDigit(this.peek())) {
+            this.advance();
+        }
+        if (this.peek() === '.' && this.isDigit(this.peek(1))) {
+            this.advance(); // .
+            while (!this.isAtEnd() && this.isDigit(this.peek())) {
+                this.advance();
+            }
+        }
+        if (this.peek() === 'e' || this.peek() === 'E') {
+            this.advance();
+            if (this.peek() === '+' || this.peek() === '-') {
+                this.advance();
+            }
+            while (!this.isAtEnd() && this.isDigit(this.peek())) {
+                this.advance();
+            }
+        }
+        const end = this.pos();
+        return {
+            kind: TokenKind.NumberLiteral,
+            text: this.text.substring(start.offset, end.offset),
+            start,
+            end,
+        };
+    }
+    lexString() {
+        const start = this.pos();
+        // Detect triple-quoted multi-line first
+        if (this.matchSeq("'''")) {
+            this.advance();
+            this.advance();
+            this.advance();
+            const bodyStart = this.offset;
+            while (!this.isAtEnd() && !this.matchSeq("'''")) {
+                if (this.peek() === '\\' && this.peek(1) !== '') {
+                    this.advance();
+                    this.advance();
+                }
+                else {
+                    this.advance();
+                }
+            }
+            if (this.isAtEnd()) {
+                throw new LexError('Unterminated triple-quoted string', start);
+            }
+            const bodyEnd = this.offset;
+            this.advance();
+            this.advance();
+            this.advance();
+            const end = this.pos();
+            const raw = this.text.substring(bodyStart, bodyEnd);
+            return {
+                kind: TokenKind.MultilineString,
+                text: this.text.substring(start.offset, end.offset),
+                value: normalizeMultiline(raw),
+                start,
+                end,
+            };
+        }
+        // single-quoted
+        this.advance(); // opening '
+        let value = '';
+        while (!this.isAtEnd() && this.peek() !== "'") {
+            if (this.peek() === '\\' && this.peek(1) !== '') {
+                const esc = this.peek(1);
+                if (esc === 'n') {
+                    value += '\n';
+                }
+                else if (esc === 't') {
+                    value += '\t';
+                }
+                else if (esc === 'r') {
+                    value += '\r';
+                }
+                else if (esc === '\\') {
+                    value += '\\';
+                }
+                else if (esc === "'") {
+                    value += "'";
+                }
+                else {
+                    value += esc;
+                }
+                this.advance();
+                this.advance();
+            }
+            else if (this.peek() === '\n') {
+                throw new LexError('Unterminated string (newline in single-quoted string)', start);
+            }
+            else {
+                value += this.advance();
+            }
+        }
+        if (this.isAtEnd()) {
+            throw new LexError('Unterminated string', start);
+        }
+        this.advance(); // closing '
+        const end = this.pos();
+        return {
+            kind: TokenKind.StringLiteral,
+            text: this.text.substring(start.offset, end.offset),
+            value,
+            start,
+            end,
+        };
+    }
+    lexQuotedIdent() {
+        const start = this.pos();
+        this.advance(); // opening "
+        let value = '';
+        while (!this.isAtEnd() && this.peek() !== '"') {
+            if (this.peek() === '\\' && this.peek(1) !== '') {
+                value += this.peek(1);
+                this.advance();
+                this.advance();
+            }
+            else if (this.peek() === '\n') {
+                throw new LexError('Unterminated quoted identifier', start);
+            }
+            else {
+                value += this.advance();
+            }
+        }
+        if (this.isAtEnd()) {
+            throw new LexError('Unterminated quoted identifier', start);
+        }
+        this.advance(); // closing "
+        const end = this.pos();
+        return {
+            kind: TokenKind.QuotedIdentifier,
+            text: this.text.substring(start.offset, end.offset),
+            value,
+            start,
+            end,
+        };
+    }
+    lexBacktick() {
+        const start = this.pos();
+        this.advance(); // `
+        const bodyStart = this.offset;
+        while (!this.isAtEnd() && this.peek() !== '`') {
+            this.advance();
+        }
+        if (this.isAtEnd()) {
+            throw new LexError('Unterminated expression literal', start);
+        }
+        const bodyEnd = this.offset;
+        this.advance(); // closing `
+        const end = this.pos();
+        return {
+            kind: TokenKind.ExpressionLiteral,
+            text: this.text.substring(start.offset, end.offset),
+            value: this.text.substring(bodyStart, bodyEnd),
+            start,
+            end,
+        };
+    }
+    /** Read a single token. Returns EOF when out of input. */
+    nextToken() {
+        this.skipTrivia();
+        if (this.isAtEnd()) {
+            const p = this.pos();
+            return {
+                kind: TokenKind.EOF,
+                text: '',
+                start: p,
+                end: p,
+            };
+        }
+        const start = this.pos();
+        const c = this.peek();
+        // Punctuation and multi-character operators
+        if (c === '{') {
+            this.advance();
+            return {
+                kind: TokenKind.LBrace,
+                text: '{',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === '}') {
+            this.advance();
+            return {
+                kind: TokenKind.RBrace,
+                text: '}',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === '[') {
+            // Greedy [*] match for wildcards in paths
+            if (this.peek(1) === '*' && this.peek(2) === ']') {
+                this.advance();
+                this.advance();
+                this.advance();
+                return {
+                    kind: TokenKind.ArrayWildcard,
+                    text: '[*]',
+                    start,
+                    end: this.pos(),
+                };
+            }
+            this.advance();
+            return {
+                kind: TokenKind.LBracket,
+                text: '[',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === ']') {
+            this.advance();
+            return {
+                kind: TokenKind.RBracket,
+                text: ']',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === '(') {
+            this.advance();
+            return {
+                kind: TokenKind.LParen,
+                text: '(',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === ')') {
+            this.advance();
+            return {
+                kind: TokenKind.RParen,
+                text: ')',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === ',') {
+            this.advance();
+            return {
+                kind: TokenKind.Comma,
+                text: ',',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === ':') {
+            this.advance();
+            return {
+                kind: TokenKind.Colon,
+                text: ':',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === ';') {
+            this.advance();
+            return {
+                kind: TokenKind.Semicolon,
+                text: ';',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === '.') {
+            this.advance();
+            return {
+                kind: TokenKind.Dot,
+                text: '.',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === '~') {
+            this.advance();
+            return {
+                kind: TokenKind.Tilde,
+                text: '~',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === '*') {
+            this.advance();
+            return {
+                kind: TokenKind.Star,
+                text: '*',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === '<') {
+            if (this.peek(1) === '>') {
+                this.advance();
+                this.advance();
+                return {
+                    kind: TokenKind.ManyToMany,
+                    text: '<>',
+                    start,
+                    end: this.pos(),
+                };
+            }
+            this.advance();
+            return {
+                kind: TokenKind.LAngle,
+                text: '<',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === '>') {
+            this.advance();
+            return {
+                kind: TokenKind.RAngle,
+                text: '>',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === '-') {
+            // Could be a negative number or the one-to-one operator.
+            // Disambiguation is positional, so we always emit Minus and let the parser decide.
+            this.advance();
+            return {
+                kind: TokenKind.Minus,
+                text: '-',
+                start,
+                end: this.pos(),
+            };
+        }
+        if (c === "'") {
+            return this.lexString();
+        }
+        if (c === '"') {
+            return this.lexQuotedIdent();
+        }
+        if (c === '`') {
+            return this.lexBacktick();
+        }
+        if (this.isDigit(c)) {
+            return this.lexNumber();
+        }
+        if (this.isIdentStart(c)) {
+            return this.lexIdentifier();
+        }
+        throw new LexError(`Unexpected character: ${JSON.stringify(c)}`, start);
+    }
+    tokenize() {
+        const out = [];
+        while (true) {
+            const t = this.nextToken();
+            out.push(t);
+            if (t.kind === TokenKind.EOF) {
+                break;
+            }
+        }
+        return out;
+    }
+}
+/**
+ * Normalize triple-quoted multi-line string content per spec §3.3.
+ * Strips leading newline if present, then de-indents based on the
+ * minimum indent of non-empty lines.
+ */
+function normalizeMultiline(raw) {
+    let s = raw;
+    if (s.startsWith('\n')) {
+        s = s.slice(1);
+    }
+    else if (s.startsWith('\r\n')) {
+        s = s.slice(2);
+    }
+    // strip trailing whitespace-only on last line that prefixed the closing '''
+    s = s.replace(/[ \t]*$/, '');
+    const lines = s.split('\n');
+    let minIndent = Infinity;
+    for (const line of lines) {
+        if (line.trim() === '')
+            continue;
+        const m = line.match(/^[ \t]*/);
+        if (m && m[0].length < minIndent) {
+            minIndent = m[0].length;
+        }
+    }
+    if (minIndent === Infinity || minIndent === 0) {
+        return lines.join('\n');
+    }
+    return lines.map((l) => l.slice(minIndent)).join('\n');
+}
+export function tokenize(text) {
+    return new Lexer(text).tokenize();
+}

package/dist/module-resolver.d.ts

@@ -0,0 +1,115 @@
+/**
+ * Module-system support utilities.
+ *
+ * Per parser-design v2, the parser produces a provenance-preserving
+ * "Shape B" AST: `ModuleImportDirective` nodes keep imported declarations
+ * inside their `clone.statements` field rather than splicing them into the
+ * parent's statement list. This preserves the information about which file
+ * each declaration came from, useful for navigation, inspector panels,
+ * round-tripping back to source text, etc.
+ *
+ * Many downstream consumers (code generators, diagram renderers, simple
+ * walkers) just want a flat list of declarations and don't care about
+ * provenance. The `flatten()` helper produces a new XDbmlDocument where
+ * `ModuleImportDirective` nodes have been replaced by their `clone.statements`,
+ * recursively.
+ *
+ * Currently P4-only: handles clone blocks. Reference-only directives are
+ * rejected at parse time, so `flatten()` doesn't need to do file resolution.
+ * In P5, a separate `resolveModules()` function will populate clone blocks
+ * from referenced files before `flatten()` runs.
+ */
+import type { CloneBlock, ModuleImportDirective, ParseOptions, XDbmlDocument } from './ast.ts';
+/**
+ * Produce a new XDbmlDocument with all module-system directives replaced
+ * by their clone-block content, recursively.
+ *
+ * At the top level, each `ModuleImportDirective` is replaced by its
+ * `clone.statements` (each statement appears at the same position the
+ * directive used to occupy).
+ *
+ * Inside a Container body, each `ModuleImportDirective` is replaced by its
+ * `clone.statements` as `ContainerBodyItem`s. Note that the spec table in
+ * §26.6 guarantees that clone-block content for entity/edge/view/enum
+ * imports is shape-compatible with `ContainerBodyItem`; other shapes
+ * (e.g., a TablePartial clone inside a Container directive) would be
+ * semantically invalid per the spec and would surface as a downstream
+ * type error rather than being caught here.
+ *
+ * Field-level imports (spec §26.8) get a special transform: the clone
+ * block holds a bare `FieldDeclaration`, which `flatten()` lifts into a
+ * synthetic `TypeDeclaration` at file scope. Downstream consumers see
+ * a normal Named Type and can use it as a field type without learning
+ * about the field-import construct.
+ *
+ * Provenance information is lost in the flattened view. Consumers that
+ * want to know where each declaration came from should walk the original
+ * (non-flattened) AST instead.
+ */
+export declare function flatten(doc: XDbmlDocument): XDbmlDocument;
+/**
+ * A `parse`-like callback. Used by `resolveImport` to recursively parse
+ * a referenced file with the same parser configuration and an extended
+ * resolution stack. The Parser injects its own bound parse function.
+ */
+export type ParseFn = (source: string, options: ParseOptions, resolutionStack: ReadonlySet<string>, depth: number) => XDbmlDocument;
+/**
+ * Result of attempting to resolve a directive's referenced file.
+ *
+ *   - `kind: 'resolved'`  -- the file was opened, parsed, and a clone
+ *     block was synthesized
+ *   - `kind: 'cycle'`     -- the resolution chain already contains this
+ *     file; per spec §26.15 cycles are allowed, so we return an empty
+ *     clone block and let name resolution (P6+) handle the actual
+ *     cross-file linking
+ *   - `kind: 'no-resolver'` -- no `readFile` was supplied; caller should
+ *     fall back to the P4 rejection
+ */
+export type ImportResolution = {
+    kind: 'resolved';
+    clone: CloneBlock;
+    resolvedPath: string;
+} | {
+    kind: 'cycle';
+    resolvedPath: string;
+} | {
+    kind: 'no-resolver';
+};
+/**
+ * Resolve a directive's reference. Returns an `ImportResolution` indicating
+ * what happened. The caller decides how to act (set `clone`, fail, etc).
+ *
+ * Path resolution: the directive's `from` is relative to the importer's
+ * `filePath`. If `from` doesn't end with `.xdbml`, that extension is
+ * appended. Returns the absolute resolved path on success.
+ *
+ * Cycles: if the resolved path is already in `resolutionStack`, returns
+ * `kind: 'cycle'` without recursing.
+ *
+ * Depth: if `depth + 1 > maxDepth`, throws.
+ */
+export declare function resolveImport(directive: ModuleImportDirective, options: ParseOptions, resolutionStack: ReadonlySet<string>, depth: number, parseFn: ParseFn): ImportResolution;
+/** Raised when a `from` source string is structurally disallowed. */
+export declare class ModuleSourceError extends Error {
+    constructor(message: string);
+}
+export type ModuleSource = {
+    kind: 'relative';
+    from: string;
+} | {
+    kind: 'url';
+    href: string;
+};
+/** True when a resolved key is a remote (https) URL rather than a path. */
+export declare function isUrlKey(s: string): boolean;
+/**
+ * Classify a directive's `from` source string. Returns a discriminated
+ * union; throws ModuleSourceError for disallowed forms. Pure and sync.
+ */
+export declare function classifyModuleSource(from: string): ModuleSource;
+/**
+ * Default recursion depth limit for module resolution. Enough for any
+ * realistic module graph; small enough to bound stack usage on
+ * pathological inputs.
+ */
+export declare const DEFAULT_MAX_DEPTH = 8;