@idfkit/core 0.2.0-rc.2 → 0.2.0

package/dist/parse/scan.js

@@ -0,0 +1,177 @@
+/**
+ * The IDF character rules, in one place.
+ *
+ * The grammar is small:
+ *   - `!` starts a comment running to the end of its line
+ *   - `,` separates fields
+ *   - `;` terminates a statement
+ *   - everything else is field text
+ *
+ * There are no string literals and no escape sequences, so a comma cannot occur inside a field
+ * value and an `!` always starts a comment. Real files depend on both.
+ *
+ * A hand-written character walk rather than a regex. The Python library matches objects with a
+ * `(?:[^;!]*(?:![^\n]*\n)?)*?` inner loop; that is a nested quantifier, so it backtracks badly on
+ * malformed input and cannot report where the problem was. A scan is about the same amount of
+ * code, is linear in the input, and always knows its line number.
+ *
+ * **Why this file exists at all.** Every position this library reports depends on the syntax layer
+ * and the model-building read agreeing about where a field begins and ends. Two implementations of
+ * "step over a comment between a separator and its value" that differ by one character put a
+ * finding on the wrong field, and that class of bug stays invisible until a file puts a comment
+ * somewhere unusual. The rules used to exist twice, once inside `lex` and once inside the
+ * `fieldLine` helper of `idf.ts`; both are now callers of this scan (research R3).
+ *
+ * **This module is internal and stays internal.** It is not exported from the package root. The
+ * syntax layer, which is public, is what a consumer outside the package reaches for; keeping the
+ * scan itself unexported is the reason the layer ships in core rather than beside the language
+ * service, because a package boundary drawn around both halves would have forced this file into a
+ * published export purely so the other half could reach it.
+ *
+ * @internal
+ */
+const EXCLAMATION = 0x21;
+const COMMA = 0x2c;
+const SEMICOLON = 0x3b;
+const LINE_FEED = 0x0a;
+/**
+ * Walk IDF text, reporting what the handler asked for.
+ *
+ * One linear pass, no allocation of its own, and it never throws: text that violates the grammar
+ * is reported as what it is rather than stopped at.
+ *
+ * @internal
+ */
+export function scan(text, handler, options = {}) {
+    const length = text.length;
+    let index = options.from ?? 0;
+    let line = options.line ?? 1;
+    /** Offset of the current line's first character, which is what turns an offset into a column. */
+    let lineStart = index - ((options.column ?? 1) - 1);
+    /** Start of the current field's text run, moved past every comment that interrupts it. */
+    let fieldStart = index;
+    /** 0 for the type name, 1 for the first field after it. */
+    let fieldIndex = 0;
+    /** True once the current statement's first non-blank, non-comment character has been seen. */
+    let open = false;
+    /** First non-blank character of the current field's value, or -1 while it has none. */
+    let valueStart = -1;
+    /** One past the last non-blank character of the run `valueStart` falls in. */
+    let valueEnd = -1;
+    /** Line `valueStart` falls on. */
+    let valueLine = 1;
+    /** True once a comment has ended the run holding `valueStart`. */
+    let valueClosed = false;
+    /**
+     * Report the field the scan is inside, closed at `at`. False when a handler asked to stop.
+     *
+     * A field with no value text is positioned at `at`, which is the separator, the terminator, or
+     * the end of input that closed it. That is the offset a value would have begun at, because
+     * everything between the field's start and `at` was whitespace or comment.
+     */
+    const closeField = (at) => {
+        if (at > fieldStart && handler.fieldText?.(fieldStart, at) === false)
+            return false;
+        const empty = valueStart < 0;
+        const stop = handler.fieldEnd?.(fieldIndex, empty ? at : valueStart, empty ? at : valueEnd, empty ? line : valueLine) === false;
+        valueStart = -1;
+        valueEnd = -1;
+        valueClosed = false;
+        return !stop;
+    };
+    while (index < length) {
+        const code = text.charCodeAt(index);
+        if (code === EXCLAMATION) {
+            // Text seen before the comment still belongs to the field, and so does text after the line
+            // feed: the comment interrupts the field without ending it. This is what lets
+            // `Zone1,  !- Name` work, where the comment is no part of the value and the value is no part
+            // of the comment.
+            if (index > fieldStart && handler.fieldText?.(fieldStart, index) === false)
+                return;
+            const feed = text.indexOf('\n', index);
+            const end = feed === -1 ? length : feed;
+            if (handler.comment?.(index, end) === false)
+                return;
+            if (valueStart >= 0)
+                valueClosed = true;
+            if (feed === -1) {
+                fieldStart = length;
+                break;
+            }
+            index = feed + 1;
+            fieldStart = index;
+            line += 1;
+            lineStart = index;
+            continue;
+        }
+        const blank = isSpace(code);
+        if (!open && !blank) {
+            open = true;
+            if (handler.statementStart?.(index, line, index - lineStart + 1) === false)
+                return;
+        }
+        if (code === COMMA || code === SEMICOLON) {
+            if (!closeField(index))
+                return;
+            if (code === COMMA) {
+                if (handler.separator?.(index) === false)
+                    return;
+                fieldIndex += 1;
+            }
+            else {
+                if (handler.terminator?.(index) === false)
+                    return;
+                if (handler.statementEnd?.(index + 1, false) === false)
+                    return;
+                fieldIndex = 0;
+                open = false;
+            }
+            index += 1;
+            fieldStart = index;
+            continue;
+        }
+        if (code === LINE_FEED) {
+            line += 1;
+            lineStart = index + 1;
+        }
+        else if (!blank) {
+            if (valueStart < 0) {
+                valueStart = index;
+                valueEnd = index + 1;
+                valueLine = line;
+            }
+            else if (!valueClosed) {
+                valueEnd = index + 1;
+            }
+        }
+        index += 1;
+    }
+    // Input ran out inside a statement. Its last field is reported like any other, so a caller reads
+    // what was written rather than having to reconstruct it from the leftovers.
+    if (!open)
+        return;
+    if (!closeField(length))
+        return;
+    handler.statementEnd?.(length, true);
+}
+/**
+ * Whether a character is whitespace, by the same definition `String.prototype.trim` uses.
+ *
+ * A field's value is trimmed, so a scan that disagreed with `trim` about one character would put
+ * a region beside the value rather than on it. ASCII is one comparison; the rest of the set costs
+ * a branch nothing outside a comment ever takes.
+ */
+function isSpace(code) {
+    if (code < 0x80)
+        return code === 0x20 || (code >= 0x09 && code <= 0x0d);
+    return (code === 0xa0 ||
+        code === 0x1680 ||
+        (code >= 0x2000 && code <= 0x200a) ||
+        code === 0x2028 ||
+        code === 0x2029 ||
+        code === 0x202f ||
+        code === 0x205f ||
+        code === 0x3000 ||
+        code === 0xfeff);
+}
+//# sourceMappingURL=scan.js.map

package/dist/parse/scan.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scan.js","sourceRoot":"","sources":["../../src/parse/scan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,MAAM,WAAW,GAAG,IAAI,CAAC;AACzB,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,SAAS,GAAG,IAAI,CAAC;AACvB,MAAM,SAAS,GAAG,IAAI,CAAC;AAwGvB;;;;;;;GAOG;AACH,MAAM,UAAU,IAAI,CAAC,IAAY,EAAE,OAAoB,EAAE,UAAuB,EAAE;IAChF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAE3B,IAAI,KAAK,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAC9B,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAC7B,iGAAiG;IACjG,IAAI,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpD,0FAA0F;IAC1F,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,2DAA2D;IAC3D,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,8FAA8F;IAC9F,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,uFAAuF;IACvF,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,8EAA8E;IAC9E,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,kCAAkC;IAClC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,kEAAkE;IAClE,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB;;;;;;OAMG;IACH,MAAM,UAAU,GAAG,CAAC,EAAU,EAAW,EAAE;QACzC,IAAI,EAAE,GAAG,UAAU,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QACnF,MAAM,KAAK,GAAG,UAAU,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,GACR,OAAO,CAAC,QAAQ,EAAE,CAChB,UAAU,EACV,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,EACvB,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,EACrB,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CACzB,KAAK,KAAK,CAAC;QACd,UAAU,GAAG,CAAC,CAAC,CAAC;QAChB,QAAQ,GAAG,CAAC,CAAC,CAAC;QACd,WAAW,GAAG,KAAK,CAAC;QACpB,OAAO,CAAC,IAAI,CAAC;IACf,CAAC,CAAC;IAEF,OAAO,KAAK,GAAG,MAAM,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAEpC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YACzB,2FAA2F;YAC3F,8EAA8E;YAC9E,6FAA6F;YAC7F,kBAAkB;YAClB,IAAI,KAAK,GAAG,UAAU,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,KAAK;gBAAE,OAAO;YACnF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACvC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YACxC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,KAAK;gBAAE,OAAO;YACpD,IAAI,UAAU,IAAI,CAAC;gBAAE,WAAW,GAAG,IAAI,CAAC;YACxC,IAAI,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC;gBAChB,UAAU,GAAG,MAAM,CAAC;gBACpB,MAAM;YACR,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;YACjB,UAAU,GAAG,KAAK,CAAC;YACnB,IAAI,IAAI,CAAC,CAAC;YACV,SAAS,GAAG,KAAK,CAAC;YAClB,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAE5B,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,GAAG,IAAI,CAAC;YACZ,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK;gBAAE,OAAO;QACrF,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO;YAC/B,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACnB,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK;oBAAE,OAAO;gBACjD,UAAU,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK;oBAAE,OAAO;gBAClD,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,KAAK,KAAK;oBAAE,OAAO;gBAC/D,UAAU,GAAG,CAAC,CAAC;gBACf,IAAI,GAAG,KAAK,CAAC;YACf,CAAC;YACD,KAAK,IAAI,CAAC,CAAC;YACX,UAAU,GAAG,KAAK,CAAC;YACnB,SAAS;QACX,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,IAAI,IAAI,CAAC,CAAC;YACV,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAClB,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;gBACnB,UAAU,GAAG,KAAK,CAAC;gBACnB,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC;gBACrB,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;iBAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxB,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,iGAAiG;IACjG,4EAA4E;IAC5E,IAAI,CAAC,IAAI;QAAE,OAAO;IAClB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO;IAChC,OAAO,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;GAMG;AACH,SAAS,OAAO,CAAC,IAAY;IAC3B,IAAI,IAAI,GAAG,IAAI;QAAE,OAAO,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC;IACxE,OAAO,CACL,IAAI,KAAK,IAAI;QACb,IAAI,KAAK,MAAM;QACf,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;QAClC,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM,CAChB,CAAC;AACJ,CAAC"}

package/dist/syntax/classify.d.ts

@@ -0,0 +1,25 @@
+import type { SyntaxLayer } from './layer.js';
+import type { Token } from './tokens.js';
+/**
+ * Every character of the text, as tokens, in source order.
+ *
+ * Two things happen here that the layer deliberately does not pay to store.
+ *
+ * The first is `trivia`. Whitespace is the complement of the stored tokens, so storing it would
+ * roughly double the token count to hold something derivable by subtraction. This function fills
+ * each gap as it walks, which is what makes the layer's complete coverage observable (FR-016)
+ * without the layer carrying it: the sequence begins at offset 0, ends at `text.length`, and has
+ * no gap and no overlap anywhere between.
+ *
+ * The second is the line split. No yielded token crosses a line boundary (FR-047), while a stored
+ * `value` region may, because the format lets a field be written across two lines and real files
+ * do it. Every token encoding in use, the Language Server Protocol's included, expresses a token
+ * as a length on a single line, so a token spanning one cannot be encoded at all. The split adds
+ * tokens and moves no boundary, so the tiling holds exactly as before, and the stored region stays
+ * whole for everything that is not being drawn.
+ *
+ * A generator, so a consumer colouring a viewport stops where it stops rather than materialising
+ * four hundred thousand tokens to read the first fifty.
+ */
+export declare function classify(layer: SyntaxLayer): Iterable<Token>;
+//# sourceMappingURL=classify.d.ts.map

package/dist/syntax/classify.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify.d.ts","sourceRoot":"","sources":["../../src/syntax/classify.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,OAAO,KAAK,EAAE,KAAK,EAAa,MAAM,aAAa,CAAC;AAEpD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAiB,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAkC7D"}

package/dist/syntax/classify.js

@@ -0,0 +1,59 @@
+import { lineStartsOf } from './region.js';
+/**
+ * Every character of the text, as tokens, in source order.
+ *
+ * Two things happen here that the layer deliberately does not pay to store.
+ *
+ * The first is `trivia`. Whitespace is the complement of the stored tokens, so storing it would
+ * roughly double the token count to hold something derivable by subtraction. This function fills
+ * each gap as it walks, which is what makes the layer's complete coverage observable (FR-016)
+ * without the layer carrying it: the sequence begins at offset 0, ends at `text.length`, and has
+ * no gap and no overlap anywhere between.
+ *
+ * The second is the line split. No yielded token crosses a line boundary (FR-047), while a stored
+ * `value` region may, because the format lets a field be written across two lines and real files
+ * do it. Every token encoding in use, the Language Server Protocol's included, expresses a token
+ * as a length on a single line, so a token spanning one cannot be encoded at all. The split adds
+ * tokens and moves no boundary, so the tiling holds exactly as before, and the stored region stays
+ * whole for everything that is not being drawn.
+ *
+ * A generator, so a consumer colouring a viewport stops where it stops rather than materialising
+ * four hundred thousand tokens to read the first fifty.
+ */
+export function* classify(layer) {
+    const text = layer.text;
+    const tokens = layer.tokens;
+    const starts = tokens.starts;
+    const ends = tokens.ends;
+    // The layer's own line index, built once and shared with every position query made against it.
+    const lineStarts = lineStartsOf(layer);
+    /** The line the walk is on. It only ever moves forward, so the whole pass stays linear. */
+    let line = 0;
+    /** One token per line the span touches, split at each line start strictly inside it. */
+    function* perLine(from, to, kind) {
+        while (line + 1 < lineStarts.length && lineStarts[line + 1] <= from)
+            line += 1;
+        let at = from;
+        while (line + 1 < lineStarts.length && lineStarts[line + 1] < to) {
+            line += 1;
+            const boundary = lineStarts[line];
+            yield { start: at, end: boundary, kind };
+            at = boundary;
+        }
+        if (at < to)
+            yield { start: at, end: to, kind };
+    }
+    /** One past the last character yielded so far, which is where the next gap would begin. */
+    let covered = 0;
+    for (let index = 0; index < tokens.length; index += 1) {
+        const start = starts[index];
+        if (start > covered)
+            yield* perLine(covered, start, 'trivia');
+        const end = ends[index];
+        yield* perLine(start, end, tokens.kindAt(index));
+        covered = end;
+    }
+    if (covered < text.length)
+        yield* perLine(covered, text.length, 'trivia');
+}
+//# sourceMappingURL=classify.js.map

package/dist/syntax/classify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify.js","sourceRoot":"","sources":["../../src/syntax/classify.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAkB;IAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,+FAA+F;IAC/F,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAEvC,2FAA2F;IAC3F,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,wFAAwF;IACxF,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAY,EAAE,EAAU,EAAE,IAAe;QACzD,OAAO,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,CAAE,IAAI,IAAI;YAAE,IAAI,IAAI,CAAC,CAAC;QAChF,IAAI,EAAE,GAAG,IAAI,CAAC;QACd,OAAO,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,EAAE,EAAE,CAAC;YAClE,IAAI,IAAI,CAAC,CAAC;YACV,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAE,CAAC;YACnC,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACzC,EAAE,GAAG,QAAQ,CAAC;QAChB,CAAC;QACD,IAAI,EAAE,GAAG,EAAE;YAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IAClD,CAAC;IAED,2FAA2F;IAC3F,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;QAC7B,IAAI,KAAK,GAAG,OAAO;YAAE,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAE,CAAC;QACzB,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACjD,OAAO,GAAG,GAAG,CAAC;IAChB,CAAC;IACD,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM;QAAE,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;AAC5E,CAAC"}

package/dist/syntax/layer.d.ts

@@ -0,0 +1,67 @@
+import type { Region } from './region.js';
+import { TokenStore } from './tokens.js';
+/**
+ * One statement as the text writes it, terminated or not.
+ *
+ * Not an object in the model and carrying no schema meaning (FR-006): a statement whose type the
+ * schema never heard of, or which carries twice the fields its type defines, is still a statement.
+ * Deciding what any of it means is the reader's job, and the layer is what a reader positions
+ * against.
+ */
+export interface Statement {
+    /** The whole statement, from its first non-blank character through its terminator. */
+    readonly region: Region;
+    /** Just the type name. Empty, at the offset one would have begun, when none was written. */
+    readonly typeName: Region;
+    /** Type name text, exactly as written and not case-folded. */
+    readonly typeNameText: string;
+    /**
+     * Fields after the type name, in positional order, one region each.
+     *
+     * Index 0 is the first field after the type name, matching `RawObject.values`, which has already
+     * had the type name shifted off. A field written empty still gets a region, an empty one
+     * positioned where its value would have begun, so that positional indexing never shifts and a
+     * blank slot in the middle of an extensible group can still be pointed at.
+     *
+     * The count is what was written rather than what the type defines. A statement carrying more or
+     * fewer fields than its type is a finding, not a representation problem.
+     */
+    readonly fields: readonly Region[];
+    /** True when no terminator was found, meaning the statement runs to end of input. */
+    readonly unterminated: boolean;
+}
+/**
+ * Everything the text contains and where it was.
+ *
+ * The layer holds the text it was built from, which is what makes byte-identical reconstruction a
+ * consequence rather than a feature: a reconstruction defined as concatenating slices of that text
+ * returns the text by construction. What can actually break is the tiling, so that is what the
+ * corpus test asserts and what {@link classify} makes observable.
+ *
+ * Nothing builds a layer implicitly (FR-005). `lex` and `parseIdf` read the same characters
+ * through the same scan and construct none of this, so a caller who never names `scanIdf` pays
+ * neither its time nor its memory.
+ */
+export interface SyntaxLayer {
+    /** The text this layer was built from. Held so regions can be resolved. */
+    readonly text: string;
+    /** Statements in source order. */
+    readonly statements: readonly Statement[];
+    /** Every meaningful token in source order, packed. Whitespace is not among them. */
+    readonly tokens: TokenStore;
+}
+/**
+ * Scan IDF text into a syntax layer.
+ *
+ * Text and nothing else: no schema, because the layer records what the text says and never what it
+ * means (FR-006). It never throws, for any input, and text that violates the grammar is
+ * represented rather than stopped at (FR-004): an unterminated final statement runs to end of
+ * input and says so, a statement written without a type name still tiles, and empty text produces
+ * a layer with no statements and no tokens, which satisfies the tiling invariant vacuously.
+ *
+ * One linear pass over the shared scan, which is the same pass `lex` makes. The budget is a
+ * quarter over a plain read of the same text; the work above the scan is a push per token and a
+ * record per statement.
+ */
+export declare function scanIdf(text: string): SyntaxLayer;
+//# sourceMappingURL=layer.d.ts.map

package/dist/syntax/layer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"layer.d.ts","sourceRoot":"","sources":["../../src/syntax/layer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAkB,MAAM,aAAa,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACxB,sFAAsF;IACtF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,4FAA4F;IAC5F,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,qFAAqF;IACrF,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAChC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,QAAQ,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,CAAC;IAC1C,oFAAoF;IACpF,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC7B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAiHjD"}

package/dist/syntax/layer.js

@@ -0,0 +1,134 @@
+import { scan } from '../parse/scan.js';
+import { TokenStore } from './tokens.js';
+/**
+ * Scan IDF text into a syntax layer.
+ *
+ * Text and nothing else: no schema, because the layer records what the text says and never what it
+ * means (FR-006). It never throws, for any input, and text that violates the grammar is
+ * represented rather than stopped at (FR-004): an unterminated final statement runs to end of
+ * input and says so, a statement written without a type name still tiles, and empty text produces
+ * a layer with no statements and no tokens, which satisfies the tiling invariant vacuously.
+ *
+ * One linear pass over the shared scan, which is the same pass `lex` makes. The budget is a
+ * quarter over a plain read of the same text; the work above the scan is a push per token and a
+ * record per statement.
+ */
+export function scanIdf(text) {
+    const tokens = new TokenStore(initialCapacity(text.length));
+    const statements = [];
+    /**
+     * Comments seen since the last field closed, as flat `[start, end]` pairs in source order.
+     *
+     * They are buffered rather than pushed on sight because a comment can interrupt a field, in
+     * which case the scan reports it before the field it sits inside: the field's own bounds are
+     * only known once the field closes. Emitting them at that point, ordered against the value,
+     * is what keeps the store in source order.
+     */
+    const comments = [];
+    /** Index into {@link comments} of the first pair not yet pushed. */
+    let nextComment = 0;
+    /** Raw text runs of the field being read, as flat `[start, end]` pairs in source order. */
+    const runs = [];
+    /** Offset the current statement opened at. */
+    let openedAt = 0;
+    /** The current statement's type name, replaced when its field closes. */
+    let typeName = EMPTY_REGION;
+    /** The current statement's fields. Reassigned per statement, so a pushed record keeps its own. */
+    let fields = [];
+    /** Push every buffered comment that begins before `offset`, keeping the store in source order. */
+    const flushComments = (offset) => {
+        while (nextComment < comments.length && comments[nextComment] < offset) {
+            tokens.push(comments[nextComment], comments[nextComment + 1], 'comment');
+            nextComment += 2;
+        }
+    };
+    scan(text, {
+        statementStart(offset) {
+            openedAt = offset;
+            fields = [];
+        },
+        /**
+         * The runs are recorded only for the sake of a field written on both sides of a comment that
+         * interrupts it. Every other reader of this scan wants the field's value, which arrives whole
+         * below.
+         */
+        fieldText(start, end) {
+            runs.push(start, end);
+        },
+        fieldEnd(index, start, end) {
+            const region = { start, end };
+            if (index === 0)
+                typeName = region;
+            else
+                fields.push(region);
+            const kind = index === 0 ? 'typeName' : 'value';
+            flushComments(start);
+            // A field written empty has an empty region, which is a position rather than a span, so it
+            // is recorded above and is no token.
+            if (end > start)
+                tokens.push(start, end, kind);
+            // Only a comment can separate two runs of one field, so a run beginning at or after the
+            // value ends is text written after an interrupting comment: `A !- why\n B,` writes both `A`
+            // and `B` into one field. The scan bounds the value at `A` deliberately, so that a value
+            // region and a comment region can never overlap, which leaves `B` reached by no region at
+            // all, and a gap holding something other than whitespace is what the tiling invariant
+            // forbids. Every other run holds the value itself or the whitespace before it, and both
+            // begin before it ends, so a well-formed field never enters the body and never allocates.
+            for (let at = 0; at < runs.length; at += 2) {
+                const runStart = runs[at];
+                if (runStart < end)
+                    continue;
+                const runEnd = runs[at + 1];
+                const raw = text.slice(runStart, runEnd);
+                const from = runStart + (raw.length - raw.trimStart().length);
+                const to = runEnd - (raw.length - raw.trimEnd().length);
+                if (from >= to)
+                    continue;
+                flushComments(from);
+                tokens.push(from, to, kind);
+            }
+            flushComments(Infinity);
+            comments.length = 0;
+            nextComment = 0;
+            runs.length = 0;
+        },
+        separator(offset) {
+            tokens.push(offset, offset + 1, 'separator');
+        },
+        terminator(offset) {
+            tokens.push(offset, offset + 1, 'terminator');
+        },
+        comment(start, end) {
+            comments.push(start, end);
+        },
+        statementEnd(end, unterminated) {
+            statements.push({
+                region: { start: openedAt, end },
+                typeName,
+                typeNameText: text.slice(typeName.start, typeName.end),
+                fields,
+                unterminated,
+            });
+        },
+    });
+    // Comments after the last terminator close no field, so nothing has flushed them. Text that is
+    // only comments reaches here having reported no statement at all.
+    flushComments(Infinity);
+    return { text, statements, tokens };
+}
+/**
+ * Stands in until a statement's first field closes, which it always does before the statement
+ * ends, so no statement is ever recorded holding this.
+ */
+const EMPTY_REGION = { start: 0, end: 0 };
+/**
+ * A starting size for the token store, so a large file does not grow it a dozen times.
+ *
+ * A commented IDF line runs to about a dozen characters per token, so one per sixteen undershoots
+ * slightly and costs at most one doubling, which is the safer side to be wrong on: a file that is
+ * mostly whitespace or mostly comment would otherwise pay for capacity it never fills.
+ */
+function initialCapacity(length) {
+    return length < 1024 ? 64 : length >> 4;
+}
+//# sourceMappingURL=layer.js.map

package/dist/syntax/layer.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"layer.js","sourceRoot":"","sources":["../../src/syntax/layer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAC;AAExC,OAAO,EAAE,UAAU,EAAkB,MAAM,aAAa,CAAC;AAsDzD;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,OAAO,CAAC,IAAY;IAClC,MAAM,MAAM,GAAG,IAAI,UAAU,CAAC,eAAe,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAC5D,MAAM,UAAU,GAAgB,EAAE,CAAC;IAEnC;;;;;;;OAOG;IACH,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,oEAAoE;IACpE,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,2FAA2F;IAC3F,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,8CAA8C;IAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,yEAAyE;IACzE,IAAI,QAAQ,GAAW,YAAY,CAAC;IACpC,kGAAkG;IAClG,IAAI,MAAM,GAAa,EAAE,CAAC;IAE1B,kGAAkG;IAClG,MAAM,aAAa,GAAG,CAAC,MAAc,EAAQ,EAAE;QAC7C,OAAO,WAAW,GAAG,QAAQ,CAAC,MAAM,IAAI,QAAQ,CAAC,WAAW,CAAE,GAAG,MAAM,EAAE,CAAC;YACxE,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAE,EAAE,QAAQ,CAAC,WAAW,GAAG,CAAC,CAAE,EAAE,SAAS,CAAC,CAAC;YAC3E,WAAW,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC,CAAC;IAEF,IAAI,CAAC,IAAI,EAAE;QACT,cAAc,CAAC,MAAM;YACnB,QAAQ,GAAG,MAAM,CAAC;YAClB,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED;;;;WAIG;QACH,SAAS,CAAC,KAAK,EAAE,GAAG;YAClB,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxB,CAAC;QAED,QAAQ,CAAC,KAAK,EAAE,KAAK,EAAE,GAAG;YACxB,MAAM,MAAM,GAAW,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;YACtC,IAAI,KAAK,KAAK,CAAC;gBAAE,QAAQ,GAAG,MAAM,CAAC;;gBAC9B,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YAEzB,MAAM,IAAI,GAAc,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,OAAO,CAAC;YAC3D,aAAa,CAAC,KAAK,CAAC,CAAC;YACrB,2FAA2F;YAC3F,qCAAqC;YACrC,IAAI,GAAG,GAAG,KAAK;gBAAE,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC;YAE/C,wFAAwF;YACxF,4FAA4F;YAC5F,yFAAyF;YACzF,0FAA0F;YAC1F,sFAAsF;YACtF,wFAAwF;YACxF,0FAA0F;YAC1F,KAAK,IAAI,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,IAAI,CAAC,MAAM,EAAE,EAAE,IAAI,CAAC,EAAE,CAAC;gBAC3C,MAAM,QAAQ,GAAG,IAAI,CAAC,EAAE,CAAE,CAAC;gBAC3B,IAAI,QAAQ,GAAG,GAAG;oBAAE,SAAS;gBAC7B,MAAM,MAAM,GAAG,IAAI,CAAC,EAAE,GAAG,CAAC,CAAE,CAAC;gBAC7B,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;gBACzC,MAAM,IAAI,GAAG,QAAQ,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,SAAS,EAAE,CAAC,MAAM,CAAC,CAAC;gBAC9D,MAAM,EAAE,GAAG,MAAM,GAAG,CAAC,GAAG,CAAC,MAAM,GAAG,GAAG,CAAC,OAAO,EAAE,CAAC,MAAM,CAAC,CAAC;gBACxD,IAAI,IAAI,IAAI,EAAE;oBAAE,SAAS;gBACzB,aAAa,CAAC,IAAI,CAAC,CAAC;gBACpB,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,CAAC;YAC9B,CAAC;YAED,aAAa,CAAC,QAAQ,CAAC,CAAC;YAExB,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC;YACpB,WAAW,GAAG,CAAC,CAAC;YAChB,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC;QAClB,CAAC;QAED,SAAS,CAAC,MAAM;YACd,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAC/C,CAAC;QAED,UAAU,CAAC,MAAM;YACf,MAAM,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,CAAC,EAAE,YAAY,CAAC,CAAC;QAChD,CAAC;QAED,OAAO,CAAC,KAAK,EAAE,GAAG;YAChB,QAAQ,CAAC,IAAI,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAC5B,CAAC;QAED,YAAY,CAAC,GAAG,EAAE,YAAY;YAC5B,UAAU,CAAC,IAAI,CAAC;gBACd,MAAM,EAAE,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,EAAE;gBAChC,QAAQ;gBACR,YAAY,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC;gBACtD,MAAM;gBACN,YAAY;aACb,CAAC,CAAC;QACL,CAAC;KACF,CAAC,CAAC;IAEH,+FAA+F;IAC/F,kEAAkE;IAClE,aAAa,CAAC,QAAQ,CAAC,CAAC;IAExB,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,MAAM,EAAE,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,YAAY,GAAW,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;AAElD;;;;;;GAMG;AACH,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC;AAC1C,CAAC"}

package/dist/syntax/region.d.ts

@@ -0,0 +1,73 @@
+/**
+ * A span of source text, half-open.
+ *
+ * `end` is one past the last character, so `text.slice(start, end)` is exactly the text the
+ * region selects, an empty region is `start === end`, and two adjacent regions share a number
+ * rather than straddling one. An empty region is what a cursor between two characters is.
+ *
+ * Offsets are indices into the JavaScript string, which are UTF-16 code units. Nothing converts
+ * them, because no consumer wants them converted: the Language Server Protocol's default
+ * `positionEncoding` is UTF-16, Monaco measures columns in UTF-16 code units, and CodeMirror
+ * addresses its document by the same offsets. The Python library counts code points instead,
+ * because Python string indices are code points; the two agree for every character below the
+ * astral planes and differ for anything above them, which in practice means an emoji in a
+ * comment. That divergence is registered rather than papered over.
+ */
+export interface Region {
+    /** Offset of the first character, into the source string. */
+    readonly start: number;
+    /** Offset one past the last character. */
+    readonly end: number;
+}
+/**
+ * A position as a human reads it, derived from an offset rather than stored.
+ *
+ * Storing line and column on every region would double its size to hold two numbers that are a
+ * function of one. Both count from one, which is what the existing findings already report; a
+ * consumer whose own convention counts a column from zero subtracts one, and that subtraction is
+ * the consumer's, because a service that guessed which convention its caller wanted would be
+ * modelling a protocol.
+ */
+export interface LineColumn {
+    /** 1-based, counting from the start of the text. */
+    readonly line: number;
+    /** 1-based, in UTF-16 code units, from the start of the line. */
+    readonly column: number;
+}
+/**
+ * The minimum a position query needs: the text its offsets index into.
+ *
+ * Declared structurally rather than by importing `SyntaxLayer`, so that positions stay usable
+ * without one and this module depends on nothing. A `SyntaxLayer` holds its text and therefore
+ * satisfies this, which is what lets `lineColumnAt(layer, offset)` read as the contract writes it.
+ */
+export interface TextSource {
+    /** The text offsets are measured into. */
+    readonly text: string;
+}
+/**
+ * The line and column an offset falls on.
+ *
+ * Binary search over the line index, so a query costs the logarithm of the line count rather than
+ * a scan of the text. An offset outside `[0, text.length]` is clamped into range rather than
+ * throwing: a cursor arrives from an editor that may be a keystroke ahead of the text this layer
+ * was built from, and refusing to answer is worse than answering about the nearest character.
+ *
+ * The column of a *statement* is the column of its first non-blank character, not of the
+ * whitespace indenting it. This function does not enforce that rule, because a statement's region
+ * already starts at that character; it is stated here because every column this feature reports
+ * obeys it, and because the conformance corpus compares findings on their line and type name and
+ * would notice the number moving.
+ */
+export declare function lineColumnAt(source: TextSource, offset: number): LineColumn;
+/**
+ * The offset a line and column names.
+ *
+ * The inverse of {@link lineColumnAt} for every position that exists, and clamped for every one
+ * that does not: a line past the end resolves on the last line, a column past the end of its line
+ * resolves at the line's last position, which is the offset of the break that ends it. Round
+ * tripping an out-of-range position therefore returns the clamped position rather than the one
+ * asked for, which is the only honest answer available.
+ */
+export declare function offsetAt(source: TextSource, position: LineColumn): number;
+//# sourceMappingURL=region.d.ts.map

package/dist/syntax/region.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"region.d.ts","sourceRoot":"","sources":["../../src/syntax/region.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AACH,MAAM,WAAW,MAAM;IACrB,6DAA6D;IAC7D,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,0CAA0C;IAC1C,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;CACtB;AAED;;;;;;;;GAQG;AACH,MAAM,WAAW,UAAU;IACzB,oDAAoD;IACpD,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,iEAAiE;IACjE,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB;AAED;;;;;;GAMG;AACH,MAAM,WAAW,UAAU;IACzB,0CAA0C;IAC1C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;CACvB;AAgCD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,CAK3E;AAED;;;;;;;;GAQG;AACH,wBAAgB,QAAQ,CAAC,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,UAAU,GAAG,MAAM,CAUzE"}