@idfkit/core 0.2.0-rc.2 → 0.3.0-rc.1

package/dist/parse/lexer.js.map

@@ -1 +1 @@
-{"version":3,"file":"lexer.js","sourceRoot":"","sources":["../../src/parse/lexer.ts"],"names":[],"mappings":"AAiEA;;;;;;;;;;;;;;;;;GAiBG;AACH,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,UAAsB,EAAE;IACxD,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IACpC,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAE3B,sEAAsE;IACtE,IAAI,MAAM,GAAa,EAAE,CAAC;IAC1B,4EAA4E;IAC5E,IAAI,MAAM,GAAa,EAAE,CAAC;IAE1B,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,aAAa,GAAG,KAAK,CAAC;IAC1B,8FAA8F;IAC9F,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB;;;;;;OAMG;IACH,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,yEAAyE;IACzE,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IAEtB,gGAAgG;IAChG,MAAM,QAAQ,GAAG,CAAC,MAAc,EAAU,EAAE,CAAC,MAAM,GAAG,SAAS,GAAG,CAAC,CAAC;IAEpE,MAAM,QAAQ,GAAG,CAAC,GAAW,EAAU,EAAE;QACvC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC;QACzC,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,MAAM,GAAG,EAAE,CAAC;QACZ,OAAO,KAAK,CAAC;IACf,CAAC,CAAC;IAEF,OAAO,KAAK,GAAG,MAAM,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;QAEzB,0FAA0F;QAC1F,6FAA6F;QAC7F,yEAAyE;QACzE,IAAI,YAAY,KAAK,CAAC,IAAI,IAAI,KAAK,SAAS,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACjE,UAAU,GAAG,IAAI,CAAC;YAClB,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC/B,YAAY,GAAG,KAAK,CAAC;QACvB,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,yEAAyE;YACzE,wEAAwE;YACxE,2DAA2D;YAC3D,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3C,MAAM,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YAC1C,IAAI,OAAO,KAAK,CAAC,CAAC,EAAE,CAAC;gBACnB,KAAK,GAAG,MAAM,CAAC;gBACf,UAAU,GAAG,MAAM,CAAC;gBACpB,MAAM;YACR,CAAC;YACD,KAAK,GAAG,OAAO,GAAG,CAAC,CAAC;YACpB,UAAU,GAAG,KAAK,CAAC;YACnB,IAAI,IAAI,CAAC,CAAC;YACV,SAAS,GAAG,KAAK,CAAC;YAClB,IAAI,CAAC,aAAa,IAAI,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC;gBACpD,MAAM,GAAG,EAAE,CAAC;gBACZ,UAAU,GAAG,IAAI,CAAC;gBAClB,YAAY,GAAG,CAAC,CAAC;gBACjB,YAAY,GAAG,CAAC,CAAC,CAAC;YACpB,CAAC;YACD,SAAS;QACX,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7B,aAAa,GAAG,IAAI,CAAC;YACrB,KAAK,IAAI,CAAC,CAAC;YACX,UAAU,GAAG,KAAK,CAAC;YACnB,SAAS;QACX,CAAC;QAED,IAAI,IAAI,KAAK,GAAG,EAAE,CAAC;YACjB,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;YAC7B,KAAK,IAAI,CAAC,CAAC;YACX,UAAU,GAAG,KAAK,CAAC;YAEnB,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;YACtC,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;gBACpB,MAAM,EAAE,CAAC;oBACP,OAAO,EAAE,0BAA0B;oBACnC,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,YAAY,IAAI,SAAS;oBACjC,IAAI,EAAE,YAAY;iBACnB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,QAAQ;oBACR,MAAM;oBACN,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,YAAY,IAAI,SAAS;oBACjC,MAAM,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;iBACrD,CAAC,CAAC;YACL,CAAC;YACD,MAAM,GAAG,EAAE,CAAC;YACZ,aAAa,GAAG,KAAK,CAAC;YACtB,UAAU,GAAG,IAAI,CAAC;YAClB,YAAY,GAAG,CAAC,CAAC;YACjB,YAAY,GAAG,CAAC,CAAC,CAAC;YAClB,SAAS;QACX,CAAC;QAED,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,IAAI,IAAI,CAAC,CAAC;YACV,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC;YACtB,IACE,CAAC,aAAa;gBACd,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE;gBAC7B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAC3C,CAAC;gBACD,qEAAqE;gBACrE,MAAM,GAAG,EAAE,CAAC;gBACZ,UAAU,GAAG,KAAK,GAAG,CAAC,CAAC;gBACvB,UAAU,GAAG,IAAI,CAAC;gBAClB,YAAY,GAAG,CAAC,CAAC;gBACjB,YAAY,GAAG,CAAC,CAAC,CAAC;YACpB,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,MAAM,QAAQ,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3E,IAAI,QAAQ,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,EAAE,CAAC;YACP,OAAO,EAAE,6BAA6B,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,iBAAiB;YACzF,IAAI,EAAE,UAAU;YAChB,IAAI,EAAE,YAAY;YAClB,MAAM,EAAE,YAAY,IAAI,SAAS;YACjC,6FAA6F;YAC7F,6FAA6F;YAC7F,gDAAgD;YAChD,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;SACpB,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC"}
+{"version":3,"file":"lexer.js","sourceRoot":"","sources":["../../src/parse/lexer.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAoB,MAAM,WAAW,CAAC;AAmEnD;;;;;;;;;;GAUG;AACH,MAAM,UAAU,GAAG,CAAC,IAAY,EAAE,UAAsB,EAAE;IACxD,MAAM,SAAS,GAAG,eAAe,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACjD,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,OAAO,CAAC,CAAC;IAC9B,OAAO,SAAS,CAAC,OAAO,CAAC;AAC3B,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,eAAe,CAC7B,IAAY,EACZ,UAAsB,EAAE;IAExB,MAAM,OAAO,GAAgB,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAEpC,qFAAqF;IACrF,IAAI,MAAM,GAAa,EAAE,CAAC;IAC1B,4EAA4E;IAC5E,IAAI,MAAM,GAAa,EAAE,CAAC;IAE1B;;;;;;OAMG;IACH,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,YAAY,GAAG,CAAC,CAAC,CAAC;IAEtB,MAAM,OAAO,GAAgB;QAC3B,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,MAAM;YACjC,UAAU,GAAG,IAAI,CAAC;YAClB,YAAY,GAAG,MAAM,CAAC;YACtB,YAAY,GAAG,MAAM,CAAC;QACxB,CAAC;QAED,SAAS,CAAC,KAAK,EAAE,GAAG;YAClB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;QACtC,CAAC;QAED,QAAQ;YACN,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YACpC,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;QAED,YAAY,CAAC,IAAI,EAAE,YAAY;YAC7B,IAAI,YAAY,EAAE,CAAC;gBACjB,0FAA0F;gBAC1F,0FAA0F;gBAC1F,8EAA8E;gBAC9E,MAAM,QAAQ,GAAG,MAAM,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC;gBACpC,IAAI,QAAQ,KAAK,EAAE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBACzC,MAAM,EAAE,CAAC;wBACP,OAAO,EAAE,6BAA6B,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,iBAAiB;wBACzF,IAAI,EAAE,UAAU;wBAChB,IAAI,EAAE,YAAY;wBAClB,MAAM,EAAE,YAAY,IAAI,SAAS;wBACjC,sFAAsF;wBACtF,uFAAuF;wBACvF,6DAA6D;wBAC7D,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;qBACpB,CAAC,CAAC;gBACL,CAAC;gBACD,MAAM,GAAG,EAAE,CAAC;gBACZ,OAAO;YACT,CAAC;YAED,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;YACtC,IAAI,QAAQ,KAAK,EAAE,EAAE,CAAC;gBACpB,MAAM,EAAE,CAAC;oBACP,OAAO,EAAE,0BAA0B;oBACnC,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,YAAY,IAAI,SAAS;oBACjC,IAAI,EAAE,YAAY;iBACnB,CAAC,CAAC;YACL,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,QAAQ;oBACR,MAAM;oBACN,IAAI,EAAE,UAAU;oBAChB,MAAM,EAAE,YAAY,IAAI,SAAS;oBACjC,MAAM,EAAE,YAAY,IAAI,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,SAAS;iBACrD,CAAC,CAAC;YACL,CAAC;YACD,MAAM,GAAG,EAAE,CAAC;QACd,CAAC;KACF,CAAC;IAEF,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC"}

package/dist/parse/scan.d.ts

@@ -0,0 +1,34 @@
+/**
+ * The IDF character rules, in one place.
+ *
+ * The grammar is small:
+ *   - `!` starts a comment running to the end of its line
+ *   - `,` separates fields
+ *   - `;` terminates a statement
+ *   - everything else is field text
+ *
+ * There are no string literals and no escape sequences, so a comma cannot occur inside a field
+ * value and an `!` always starts a comment. Real files depend on both.
+ *
+ * A hand-written character walk rather than a regex. The Python library matches objects with a
+ * `(?:[^;!]*(?:![^\n]*\n)?)*?` inner loop; that is a nested quantifier, so it backtracks badly on
+ * malformed input and cannot report where the problem was. A scan is about the same amount of
+ * code, is linear in the input, and always knows its line number.
+ *
+ * **Why this file exists at all.** Every position this library reports depends on the syntax layer
+ * and the model-building read agreeing about where a field begins and ends. Two implementations of
+ * "step over a comment between a separator and its value" that differ by one character put a
+ * finding on the wrong field, and that class of bug stays invisible until a file puts a comment
+ * somewhere unusual. The rules used to exist twice, once inside `lex` and once inside the
+ * `fieldLine` helper of `idf.ts`; both are now callers of this scan (research R3).
+ *
+ * **This module is internal and stays internal.** It is not exported from the package root. The
+ * syntax layer, which is public, is what a consumer outside the package reaches for; keeping the
+ * scan itself unexported is the reason the layer ships in core rather than beside the language
+ * service, because a package boundary drawn around both halves would have forced this file into a
+ * published export purely so the other half could reach it.
+ *
+ * @internal
+ */
+export {};
+//# sourceMappingURL=scan.d.ts.map

package/dist/parse/scan.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"scan.d.ts","sourceRoot":"","sources":["../../src/parse/scan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG"}

package/dist/parse/scan.js

@@ -0,0 +1,177 @@
+/**
+ * The IDF character rules, in one place.
+ *
+ * The grammar is small:
+ *   - `!` starts a comment running to the end of its line
+ *   - `,` separates fields
+ *   - `;` terminates a statement
+ *   - everything else is field text
+ *
+ * There are no string literals and no escape sequences, so a comma cannot occur inside a field
+ * value and an `!` always starts a comment. Real files depend on both.
+ *
+ * A hand-written character walk rather than a regex. The Python library matches objects with a
+ * `(?:[^;!]*(?:![^\n]*\n)?)*?` inner loop; that is a nested quantifier, so it backtracks badly on
+ * malformed input and cannot report where the problem was. A scan is about the same amount of
+ * code, is linear in the input, and always knows its line number.
+ *
+ * **Why this file exists at all.** Every position this library reports depends on the syntax layer
+ * and the model-building read agreeing about where a field begins and ends. Two implementations of
+ * "step over a comment between a separator and its value" that differ by one character put a
+ * finding on the wrong field, and that class of bug stays invisible until a file puts a comment
+ * somewhere unusual. The rules used to exist twice, once inside `lex` and once inside the
+ * `fieldLine` helper of `idf.ts`; both are now callers of this scan (research R3).
+ *
+ * **This module is internal and stays internal.** It is not exported from the package root. The
+ * syntax layer, which is public, is what a consumer outside the package reaches for; keeping the
+ * scan itself unexported is the reason the layer ships in core rather than beside the language
+ * service, because a package boundary drawn around both halves would have forced this file into a
+ * published export purely so the other half could reach it.
+ *
+ * @internal
+ */
+const EXCLAMATION = 0x21;
+const COMMA = 0x2c;
+const SEMICOLON = 0x3b;
+const LINE_FEED = 0x0a;
+/**
+ * Walk IDF text, reporting what the handler asked for.
+ *
+ * One linear pass, no allocation of its own, and it never throws: text that violates the grammar
+ * is reported as what it is rather than stopped at.
+ *
+ * @internal
+ */
+export function scan(text, handler, options = {}) {
+    const length = text.length;
+    let index = options.from ?? 0;
+    let line = options.line ?? 1;
+    /** Offset of the current line's first character, which is what turns an offset into a column. */
+    let lineStart = index - ((options.column ?? 1) - 1);
+    /** Start of the current field's text run, moved past every comment that interrupts it. */
+    let fieldStart = index;
+    /** 0 for the type name, 1 for the first field after it. */
+    let fieldIndex = 0;
+    /** True once the current statement's first non-blank, non-comment character has been seen. */
+    let open = false;
+    /** First non-blank character of the current field's value, or -1 while it has none. */
+    let valueStart = -1;
+    /** One past the last non-blank character of the run `valueStart` falls in. */
+    let valueEnd = -1;
+    /** Line `valueStart` falls on. */
+    let valueLine = 1;
+    /** True once a comment has ended the run holding `valueStart`. */
+    let valueClosed = false;
+    /**
+     * Report the field the scan is inside, closed at `at`. False when a handler asked to stop.
+     *
+     * A field with no value text is positioned at `at`, which is the separator, the terminator, or
+     * the end of input that closed it. That is the offset a value would have begun at, because
+     * everything between the field's start and `at` was whitespace or comment.
+     */
+    const closeField = (at) => {
+        if (at > fieldStart && handler.fieldText?.(fieldStart, at) === false)
+            return false;
+        const empty = valueStart < 0;
+        const stop = handler.fieldEnd?.(fieldIndex, empty ? at : valueStart, empty ? at : valueEnd, empty ? line : valueLine) === false;
+        valueStart = -1;
+        valueEnd = -1;
+        valueClosed = false;
+        return !stop;
+    };
+    while (index < length) {
+        const code = text.charCodeAt(index);
+        if (code === EXCLAMATION) {
+            // Text seen before the comment still belongs to the field, and so does text after the line
+            // feed: the comment interrupts the field without ending it. This is what lets
+            // `Zone1,  !- Name` work, where the comment is no part of the value and the value is no part
+            // of the comment.
+            if (index > fieldStart && handler.fieldText?.(fieldStart, index) === false)
+                return;
+            const feed = text.indexOf('\n', index);
+            const end = feed === -1 ? length : feed;
+            if (handler.comment?.(index, end) === false)
+                return;
+            if (valueStart >= 0)
+                valueClosed = true;
+            if (feed === -1) {
+                fieldStart = length;
+                break;
+            }
+            index = feed + 1;
+            fieldStart = index;
+            line += 1;
+            lineStart = index;
+            continue;
+        }
+        const blank = isSpace(code);
+        if (!open && !blank) {
+            open = true;
+            if (handler.statementStart?.(index, line, index - lineStart + 1) === false)
+                return;
+        }
+        if (code === COMMA || code === SEMICOLON) {
+            if (!closeField(index))
+                return;
+            if (code === COMMA) {
+                if (handler.separator?.(index) === false)
+                    return;
+                fieldIndex += 1;
+            }
+            else {
+                if (handler.terminator?.(index) === false)
+                    return;
+                if (handler.statementEnd?.(index + 1, false) === false)
+                    return;
+                fieldIndex = 0;
+                open = false;
+            }
+            index += 1;
+            fieldStart = index;
+            continue;
+        }
+        if (code === LINE_FEED) {
+            line += 1;
+            lineStart = index + 1;
+        }
+        else if (!blank) {
+            if (valueStart < 0) {
+                valueStart = index;
+                valueEnd = index + 1;
+                valueLine = line;
+            }
+            else if (!valueClosed) {
+                valueEnd = index + 1;
+            }
+        }
+        index += 1;
+    }
+    // Input ran out inside a statement. Its last field is reported like any other, so a caller reads
+    // what was written rather than having to reconstruct it from the leftovers.
+    if (!open)
+        return;
+    if (!closeField(length))
+        return;
+    handler.statementEnd?.(length, true);
+}
+/**
+ * Whether a character is whitespace, by the same definition `String.prototype.trim` uses.
+ *
+ * A field's value is trimmed, so a scan that disagreed with `trim` about one character would put
+ * a region beside the value rather than on it. ASCII is one comparison; the rest of the set costs
+ * a branch nothing outside a comment ever takes.
+ */
+function isSpace(code) {
+    if (code < 0x80)
+        return code === 0x20 || (code >= 0x09 && code <= 0x0d);
+    return (code === 0xa0 ||
+        code === 0x1680 ||
+        (code >= 0x2000 && code <= 0x200a) ||
+        code === 0x2028 ||
+        code === 0x2029 ||
+        code === 0x202f ||
+        code === 0x205f ||
+        code === 0x3000 ||
+        code === 0xfeff);
+}
+//# sourceMappingURL=scan.js.map

package/dist/parse/scan.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"scan.js","sourceRoot":"","sources":["../../src/parse/scan.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA+BG;AAEH,MAAM,WAAW,GAAG,IAAI,CAAC;AACzB,MAAM,KAAK,GAAG,IAAI,CAAC;AACnB,MAAM,SAAS,GAAG,IAAI,CAAC;AACvB,MAAM,SAAS,GAAG,IAAI,CAAC;AAwGvB;;;;;;;GAOG;AACH,MAAM,UAAU,IAAI,CAAC,IAAY,EAAE,OAAoB,EAAE,UAAuB,EAAE;IAChF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;IAE3B,IAAI,KAAK,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAC9B,IAAI,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC;IAC7B,iGAAiG;IACjG,IAAI,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAEpD,0FAA0F;IAC1F,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,2DAA2D;IAC3D,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,8FAA8F;IAC9F,IAAI,IAAI,GAAG,KAAK,CAAC;IACjB,uFAAuF;IACvF,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC;IACpB,8EAA8E;IAC9E,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,kCAAkC;IAClC,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,kEAAkE;IAClE,IAAI,WAAW,GAAG,KAAK,CAAC;IAExB;;;;;;OAMG;IACH,MAAM,UAAU,GAAG,CAAC,EAAU,EAAW,EAAE;QACzC,IAAI,EAAE,GAAG,UAAU,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,UAAU,EAAE,EAAE,CAAC,KAAK,KAAK;YAAE,OAAO,KAAK,CAAC;QACnF,MAAM,KAAK,GAAG,UAAU,GAAG,CAAC,CAAC;QAC7B,MAAM,IAAI,GACR,OAAO,CAAC,QAAQ,EAAE,CAChB,UAAU,EACV,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,EACvB,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,QAAQ,EACrB,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CACzB,KAAK,KAAK,CAAC;QACd,UAAU,GAAG,CAAC,CAAC,CAAC;QAChB,QAAQ,GAAG,CAAC,CAAC,CAAC;QACd,WAAW,GAAG,KAAK,CAAC;QACpB,OAAO,CAAC,IAAI,CAAC;IACf,CAAC,CAAC;IAEF,OAAO,KAAK,GAAG,MAAM,EAAE,CAAC;QACtB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;QAEpC,IAAI,IAAI,KAAK,WAAW,EAAE,CAAC;YACzB,2FAA2F;YAC3F,8EAA8E;YAC9E,6FAA6F;YAC7F,kBAAkB;YAClB,IAAI,KAAK,GAAG,UAAU,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,UAAU,EAAE,KAAK,CAAC,KAAK,KAAK;gBAAE,OAAO;YACnF,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;YACvC,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YACxC,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,KAAK;gBAAE,OAAO;YACpD,IAAI,UAAU,IAAI,CAAC;gBAAE,WAAW,GAAG,IAAI,CAAC;YACxC,IAAI,IAAI,KAAK,CAAC,CAAC,EAAE,CAAC;gBAChB,UAAU,GAAG,MAAM,CAAC;gBACpB,MAAM;YACR,CAAC;YACD,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC;YACjB,UAAU,GAAG,KAAK,CAAC;YACnB,IAAI,IAAI,CAAC,CAAC;YACV,SAAS,GAAG,KAAK,CAAC;YAClB,SAAS;QACX,CAAC;QAED,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;QAE5B,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,EAAE,CAAC;YACpB,IAAI,GAAG,IAAI,CAAC;YACZ,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,GAAG,SAAS,GAAG,CAAC,CAAC,KAAK,KAAK;gBAAE,OAAO;QACrF,CAAC;QAED,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACzC,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;gBAAE,OAAO;YAC/B,IAAI,IAAI,KAAK,KAAK,EAAE,CAAC;gBACnB,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK;oBAAE,OAAO;gBACjD,UAAU,IAAI,CAAC,CAAC;YAClB,CAAC;iBAAM,CAAC;gBACN,IAAI,OAAO,CAAC,UAAU,EAAE,CAAC,KAAK,CAAC,KAAK,KAAK;oBAAE,OAAO;gBAClD,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC,KAAK,GAAG,CAAC,EAAE,KAAK,CAAC,KAAK,KAAK;oBAAE,OAAO;gBAC/D,UAAU,GAAG,CAAC,CAAC;gBACf,IAAI,GAAG,KAAK,CAAC;YACf,CAAC;YACD,KAAK,IAAI,CAAC,CAAC;YACX,UAAU,GAAG,KAAK,CAAC;YACnB,SAAS;QACX,CAAC;QAED,IAAI,IAAI,KAAK,SAAS,EAAE,CAAC;YACvB,IAAI,IAAI,CAAC,CAAC;YACV,SAAS,GAAG,KAAK,GAAG,CAAC,CAAC;QACxB,CAAC;aAAM,IAAI,CAAC,KAAK,EAAE,CAAC;YAClB,IAAI,UAAU,GAAG,CAAC,EAAE,CAAC;gBACnB,UAAU,GAAG,KAAK,CAAC;gBACnB,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC;gBACrB,SAAS,GAAG,IAAI,CAAC;YACnB,CAAC;iBAAM,IAAI,CAAC,WAAW,EAAE,CAAC;gBACxB,QAAQ,GAAG,KAAK,GAAG,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,KAAK,IAAI,CAAC,CAAC;IACb,CAAC;IAED,iGAAiG;IACjG,4EAA4E;IAC5E,IAAI,CAAC,IAAI;QAAE,OAAO;IAClB,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;QAAE,OAAO;IAChC,OAAO,CAAC,YAAY,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;GAMG;AACH,SAAS,OAAO,CAAC,IAAY;IAC3B,IAAI,IAAI,GAAG,IAAI;QAAE,OAAO,IAAI,KAAK,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC,CAAC;IACxE,OAAO,CACL,IAAI,KAAK,IAAI;QACb,IAAI,KAAK,MAAM;QACf,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;QAClC,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM;QACf,IAAI,KAAK,MAAM,CAChB,CAAC;AACJ,CAAC"}

package/dist/preserve/source.d.ts

@@ -0,0 +1,3 @@
+import type { IdfObject } from '../object.js';
+export declare function isUntouched(obj: IdfObject, source: PreservedSource | undefined): boolean;
+//# sourceMappingURL=source.d.ts.map

package/dist/preserve/source.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"source.d.ts","sourceRoot":"","sources":["../../src/preserve/source.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AA8E9C,wBAAgB,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,GAAG,SAAS,GAAG,OAAO,CAKxF"}

package/dist/preserve/source.js

@@ -0,0 +1,60 @@
+import { SOURCE } from '../internal.js';
+/**
+ * Whether an object is still exactly the characters it was read from.
+ *
+ * The third clause is not redundant: an object carrying an index from a file it is no longer in
+ * fails the identity check, which is what makes a document assembled from more than one source
+ * fall out rather than need handling.
+ *
+ * @internal
+ */
+export function isWholeDocumentUntouched(document, source) {
+    if (document.size !== source.countAtRead)
+        return false;
+    for (const obj of document.objects()) {
+        if (!isUntouched(obj, source))
+            return false;
+    }
+    return true;
+}
+export function isUntouched(obj, source) {
+    if (source === undefined)
+        return false;
+    const at = obj[SOURCE];
+    if (at === undefined)
+        return false;
+    return source.anchors[at] === obj;
+}
+/**
+ * The statement each raw object was read from, by index into `objects`.
+ *
+ * The lexer records one offset per object and the layer records the same offset as the start of
+ * that statement's region, so the two match on a number both already hold. One pass with two
+ * cursors: the lexer's sequence is a subsequence of the layer's, because some statements produce
+ * no object.
+ *
+ * An object matching no statement gets `undefined` and is written by formatting, which is the
+ * honest answer for one whose characters cannot be located.
+ *
+ * @internal
+ */
+export function statementIndexes(layer, objects) {
+    const found = new Array(objects.length).fill(undefined);
+    const statements = layer.statements;
+    let at = 0;
+    for (let index = 0; index < objects.length; index += 1) {
+        const offset = objects[index]?.offset;
+        if (offset === undefined)
+            continue;
+        while (at < statements.length && statements[at].region.start < offset)
+            at += 1;
+        if (at >= statements.length)
+            break;
+        if (statements[at].region.start === offset) {
+            found[index] = at;
+            at += 1;
+        }
+    }
+    return found;
+}
+//# sourceMappingURL=source.js.map

package/dist/preserve/source.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"source.js","sourceRoot":"","sources":["../../src/preserve/source.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AA2DxC;;;;;;;;GAQG;AACH,MAAM,UAAU,wBAAwB,CACtC,QAA8D,EAC9D,MAAuB;IAEvB,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,CAAC,WAAW;QAAE,OAAO,KAAK,CAAC;IACvD,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACrC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,MAAM,CAAC;YAAE,OAAO,KAAK,CAAC;IAC9C,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,MAAM,UAAU,WAAW,CAAC,GAAc,EAAE,MAAmC;IAC7E,IAAI,MAAM,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACvC,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC;IACvB,IAAI,EAAE,KAAK,SAAS;QAAE,OAAO,KAAK,CAAC;IACnC,OAAO,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,KAAK,GAAG,CAAC;AACpC,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,gBAAgB,CAC9B,KAAkB,EAClB,OAA6B;IAE7B,MAAM,KAAK,GAA2B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAChF,MAAM,UAAU,GAAG,KAAK,CAAC,UAAU,CAAC;IACpC,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACvD,MAAM,MAAM,GAAG,OAAO,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;QACtC,IAAI,MAAM,KAAK,SAAS;YAAE,SAAS;QACnC,OAAO,EAAE,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,EAAE,CAAE,CAAC,MAAM,CAAC,KAAK,GAAG,MAAM;YAAE,EAAE,IAAI,CAAC,CAAC;QAChF,IAAI,EAAE,IAAI,UAAU,CAAC,MAAM;YAAE,MAAM;QACnC,IAAI,UAAU,CAAC,EAAE,CAAE,CAAC,MAAM,CAAC,KAAK,KAAK,MAAM,EAAE,CAAC;YAC5C,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;YAClB,EAAE,IAAI,CAAC,CAAC;QACV,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/preserve/write.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=write.d.ts.map

package/dist/preserve/write.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"write.d.ts","sourceRoot":"","sources":["../../src/preserve/write.ts"],"names":[],"mappings":""}

package/dist/preserve/write.js

@@ -0,0 +1,85 @@
+import { OWNER } from '../internal.js';
+import { writeObject } from '../write/idf.js';
+import { isUntouched } from './source.js';
+/**
+ * Reproduce the text a document was read from, per object.
+ *
+ * A walk over the statements the read scanned and the gaps between them. The gaps are
+ * unconditional and only the statements are decided: every character is either inside a statement
+ * or in a gap, because the layer tiles the text, and nothing in a gap belongs to an object. That
+ * is what makes the one-object diff structural rather than careful, and it is why nothing is
+ * reordered, no header is added and the version statement is not moved.
+ *
+ * A statement's region ends at its terminator, so a comment trailing the semicolon on the same
+ * line is in the gap: removing an object leaves it, and reformatting one leaves it below the new
+ * text. Behaviour rather than defects, and the alternative is a writer that guesses which comments
+ * are about which object.
+ *
+ * @internal
+ */
+export function writePreserved(document, source, options) {
+    const text = source.layer.text;
+    const statements = source.layer.statements;
+    const parts = [];
+    // Everything before the first statement, which for a file with none is the whole text: an empty
+    // file and a file of comments are both reproduced by this line alone.
+    parts.push(text.slice(0, statements[0]?.region.start ?? text.length));
+    for (let index = 0; index < statements.length; index += 1) {
+        const statement = statements[index];
+        parts.push(statementPart(source, index, text, options));
+        // Unconditional: the gap is emitted whether the statement was copied, reformatted or dropped.
+        parts.push(text.slice(statement.region.end, statements[index + 1]?.region.start ?? text.length));
+    }
+    appendNewObjects(document, source, parts, options);
+    return parts.join('');
+}
+/**
+ * An anchor that is `undefined` is a statement the read rejected, an unknown type or a duplicate
+ * name. Its characters are reproduced: the read already reported a diagnostic, and the write is
+ * not the place to delete text the author wrote.
+ */
+function statementPart(source, index, text, options) {
+    const statement = source.layer.statements[index];
+    const verbatim = text.slice(statement.region.start, statement.region.end);
+    const anchored = source.anchors[index];
+    if (anchored === undefined)
+        return verbatim;
+    // Removal is answered from ownership: `remove` already clears the owner, and recording it on
+    // the object would mean holding a reference to something the document has let go.
+    if (anchored[OWNER] === undefined)
+        return '';
+    if (isUntouched(anchored, source))
+        return verbatim;
+    return writeObject(anchored, options);
+}
+/**
+ * Objects no anchor names, formatted and appended after everything the layer holds.
+ *
+ * The end is the only placement that cannot disturb text the author wrote. The newline guard is
+ * the file-with-no-trailing-newline case: appending must not run onto the author's last line.
+ */
+function appendNewObjects(document, source, parts, options) {
+    const anchored = new Set(source.anchors);
+    // The last non-empty part rather than a join of everything so far, which would make appending N
+    // objects cost N passes over the whole file.
+    let tail = lastNonEmpty(parts);
+    for (const obj of document.objects()) {
+        if (anchored.has(obj))
+            continue;
+        if (tail !== '' && !tail.endsWith('\n'))
+            parts.push('\n');
+        parts.push(writeObject(obj, options));
+        parts.push('\n');
+        tail = '\n';
+    }
+}
+/** The last part that carries a character, which is what decides whether the output ends in one. */
+function lastNonEmpty(parts) {
+    for (let index = parts.length - 1; index >= 0; index -= 1) {
+        const part = parts[index];
+        if (part !== '')
+            return part;
+    }
+    return '';
+}
+//# sourceMappingURL=write.js.map

package/dist/preserve/write.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"write.js","sourceRoot":"","sources":["../../src/preserve/write.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAEvC,OAAO,EAAE,WAAW,EAA2B,MAAM,iBAAiB,CAAC;AACvE,OAAO,EAAE,WAAW,EAAwB,MAAM,aAAa,CAAC;AAEhE;;;;;;;;;;;;;;;GAeG;AACH,MAAM,UAAU,cAAc,CAC5B,QAAgD,EAChD,MAAuB,EACvB,OAA2B;IAE3B,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;IAC/B,MAAM,UAAU,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC;IAC3C,MAAM,KAAK,GAAa,EAAE,CAAC;IAE3B,gGAAgG;IAChG,sEAAsE;IACtE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;IAEtE,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,UAAU,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,SAAS,GAAG,UAAU,CAAC,KAAK,CAAE,CAAC;QACrC,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC,CAAC;QACxD,8FAA8F;QAC9F,KAAK,CAAC,IAAI,CACR,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,GAAG,EAAE,UAAU,CAAC,KAAK,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,CACrF,CAAC;IACJ,CAAC;IAED,gBAAgB,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;IACnD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CACpB,MAAuB,EACvB,KAAa,EACb,IAAY,EACZ,OAA2B;IAE3B,MAAM,SAAS,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,KAAK,CAAE,CAAC;IAClD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC1E,MAAM,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IAC5C,6FAA6F;IAC7F,kFAAkF;IAClF,IAAI,QAAQ,CAAC,KAAK,CAAC,KAAK,SAAS;QAAE,OAAO,EAAE,CAAC;IAC7C,IAAI,WAAW,CAAC,QAAQ,EAAE,MAAM,CAAC;QAAE,OAAO,QAAQ,CAAC;IACnD,OAAO,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;AACxC,CAAC;AAED;;;;;GAKG;AACH,SAAS,gBAAgB,CACvB,QAAgD,EAChD,MAAuB,EACvB,KAAe,EACf,OAA2B;IAE3B,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACzC,gGAAgG;IAChG,6CAA6C;IAC7C,IAAI,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAC/B,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC;QACrC,IAAI,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAChC,IAAI,IAAI,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1D,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,GAAG,EAAE,OAAO,CAAC,CAAC,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjB,IAAI,GAAG,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,oGAAoG;AACpG,SAAS,YAAY,CAAC,KAAwB;IAC5C,KAAK,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAE,CAAC;QAC3B,IAAI,IAAI,KAAK,EAAE;YAAE,OAAO,IAAI,CAAC;IAC/B,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC"}

package/dist/syntax/classify.d.ts

@@ -0,0 +1,25 @@
+import type { SyntaxLayer } from './layer.js';
+import type { Token } from './tokens.js';
+/**
+ * Every character of the text, as tokens, in source order.
+ *
+ * Two things happen here that the layer deliberately does not pay to store.
+ *
+ * The first is `trivia`. Whitespace is the complement of the stored tokens, so storing it would
+ * roughly double the token count to hold something derivable by subtraction. This function fills
+ * each gap as it walks, which is what makes the layer's complete coverage observable (FR-016)
+ * without the layer carrying it: the sequence begins at offset 0, ends at `text.length`, and has
+ * no gap and no overlap anywhere between.
+ *
+ * The second is the line split. No yielded token crosses a line boundary (FR-047), while a stored
+ * `value` region may, because the format lets a field be written across two lines and real files
+ * do it. Every token encoding in use, the Language Server Protocol's included, expresses a token
+ * as a length on a single line, so a token spanning one cannot be encoded at all. The split adds
+ * tokens and moves no boundary, so the tiling holds exactly as before, and the stored region stays
+ * whole for everything that is not being drawn.
+ *
+ * A generator, so a consumer colouring a viewport stops where it stops rather than materialising
+ * four hundred thousand tokens to read the first fifty.
+ */
+export declare function classify(layer: SyntaxLayer): Iterable<Token>;
+//# sourceMappingURL=classify.d.ts.map

package/dist/syntax/classify.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify.d.ts","sourceRoot":"","sources":["../../src/syntax/classify.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAE9C,OAAO,KAAK,EAAE,KAAK,EAAa,MAAM,aAAa,CAAC;AAEpD;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,wBAAiB,QAAQ,CAAC,KAAK,EAAE,WAAW,GAAG,QAAQ,CAAC,KAAK,CAAC,CAkC7D"}

package/dist/syntax/classify.js

@@ -0,0 +1,59 @@
+import { lineStartsOf } from './region.js';
+/**
+ * Every character of the text, as tokens, in source order.
+ *
+ * Two things happen here that the layer deliberately does not pay to store.
+ *
+ * The first is `trivia`. Whitespace is the complement of the stored tokens, so storing it would
+ * roughly double the token count to hold something derivable by subtraction. This function fills
+ * each gap as it walks, which is what makes the layer's complete coverage observable (FR-016)
+ * without the layer carrying it: the sequence begins at offset 0, ends at `text.length`, and has
+ * no gap and no overlap anywhere between.
+ *
+ * The second is the line split. No yielded token crosses a line boundary (FR-047), while a stored
+ * `value` region may, because the format lets a field be written across two lines and real files
+ * do it. Every token encoding in use, the Language Server Protocol's included, expresses a token
+ * as a length on a single line, so a token spanning one cannot be encoded at all. The split adds
+ * tokens and moves no boundary, so the tiling holds exactly as before, and the stored region stays
+ * whole for everything that is not being drawn.
+ *
+ * A generator, so a consumer colouring a viewport stops where it stops rather than materialising
+ * four hundred thousand tokens to read the first fifty.
+ */
+export function* classify(layer) {
+    const text = layer.text;
+    const tokens = layer.tokens;
+    const starts = tokens.starts;
+    const ends = tokens.ends;
+    // The layer's own line index, built once and shared with every position query made against it.
+    const lineStarts = lineStartsOf(layer);
+    /** The line the walk is on. It only ever moves forward, so the whole pass stays linear. */
+    let line = 0;
+    /** One token per line the span touches, split at each line start strictly inside it. */
+    function* perLine(from, to, kind) {
+        while (line + 1 < lineStarts.length && lineStarts[line + 1] <= from)
+            line += 1;
+        let at = from;
+        while (line + 1 < lineStarts.length && lineStarts[line + 1] < to) {
+            line += 1;
+            const boundary = lineStarts[line];
+            yield { start: at, end: boundary, kind };
+            at = boundary;
+        }
+        if (at < to)
+            yield { start: at, end: to, kind };
+    }
+    /** One past the last character yielded so far, which is where the next gap would begin. */
+    let covered = 0;
+    for (let index = 0; index < tokens.length; index += 1) {
+        const start = starts[index];
+        if (start > covered)
+            yield* perLine(covered, start, 'trivia');
+        const end = ends[index];
+        yield* perLine(start, end, tokens.kindAt(index));
+        covered = end;
+    }
+    if (covered < text.length)
+        yield* perLine(covered, text.length, 'trivia');
+}
+//# sourceMappingURL=classify.js.map

package/dist/syntax/classify.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"classify.js","sourceRoot":"","sources":["../../src/syntax/classify.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAG3C;;;;;;;;;;;;;;;;;;;;GAoBG;AACH,MAAM,SAAS,CAAC,CAAC,QAAQ,CAAC,KAAkB;IAC1C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;IACxB,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC;IAC7B,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC;IACzB,+FAA+F;IAC/F,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC;IAEvC,2FAA2F;IAC3F,IAAI,IAAI,GAAG,CAAC,CAAC;IAEb,wFAAwF;IACxF,QAAQ,CAAC,CAAC,OAAO,CAAC,IAAY,EAAE,EAAU,EAAE,IAAe;QACzD,OAAO,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,CAAE,IAAI,IAAI;YAAE,IAAI,IAAI,CAAC,CAAC;QAChF,IAAI,EAAE,GAAG,IAAI,CAAC;QACd,OAAO,IAAI,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,IAAI,UAAU,CAAC,IAAI,GAAG,CAAC,CAAE,GAAG,EAAE,EAAE,CAAC;YAClE,IAAI,IAAI,CAAC,CAAC;YACV,MAAM,QAAQ,GAAG,UAAU,CAAC,IAAI,CAAE,CAAC;YACnC,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;YACzC,EAAE,GAAG,QAAQ,CAAC;QAChB,CAAC;QACD,IAAI,EAAE,GAAG,EAAE;YAAE,MAAM,EAAE,KAAK,EAAE,EAAE,EAAE,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC;IAClD,CAAC;IAED,2FAA2F;IAC3F,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACtD,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAE,CAAC;QAC7B,IAAI,KAAK,GAAG,OAAO;YAAE,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,KAAK,EAAE,QAAQ,CAAC,CAAC;QAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,KAAK,CAAE,CAAC;QACzB,KAAK,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QACjD,OAAO,GAAG,GAAG,CAAC;IAChB,CAAC;IACD,IAAI,OAAO,GAAG,IAAI,CAAC,MAAM;QAAE,KAAK,CAAC,CAAC,OAAO,CAAC,OAAO,EAAE,IAAI,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;AAC5E,CAAC"}

package/dist/syntax/layer.d.ts

@@ -0,0 +1,67 @@
+import type { Region } from './region.js';
+import { TokenStore } from './tokens.js';
+/**
+ * One statement as the text writes it, terminated or not.
+ *
+ * Not an object in the model and carrying no schema meaning (FR-006): a statement whose type the
+ * schema never heard of, or which carries twice the fields its type defines, is still a statement.
+ * Deciding what any of it means is the reader's job, and the layer is what a reader positions
+ * against.
+ */
+export interface Statement {
+    /** The whole statement, from its first non-blank character through its terminator. */
+    readonly region: Region;
+    /** Just the type name. Empty, at the offset one would have begun, when none was written. */
+    readonly typeName: Region;
+    /** Type name text, exactly as written and not case-folded. */
+    readonly typeNameText: string;
+    /**
+     * Fields after the type name, in positional order, one region each.
+     *
+     * Index 0 is the first field after the type name, matching `RawObject.values`, which has already
+     * had the type name shifted off. A field written empty still gets a region, an empty one
+     * positioned where its value would have begun, so that positional indexing never shifts and a
+     * blank slot in the middle of an extensible group can still be pointed at.
+     *
+     * The count is what was written rather than what the type defines. A statement carrying more or
+     * fewer fields than its type is a finding, not a representation problem.
+     */
+    readonly fields: readonly Region[];
+    /** True when no terminator was found, meaning the statement runs to end of input. */
+    readonly unterminated: boolean;
+}
+/**
+ * Everything the text contains and where it was.
+ *
+ * The layer holds the text it was built from, which is what makes byte-identical reconstruction a
+ * consequence rather than a feature: a reconstruction defined as concatenating slices of that text
+ * returns the text by construction. What can actually break is the tiling, so that is what the
+ * corpus test asserts and what {@link classify} makes observable.
+ *
+ * Nothing builds a layer implicitly (FR-005). `lex` and `parseIdf` read the same characters
+ * through the same scan and construct none of this, so a caller who never names `scanIdf` pays
+ * neither its time nor its memory.
+ */
+export interface SyntaxLayer {
+    /** The text this layer was built from. Held so regions can be resolved. */
+    readonly text: string;
+    /** Statements in source order. */
+    readonly statements: readonly Statement[];
+    /** Every meaningful token in source order, packed. Whitespace is not among them. */
+    readonly tokens: TokenStore;
+}
+/**
+ * Scan IDF text into a syntax layer.
+ *
+ * Text and nothing else: no schema, because the layer records what the text says and never what it
+ * means (FR-006). It never throws, for any input, and text that violates the grammar is
+ * represented rather than stopped at (FR-004): an unterminated final statement runs to end of
+ * input and says so, a statement written without a type name still tiles, and empty text produces
+ * a layer with no statements and no tokens, which satisfies the tiling invariant vacuously.
+ *
+ * One linear pass over the shared scan, which is the same pass `lex` makes. The budget is a
+ * quarter over a plain read of the same text; the work above the scan is a push per token and a
+ * record per statement.
+ */
+export declare function scanIdf(text: string): SyntaxLayer;
+//# sourceMappingURL=layer.d.ts.map

package/dist/syntax/layer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"layer.d.ts","sourceRoot":"","sources":["../../src/syntax/layer.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAC1C,OAAO,EAAE,UAAU,EAAkB,MAAM,aAAa,CAAC;AAEzD;;;;;;;GAOG;AACH,MAAM,WAAW,SAAS;IACxB,sFAAsF;IACtF,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,4FAA4F;IAC5F,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,8DAA8D;IAC9D,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B;;;;;;;;;;OAUG;IACH,QAAQ,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,CAAC;IACnC,qFAAqF;IACrF,QAAQ,CAAC,YAAY,EAAE,OAAO,CAAC;CAChC;AAED;;;;;;;;;;;GAWG;AACH,MAAM,WAAW,WAAW;IAC1B,2EAA2E;IAC3E,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,kCAAkC;IAClC,QAAQ,CAAC,UAAU,EAAE,SAAS,SAAS,EAAE,CAAC;IAC1C,oFAAoF;IACpF,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;CAC7B;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,WAAW,CAIjD"}