@limaformat/lima 0.3.1 → 0.5.0

package/dist/reference-tokens2.d.ts

@@ -1,12 +1,43 @@
 /** Reference tokens recorded while the annotated Core tree is being built. */
-import type { StringSourceSpan } from './builder.js';
 export type ReferenceToken2 = {
     token: string;
+    /**
+     * Index of the token within the decoded scalar value — the splice point
+     * for interpolation. A UTF-16 offset (JS `String.slice` units); stays
+     * implementation-local, unlike `line`/`offset`.
+     */
     index: number;
-    offset: number;
+    /** Physical 1-based source line of the token (References 2.0 §2.4). */
     line: number;
+    /**
+     * 0-based **codepoint** offset of the token within its source line — the
+     * `column - 1` a diagnostic reports, and the tie-break key for §5 error
+     * ordering. Identical across implementations.
+     */
+    offset: number;
     documentPath?: string;
     partialPath?: string;
 };
+/**
+ * Where a scalar begins in the source: physical 1-based `line` and 0-based
+ * codepoint column `col` of its first character. A token's position is this
+ * anchor plus its codepoint offset within the value.
+ *
+ * `raw` is the value text a token's position is *read from*, before comment
+ * stripping and `\#` collapse (so `x: a\#b ${t}` reports `${t}` at its real
+ * column) — for a block scalar it is the original body lines joined with
+ * `\n` (§2.4 forbids reconstructing a token's line from the `^^`-merged
+ * string). Omitted only when it equals the decoded value.
+ *
+ * `tabAdjust[i]` is the column count Core §3 added to physical line
+ * `line + i` by leading-tab expansion; it is subtracted so the reported
+ * column is the *original*-source column (§2.4).
+ */
+export type ReferenceSource = {
+    raw?: string;
+    line: number;
+    col: number;
+    tabAdjust?: number[];
+};
 export declare const PURE_REFERENCE_2: RegExp;
-export declare const scanReferenceTokens2: (value: string, firstLine: number, sourceSpans?: StringSourceSpan[]) => ReferenceToken2[];
+export declare const scanReferenceTokens2: (value: string, source: ReferenceSource | undefined) => ReferenceToken2[];

package/dist/reference-tokens2.js

@@ -1,3 +1,4 @@
+/** Reference tokens recorded while the annotated Core tree is being built. */
 const SEGMENT = '[a-zA-Z0-9_][a-zA-Z0-9_:-]*';
 const DOC_PATH = `${SEGMENT}(?:\\.${SEGMENT})*`;
 const PARTIAL_NAME = '[a-zA-Z0-9_][a-zA-Z0-9_:/-]*';
@@ -5,33 +6,65 @@ const PARTIAL_PATH = `${PARTIAL_NAME}(?:\\.${SEGMENT})*`;
 export const PURE_REFERENCE_2 = new RegExp(`^(?:\\$\\{(${DOC_PATH})\\}|\\$\\((${PARTIAL_PATH})\\))$`);
 const REFERENCE_2 = new RegExp(`\\$\\{(${DOC_PATH})\\}|\\$\\((${PARTIAL_PATH})\\)`, 'g');
 const NO_TOKENS = [];
-export const scanReferenceTokens2 = (value, firstLine, sourceSpans) => {
-    if (!value.includes('${') && !value.includes('$('))
-        return NO_TOKENS;
+/** Codepoint length of the UTF-16 slice `[a, b)` of `s`. */
+const codepointLen = (s, a, b) => {
+    let n = 0;
+    for (let i = a; i < b;) {
+        i += s.codePointAt(i) > 0xffff ? 2 : 1;
+        n++;
+    }
+    return n;
+};
+/** Splice points (decoded value order): token text, UTF-16 index, path. */
+const scanDecoded = (value) => {
     REFERENCE_2.lastIndex = 0;
-    const result = [];
-    let line = firstLine;
-    let lineStart = 0;
+    const out = [];
+    for (const m of value.matchAll(REFERENCE_2)) {
+        out.push({
+            token: m[0], index: m.index ?? 0,
+            ...(m[1] !== undefined ? { documentPath: m[1] } : { partialPath: m[2] }),
+        });
+    }
+    return out;
+};
+/** Physical `(line, offset)` of each token, in source order, from the raw text. */
+const scanPhysical = (raw, firstLine, firstCol, tabAdjust) => {
+    REFERENCE_2.lastIndex = 0;
+    const out = [];
     let scanned = 0;
-    let spanIndex = 0;
-    for (const match of value.matchAll(REFERENCE_2)) {
-        const index = match.index ?? 0;
-        while (scanned < index) {
-            if (value.charCodeAt(scanned) === 10) {
+    let line = firstLine;
+    let col = firstCol;
+    let lineIndex = 0;
+    for (const m of raw.matchAll(REFERENCE_2)) {
+        const idx = m.index ?? 0;
+        while (scanned < idx) {
+            const cp = raw.codePointAt(scanned);
+            if (cp === 10) {
                 line++;
-                lineStart = scanned + 1;
+                col = 0;
+                lineIndex++;
             }
-            scanned++;
+            else
+                col++;
+            scanned += cp > 0xffff ? 2 : 1;
         }
-        while (sourceSpans && spanIndex + 1 < sourceSpans.length && sourceSpans[spanIndex + 1].start <= index)
-            spanIndex++;
-        const source = sourceSpans?.[spanIndex];
-        result.push({
-            token: match[0], index,
-            line: source?.line ?? line,
-            offset: source ? source.sourceOffset + index - source.start : index - lineStart,
-            ...(match[1] !== undefined ? { documentPath: match[1] } : { partialPath: match[2] }),
-        });
+        out.push({ line, offset: col - (tabAdjust?.[lineIndex] ?? 0) });
+    }
+    return out;
+};
+export const scanReferenceTokens2 = (value, source) => {
+    if (!value.includes('${') && !value.includes('$('))
+        return NO_TOKENS;
+    const decoded = scanDecoded(value);
+    if (decoded.length === 0)
+        return NO_TOKENS;
+    // The raw and decoded scans see the same tokens in the same order —
+    // `\#` collapse and `^^` merge never add or remove a `${…}` / `$(…)`.
+    const physical = source === undefined
+        ? scanPhysical(value, 1, 0)
+        : scanPhysical(source.raw ?? value, source.line, source.col, source.tabAdjust);
+    if (physical.length !== decoded.length) {
+        throw new Error(`Lima internal: reference token count mismatch (raw ${physical.length}, decoded ${decoded.length})`);
     }
-    return result;
+    return decoded.map((d, i) => ({ ...d, line: physical[i].line, offset: physical[i].offset }));
 };

package/dist/references2.d.ts

@@ -8,6 +8,8 @@ export type ParseOptions = {
     onWarning?: (diagnostic: Diagnostic) => void;
 };
 export declare const parse: <T extends Record<string, unknown> = Meta>(frontMatter: string, options?: ParseOptions) => T;
+/** Test oracle: exercises the resolver without its dependency-result cache. */
+export declare const __parseWithoutResolveCacheForTest: <T extends Record<string, unknown> = Meta>(frontMatter: string, options?: ParseOptions) => T;
 /** @deprecated Use parse(). */
 export declare const parseReferences: typeof parse;
 export {};

package/dist/references2.js

@@ -3,11 +3,19 @@ import { canonicalString, codepointLength, countNodes, ingestPartialValue, PARTI
 import { NESTING_DEPTH_LIMIT, parseCore, parseCoreWithPositions, toPlainValue, } from './core.js';
 import { hasActiveReferences2 } from './scalars.js';
 import { LimaError } from './errors.js';
-import { collectAllParticipants, deepCopyPositioned, earliestParticipant, finalizePositioned, partialToPositioned, } from './references.js';
-const emptyMapping = () => Object.create(null);
+import { collectAllParticipants, deepCopyPositioned, earliestParticipant, emptyMapping, finalizePositioned, partialToPositioned, } from './positioned-tree.js';
 const PARTIAL_NAME = '[a-zA-Z0-9_][a-zA-Z0-9_:/-]*';
 const PARTIAL_NAME_RE = new RegExp(`^${PARTIAL_NAME}$`);
 const MAX_EDGES = 3;
+/** Copies a pure-reference target and stamps the new insertion without
+ * discarding provenance already attached to the copied root (R-112). */
+const copyWithInsertion = (target, insertedAt) => {
+    const copy = deepCopyPositioned(target);
+    const priorInsertions = copy.insertedAt === undefined
+        ? copy.priorInsertions
+        : [...(copy.priorInsertions ?? []), copy.insertedAt];
+    return { ...copy, insertedAt, priorInsertions };
+};
 const addDiagnostic = (ctx, diagnostic, offset = 0) => {
     ctx.diagnostics.push({ ...diagnostic, offset });
 };
@@ -75,9 +83,9 @@ const resolveNodeUncached = (node, document, partials, ctx, remainingEdges, stac
                 const match = tokenMatches(item)[0];
                 const token = match?.token ?? item.value;
                 addDiagnostic(ctx, {
-                    code: 'INVALID_REFERENCE_SHAPE', line: item.line, token,
-                    message: `Lima: reference "${token}" resolves to an array, which cannot be inserted as a sequence item at line ${item.line}`,
-                });
+                    code: 'INVALID_REFERENCE_SHAPE', line: match?.line ?? item.line, token,
+                    message: `Lima: reference "${token}" resolves to an array, which cannot be inserted as a sequence item at line ${match?.line ?? item.line}`,
+                }, match?.offset ?? 0);
                 return item;
             }
             return result.value;
@@ -120,8 +128,10 @@ const resolveNodeUncached = (node, document, partials, ctx, remainingEdges, stac
         if (target === undefined || stack.has(target))
             return { value: node, complete: false };
         if (pure.partialPath !== undefined) {
+            // `partialToPositioned` never annotates partial targets: partials are
+            // inert and have no source token before this document insertion.
             return {
-                value: { ...deepCopyPositioned(target), insertedAt: { line: pure.line, token: pure.token } },
+                value: copyWithInsertion(target, { line: pure.line, offset: pure.offset, token: pure.token }),
                 complete: true,
             };
         }
@@ -134,7 +144,9 @@ const resolveNodeUncached = (node, document, partials, ctx, remainingEdges, stac
         if (selected === undefined)
             return { value: node, complete: false };
         return {
-            value: { ...deepCopyPositioned(selected), insertedAt: { line: pure.line, token: pure.token } },
+            // `selected` may itself be the root inserted by an earlier reference;
+            // both it and this outer copy participate in final-structure errors.
+            value: copyWithInsertion(selected, { line: pure.line, offset: pure.offset, token: pure.token }),
             complete: true,
         };
     }
@@ -182,13 +194,24 @@ const resolveNodeUncached = (node, document, partials, ctx, remainingEdges, stac
     }
     return { value: { ...node, value: output, references2: unresolvedTokens }, complete };
 };
-/** Successful and failed dependency results are immutable and budget-specific. */
+/**
+ * The cache key is complete without `stack`: for a fixed node and edge
+ * budget, its outgoing targets and traversal order are fixed by the parsed
+ * document. If one of those targets is already on the current stack, that
+ * target is on a dependency cycle reachable from this node; entering from a
+ * different ancestor cannot make the cycle complete, and the monotonically
+ * decreasing edge budget fixes where an overlong path becomes unresolved.
+ * Thus `stack` only detects an already-inevitable incomplete dependency; it
+ * does not select a different successful value. Top-level calls are excluded
+ * (`stack.size > 1`) and cached Resolution trees are immutable, so sharing a
+ * successful or incomplete result at the same remaining budget is safe.
+ */
 const resolveNode = (node, document, partials, ctx, remainingEdges, stack) => {
-    const cacheable = stack.size > 1 && (node.kind === 'array' || node.kind === 'mapping' ||
-        (isActiveString(node) && tokenMatches(node).length > 0));
-    if (!cacheable)
+    const cache = ctx.cache;
+    if (cache === undefined || stack.size <= 1 || !(node.kind === 'array' || node.kind === 'mapping' ||
+        (isActiveString(node) && tokenMatches(node).length > 0)))
         return resolveNodeUncached(node, document, partials, ctx, remainingEdges, stack);
-    const byBudget = ctx.cache.get(node);
+    const byBudget = cache.get(node);
     const cached = byBudget?.get(remainingEdges);
     if (cached !== undefined)
         return cached;
@@ -196,7 +219,7 @@ const resolveNode = (node, document, partials, ctx, remainingEdges, stack) => {
     if (byBudget !== undefined)
         byBudget.set(remainingEdges, result);
     else
-        ctx.cache.set(node, new Map([[remainingEdges, result]]));
+        cache.set(node, new Map([[remainingEdges, result]]));
     return result;
 };
 const validatePartials = (raw) => {
@@ -243,7 +266,7 @@ const scanUnresolved = (node, ctx) => {
         for (const child of node.entries.values())
             scanUnresolved(child, ctx);
 };
-export const parse = (frontMatter, options) => {
+const parseInternal = (frontMatter, options, useResolveCache) => {
     if (options?.mode !== undefined && options.mode !== 'core' && options.mode !== 'references') {
         throw new TypeError(`Lima: invalid parse mode "${String(options.mode)}"`);
     }
@@ -260,7 +283,7 @@ export const parse = (frontMatter, options) => {
         return parseCore(frontMatter, options);
     }
     const document = parseCoreWithPositions(frontMatter, { strict: options?.strict ?? false, onWarning: options?.onWarning });
-    const ctx = { diagnostics: [], cache: new WeakMap() };
+    const ctx = { diagnostics: [], ...(useResolveCache ? { cache: new WeakMap() } : {}) };
     const resolved = new Map();
     for (const [key, value] of document) {
         const stack = new Set([value]);
@@ -271,7 +294,10 @@ export const parse = (frontMatter, options) => {
             scanUnresolved(value, ctx);
     if (ctx.diagnostics.length > 0) {
         ctx.diagnostics.sort((a, b) => a.line - b.line || a.offset - b.offset);
-        throw new LimaError(ctx.diagnostics[0]);
+        const winner = ctx.diagnostics[0];
+        // §2.4: expose the token's physical position (1-based) — Rust and Go
+        // already do; `offset` was internal-only here.
+        throw new LimaError({ ...winner, column: winner.offset + 1 });
     }
     const finalized = [...resolved].map(([key, value]) => [key, finalizePositioned(value)]);
     const depth = finalized.length === 0 ? 0 : Math.max(...finalized.map(([, value]) => value.depth));
@@ -280,6 +306,7 @@ export const parse = (frontMatter, options) => {
         const winner = earliestParticipant(participants);
         throw new LimaError({
             code: 'RESOURCE_LIMIT', line: winner?.line ?? 1, token: winner?.token,
+            ...(winner?.offset !== undefined ? { column: winner.offset + 1 } : {}),
             message: `Lima: nesting depth exceeds maximum of ${NESTING_DEPTH_LIMIT} at line ${winner?.line ?? 1}`,
         });
     }
@@ -293,6 +320,7 @@ export const parse = (frontMatter, options) => {
         const winner = earliestParticipant(participants);
         throw new LimaError({
             code: 'RESOURCE_LIMIT', line: winner?.line ?? 1, token: winner?.token,
+            ...(winner?.offset !== undefined ? { column: winner.offset + 1 } : {}),
             message: `Lima: result exceeds maximum size of ${RESULT_NODE_LIMIT} total nodes at line ${winner?.line ?? 1}`,
         });
     }
@@ -301,5 +329,8 @@ export const parse = (frontMatter, options) => {
         out[key] = value.native;
     return out;
 };
+export const parse = (frontMatter, options) => parseInternal(frontMatter, options, true);
+/** Test oracle: exercises the resolver without its dependency-result cache. */
+export const __parseWithoutResolveCacheForTest = (frontMatter, options) => parseInternal(frontMatter, options, false);
 /** @deprecated Use parse(). */
 export const parseReferences = parse;

package/dist/scalars.d.ts

@@ -7,40 +7,47 @@
 import { type LimaValue } from './value.js';
 import { type ParseContext } from './normalize.js';
 import type { ValueBuilder } from './builder.js';
-import { type ReferenceToken2 } from './reference-tokens2.js';
+import { type ReferenceSource, type ReferenceToken2 } from './reference-tokens2.js';
 /**
  * `insertedAt` is never set by Core — it's a References-only annotation
  * (see references.ts's `resolveTree`), stamped on the root of a value
  * copied in by a successful pure-reference resolution, with the source
- * token and line that caused the insertion. It powers References §5's
- * global-error attribution (R-112): when a final-result limit (nesting
- * depth, total node count) is violated, the lowest-line `insertedAt` among
- * the participating nodes identifies which reference token to blame — the
- * spec requires the error message to include both the token and the line.
+ * token, line, and character offset that caused the insertion. It powers
+ * References §5's global-error attribution (R-112): when a final-result
+ * limit (nesting depth, total node count) is violated, the lowest source
+ * position (`line`, then `offset`) among the participating nodes identifies
+ * which reference token to blame — the spec requires the error message to
+ * include both the token and the line. `priorInsertions` retains older
+ * root provenance when a resolved pure-reference result is copied again.
  */
 export type InsertedAt = {
     line: number;
     token: string;
+    offset?: number;
 };
 export type PositionedValue = {
     kind: 'null';
     line: number;
     insertedAt?: InsertedAt;
+    priorInsertions?: InsertedAt[];
 } | {
     kind: 'bool';
     value: boolean;
     line: number;
     insertedAt?: InsertedAt;
+    priorInsertions?: InsertedAt[];
 } | {
     kind: 'int';
     value: number;
     line: number;
     insertedAt?: InsertedAt;
+    priorInsertions?: InsertedAt[];
 } | {
     kind: 'float';
     value: number;
     line: number;
     insertedAt?: InsertedAt;
+    priorInsertions?: InsertedAt[];
 } | {
     kind: 'string';
     value: string;
@@ -48,23 +55,27 @@ export type PositionedValue = {
     quoted: boolean;
     references2?: ReferenceToken2[];
     insertedAt?: InsertedAt;
+    priorInsertions?: InsertedAt[];
 } | {
     kind: 'instant';
     value: Date;
     line: number;
     insertedAt?: InsertedAt;
+    priorInsertions?: InsertedAt[];
 } | {
     kind: 'array';
     items: PositionedValue[];
     line: number;
     references2Active?: boolean;
     insertedAt?: InsertedAt;
+    priorInsertions?: InsertedAt[];
 } | {
     kind: 'mapping';
     entries: Map<string, PositionedValue>;
     line: number;
     references2Active?: boolean;
     insertedAt?: InsertedAt;
+    priorInsertions?: InsertedAt[];
 };
 export declare const hasActiveReferences2: (value: PositionedValue) => boolean;
 /** The `ValueBuilder<PositionedValue>` — reconstructs today's annotated tree exactly, for References. */
@@ -78,17 +89,76 @@ export declare const NO_SPAN_VALUE: unique symbol;
  * handled by the full string-based scalar/flow grammar.
  */
 export declare const parseSimpleScalarSpan: <V, M>(source: string, start: number, end: number, line: number, strict: boolean, builder: ValueBuilder<V, M>) => V | typeof NO_SPAN_VALUE;
+/**
+ * Index of the quote that closes a quoted scalar opening at `s[0]`, or -1
+ * if it is never closed. Escape-aware, so `"a\""` closes at its last
+ * character rather than the escaped inner `"`:
+ *   - double quotes: a backslash escapes the next character (§6.1.2);
+ *   - single-quoted *values* (§6.1.3 "Backslash pairing"): scanning left
+ *     to right, `\\` (two literal backslashes) and `\'` (an escaped quote)
+ *     are each consumed as a two-character unit, so a lone `\` is literal
+ *     and a closing `'` after an even backslash run closes the string.
+ *     Pass `singleQuoteEscape = false` for single-quoted *keys*, which are
+ *     fully literal (§5.2), where the first `'` closes.
+ * `s[0]` is assumed to be `'` or `"`.
+ */
+export declare const closingQuoteIndex: (s: string, singleQuoteEscape?: boolean) => number;
+/**
+ * Whether `raw` (a key candidate as written, before quote stripping) is a
+ * usable Lima key:
+ *   - a quoted string that is properly closed at its final character, or
+ *   - an unquoted token matching the Core §5.1 grammar.
+ *
+ * §5.1 is a closed pattern: a key with a space, a dot, a slash, a paren, or
+ * any other character outside `[a-zA-Z0-9_:\-]` (a surviving NBSP included)
+ * is not an unquoted key. Per §4, such a line/item is unrecognised and
+ * skipped in *both* modes — §10's strict error list stays closed. The three
+ * mapping contexts (top level, block-nested, flow) now share this one
+ * check; `docs/decisions/structural-indentation-unicode-whitespace.md`
+ * governs only whether Unicode whitespace counts as *indentation*, not
+ * whether it may appear in a key.
+ */
+export declare const isValidKey: (raw: string) => boolean;
 export declare const unescapeDQ: (s: string, strict?: boolean, line?: number) => string;
+/**
+ * The value text with a trailing `#` comment removed, but with `\#`
+ * escapes left as written. This is the *physical* form a References 2.0
+ * token's column is measured against (§2.4): the comment is not part of
+ * the value (so its `${…}`-shaped text must not be scanned as a token),
+ * but a `\#` before a token still occupies its two source columns.
+ *
+ * `trimEnd()` is exactly the project whitespace set (ECMAScript WhiteSpace
+ * + LineTerminator = `isTrimWhitespace`) — a U+00A0 / FEFF between the
+ * value and the `#` is not part of the value. The Rust and Go ports must
+ * use `is_trim_whitespace` here, not a host `trim_end` or `" \t"`.
+ */
+export declare const stripCommentKeepEscapes: (val: string) => string;
+/**
+ * The value text with a trailing `#` comment removed and each `\#` *outside*
+ * a quoted string collapsed to `#` (Core §6.1.4 — the escaped-hash rule is
+ * defined for unquoted values only). Inside `"..."` / `'...'` the backslash
+ * is left intact: `\#` there is an unknown double-quoted escape (§6.1.2:
+ * strict throws, non-strict keeps it) or a literal backslash in a
+ * single-quoted string (§6.1.3). Finds the same comment boundary as
+ * `stripCommentKeepEscapes` — the two must agree, they annotate the same
+ * value.
+ */
 export declare const stripComment: (val: string) => string;
-/** Strips a key's surrounding quotes (unescaping double-quoted keys), or returns it unchanged. */
-export declare const stripKeyQuotes: (s: string) => string;
+/**
+ * Strips a key's surrounding quotes — unescaping a double-quoted key
+ * (§6.1.2 escapes, strict-checked), taking a single-quoted key literally
+ * (§5.2). Returns `s` unchanged when it is not a properly-closed quoted
+ * string. Callers gate on `isValidKey` first, so an unterminated or
+ * trailing-content key never reaches here as a real key.
+ */
+export declare const stripKeyQuotes: (s: string, strict?: boolean, line?: number) => string;
 /**
  * Quoted-or-typed scalar, shared by every value position (top-level inline
  * values, flow-sequence/flow-mapping items, block-array scalar items).
- * `topLevel` gates two checks that only apply at the outermost resolveValue
- * call site in the legacy parser and are deliberately not extended to flow
- * items here, to keep this a faithful behavioral port: the "unclosed flow
- * bracket" throw and the "non-whitespace after closing quote" strict throw.
+ * §10.1's two quoted-string strict checks — "unterminated quoted string"
+ * and "non-whitespace content after closing quote in an inline value" —
+ * apply in every one of those positions, so they are enforced here rather
+ * than gated to the top level.
  */
-export declare const parseQuotedOrTyped: <V, M>(raw: string, ctx: ParseContext, line: number, topLevel: boolean, builder: ValueBuilder<V, M>) => V;
-export declare const parseScalarValue: <V, M>(raw: string, ctx: ParseContext, line: number, builder: ValueBuilder<V, M>) => V;
+export declare const parseQuotedOrTyped: <V, M>(raw: string, ctx: ParseContext, line: number, builder: ValueBuilder<V, M>, source?: ReferenceSource) => V;
+export declare const parseScalarValue: <V, M>(raw: string, ctx: ParseContext, line: number, builder: ValueBuilder<V, M>, source?: ReferenceSource) => V;