@limaformat/lima 0.3.1 → 0.5.0

package/dist/scalars.js

@@ -13,13 +13,12 @@ export const hasActiveReferences2 = (value) => value.kind === 'string' ? (value.
         : false;
 /** The `ValueBuilder<PositionedValue>` — reconstructs today's annotated tree exactly, for References. */
 export const positionedBuilder = {
-    tracksStringSourcePositions: true,
     null: (line) => ({ kind: 'null', line }),
     bool: (value, line) => ({ kind: 'bool', value, line }),
     int: (value, line) => ({ kind: 'int', value, line }),
     float: (value, line) => ({ kind: 'float', value, line }),
-    string: (value, line, quoted, sourceSpans) => {
-        const references2 = quoted ? undefined : scanReferenceTokens2(value, line, sourceSpans);
+    string: (value, line, quoted, source) => {
+        const references2 = quoted ? undefined : scanReferenceTokens2(value, source);
         return {
             kind: 'string', value, line, quoted,
             ...(references2 && references2.length > 0 ? { references2 } : {}),
@@ -239,7 +238,7 @@ export const parseSimpleScalarSpan = (source, start, end, line, strict, builder)
  * an intermediate tagged `LimaValue` keeps both builders on one grammar while
  * eliminating an allocate-then-switch conversion for every scalar.
  */
-const buildTyped = (str, strict, line, builder) => {
+const buildTyped = (str, strict, line, builder, source) => {
     if (str === '' || str === 'null' || str === '~')
         return builder.null(line);
     if (str === 'true')
@@ -253,7 +252,7 @@ const buildTyped = (str, strict, line, builder) => {
     // the email/date prechecks for ordinary words, URLs and identifiers.
     if (!((first >= 48 && first <= 57) || first === 45 || first === 46)) {
         checkStringLimit(str, line);
-        return builder.string(str, line, false);
+        return builder.string(str, line, false, source);
     }
     // Hex (0x/0X), octal (0o/0O), binary (0b/0B) — kept as strings (YAML 1.2 compatible).
     if (str.length > 2 && str.charCodeAt(0) === 48 &&
@@ -261,7 +260,7 @@ const buildTyped = (str, strict, line, builder) => {
             str.charCodeAt(1) === 111 || str.charCodeAt(1) === 79 ||
             str.charCodeAt(1) === 98 || str.charCodeAt(1) === 66)) {
         checkStringLimit(str, line);
-        return builder.string(str, line, false);
+        return builder.string(str, line, false, source);
     }
     const exactIsoShape = (str.length === 10 && str.charCodeAt(4) === 45 && str.charCodeAt(7) === 45) ||
         (str.length === 20 && str.charCodeAt(4) === 45 && str.charCodeAt(7) === 45 &&
@@ -272,7 +271,7 @@ const buildTyped = (str, strict, line, builder) => {
         if (date !== null)
             return builder.instant(date, line);
         checkStringLimit(str, line);
-        return builder.string(str, line, false);
+        return builder.string(str, line, false, source);
     }
     if (NUMBER_RE.test(str)) {
         const n = Number(str);
@@ -307,10 +306,82 @@ const buildTyped = (str, strict, line, builder) => {
             return builder.instant(date, line);
     }
     checkStringLimit(str, line);
-    return builder.string(str, line, false);
+    return builder.string(str, line, false, source);
 };
 // ─── Scalar / quoting ──────────────────────────────────────────────────────
-const ESCAPED_HASH_RE = /\\#/g;
+/**
+ * Index of the quote that closes a quoted scalar opening at `s[0]`, or -1
+ * if it is never closed. Escape-aware, so `"a\""` closes at its last
+ * character rather than the escaped inner `"`:
+ *   - double quotes: a backslash escapes the next character (§6.1.2);
+ *   - single-quoted *values* (§6.1.3 "Backslash pairing"): scanning left
+ *     to right, `\\` (two literal backslashes) and `\'` (an escaped quote)
+ *     are each consumed as a two-character unit, so a lone `\` is literal
+ *     and a closing `'` after an even backslash run closes the string.
+ *     Pass `singleQuoteEscape = false` for single-quoted *keys*, which are
+ *     fully literal (§5.2), where the first `'` closes.
+ * `s[0]` is assumed to be `'` or `"`.
+ */
+export const closingQuoteIndex = (s, singleQuoteEscape = true) => {
+    const q = s.charCodeAt(0);
+    for (let i = 1; i < s.length; i++) {
+        const c = s.charCodeAt(i);
+        if (q === 34) {
+            if (c === 92) {
+                i++;
+                continue;
+            }
+            if (c === 34)
+                return i;
+        }
+        else {
+            if (singleQuoteEscape && c === 92) {
+                const n = s.charCodeAt(i + 1);
+                if (n === 92 || n === 39) {
+                    i++;
+                    continue;
+                }
+            }
+            if (c === 39)
+                return i;
+        }
+    }
+    return -1;
+};
+/**
+ * The unquoted-key grammar of Core §5.1: `[a-zA-Z0-9_][a-zA-Z0-9_:\-]*`.
+ * Kept byte-for-byte identical to `scanner.ts`'s `isKeyStartChar` /
+ * `isKeyContinueChar`, which enforce the same grammar at the top level.
+ */
+const isUnquotedKeyStart = (c) => (c >= 97 && c <= 122) || (c >= 65 && c <= 90) || (c >= 48 && c <= 57) || c === 95;
+const isUnquotedKeyContinue = (c) => isUnquotedKeyStart(c) || c === 58 || c === 45;
+/**
+ * Whether `raw` (a key candidate as written, before quote stripping) is a
+ * usable Lima key:
+ *   - a quoted string that is properly closed at its final character, or
+ *   - an unquoted token matching the Core §5.1 grammar.
+ *
+ * §5.1 is a closed pattern: a key with a space, a dot, a slash, a paren, or
+ * any other character outside `[a-zA-Z0-9_:\-]` (a surviving NBSP included)
+ * is not an unquoted key. Per §4, such a line/item is unrecognised and
+ * skipped in *both* modes — §10's strict error list stays closed. The three
+ * mapping contexts (top level, block-nested, flow) now share this one
+ * check; `docs/decisions/structural-indentation-unicode-whitespace.md`
+ * governs only whether Unicode whitespace counts as *indentation*, not
+ * whether it may appear in a key.
+ */
+export const isValidKey = (raw) => {
+    const f = raw.charCodeAt(0);
+    if (f === 34 || f === 39)
+        return closingQuoteIndex(raw, false) === raw.length - 1;
+    if (!isUnquotedKeyStart(f))
+        return false;
+    for (let i = 1; i < raw.length; i++) {
+        if (!isUnquotedKeyContinue(raw.charCodeAt(i)))
+            return false;
+    }
+    return true;
+};
 const ANY_ESCAPE_RE = /\\(u[0-9a-fA-F]{0,4}|U[0-9a-fA-F]{0,8}|x[0-9a-fA-F]{0,2}|.)/gs;
 const SINGLE_CHAR_ESCAPES = '"\\/bfnrt'; // deliberately excludes '0' — Core Appendix A: \0 is unknown, not a null shorthand.
 const U_ESCAPE_RE = /^u([0-9a-fA-F]{4})$/;
@@ -359,8 +430,54 @@ export const unescapeDQ = (s, strict = false, line = 0) => {
         }
     });
 };
+/**
+ * The value text with a trailing `#` comment removed, but with `\#`
+ * escapes left as written. This is the *physical* form a References 2.0
+ * token's column is measured against (§2.4): the comment is not part of
+ * the value (so its `${…}`-shaped text must not be scanned as a token),
+ * but a `\#` before a token still occupies its two source columns.
+ *
+ * `trimEnd()` is exactly the project whitespace set (ECMAScript WhiteSpace
+ * + LineTerminator = `isTrimWhitespace`) — a U+00A0 / FEFF between the
+ * value and the `#` is not part of the value. The Rust and Go ports must
+ * use `is_trim_whitespace` here, not a host `trim_end` or `" \t"`.
+ */
+export const stripCommentKeepEscapes = (val) => {
+    let quote = 0;
+    for (let i = 0; i < val.length; i++) {
+        const cc = val.charCodeAt(i);
+        if (quote) {
+            if (cc === 92)
+                i++;
+            else if (cc === quote)
+                quote = 0;
+        }
+        else if (cc === 34 || cc === 39) {
+            quote = cc;
+        }
+        else if (cc === 92 && val.charCodeAt(i + 1) === 35) {
+            i++;
+        }
+        else if (cc === 35) {
+            return val.slice(0, i).trimEnd();
+        }
+    }
+    return val;
+};
+/**
+ * The value text with a trailing `#` comment removed and each `\#` *outside*
+ * a quoted string collapsed to `#` (Core §6.1.4 — the escaped-hash rule is
+ * defined for unquoted values only). Inside `"..."` / `'...'` the backslash
+ * is left intact: `\#` there is an unknown double-quoted escape (§6.1.2:
+ * strict throws, non-strict keeps it) or a literal backslash in a
+ * single-quoted string (§6.1.3). Finds the same comment boundary as
+ * `stripCommentKeepEscapes` — the two must agree, they annotate the same
+ * value.
+ */
 export const stripComment = (val) => {
     let quote = 0;
+    let out = '';
+    let seg = 0;
     for (let i = 0; i < val.length; i++) {
         const cc = val.charCodeAt(i);
         if (quote) {
@@ -373,52 +490,64 @@ export const stripComment = (val) => {
             quote = cc;
         }
         else if (cc === 92 && val.charCodeAt(i + 1) === 35) {
+            out += val.slice(seg, i) + '#';
             i++;
+            seg = i + 1;
         }
         else if (cc === 35) {
-            return val.slice(0, i).trimEnd().replace(ESCAPED_HASH_RE, '#');
+            return (out + val.slice(seg, i)).trimEnd();
         }
     }
-    return val.replace(ESCAPED_HASH_RE, '#');
+    return out + val.slice(seg);
 };
-/** Strips a key's surrounding quotes (unescaping double-quoted keys), or returns it unchanged. */
-export const stripKeyQuotes = (s) => {
+/**
+ * Strips a key's surrounding quotes — unescaping a double-quoted key
+ * (§6.1.2 escapes, strict-checked), taking a single-quoted key literally
+ * (§5.2). Returns `s` unchanged when it is not a properly-closed quoted
+ * string. Callers gate on `isValidKey` first, so an unterminated or
+ * trailing-content key never reaches here as a real key.
+ */
+export const stripKeyQuotes = (s, strict = false, line = 0) => {
     const f = s.charCodeAt(0);
-    if (f === 39 && s.charCodeAt(s.length - 1) === 39)
-        return s.slice(1, -1);
-    if (f === 34 && s.charCodeAt(s.length - 1) === 34)
-        return unescapeDQ(s.slice(1, -1));
+    if ((f === 34 || f === 39) && closingQuoteIndex(s, false) === s.length - 1) {
+        return f === 34 ? unescapeDQ(s.slice(1, -1), strict, line) : s.slice(1, -1);
+    }
     return s;
 };
 /**
  * Quoted-or-typed scalar, shared by every value position (top-level inline
  * values, flow-sequence/flow-mapping items, block-array scalar items).
- * `topLevel` gates two checks that only apply at the outermost resolveValue
- * call site in the legacy parser and are deliberately not extended to flow
- * items here, to keep this a faithful behavioral port: the "unclosed flow
- * bracket" throw and the "non-whitespace after closing quote" strict throw.
+ * §10.1's two quoted-string strict checks — "unterminated quoted string"
+ * and "non-whitespace content after closing quote in an inline value" —
+ * apply in every one of those positions, so they are enforced here rather
+ * than gated to the top level.
  */
-export const parseQuotedOrTyped = (raw, ctx, line, topLevel, builder) => {
+export const parseQuotedOrTyped = (raw, ctx, line, builder, source) => {
     const first = raw.charCodeAt(0);
     if (first === 34 || first === 39) {
-        if (raw.charCodeAt(raw.length - 1) === first) {
+        const close = closingQuoteIndex(raw);
+        if (close === raw.length - 1) {
             const unquoted = raw.slice(1, -1);
             const value = first === 34 ? unescapeDQ(unquoted, ctx.strict, line) : unquoted.replace(/\\'/g, "'");
             checkStringLimit(value, line);
             return builder.string(value, line, true);
         }
-        if (topLevel && ctx.strict) {
-            throw new LimaError({ code: 'INVALID_QUOTE', line, message: `Lima: non-whitespace content after closing quote at line ${line}` });
+        if (ctx.strict) {
+            throw close === -1
+                ? new LimaError({ code: 'INVALID_QUOTE', line, message: `Lima: unterminated quoted string at line ${line}` })
+                : new LimaError({ code: 'INVALID_QUOTE', line, message: `Lima: non-whitespace content after closing quote at line ${line}` });
         }
+        // Non-strict: §6.1.2/§6.1.3 + §10.1 — the entire value falls back to
+        // a literal string (opening quote included), handled below.
     }
     if (raw !== '' && raw !== 'null' && raw !== '~' && raw !== 'true' && raw !== 'false' &&
         !((first >= 48 && first <= 57) || first === 45 || first === 46)) {
         checkStringLimit(raw, line);
-        return builder.string(raw, line, false);
+        return builder.string(raw, line, false, source);
     }
-    return buildTyped(raw, ctx.strict, line, builder);
+    return buildTyped(raw, ctx.strict, line, builder, source);
 };
-export const parseScalarValue = (raw, ctx, line, builder) => {
+export const parseScalarValue = (raw, ctx, line, builder, source) => {
     const first = raw.charCodeAt(0);
     if (ctx.strict && (first === 91 || first === 123)) {
         throw new LimaError({
@@ -426,5 +555,5 @@ export const parseScalarValue = (raw, ctx, line, builder) => {
             message: `Lima: unclosed flow ${first === 91 ? 'sequence' : 'mapping'} at line ${line}`,
         });
     }
-    return parseQuotedOrTyped(raw, ctx, line, true, builder);
+    return parseQuotedOrTyped(raw, ctx, line, builder, source);
 };

package/dist/scanner.d.ts

@@ -9,15 +9,23 @@
  * standard `regex` crate is itself RE2-derived and cannot express
  * backtracking-dependent patterns at all.
  *
- * Reproduces (verified by extensive differential testing against the
- * regex it replaced — over 100,000 fuzzed and structured cases, the full
- * conformance corpus, and hand-built adversarial edge cases, zero
- * divergences) the exact matching behavior of:
+ * Equivalent to this regex:
  *
- *   /^(?:([a-zA-Z\d_][a-zA-Z\d_:-]*)|'([^']*)'|"((?:[^"\\]|\\.)*)"):( *\n| )/gm
+ *   /^(?:([a-zA-Z\d_][a-zA-Z\d_:-]*)|'([^'\n]*)'|"((?:[^"\\\n]|\\[^\n])*)"):( *\n| )/gm
  *
- * including subtle backtracking-dependent cases that are easy to get
- * wrong without empirical verification against the real parser:
+ * The scanner replaced an actual regex of this shape; the differential
+ * testing that verified the port — over 100,000 fuzzed and structured
+ * cases, the full conformance corpus, hand-built adversarial edge cases,
+ * zero divergences — was against that original. The quoted-key character
+ * classes have since been tightened to exclude only U+000A (Core errata
+ * 1.0.8 / 1.0.9), reflected above: `[^'\n]`, `[^"\\\n]`, and `\\[^\n]` (a
+ * backslash escapes any character except U+000A; the resulting sequence is
+ * then validated as an escape at decode time — an unknown escape like
+ * `\` + U+2028 throws in strict, is kept literally in non-strict, exactly
+ * like `\q`).
+ *
+ * Subtle backtracking-dependent cases that are easy to get wrong without
+ * empirical verification against the real parser:
  *   - `a:b: value` → key "a:b" (colon is a legal mid-key character; the
  *     mandatory separator-introducing `:` is whichever colon within the
  *     greedily-matched run is the RIGHTMOST one for which a valid
@@ -30,12 +38,17 @@
  *     at all (the separator alternative `( *\n| )` requires an actual
  *     trailing newline or a literal space; end-of-string alone satisfies
  *     neither).
- *   - Quoted keys (`'...'`, `"..."`) may contain a literal `\n` — the
- *     character classes inside quotes don't exclude it.
- *   - A backslash directly followed by a line terminator inside a
- *     double-quoted key is NOT a valid `\\.` escape — the source regex's
- *     `.` never matches a line terminator without the `s` flag (not set),
- *     so this fails the match entirely rather than consuming the pair.
+ *   - A raw U+000A in a quoted key aborts the match — and only U+000A:
+ *     Core §15.6's single- and double-quoted-key character classes exclude
+ *     U+000A alone, so U+2028 / U+2029 (and, on already-§3-normalised
+ *     input, U+000D) are ordinary quoted-key characters here, matching the
+ *     nested and flow key paths and the Rust/Go scanners. The `\n` escape
+ *     is unaffected — it decodes to a key containing U+000A.
+ *   - A backslash directly followed by U+000A inside a double-quoted key
+ *     aborts the match (`\\[^\n]` — the backslash needs a non-U+000A
+ *     character after it). A backslash followed by any other character,
+ *     U+2028 / U+2029 included, is consumed as an escape pair here and
+ *     resolved at decode time.
  */
 export interface KeyMatch {
     /** One-based source line of the key. */

package/dist/scanner.js

@@ -9,15 +9,23 @@
  * standard `regex` crate is itself RE2-derived and cannot express
  * backtracking-dependent patterns at all.
  *
- * Reproduces (verified by extensive differential testing against the
- * regex it replaced — over 100,000 fuzzed and structured cases, the full
- * conformance corpus, and hand-built adversarial edge cases, zero
- * divergences) the exact matching behavior of:
+ * Equivalent to this regex:
  *
- *   /^(?:([a-zA-Z\d_][a-zA-Z\d_:-]*)|'([^']*)'|"((?:[^"\\]|\\.)*)"):( *\n| )/gm
+ *   /^(?:([a-zA-Z\d_][a-zA-Z\d_:-]*)|'([^'\n]*)'|"((?:[^"\\\n]|\\[^\n])*)"):( *\n| )/gm
  *
- * including subtle backtracking-dependent cases that are easy to get
- * wrong without empirical verification against the real parser:
+ * The scanner replaced an actual regex of this shape; the differential
+ * testing that verified the port — over 100,000 fuzzed and structured
+ * cases, the full conformance corpus, hand-built adversarial edge cases,
+ * zero divergences — was against that original. The quoted-key character
+ * classes have since been tightened to exclude only U+000A (Core errata
+ * 1.0.8 / 1.0.9), reflected above: `[^'\n]`, `[^"\\\n]`, and `\\[^\n]` (a
+ * backslash escapes any character except U+000A; the resulting sequence is
+ * then validated as an escape at decode time — an unknown escape like
+ * `\` + U+2028 throws in strict, is kept literally in non-strict, exactly
+ * like `\q`).
+ *
+ * Subtle backtracking-dependent cases that are easy to get wrong without
+ * empirical verification against the real parser:
  *   - `a:b: value` → key "a:b" (colon is a legal mid-key character; the
  *     mandatory separator-introducing `:` is whichever colon within the
  *     greedily-matched run is the RIGHTMOST one for which a valid
@@ -30,12 +38,17 @@
  *     at all (the separator alternative `( *\n| )` requires an actual
  *     trailing newline or a literal space; end-of-string alone satisfies
  *     neither).
- *   - Quoted keys (`'...'`, `"..."`) may contain a literal `\n` — the
- *     character classes inside quotes don't exclude it.
- *   - A backslash directly followed by a line terminator inside a
- *     double-quoted key is NOT a valid `\\.` escape — the source regex's
- *     `.` never matches a line terminator without the `s` flag (not set),
- *     so this fails the match entirely rather than consuming the pair.
+ *   - A raw U+000A in a quoted key aborts the match — and only U+000A:
+ *     Core §15.6's single- and double-quoted-key character classes exclude
+ *     U+000A alone, so U+2028 / U+2029 (and, on already-§3-normalised
+ *     input, U+000D) are ordinary quoted-key characters here, matching the
+ *     nested and flow key paths and the Rust/Go scanners. The `\n` escape
+ *     is unaffected — it decodes to a key containing U+000A.
+ *   - A backslash directly followed by U+000A inside a double-quoted key
+ *     aborts the match (`\\[^\n]` — the backslash needs a non-U+000A
+ *     character after it). A backslash followed by any other character,
+ *     U+2028 / U+2029 included, is consumed as an escape pair here and
+ *     resolved at decode time.
  */
 const isKeyStartChar = (c) => (c >= 97 && c <= 122) || (c >= 65 && c <= 90) || (c >= 48 && c <= 57) || c === 95; // a-z A-Z 0-9 _
 const isKeyContinueChar = (c) => isKeyStartChar(c) || c === 58 || c === 45; // + : -
@@ -62,6 +75,10 @@ const matchAt = (s, pos, line, out) => {
         const end = s.indexOf("'", pos + 1);
         if (end === -1 || s.charCodeAt(end + 1) !== 58)
             return false;
+        for (let i = pos + 1; i < end; i++) {
+            if (s.charCodeAt(i) === 10)
+                return false; // §15.6 excludes U+000A only
+        }
         sepStart = end + 2;
         separator = matchSeparator(s, sepStart);
         if (separator === 0)
@@ -77,8 +94,7 @@ const matchAt = (s, pos, line, out) => {
             const cc = s.charCodeAt(i);
             if (cc === 92) {
                 // `\\.` in the source regex — see the module doc comment.
-                const next = s.charCodeAt(i + 1);
-                if (i + 1 >= s.length || next === 10 || next === 13 || next === 0x2028 || next === 0x2029)
+                if (i + 1 >= s.length || s.charCodeAt(i + 1) === 10)
                     return false;
                 i += 2;
                 continue;
@@ -87,6 +103,8 @@ const matchAt = (s, pos, line, out) => {
                 closed = true;
                 break;
             }
+            if (cc === 10)
+                return false; // §15.6 excludes U+000A only
             i++;
         }
         if (!closed)

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@limaformat/lima",
-  "version": "0.3.1",
+  "version": "0.5.0",
   "description": "Lima frontmatter parser — LIMA Is Metadata Annotation",
   "keywords": ["frontmatter", "parser", "yaml", "markdown", "metadata"],
   "homepage": "https://limaformat.dev",
@@ -20,27 +20,27 @@
     }
   },
   "files": ["dist", "README.md", "LICENSE", "CHANGELOG.md"],
-  "engines": {
-  "node": ">=18"
-},
-"scripts": {
-  "build": "tsc -p tsconfig.build.json",
-  "pretest": "bun run build",
-  "test": "bun test",
-  "typecheck": "tsc --noEmit",
-  "prepublishOnly": "bun run typecheck && bun test",
-  "bench": "bun run bench/index.ts",
-  "bench:vs-yaml": "bun run bench/vs-yaml.ts",
-  "bench:vs-yaml:node": "node bench/vs-yaml-node.mjs",
-  "bench:cross-language": "bun run bench/cross-language.ts",
-  "bench:differential": "bun run bench/differential.ts",
-  "wordcount": "bun run scripts/wordcount.ts"
-},
-"devDependencies": {
-  "@types/bun": "^1.3.14",
-  "@types/js-yaml": "^4.0.9",
-  "js-yaml": "^5.2.3",
-  "typescript": "^7.0.2",
-  "typescript-classic": "npm:typescript@^5.9.3"
-}
+  "engines": { "node": ">=18" },
+  "scripts": {
+    "build": "tsc -p tsconfig.build.json",
+    "prepack": "bun run build",
+    "pretest": "bun run build",
+    "test": "bun test",
+    "pretypecheck": "bun run build",
+    "typecheck": "tsc --noEmit",
+    "prepublishOnly": "bun run typecheck && bun test",
+    "bench": "bun run bench/index.ts",
+    "bench:vs-yaml": "bun run bench/vs-yaml.ts",
+    "bench:vs-yaml:node": "node bench/vs-yaml-node.mjs",
+    "bench:cross-language": "bun run bench/cross-language.ts",
+    "bench:differential": "bun run bench/differential.ts",
+    "wordcount": "bun run scripts/wordcount.ts"
+  },
+  "devDependencies": {
+    "@types/bun": "^1.3.14",
+    "@types/js-yaml": "^4.0.9",
+    "js-yaml": "^5.2.3",
+    "typescript": "^7.0.2",
+    "typescript-classic": "npm:typescript@^5.9.3"
+  }
 }