@spintax/core 0.13.0 → 0.14.0

package/dist/index.d.cts

@@ -29,8 +29,12 @@
  * 4 — `raw` is kept on more constructs: one holding a `{?…}` conditional, and a permutation with a
  * reference anywhere in its `<config>` (#80). The shape is 3's, but an `Ast` cached by version 3
  * lacks `raw` exactly where the new rule needs it, and would render `[{?f?a|b|x}|c]` with a raw `|`.
+ *
+ * 5 — `/! … !/` verbatim spans (#92, ADR 0001) are encoded into the text as tokens at parse time.
+ * The shape is 4's, but an `Ast` parsed by version 4 carries no tokens, and would spin `/!{a|b}!/`
+ * instead of emitting it.
  */
-declare const AST_VERSION = 4;
+declare const AST_VERSION = 5;
 /** Opaque public handle (re-exported as `Ast` from the package index). */
 interface Ast {
     readonly astVersion: typeof AST_VERSION;
@@ -95,7 +99,8 @@ interface PluralIssue {
  * lets the deterministic parity fixtures assert exact output.
  *
  * Order: strip stray sentinels → build vars, roll #def, tree-walk (+ #include) →
- * cosmetic post-process (if on) → mandatory neutralize safety-restore (always).
+ * cosmetic post-process (if on) → mandatory neutralize safety-restore (always) → decode the
+ * `/! … !/` spans (always).
  */
 
 /** Default for {@link PipelineOptions.maxDepth} (#include + parse-nesting guard). */

package/dist/index.d.ts

@@ -29,8 +29,12 @@
  * 4 — `raw` is kept on more constructs: one holding a `{?…}` conditional, and a permutation with a
  * reference anywhere in its `<config>` (#80). The shape is 3's, but an `Ast` cached by version 3
  * lacks `raw` exactly where the new rule needs it, and would render `[{?f?a|b|x}|c]` with a raw `|`.
+ *
+ * 5 — `/! … !/` verbatim spans (#92, ADR 0001) are encoded into the text as tokens at parse time.
+ * The shape is 4's, but an `Ast` parsed by version 4 carries no tokens, and would spin `/!{a|b}!/`
+ * instead of emitting it.
  */
-declare const AST_VERSION = 4;
+declare const AST_VERSION = 5;
 /** Opaque public handle (re-exported as `Ast` from the package index). */
 interface Ast {
     readonly astVersion: typeof AST_VERSION;
@@ -95,7 +99,8 @@ interface PluralIssue {
  * lets the deterministic parity fixtures assert exact output.
  *
  * Order: strip stray sentinels → build vars, roll #def, tree-walk (+ #include) →
- * cosmetic post-process (if on) → mandatory neutralize safety-restore (always).
+ * cosmetic post-process (if on) → mandatory neutralize safety-restore (always) → decode the
+ * `/! … !/` spans (always).
  */
 
 /** Default for {@link PipelineOptions.maxDepth} (#include + parse-nesting guard). */

package/dist/index.js

@@ -1,5 +1,5 @@
 // src/internal/ast.ts
-var AST_VERSION = 4;
+var AST_VERSION = 5;
 function walk(nodes, visit) {
   const stack = [{ list: nodes, i: 0 }];
   while (stack.length > 0) {
@@ -43,31 +43,6 @@ function nameMap(...sources) {
   return map;
 }
 
-// src/internal/neutralize.ts
-var STRUCTURAL = ["{", "}", "[", "]", "%", "#"];
-var SENTINEL_BASE = 57344;
-var SHIELD = /* @__PURE__ */ new Map();
-var RESTORE = /* @__PURE__ */ new Map();
-STRUCTURAL.forEach((ch, i) => {
-  const sentinel = String.fromCharCode(SENTINEL_BASE + i);
-  SHIELD.set(ch, sentinel);
-  RESTORE.set(sentinel, ch);
-});
-var SHIELD_RE = /[{}[\]%#]/gu;
-var RESTORE_RE = new RegExp(
-  `[${String.fromCharCode(SENTINEL_BASE)}-${String.fromCharCode(SENTINEL_BASE + STRUCTURAL.length - 1)}]`,
-  "gu"
-);
-function neutralize(value) {
-  return value.replace(SHIELD_RE, (ch) => SHIELD.get(ch) ?? ch);
-}
-function safetyRestore(text) {
-  return text.replace(RESTORE_RE, (ch) => RESTORE.get(ch) ?? ch);
-}
-function stripSentinels(text) {
-  return text.replace(RESTORE_RE, "");
-}
-
 // src/internal/pairs.ts
 var BRACE_OPEN = 123;
 var BRACE_CLOSE = 125;
@@ -345,6 +320,126 @@ var TextIndex = class {
   }
 };
 
+// src/internal/verbatim.ts
+var OPEN = 983040;
+var CLOSE = 983041;
+var DIGIT = 987136;
+var RESERVED_RE = /[-\u{F0000}-\u{F1FFF}]/gu;
+var SPAN_RANGE_RE = /[\u{F0000}-\u{F1FFF}]/gu;
+var TOKEN_RE = /\u{F0000}((?:[\u{F1000}-\u{F1FFF}]{2})*)\u{F0001}/gu;
+var SPAN_HIGH_RE = /[\uDB80-\uDB87]/;
+function encodeVerbatim(content) {
+  let out = String.fromCodePoint(OPEN);
+  for (const ch of content) {
+    const c = ch.codePointAt(0);
+    out += String.fromCodePoint(DIGIT + (c >> 12), DIGIT + (c & 4095));
+  }
+  return out + String.fromCodePoint(CLOSE);
+}
+function decodeVerbatim(text) {
+  if (!SPAN_HIGH_RE.test(text)) return text;
+  TOKEN_RE.lastIndex = 0;
+  return text.replace(TOKEN_RE, (_m, digits) => {
+    let out = "";
+    const units = [...digits];
+    for (let i = 0; i + 1 < units.length; i += 2) {
+      out += String.fromCodePoint(units[i].codePointAt(0) - DIGIT << 12 | units[i + 1].codePointAt(0) - DIGIT);
+    }
+    return out;
+  });
+}
+function budgetLength(value) {
+  return SPAN_HIGH_RE.test(value) ? decodeVerbatim(value).length : value.length;
+}
+function stripSpanRange(value) {
+  return SPAN_HIGH_RE.test(value) ? value.replace(SPAN_RANGE_RE, "") : value;
+}
+function projectSource(src) {
+  const outStarts = [];
+  const srcStarts = [];
+  const srcEnds = [];
+  const tokens = [];
+  let out = "";
+  let spans = 0;
+  let unclosed = -1;
+  const copy = (start, end) => {
+    if (start >= end) return;
+    const chunk = src.slice(start, end);
+    let from2 = 0;
+    for (const m of chunk.matchAll(RESERVED_RE)) {
+      if (m.index > from2) push(start + from2, start + m.index, chunk.slice(from2, m.index), false);
+      from2 = m.index + m[0].length;
+    }
+    if (from2 < chunk.length) push(start + from2, end, chunk.slice(from2), false);
+  };
+  const push = (srcStart, srcEnd, piece, token) => {
+    outStarts.push(out.length);
+    srcStarts.push(srcStart);
+    srcEnds.push(srcEnd);
+    tokens.push(token);
+    out += piece;
+  };
+  let from = 0;
+  let at = 0;
+  let nextComment = -2;
+  let nextSpan = -2;
+  let commentCloserGone = false;
+  let spanCloserGone = false;
+  for (; ; ) {
+    if (nextComment !== -1 && nextComment < at) nextComment = src.indexOf("/#", at);
+    if (nextSpan !== -1 && nextSpan < at) nextSpan = src.indexOf("/!", at);
+    if (nextComment === -1 && nextSpan === -1) break;
+    const isSpan = nextComment === -1 || nextSpan !== -1 && nextSpan < nextComment;
+    const open = isSpan ? nextSpan : nextComment;
+    const gone = isSpan ? spanCloserGone : commentCloserGone;
+    const close = gone ? -1 : src.indexOf(isSpan ? "!/" : "#/", open + 2);
+    if (close === -1) {
+      if (isSpan) {
+        spanCloserGone = true;
+        if (unclosed === -1) unclosed = open;
+      } else {
+        commentCloserGone = true;
+      }
+      at = open + 2;
+      continue;
+    }
+    copy(from, open);
+    if (isSpan) {
+      spans += 1;
+      if (close > open + 2) push(open, close + 2, encodeVerbatim(src.slice(open + 2, close)), true);
+    }
+    from = at = close + 2;
+  }
+  copy(from, src.length);
+  const segmentAt = (offset) => {
+    let lo = 0;
+    let hi = outStarts.length - 1;
+    if (hi < 0 || outStarts[0] > offset) return -1;
+    while (lo < hi) {
+      const mid = lo + hi + 1 >> 1;
+      if (outStarts[mid] <= offset) lo = mid;
+      else hi = mid - 1;
+    }
+    return lo;
+  };
+  const toSource = (offset) => {
+    const at2 = segmentAt(offset);
+    if (at2 === -1) return 0;
+    const delta = offset - outStarts[at2];
+    if (delta <= 0) return srcStarts[at2];
+    if (tokens[at2]) return srcEnds[at2];
+    return Math.min(srcStarts[at2] + delta, srcEnds[at2]);
+  };
+  const toSourceEnd = (offset) => {
+    if (offset <= 0) return toSource(0);
+    const at2 = segmentAt(offset - 1);
+    if (at2 === -1) return 0;
+    if (tokens[at2]) return srcEnds[at2];
+    return Math.min(srcStarts[at2] + (offset - outStarts[at2]), srcEnds[at2]);
+  };
+  return { text: out, spans, unclosed, toSource, toSourceEnd };
+}
+
 // src/internal/parser.ts
 var VARIABLE_RE = /%(\w+)%/y;
 var PERCENT = 37;
@@ -356,7 +451,7 @@ var DIRECTIVE_RE = /^[ \t]*#(set|def)[ \t]+%(\w+)%[ \t]*=[ \t]*((?:.*[^ \t\n\r\u
 var CONDITIONAL_NAME_RE2 = /[A-Za-z_]\w*/y;
 var PLURAL_PREFIX = "plural ";
 function parseTemplate(src) {
-  const { body, setDefs, defDefs } = extractDirectives(stripComments(stripSentinels(src)));
+  const { body, setDefs, defDefs } = extractDirectives(projectSource(src).text);
   return { astVersion: AST_VERSION, source: src, setDefs, defDefs, nodes: parseSequence(body) };
 }
 function extractDirectives(text) {
@@ -390,19 +485,6 @@ function extractDirectives(text) {
   );
   return { body: stripped.replace(/\n{3,}/gu, "\n\n"), setDefs, defDefs, occurrences };
 }
-function stripComments(text) {
-  let out = "";
-  let from = 0;
-  for (; ; ) {
-    const open = text.indexOf("/#", from);
-    if (open === -1) break;
-    const close = text.indexOf("#/", open + 2);
-    if (close === -1) break;
-    out += text.slice(from, open);
-    from = close + 2;
-  }
-  return from === 0 ? text : out + text.slice(from);
-}
 function parseSequence(text) {
   const index = new TextIndex(text);
   const frame = (start, end) => ({
@@ -787,7 +869,8 @@ var KNOWN_CONFIG_KEYS = /* @__PURE__ */ new Set(["minsize", "maxsize", "sep", "l
 var INCLUDE_RE = /^[ \t]*#include[ \t\n\r\f\x0B]+"([^"]+)"[ \t\n\r\f\x0B]*$/gmu;
 function validateTemplate(src, opts = {}) {
   const diagnostics = [];
-  const text = stripComments(src);
+  const projection = projectSource(src);
+  const text = projection.text;
   const idx = buildLineIndex(text);
   checkBrackets(text, diagnostics);
   checkDirectives(text, diagnostics);
@@ -797,7 +880,62 @@ function validateTemplate(src, opts = {}) {
   if (opts.knownIncludes && opts.knownIncludes.length > 0) {
     checkIncludeTargets(text, idx, opts.knownIncludes, diagnostics);
   }
-  return diagnostics;
+  const srcIdx = text === src ? idx : buildLineIndex(src);
+  const remap = { text, idx, src, srcIdx, toSource: projection.toSource, toSourceEnd: projection.toSourceEnd };
+  const placed = text === src ? diagnostics : diagnostics.map((d) => toSourcePositions(d, remap));
+  if (projection.unclosed !== -1) {
+    placed.push(warn("verbatim.unclosed", "Unclosed '/!' \u2014 the text after it is parsed as spintax.", span(srcIdx, projection.unclosed, 2)));
+  }
+  return placed;
+}
+function toSourcePositions(d, r) {
+  const codePoints = d.code.startsWith("bracket.");
+  const move = (line, column) => {
+    const offset = r.toSource(lineColToOffset(r, line, column, codePoints));
+    return lineColAt(r, offset, codePoints);
+  };
+  const start = move(d.line, d.column);
+  const data = d.data === void 0 ? void 0 : Object.fromEntries(
+    Object.entries(d.data).map(([key, value]) => [key, typeof value === "string" ? decodeVerbatim(value) : value])
+  );
+  const moved = { ...d, message: decodeVerbatim(d.message), line: start.line, column: start.column, ...data === void 0 ? {} : { data } };
+  if (d.endLine !== void 0 && d.endColumn !== void 0) {
+    const end = lineColAt(r, r.toSourceEnd(lineColToOffset(r, d.endLine, d.endColumn, codePoints)), codePoints);
+    return { ...moved, endLine: end.line, endColumn: end.column };
+  }
+  return moved;
+}
+function astralPrefix(text) {
+  const prefix = new Uint32Array(text.length + 1);
+  for (let i = 0; i < text.length; i += 1) {
+    const c = text.charCodeAt(i);
+    const pair = c >= 55296 && c <= 56319 && (text.charCodeAt(i + 1) & 64512) === 56320;
+    prefix[i + 1] = prefix[i] + (pair ? 1 : 0);
+  }
+  return prefix;
+}
+function codePointsBetween(prefix, from, to) {
+  return to - from - (prefix[to] - prefix[from]);
+}
+function lineColToOffset(r, line, column, codePoints) {
+  const start = r.idx.starts[Math.min(Math.max(line, 1), r.idx.starts.length) - 1] ?? 0;
+  if (!codePoints) return Math.min(start + column - 1, r.text.length);
+  const prefix = r.textAstral ??= astralPrefix(r.text);
+  let lo = start;
+  let hi = Math.min(start + 2 * (column - 1), r.text.length);
+  while (lo < hi) {
+    const mid = lo + hi >> 1;
+    if (codePointsBetween(prefix, start, mid) < column - 1) lo = mid + 1;
+    else hi = mid;
+  }
+  return lo;
+}
+function lineColAt(r, offset, codePoints) {
+  const pos = offsetToLineCol(r.srcIdx, offset);
+  if (!codePoints) return pos;
+  const start = r.srcIdx.starts[pos.line - 1] ?? 0;
+  const prefix = r.srcAstral ??= astralPrefix(r.src);
+  return { line: pos.line, column: codePointsBetween(prefix, start, Math.min(offset, r.src.length)) + 1 };
 }
 function err(code, message, pos) {
   return { severity: "error", code, message, ...pos };
@@ -1223,13 +1361,14 @@ function checkIncludeTargets(text, idx, known, out) {
   const set = new Set(known);
   INCLUDE_RE.lastIndex = 0;
   for (const m of text.matchAll(INCLUDE_RE)) {
-    const ref = m[1] ?? "";
+    const raw = m[1] ?? "";
+    const ref = decodeVerbatim(raw);
     if (!set.has(ref)) {
       const refOffset = (m.index ?? 0) + m[0].indexOf('"') + 1;
       out.push(err(
         "include.unknown-target",
         `#include target '${ref}' does not match any known template.`,
-        { ...span(idx, refOffset, ref.length), data: { target: ref } }
+        { ...span(idx, refOffset, raw.length), data: { target: ref } }
       ));
     }
   }
@@ -1243,10 +1382,10 @@ var INCLUDE_RE2 = /^[ \t]*#include[ \t\n\r\f\x0B]+"([^"]+)"[ \t\n\r\f\x0B]*$/gmu
 var VARIABLE_RE2 = /%(\w+)%/gu;
 var CONDITIONAL_REF_RE = /\{\?!?([A-Za-z_]\w*)\?/gu;
 function extractFromSource(src) {
-  const text = stripComments(src);
+  const text = projectSource(src).text;
   const sets = collect(text, SET_DEF_RE, true);
   const defs = collect(text, DEF_DEF_RE, true);
-  const includes = collect(text, INCLUDE_RE2, false);
+  const includes = [...new Set(collect(text, INCLUDE_RE2, false).map(decodeVerbatim))];
   const body = text.replace(DEFINITION_LHS_RE, "");
   const refs = /* @__PURE__ */ new Set();
   addAll(refs, body, VARIABLE_RE2, true);
@@ -1465,7 +1604,8 @@ function renderAst(ast, ctx) {
 }
 function resolveIncludes(text, ctx) {
   INCLUDE_LINE_RE.lastIndex = 0;
-  return text.replace(INCLUDE_LINE_RE, (_m, ref) => {
+  return text.replace(INCLUDE_LINE_RE, (_m, raw) => {
+    const ref = decodeVerbatim(raw);
     if (ctx.includeStack.includes(ref) || ctx.includeStack.length >= ctx.maxDepth) return "";
     let included;
     try {
@@ -1619,7 +1759,7 @@ function resolveVariable(name, opts) {
   const value = opts.vars[name.toLowerCase()];
   if (value === void 0 || opts.frozen === true) return `%${name}%`;
   if (opts.budget.left <= 0) return `%${name}%`;
-  opts.budget.left -= value.length;
+  opts.budget.left -= budgetLength(value);
   if (opts.depth >= MAX_VARIABLE_DEPTH || !/[{[%]/u.test(value)) return value;
   return renderFragment(parseSequence(value), { ...opts, depth: opts.depth + 1 });
 }
@@ -1636,7 +1776,7 @@ function expandVarsFixpoint(text, opts, passes) {
       const value = opts.vars[name.toLowerCase()];
       if (value === void 0) return m;
       if (opts.budget.left <= 0) return m;
-      opts.budget.left -= value.length;
+      opts.budget.left -= budgetLength(value);
       changed = true;
       return value;
     });
@@ -2056,10 +2196,10 @@ var afterLineBreak = (text, from) => {
   return at === -1 ? null : { leadStart: at + 1, resume: at + 1 };
 };
 var SHIELD_PREFIXES = ["URL", "URI", "EMAIL", "DOTTOK", "DOM", "NUM", "ABBR"];
-var RESTORE_RE2 = new RegExp(`\\x00(?:${SHIELD_PREFIXES.join("|")})_\\d+\\x00`, "gu");
+var RESTORE_RE = new RegExp(`\\x00(?:${SHIELD_PREFIXES.join("|")})_\\d+\\x00`, "gu");
 function restore(text, input, placeholders) {
   if (!input.includes("\0")) {
-    return text.replace(RESTORE_RE2, (key) => placeholders.get(key) ?? key);
+    return text.replace(RESTORE_RE, (key) => placeholders.get(key) ?? key);
   }
   return restoreAsTheLoopDoes(text, placeholders);
 }
@@ -2129,11 +2269,33 @@ function postProcess(input, locale = "") {
   return restore(text, input, placeholders).trim();
 }
 
+// src/internal/neutralize.ts
+var STRUCTURAL = ["{", "}", "[", "]", "%", "#"];
+var SENTINEL_BASE = 57344;
+var SHIELD = /* @__PURE__ */ new Map();
+var RESTORE = /* @__PURE__ */ new Map();
+STRUCTURAL.forEach((ch, i) => {
+  const sentinel = String.fromCharCode(SENTINEL_BASE + i);
+  SHIELD.set(ch, sentinel);
+  RESTORE.set(sentinel, ch);
+});
+var SHIELD_RE = /[{}[\]%#]/gu;
+var RESTORE_RE2 = new RegExp(
+  `[${String.fromCharCode(SENTINEL_BASE)}-${String.fromCharCode(SENTINEL_BASE + STRUCTURAL.length - 1)}]`,
+  "gu"
+);
+function neutralize(value) {
+  return value.replace(SHIELD_RE, (ch) => SHIELD.get(ch) ?? ch);
+}
+function safetyRestore(text) {
+  return text.replace(RESTORE_RE2, (ch) => RESTORE.get(ch) ?? ch);
+}
+
 // src/internal/pipeline.ts
 var DEFAULT_MAX_DEPTH = 20;
 function renderWith(input, rng, opts = {}) {
   const ctx = {
-    runtimeContext: opts.context ?? {},
+    runtimeContext: runtimeValues(opts.context),
     rng,
     locale: opts.locale ?? "",
     resolver: opts.includeResolver,
@@ -2147,7 +2309,13 @@ function renderWith(input, rng, opts = {}) {
   const ast = resolveAst(input);
   let out = renderAst(ast, ctx);
   if (opts.postProcess !== false) out = postProcess(out, ctx.locale);
-  return safetyRestore(out);
+  return decodeVerbatim(safetyRestore(out));
+}
+function runtimeValues(context) {
+  if (context === void 0) return {};
+  const entries = Object.entries(context);
+  if (entries.every(([, value]) => stripSpanRange(value) === value)) return context;
+  return Object.fromEntries(entries.map(([key, value]) => [key, stripSpanRange(value)]));
 }
 function resolveAst(input) {
   if (typeof input === "string") return parseTemplate(input);
@@ -2229,6 +2397,7 @@ function countConstructs(ast, includeCount) {
   });
   counts.set = Object.keys(ast.setDefs).length;
   counts.include = includeCount;
+  counts.verbatim = projectSource(ast.source).spans;
   return counts;
 }
 function neutralize2(value) {