@trazum/core 1.86.0 → 2.1.0

package/src/segment.ts

@@ -23,8 +23,102 @@ interface PatternDef {
 const PATTERNS: PatternDef[] = [
   // Fenced code blocks with ``` or ~~~ (unclosed blocks run to the end).
   { kind: 'fenced-code', regex: /(?:```|~~~)[\s\S]*?(?:```|~~~|$)/g },
-  // Indented blocks of 4+ spaces are left to the rules: they are ambiguous in markdown.
+  /*
+    Indented code blocks, which this file used to leave to the rules on the
+    grounds that they are *"ambiguous in markdown"*. The ambiguity is real. The
+    consequence of acting on it was not proportionate to it.
+
+    **What the rules did to four lines of indented code**, at the aggressive
+    level, every one measured rather than imagined:
+
+      const label = "please keep";   ->  Const label = " keep";
+      def run(x):                    ->  Def run(x):
+      SELECT … WHERE note = 'please refund';  ->  … = ' refund';
+      {"reason": "please cancel"}    ->  {"reason": " cancel"}
+
+    Three separate failures at once. The indentation goes, which by itself
+    makes the Python a syntax error. Keywords get sentence-capitalised, which
+    makes the rest of them syntax errors. And **string literals are edited** —
+    that SQL now matches a different value and that payload carries a different
+    reason. The report says tokens were saved.
+
+    The blank line before the block is what makes this checkable rather than a
+    guess: CommonMark says an indented code block cannot interrupt a paragraph,
+    so a run of indented lines after a blank line is code by the specification
+    rather than by anybody's judgement here.
+
+    **What it still gets wrong, stated rather than discovered later.** Content
+    indented four or more spaces inside a deeply nested list is continuation,
+    not code, and this protects it. That costs unsaved tokens in a rare shape.
+    Not protecting code costs a broken prompt in a common one, and those are not
+    the same kind of wrong.
+
+    Every quantifier is bounded or consumes a whole line exactly once, for the
+    reason the email mask below now carries in full.
+  */
+  {
+    kind: 'indented-code',
+    /*
+      **The lookbehind requires real content before the blank line, and the
+      fuzzer is why.** Its first version asked only for a blank line, which a
+      document *beginning* with one satisfies — and `optimize` ends with a
+      `.trim()` on the whole reassembled string, outside every mask, which then
+      removes the leading newlines and the first line's indentation. The block
+      stopped being a block, the second pass no longer protected it, and 54 of
+      1,500 corpus inputs failed the idempotence property.
+
+      The trim is fixed at its source now — `optimize` trims the *masked* string,
+      where a protected span is a placeholder nothing can edit — so the mask can
+      also claim a block at the very start of a document. It has to: at the
+      aggressive level a rule can delete the sentence before the blank line, and
+      a block that was protected on the first pass has to still be protected on
+      the second or the output never settles.
+    */
+    regex: /(?<=\S[ \t]*\n[ \t]*\n)(?:(?: {4,}|\t)[^\n]*(?:\n|$))+/g,
+  },
   { kind: 'url', regex: /\b(?:https?|ftp):\/\/[^\s<>"')\]]+/g, trimTrailing: /[.,;:!?]+$/ },
+  /*
+    Email addresses, for the reason at the top of this file.
+
+    **Five of ten realistic addresses came out corrupted before this existed.**
+    `please@example.com` became `@example.com`, and so did `thanks@`,
+    `basically@`, `essentially.ops@` and `very.important@`: the politeness,
+    filler and intensifier rules match the local part as ordinary prose and cut
+    it out. What is left is not a wrong address, it is not an address — and the
+    report says tokens were saved.
+
+    That is the failure this module's own first paragraph names. A code block, a
+    URL and a placeholder were on the list; the thing every support prompt in
+    the world carries was not. `support@` and `no-reply@` survived, and
+    `por.favor@ejemplo.es` survived only because the Spanish politeness entry is
+    written with a space — luck, not design, which is what made it invisible.
+
+    After the URL pattern so a `mailto:` link is claimed as a URL first, and
+    trailing punctuation is trimmed for the same reason it is there: the full
+    stop in "write to a@b.com." belongs to the sentence.
+  */
+  {
+    kind: 'email',
+    /*
+      **Every quantifier here is bounded, and the first version of this was not.**
+      The local part accepts `.`, so on a long dotted run with no `@` in it —
+      a dotted run behind a scheme, which is a fixture in `security.test.js`
+      for exactly this reason — an unbounded `+` matches the whole run from every starting
+      position and then fails. Quadratic: **897ms on 40,008 characters**, and
+      `optimize` runs this alongside every other pattern, which took the total
+      past the 5-second cliff detector on a CI runner while passing on a faster
+      machine. That test's own note had already recorded the lesson from the
+      last time — *"quadratic until the label quantifiers were bounded"* — and
+      this pattern arrived a week later without them.
+
+      The bounds are RFC 5321's, not invented: 64 octets for a local part, 63
+      for a domain label. The TLD's 24 is generous against the longest real one
+      (`.travelersinsurance`, 19). Bounded, the same input takes **8ms**, and no
+      address in the corpus is lost.
+    */
+    regex: /\b[A-Za-z0-9._%+-]{1,64}@[A-Za-z0-9-]{1,63}(?:\.[A-Za-z0-9-]{1,63})*\.[A-Za-z]{2,24}/g,
+    trimTrailing: /[.,;:!?]+$/,
+  },
   // Template placeholders: {{var}}, {var}, ${var}, {% tag %}, <<VAR>>, %(var)s
   {
     kind: 'placeholder',

package/src/structure.ts

@@ -54,6 +54,31 @@ interface AxisDefinition {
  */
 const RESPOND = String.raw`(?:respond|reply|answer|write|speak|output|responde|contesta|responder|escribe|redacta)`;
 
+/**
+ * How a prompt refers to the reader having produced their message.
+ *
+ * `used|wrote|speaks` missed on a tense: *"respond in the same language the
+ * user writes in"* — present, which is how a standing instruction is normally
+ * written — was invisible, so a prompt pinning English and mirroring the user
+ * in the same breath reported no conflict at all.
+ */
+const SPOKE = String.raw`(?:used?|uses|wrote|writes|writing|speaks|spoke|sent|asked)`;
+
+/** Nouns that mean *the thing the model produces*, as opposed to any other text. */
+const THE_ANSWER = String.raw`(?:answers?|responses?|repl(?:y|ies)|explanations?|respuestas?)`;
+
+/**
+ * A numeric ceiling on length, without the unit.
+ *
+ * Kept separate because it is only ever safe next to an anchor: *"the title
+ * field takes at most five words"* is the same phrase about a form field, and
+ * reading it as an instruction about the answer is how a widening turns a
+ * detector into a nuisance.
+ */
+const REQUIRE_SHORT = String.raw`keep\s+(?:it|them|${THE_ANSWER})\s+(?:short|brief|concise)`;
+
+const AT_MOST = String.raw`(?:at\s+most|no\s+more\s+than|under|within|máximo\s+de?)\s+(?:one|two|three|four|five|\d+)\s+(?:sentences?|words?|frases?|palabras?)`;
+
 const AXES: readonly AxisDefinition[] = [
   {
     axis: 'response-language',
@@ -71,7 +96,7 @@ const AXES: readonly AxisDefinition[] = [
           // The qualifier slot ("own", "native", "preferred") is what makes
           // this survive real prompts: "the customer's own language" is at
           // least as common as the bare form.
-          String.raw`\bin\s+the\s+(?:same\s+)?language\s+(?:of|as|the\s+\w+\s+(?:used|wrote|speaks))|\bin\s+(?:the\s+(?:user|customer|client)'?s?|their)\s+(?:own\s+|native\s+|preferred\s+)?language\b|\bmatch\s+the\s+language\b|\ben\s+(?:el\s+)?(?:mismo\s+)?idioma\s+(?:del?\s+|en\s+que\s+)?(?:usuario|cliente|consulta|mensaje|escrib)`,
+          String.raw`\bin\s+the\s+(?:same\s+)?language\s+(?:of|as|the\s+\w+\s+(?:used|wrote|speaks))|\bin\s+(?:the\s+(?:user|customer|client)'?s?|their)\s+(?:own\s+|native\s+|preferred\s+)?language\b|\bmatch\s+the\s+language\b|\b${RESPOND}\b[^.!?\n]{0,40}?\bin\s+(?:the\s+(?:same\s+)?|whatever\s+|whichever\s+)language\s+(?:of|as|that\s+|the\s+\w+\s+)?\w*\s*${SPOKE}|\ben\s+(?:el\s+)?(?:mismo\s+)?idioma\s+(?:del?\s+|en\s+que\s+)?(?:usuario|cliente|consulta|mensaje|escrib)`,
           'i',
         ),
       ],
@@ -109,14 +134,14 @@ const AXES: readonly AxisDefinition[] = [
       [
         'length-brief',
         new RegExp(
-          String.raw`\b(?:be\s+(?:brief|concise|succinct|terse)|keep\s+it\s+(?:short|brief)|as\s+short\s+as\s+possible|in\s+(?:one|a\s+single)\s+sentence|s[ée]\s+(?:breve|conciso)|de\s+forma\s+(?:breve|concisa)|brevemente)\b`,
+          String.raw`\b(?:be\s+(?:brief|concise|succinct|terse)|keep\s+it\s+(?:short|brief)|as\s+short\s+as\s+possible|${REQUIRE_SHORT}|\b${RESPOND}\b[^.!?\n]{0,40}?${AT_MOST}|in\s+(?:one|a\s+single)\s+sentence|s[ée]\s+(?:breve|conciso)|de\s+forma\s+(?:breve|concisa)|brevemente)\b`,
           'i',
         ),
       ],
       [
         'length-detailed',
         new RegExp(
-          String.raw`\b(?:be\s+(?:detailed|comprehensive|thorough|exhaustive)|in\s+(?:great\s+)?depth|as\s+much\s+detail\s+as\s+possible|s[ée]\s+(?:exhaustivo|detallado)|de\s+forma\s+(?:detallada|exhaustiva)|con\s+todo\s+detalle|detalladamente)\b`,
+          String.raw`\b(?:be\s+(?:detailed|comprehensive|thorough|exhaustive)|in\s+(?:great\s+)?depth|(?:detailed|comprehensive|thorough|exhaustive|in-depth)[^.!?\n]{0,30}?\b${THE_ANSWER}\b|as\s+much\s+detail\s+as\s+possible|s[ée]\s+(?:exhaustivo|detallado)|de\s+forma\s+(?:detallada|exhaustiva)|con\s+todo\s+detalle|detalladamente)\b`,
           'i',
         ),
       ],
@@ -128,14 +153,14 @@ const AXES: readonly AxisDefinition[] = [
       [
         'reasoning-shown',
         new RegExp(
-          String.raw`\b(?:explain\s+your\s+(?:reasoning|answer|thinking)|show\s+your\s+(?:work|reasoning|thinking)|think\s+step[\s-]by[\s-]step|justify\s+your\s+answer|explica\s+tu\s+razonamiento|razona\s+paso\s+a\s+paso|justifica\s+tu\s+respuesta)\b`,
+          String.raw`\b(?:explain\s+your\s+(?:reasoning|answer|thinking)|show\s+your\s+(?:work|reasoning|thinking)|show\s+your\s+chain\s+of\s+thought|walk\s+(?:me\s+)?through\s+your\s+(?:reasoning|thinking|logic)|think\s+step[\s-]by[\s-]step|justify\s+your\s+answer|explica\s+tu\s+razonamiento|razona\s+paso\s+a\s+paso|justifica\s+tu\s+respuesta)\b`,
           'i',
         ),
       ],
       [
         'reasoning-hidden',
         new RegExp(
-          String.raw`\b(?:no\s+(?:explanation|commentary|preamble)|without\s+(?:explanation|commentary)|do\s+not\s+explain|don'?t\s+explain|no\s+expliques|sin\s+(?:explicaciones|comentarios)|sin\s+preámbulo)\b`,
+          String.raw`\b(?:no\s+(?:explanation|commentary|preamble)|without\s+(?:explanation|commentary|justification|reasoning)|no\s+justification|do\s+not\s+explain|don'?t\s+explain|no\s+expliques|sin\s+(?:explicaciones|comentarios)|sin\s+preámbulo)\b`,
           'i',
         ),
       ],
@@ -249,10 +274,83 @@ const LABEL_SUFFIX = String.raw`[ \t]{0,4}\d{0,6}[ \t]{0,4}[:.)]`;
  * indistinguishable from ordinary prose, and guessing would make the advisory
  * fire on prompts containing no examples at all.
  */
+/**
+ * The label on the side that **asks**, in the wordings prompts actually use.
+ *
+ * **This list was three entries long and it made six analyses blind.** A
+ * support prompt labelled `Customer:` / `Agent:` — the single most common
+ * shape there is — found zero examples, so `prune` had nothing to cut,
+ * `profile` reported the examples as costing nothing, and the redundant-example
+ * advisory never fired. Measured across fourteen labellings real prompts use,
+ * nine found nothing: `Customer`, `Human`, `Question`, `Client`, `Prompt`,
+ * `Query`, `Cliente`, `Pregunta`, `REQUEST`. `Human:` is Anthropic's own
+ * historical convention and `Q:` worked while `Question:` did not, which is
+ * the arbitrariness that gives the fault away.
+ *
+ * The failure was silent, which is what made it survive: an unrecognised label
+ * produces no examples, and no examples produces no finding — the same shape
+ * as a gate that passes because it is checking nothing.
+ *
+ * Only openers belong here. `Answer:` and `Assistant:` close a turn, and
+ * splitting on those would cut every example between its question and its
+ * answer and then compare the halves.
+ */
+const ASKER_LABELS = [
+  'user',
+  'usuario',
+  'human',
+  'humano',
+  'customer',
+  'cliente',
+  'client',
+  'q',
+  'question',
+  'pregunta',
+  'query',
+  'consulta',
+  'request',
+  'petición',
+  'peticion',
+  'prompt',
+] as const;
+
+/**
+ * The label on the side that **answers**.
+ *
+ * Never a tier: these appear inside an example rather than opening one. They
+ * are here so `EXAMPLE_FIELD_LINE` can be built from the same two lists the
+ * splitter uses, which is what stops the pair drifting apart again — the field
+ * detector already knew `question` and `answer` while the splitter did not,
+ * and that disagreement is precisely how `Q:` came to work and `Question:` not.
+ */
+const ANSWERER_LABELS = [
+  'assistant',
+  'asistente',
+  'a',
+  'answer',
+  'respuesta',
+  'output',
+  'salida',
+  'response',
+  'reply',
+  'agent',
+  'agente',
+  'support',
+  'soporte',
+  'bot',
+  'result',
+  'resultado',
+] as const;
+
+/** Labels that name an example as a whole, rather than one side of it. */
+const BLOCK_LABELS = ['example', 'ejemplo', 'input', 'entrada'] as const;
+
+const anyOf = (labels: readonly string[]) => labels.join('|');
+
 const EXAMPLE_HEADER_TIERS: readonly RegExp[] = [
   new RegExp(`${LABEL_PREFIX}(?:example|ejemplo)${LABEL_SUFFIX}`, 'i'),
   new RegExp(`${LABEL_PREFIX}(?:input|entrada)${LABEL_SUFFIX}`, 'i'),
-  new RegExp(`${LABEL_PREFIX}(?:user|usuario|q)${LABEL_SUFFIX}`, 'i'),
+  new RegExp(`${LABEL_PREFIX}(?:${anyOf(ASKER_LABELS)})${LABEL_SUFFIX}`, 'i'),
 ];
 
 /**
@@ -265,7 +363,7 @@ const EXAMPLE_HEADER_TIERS: readonly RegExp[] = [
  * repetition it removed.
  */
 export const EXAMPLE_FIELD_LINE = new RegExp(
-  `${LABEL_PREFIX}(?:input|output|entrada|salida|example|ejemplo|user|usuario|assistant|asistente|q|a|question|answer|pregunta|respuesta)${LABEL_SUFFIX}`,
+  `${LABEL_PREFIX}(?:${anyOf([...BLOCK_LABELS, ...ASKER_LABELS, ...ANSWERER_LABELS])})${LABEL_SUFFIX}`,
   'i',
 );
 
@@ -307,6 +405,30 @@ export interface ExampleAnalysis {
  * "arrived fast" teach the same thing needs a model, not a word-set overlap;
  * that belongs to the optional LLM pass, not here.
  */
+/**
+ * How alike two examples must be before one is called redundant.
+ *
+ * **Do not lower this hoping to catch paraphrases: it was measured and it does
+ * not work.** Four support examples whose answers were *"Let me look that up
+ * for you. Could you share your order number?"*, *"I can check that for you.
+ * What's your order number?"*, *"Happy to help. May I have your order
+ * number?"* and *"Of course. What is your order number?"* are one example
+ * wearing four coats — any reader says so instantly. Their word overlap is
+ * 0.29–0.43, and comparing only the answers makes it **worse**, 0.21–0.39,
+ * because the shared content is three words (`your`, `order`, `number`) while
+ * the openers are all different. There is no threshold that separates those
+ * from genuinely different examples: at 0.4 this fires on every prompt whose
+ * examples share a domain vocabulary.
+ *
+ * That similarity is a semantic judgement, and this repository has a place for
+ * those — `trazum semantic`, which says what it costs and asks first. What
+ * word overlap can honestly find is a paragraph somebody pasted twice and
+ * edited lightly, which is what 0.7 is for.
+ *
+ * So the deterministic half reports **what the examples cost**, which is a
+ * fact, and leaves *whether one should go* to `prune`, which answers it by
+ * running the cases rather than by counting words.
+ */
 const EXAMPLE_SIMILARITY = 0.7;
 
 /** Splits the prompt on one header style. */
@@ -356,10 +478,70 @@ function trimToExample(block: string): string {
   return kept.join('\n\n').trim();
 }
 
+/**
+ * Examples delimited by an `<example>` tag rather than by a label.
+ *
+ * **This is Anthropic's own documented convention for few-shot prompting**, and
+ * the product whose headline model is Claude could not see it: a code-review
+ * prompt wrapping three demonstrations in `<example>` found zero examples, and
+ * the same six analyses went quiet that `Customer:` had silenced. A shape a
+ * provider tells people to use is not an edge case.
+ *
+ * Tried before the label tiers because it is the least ambiguous thing in this
+ * file. A label is a word that might be prose; `<example>` is a delimiter, so
+ * there is no threshold and no judgement — either the tag is there or it is not.
+ *
+ * Only `example` and `examples`, which are what the documentation names.
+ * `<sample>` and `<demonstration>` appear in the wild and are not added: the
+ * list would then be a guess about what people might write, and this splitter
+ * has already been burnt once by a vocabulary somebody thought was wide enough.
+ * A tag arriving here should arrive with a prompt that uses it.
+ *
+ * A tag inside a fenced block is documentation about the tag rather than a use
+ * of it — a prompt explaining how to write examples is not a prompt containing
+ * them — so fenced regions are removed before matching.
+ */
+function xmlExamples(prompt: string): string[] {
+  const outsideFences = prompt.replace(/^([ \t]{0,3})(`{3,}|~{3,})[\s\S]*?^\1\2[ \t]*$/gm, '');
+  const tag = /<example(?:\s[^>]*)?>([\s\S]*?)<\/example>/gi;
+  return [...outsideFences.matchAll(tag)].map(([, body]) => (body ?? '').trim()).filter(Boolean);
+}
+
+/**
+ * **A shape this deliberately does not detect, and the reason is the point.**
+ *
+ * Inline mappings are everywhere — `1. "package never arrived" -> shipping`,
+ * `- "Acme raised a round" => Acme` — and on the probes that found the two
+ * faults above they are the remaining silent case. They are not added, because
+ * the only thing separating a demonstration from an instruction here is
+ * judgement: `if the ticket is about delivery -> use the shipping category` is
+ * the same line shape and is a rule, not an example.
+ *
+ * A detector that got that wrong would not merely report a wrong number.
+ * `prune` *removes* examples, so a rule misread as an example becomes a
+ * proposal to delete an instruction — and it would arrive with the same
+ * confidence as a correct one.
+ *
+ * The tempting narrowing is to require the left side to be quoted. It was
+ * considered and refused: both probes are quoted **because the author of the
+ * probes wrote them that way**, and a threshold fitted to one's own fixture is
+ * a threshold that has been measured against nothing. What would settle it is
+ * real prompts using the shape, not a rule invented here.
+ *
+ * Until then the honest behaviour is the one `prune` now has: say what was
+ * looked for, so a prompt this cannot read reports that it was not seen rather
+ * than that it does not exist. That sentence used to read "this prompt has
+ * fewer than two few-shot examples", which was a confident false statement
+ * about a prompt with four of them.
+ */
+
 /** Splits the prompt into labelled example blocks. */
 export function findExamples(prompt: string, count: TokenCounter): ExampleBlock[] {
   const lines = prompt.split('\n');
 
+  const tagged = xmlExamples(prompt);
+  if (tagged.length >= 2) return tagged.map((text) => ({ text, tokens: count(text) }));
+
   for (const header of EXAMPLE_HEADER_TIERS) {
     const blocks = splitOnHeader(lines, header);
     if (blocks.length >= 2) {
@@ -447,6 +629,90 @@ function fencedBlocks(prompt: string): Array<{ lang: string; body: string }> {
  * prompts worth checking. Only keys at nesting depth 1 count, so a nested
  * field name cannot be mistaken for a top-level one.
  */
+/**
+ * Fence languages this reads a schema out of.
+ *
+ * `ts` and `typescript` are new; the rest were already here and three of them
+ * did not work. See `unquotedKeys`.
+ */
+const SCHEMA_LANGS = /^(?:json|jsonc|json5|yaml|yml|ts|typescript)$/;
+
+/**
+ * Keys written without quotes, which is most of them.
+ *
+ * **The filter above already admitted `yaml`, `yml` and `json5`, and the
+ * extractor could not read any of them.** It only accepted a quoted string
+ * followed by a colon, which is JSON and nothing else — so a prompt showing its
+ * schema as YAML matched a language this module claims to support, produced
+ * zero keys, and reported no restatement however thoroughly the prose walked
+ * the fields. Not a missing feature: a promise made in one expression and
+ * broken by the next.
+ *
+ * `ts` is added at the same time because it is the same omission wearing a
+ * different label — `interface Ticket { category: string }` is how a
+ * TypeScript codebase writes the structured output it wants back, and it was
+ * not on the list at all.
+ *
+ * Two shapes, because YAML has no braces:
+ *
+ * - **Braced** (`json5`, `jsonc`, `ts`): an identifier at depth 1 followed by
+ *   `:` or `?:`. Depth is what keeps a nested object's fields out.
+ * - **Indented** (`yaml`, `yml`): an identifier at the block's own outermost
+ *   indent, followed by `:`. Anything indented past it belongs to a parent key
+ *   and is not top-level.
+ *
+ * Deliberately not applied to a fence with **no** language. An unlabelled block
+ * is as likely to be a shell session or a log line as a schema, and
+ * `Note: something` reads as a key to any rule this simple. The quoted form
+ * stays the only thing read there, exactly as before.
+ */
+function unquotedKeys(body: string, lang: string): string[] {
+  if (!SCHEMA_LANGS.test(lang)) return [];
+
+  if (/^(?:yaml|yml)$/.test(lang)) {
+    const lines = body
+      .split('\n')
+      .filter((line) => line.trim() !== '' && !/^\s*#/.test(line));
+    const outermost = Math.min(...lines.map((line) => line.length - line.trimStart().length));
+    if (!Number.isFinite(outermost)) return [];
+    return [
+      ...new Set(
+        lines
+          .filter((line) => line.length - line.trimStart().length === outermost)
+          .map((line) => /^\s*([A-Za-z_][\w-]*)\s*:/.exec(line))
+          .filter((found): found is RegExpExecArray => found !== null)
+          .map((found) => found[1]!),
+      ),
+    ];
+  }
+
+  /*
+    Character by character rather than line by line, and that was found by
+    running it: `interface Ticket { category: string; urgency: number }` is one
+    line, so a line-oriented walk sees depth 0 where it starts and reads nothing
+    at all. Single-line objects are the common case in a `ts` fence and in the
+    JSON5 people paste, not the exception.
+  */
+  const keys: string[] = [];
+  let depth = 0;
+  let candidate = '';
+  for (let i = 0; i < body.length; i++) {
+    const ch = body[i]!;
+    if (ch === '{' || ch === '[') { depth++; candidate = ''; continue; }
+    if (ch === '}' || ch === ']') { depth--; candidate = ''; continue; }
+    if (ch === ',' || ch === ';' || ch === '\n') { candidate = ''; continue; }
+    if (/[A-Za-z0-9_-]/.test(ch)) { candidate += ch; continue; }
+    if (ch === ':' || (ch === '?' && body[i + 1] === ':')) {
+      if (depth === 1 && /^[A-Za-z_][\w-]*$/.test(candidate)) keys.push(candidate);
+      candidate = '';
+      continue;
+    }
+    // Whitespace between a name and its colon is fine; anything else is not a key.
+    if (!/\s/.test(ch)) candidate = '';
+  }
+  return [...new Set(keys)];
+}
+
 function topLevelKeys(body: string): string[] {
   const keys: string[] = [];
   let depth = 0;
@@ -505,8 +771,8 @@ export function findRestatedFormat(prompt: string, count: TokenCounter): Restate
   const keys = [
     ...new Set(
       blocks
-        .filter((b) => b.lang === '' || /json|jsonc|json5|yaml|yml/.test(b.lang))
-        .flatMap((b) => topLevelKeys(b.body)),
+        .filter((b) => b.lang === '' || SCHEMA_LANGS.test(b.lang))
+        .flatMap((b) => [...topLevelKeys(b.body), ...unquotedKeys(b.body, b.lang)]),
     ),
   ].filter((k) => k.length >= 3);
 

package/src/types.ts

@@ -363,8 +363,10 @@ export interface Segment {
 
 export type ProtectionKind =
   | 'fenced-code'
+  | 'indented-code'
   | 'inline-code'
   | 'url'
+  | 'email'
   | 'placeholder'
   | 'xml-tag';