@hypit/hypit 0.1.14 → 0.2.1

package/packages/script/README.md

@@ -10,7 +10,7 @@ know that Script, Segment or Narrative exist.
 
 ```svml
 <script id="story">
-  @answer
+  @{answer}
 
   <opening>
     <ALICE> I will make the first point.
@@ -18,7 +18,7 @@ know that Script, Segment or Narrative exist.
   </opening>
 
   <pause/>
-  @/answer
+  @{/answer}
 </script>
 ```
 
@@ -65,8 +65,8 @@ CaptionDocument contracts.
   Role Cue or the Segment close ends that turn.
 - **Dual Text**: one authored speech span with separate display and spoken projections, written
   `<display text | spoken text>`. In `<display text|>`, omitted speech inherits the displayed prose.
-- **Selection marker**: a named semantic range, written `@name ... @/name`.
-- **Moment marker**: a named semantic point, written `@name!`.
+- **Selection marker**: a named semantic range, written `@{name} ... @{/name}`.
+- **Moment marker**: a named semantic point, written `@{name!}`.
 - **CaptionDocument**: the Script-owned caption truth; it contains **Display Words**,
   **Alignment Units** and **Cue Breaks**. It contains no seconds or frames.
 - **Token attribute**: a flat postfix display-word annotation such as `really{emphasis}` or
@@ -86,7 +86,7 @@ Caption Styles choose whether to highlight, reveal or keep the text steady.
 
 In a shared side, markers and display attributes do not enter either text projection. Its speech
 Tokens retain offsets into the actually written left-hand text, so Studio can move an anchor inside
-`<组@beat!件化|>` without expanding the shorthand. Attributes still annotate the preceding display
+`<组@{beat!}件化|>` without expanding the shorthand. Attributes still annotate the preceding display
 word, not the whole group. Whitespace-only speech is omitted; a group with no spoken text on either
 side is invalid. This adds no new public value type or protocol version.
 
@@ -101,8 +101,8 @@ Units; it does not split the Segment, cut the picture or end a Selection.
 <script id="story">
   <exchange>
     <HOST> I use it || every day, || since <2012 | twenty twelve>.
-    <GUEST> Even @proof on holiday @/proof?
-    <HOST> @answer! Especially then.
+    <GUEST> Even @{proof} on holiday @{/proof}?
+    <HOST> @{answer!} Especially then.
   </exchange>
 </script>
 ```
@@ -117,19 +117,53 @@ following characters. Display punctuation attaches to neighboring words without
 These units support precise timing and highlighting. A Caption Cue can hold a whole phrase of them;
 `||` chooses its handoff independently of character counts or visual line wrapping.
 
+Annotations do not create speech boundaries. Comments are transparent (`hel<!--note-->lo`
+remains `hello`); a postfix attribute or `||` inside a word is invalid. Script analyzes a complete
+prose run before binding these constructs. An explicit Dual correspondence and a speaker/Segment
+boundary remain authored structure. Shared Dual groups still expose their internal speech anchors.
+
+A Dual display side is literal authored text, including symbols and emoji: `<😀|smile>` and
+`<.|dot>` have explicit speech correspondence and require no invented speech token for the symbol.
+A literal-only display is one display surface within that correspondence.
+
+## Display spelling and separators
+
+Script preserves the normalized display spelling independently of speech tokenization. Ordinary
+whitespace runs become one space; leading/trailing whitespace in a Turn and padding at the edges
+of a Dual Text side are omitted. No language-specific rule removes a Chinese space or inserts a
+space between numeric and Korean/Latin tokens. `是的 就是这样`, `3개월`, `3 개월`, `3D` and `3 D`
+therefore remain distinct as authored. Source newlines are prose formatting, not Caption Cue breaks.
+Use `||` for Cues and a family's layout controls for visual rows.
+
+Each `CaptionDisplayWord.separatorBefore` is `""` or `" "`, relative to the preceding displayed word
+in its Turn. Together with `text`, it carries the display spelling to consumers; it is not a speech
+Token and has no timing. A renderer suppresses the leading separator at a displayed line/Cue start.
+Shared groups retain internal separators and individual speech anchors: `<New York|>` is one
+Alignment Unit with a space inside, while `<3D|>` has none. Grouping is a creative choice, not a
+workaround for preserving spelling.
+
 ## Selection and Moment affinity
 
+Every marker starts with `@{` and ends with `}`. The complete marker is zero-width in speech and
+display; surrounding prose whitespace remains prose. All control sigils belong inside the braces:
+`@{beat!}` is a Moment, whereas `@{part}!` opens a Selection followed by a literal exclamation mark.
+Names match `[a-z][a-z0-9_-]{0,63}`; whitespace and nesting inside a marker are invalid.
+A marker cannot split a speech Token or separate it from attached punctuation: place
+`@{beat!}“测试”`, not `“@{beat!}测试”`. `hello{emphasis}` is a postfix display attribute; `@{part}`
+is consumed as one marker and cannot be mistaken for that attribute. Write a literal `@{part}` as
+`\@\{part\}`.
+
 A marker selects the adjacent semantic anchor; it does not write a timecode. Inside a spoken
 passage, its affinity normally chooses a neighboring word boundary:
 
 | Marker | Boundary |
 | --- | --- |
-| `@name` | Selection opens at the next word's start: right affinity. |
-| `~@name` | Selection opens at the previous word's end: left affinity. |
-| `@/name` | Selection closes at the previous word's end: left affinity. |
-| `@/name~` | Selection closes at the next word's start: right affinity. |
-| `@name!` | Moment at the next word's start: right affinity. |
-| `~@name!` | Moment at the previous word's end: left affinity. |
+| `@{name}` | Selection opens at the next word's start: right affinity. |
+| `@{~name}` | Selection opens at the previous word's end: left affinity. |
+| `@{/name}` | Selection closes at the previous word's end: left affinity. |
+| `@{/name~}` | Selection closes at the next word's start: right affinity. |
+| `@{name!}` | Moment at the next word's start: right affinity. |
+| `@{~name!}` | Moment at the previous word's end: left affinity. |
 
 At structural edges, the parser retains the corresponding Segment or program boundary rather than
 inventing a neighboring word. Selection and Moment ids share one namespace. Selections can overlap,
@@ -138,37 +172,50 @@ cross and span Segments; unlike tags, they do not need to nest.
 To join two visual Selections without exposing their inter-word pause, match affinity on both sides:
 
 ```text
-@coffee my coffee @/coffee ~@smoothie my smoothie @/smoothie
-@coffee my coffee @/coffee~ @smoothie my smoothie @/smoothie
+@{coffee} my coffee @{/coffee} @{~smoothie} my smoothie @{/smoothie}
+@{coffee} my coffee @{/coffee~} @{smoothie} my smoothie @{/smoothie}
 ```
 
 These are alternative spellings, not two occurrences to put in the same Script. In the first,
 “coffee” ends both the first Window and the gap's left boundary, so the smoothie Selection owns the
 pause. In the second, the next “my” starts both touching boundaries, so the coffee Selection owns it.
-Plain `@/coffee @smoothie` leaves the gap between previous word end and next word start outside both.
+Plain `@{/coffee} @{smoothie}` leaves the gap between previous word end and next word start outside both.
 The media consumer still decides playback and visual coverage inside those projected Windows.
 
 ## Marker writeback
 
 `adjustScriptSelection` and `adjustScriptMoment` accept explicit anchor identities. A Selection's
-two endpoints are written together. Script owns their legal source sites: token punctuation and
-postfix attributes stay attached, Segment boundaries stay structural, and coincident markers are
-ordered together. Writeback reads the current source, removes the markers, normalizes ordinary
-same-line prose, and places the markers at their requested anchors. Repeated spaces and tabs in
-prose become necessary separators; punctuation uses the same attachment rules as Script's text
-projection. Newlines, blank lines and each line's leading spaces/tabs remain intact. A marker at
-the start of a line is placed after its indentation; whitespace after that marker is prose, not
-additional indentation. Marker-only lines remain blank when the marker leaves. Comments, tags,
-Dual Text and display attributes retain their own syntax and are not passed through prose cleanup.
-
-Normalization keeps lexical units unchanged and does not restore earlier whitespace spellings.
-It has no persistent formatting state: the same text structure and anchor relationships produce
-the same spelling on repeated edits. Empty self-closing Segments expand when needed to make their
-distinct boundaries writable.
-
-Writeback reparses the result to retain the intended identities and unchanged narrative/caption
-content. These checks concern authored order, not frame order. A consumer's temporal projection
-determines whether the resulting Instant or Window is usable.
+two endpoints are moved together; unrelated markers and all source prose remain untouched. Names
+terminate at `}`, so writeback never inserts a separating space or normalizes surrounding prose.
+A no-op adjustment returns the original source. Script owns legal insertion sites: Tokens and
+postfix attributes stay together, Dual Text markers follow the actual speech side, and coincident
+markers written together have deterministic order. Empty self-closing Segments expand only when a
+requested boundary needs an interior insertion site.
+
+Writeback reparses the result to retain the requested bindings and unchanged speech, display and
+Narrative/Caption content. Source offsets remain parser-private; no formatting history is stored.
+
+## Explicit migration from 0.1
+
+The 0.2 parser rejects bare `@name` markers. Preview migration from the repository or installed
+Distribution root, then explicitly write the reviewed result:
+
+```sh
+node packages/script/bin/migrate-0.2.mjs /path/to/film.svml
+node packages/script/bin/migrate-0.2.mjs /path/to/film.svml --write
+```
+
+Use `--body` for a file containing a raw Script body rather than outer SVML. The tool converts
+markers only inside Script bodies, leaves comments and escapes intact, and does not touch provider
+prompt references such as `@image1`. It neither installs anything nor runs during a build.
+
+The tool changes marker spelling, not marker placement. Move a marker that separates a word from
+its attached quote or punctuation to the complete word boundary before using that source.
+
+Review authored whitespace after migration: spaces previously discarded by Chinese/punctuation
+normalization now appear. The tool preserves source spaces rather than guessing the author's intent.
+Regenerate affected Narrative, caption and Build results with the new reader/writer together;
+protocol identities remain `@1`. Existing rendered media is not modified by source migration.
 
 ## Complete authored content and narrow exports
 

package/packages/script/bin/migrate-0.2.mjs

@@ -0,0 +1,90 @@
+#!/usr/bin/env node
+/** Explicit, one-time marker migration. Never called by parsing, building, or Studio. */
+import { readFile, writeFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+
+export function migrateScriptBody(source) {
+  let result = "";
+  for (let cursor = 0; cursor < source.length;) {
+    if (source[cursor] === "\\") {
+      result += source.slice(cursor, cursor + 2);
+      cursor += 2;
+      continue;
+    }
+    if (source.startsWith("<!--", cursor)) {
+      const close = source.indexOf("-->", cursor + 4);
+      if (close < 0) throw new Error("Unclosed Script comment");
+      result += source.slice(cursor, close + 3);
+      cursor = close + 3;
+      continue;
+    }
+    if (source.startsWith("@{", cursor)) {
+      const close = source.indexOf("}", cursor + 2);
+      if (close < 0) throw new Error("Unclosed delimited Script marker");
+      result += source.slice(cursor, close + 1);
+      cursor = close + 1;
+      continue;
+    }
+    const rest = source.slice(cursor);
+    const match = /^@\/([a-z][a-z0-9_-]{0,63})(~)/u.exec(rest)
+      ?? /^@\/([a-z][a-z0-9_-]{0,63})(?![A-Za-z0-9_-])/u.exec(rest);
+    const open = match ? undefined : /^(~)?@([a-z][a-z0-9_-]{0,63})(!)/u.exec(rest)
+      ?? /^(~)?@([a-z][a-z0-9_-]{0,63})(?![A-Za-z0-9_-])/u.exec(rest);
+    if (match) {
+      result += `@{/${match[1]}${match[2] ?? ""}}`;
+      cursor += match[0].length;
+    } else if (open) {
+      result += `@{${open[1] ?? ""}${open[2]}${open[3] ?? ""}}`;
+      cursor += open[0].length;
+    } else {
+      result += source[cursor++];
+    }
+  }
+  return result;
+}
+
+/** Change Script raw bodies only; package references and generation prompt tags stay untouched. */
+export function migrateSvml(source) {
+  let result = "";
+  let cursor = 0;
+  const token = /<!--[\s\S]*?-->|<(?:[A-Za-z_][\w.-]*:)?script(?=[\s/>])(?:"[^"]*"|'[^']*'|[^'">])*>/gu;
+  for (let match = token.exec(source); match; match = token.exec(source)) {
+    if (match[0].startsWith("<!--") || match[0].endsWith("/>")) continue;
+    const name = /^<([^\s>]+)/u.exec(match[0])[1];
+    const close = `</${name}>`;
+    const start = match.index + match[0].length;
+    let end = start;
+    while (end < source.length) {
+      if (source[end] === "\\") { end += 2; continue; }
+      if (source.startsWith("<!--", end)) {
+        const stop = source.indexOf("-->", end + 4);
+        if (stop < 0) throw new Error("Unclosed Script comment");
+        end = stop + 3;
+        continue;
+      }
+      if (source.startsWith(close, end)) break;
+      end++;
+    }
+    if (end >= source.length) throw new Error(`Missing ${close}`);
+    result += source.slice(cursor, start) + migrateScriptBody(source.slice(start, end)) + close;
+    cursor = end + close.length;
+    token.lastIndex = cursor;
+  }
+  return result + source.slice(cursor);
+}
+
+if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) {
+  const args = process.argv.slice(2);
+  const flags = new Set(args.filter(arg => arg.startsWith("--")));
+  const paths = args.filter(arg => !arg.startsWith("--"));
+  if (paths.length !== 1 || [...flags].some(flag => !["--write", "--body"].includes(flag))) {
+    console.error("Usage: node migrate-0.2.mjs <source.svml> [--body] [--write]\nDefault: print migrated SVML. --body: input is a raw Script body. --write: replace the named file.");
+    process.exitCode = 2;
+  } else {
+    const source = await readFile(paths[0], "utf8");
+    const migrated = flags.has("--body") ? migrateScriptBody(source) : migrateSvml(source);
+    if (flags.has("--write")) await writeFile(paths[0], migrated);
+    else process.stdout.write(migrated);
+  }
+}

package/packages/script/src/edit.ts

@@ -3,7 +3,6 @@ import type { SourceRange } from "@hypit/protocol";
 import { canonicalStringify } from "@hypit/protocol";
 import { captionDocument, narrativeValue } from "./narrative.js";
 import { parseScript } from "./parser.js";
-import { cleanHorizontalProse } from "./lexical.js";
 import type { Affinity, ParsedNarrative, SemanticAnchor } from "./types.js";
 
 export type ScriptAnchorEditSite = {
@@ -75,8 +74,8 @@ export function scriptAnchorEditSites(parsed: ParsedNarrative): readonly ScriptA
 }
 
 function marker(id: string, edge: "open" | "close", affinity: Affinity): string {
-  if (edge === "open") return affinity === "left" ? `~@${id}` : `@${id}`;
-  return affinity === "left" ? `@/${id}` : `@/${id}~`;
+  if (edge === "open") return `@{${affinity === "left" ? "~" : ""}${id}}`;
+  return `@{/${id}${affinity === "right" ? "~" : ""}}`;
 }
 
 function applyEdits(source: string, edits: readonly Edit[]): string {
@@ -100,61 +99,32 @@ type AdjustmentInput = {
 
 type NamedAnchor = { readonly id: string; readonly edge: "open" | "close" | "moment"; readonly anchorId: string };
 
-/** Normalize only parser-owned prose. Indentation is read from this edit's input, never stored. */
-function normalizedProse(source: string, parsed: ParsedNarrative, original: string): string {
-  const bodyStarts = new Set(parsed.segments.flatMap(segment => [segment.contentRange.start,
-    ...segment.atoms.filter(atom => atom.kind === "role").map(atom => atom.range.end)]));
-  const bodyEnds = new Set(parsed.segments.map(segment => segment.contentRange.end));
-  const indentation = original.split(/\r\n|\r|\n/u).map(line => /^[ \t]*/u.exec(line)![0].length);
-  const lines: Array<{ start: number; end: number; indentEnd: number }> = [];
-  let offset = 0;
-  const parts = source.split(/(\r\n|\r|\n)/u);
-  for (let index = 0; index < parts.length; index += 2) {
-    const text = parts[index]!;
-    lines.push({ start: offset, end: offset + text.length, indentEnd: offset + indentation[index / 2]! });
-    offset += text.length + (parts[index + 1]?.length ?? 0);
-  }
-  const edits: Edit[] = [];
-  for (const range of parsed.proseRanges) for (const line of lines) {
-    const start = Math.max(range.start, line.indentEnd);
-    const end = Math.min(range.end, line.end);
-    if (start >= end) continue;
-    let text = cleanHorizontalProse(source.slice(start, end));
-    const outside = !parsed.segments.some(segment => range.start >= segment.contentRange.start && range.end <= segment.contentRange.end);
-    if (start === line.indentEnd || bodyStarts.has(start) || outside) text = text.replace(/^[ \t]+/u, "");
-    if (end === line.end || bodyEnds.has(end) || outside) text = text.replace(/[ \t]+$/u, "");
-    edits.push({ range: { start, end }, replacement: text });
-  }
-  return applyEdits(source, edits);
-}
-
-/** One canonical boundary spelling, independent of the order of previous gestures. */
+/** Move only the requested markers. Delimited names never require editing prose separators. */
 function rewrite(input: AdjustmentInput, markers: readonly NamedAnchor[]): string {
-  const ranges = [
-    ...input.parsed.moments.map((item) => item.range),
-    ...input.parsed.selections.flatMap((item) => [item.open.range, item.close.range]),
-  ].map((range) => ({ start: range.start - input.parsed.sourceRange.start, end: range.end - input.parsed.sourceRange.start }))
-    .sort((a, b) => a.start - b.start);
-  const edits: Edit[] = ranges.map(range => ({ range, replacement: "" }));
+  const original = namedAnchors(input.parsed);
+  const changedNames = new Set(markers.filter(value => original.find(item => item.id === value.id && item.edge === value.edge)?.anchorId !== value.anchorId).map(value => value.id));
+  const moved = markers.filter(value => changedNames.has(value.id));
+  if (moved.length === 0) return input.source;
+  const origin = input.parsed.sourceRange.start;
+  const edits: Edit[] = moved.map(value => {
+    const range = value.edge === "moment"
+      ? input.parsed.moments.find(item => item.id === value.id)!.range
+      : input.parsed.selections.find(item => item.id === value.id)![value.edge].range;
+    return { range: { start: range.start - origin, end: range.end - origin }, replacement: "" };
+  });
   for (const segment of input.parsed.segments) {
-    if (segment.selfClosing) edits.push({ range: { start: segment.range.start - input.parsed.sourceRange.start, end: segment.range.end - input.parsed.sourceRange.start }, replacement: `<${segment.id}></${segment.id}>` });
+    if (segment.selfClosing && moved.some(value => value.anchorId === segment.startAnchorId || value.anchorId === segment.endAnchorId)) {
+      edits.push({ range: { start: segment.range.start - origin, end: segment.range.end - origin }, replacement: `<${segment.id}></${segment.id}>` });
+    }
   }
-  const unmarked = applyEdits(input.source, edits);
-  const base = normalizedProse(unmarked, parseScript(input.sourceName, unmarked), input.source);
-  const parsed = parseScript(input.sourceName, base);
+  const base = applyEdits(input.source, edits);
+  const parsed = parseScript(input.sourceName, base, origin);
   const sites = new Map(scriptAnchorEditSites(parsed).map((site, order) => [site.anchorId, { ...site, order }]));
   const groups = new Map<number, Array<NamedAnchor & { affinity: Affinity; order: number }>>();
-  for (const value of markers) {
+  for (const value of moved) {
     const site = sites.get(value.anchorId);
     if (!site) throw new Error(`Semantic Anchor ${value.anchorId} does not exist.`);
-    let offset = site.offset;
-    const lineStart = Math.max(base.lastIndexOf("\n", offset - 1), base.lastIndexOf("\r", offset - 1)) + 1;
-    const indentEnd = lineStart + /^[ \t]*/u.exec(base.slice(lineStart))![0].length;
-    if (offset < indentEnd) offset = indentEnd;
-    // Reuse horizontal separators without moving a word marker ahead of line indentation.
-    let gapStart = offset;
-    while (gapStart > 0 && /[ \t]/u.test(base[gapStart - 1]!)) gapStart -= 1;
-    if (gapStart > 0 && !/[\r\n]/u.test(base[gapStart - 1]!)) offset = gapStart;
+    const offset = site.offset - origin;
     const group = groups.get(offset) ?? [];
     group.push({ ...value, affinity: site.affinity, order: site.order });
     groups.set(offset, group);
@@ -163,18 +133,13 @@ function rewrite(input: AdjustmentInput, markers: readonly NamedAnchor[]): strin
     group.sort((a, b) => a.order - b.order
       || (a.edge === "open" ? 0 : a.edge === "moment" ? 1 : 2) - (b.edge === "open" ? 0 : b.edge === "moment" ? 1 : 2)
       || a.id.localeCompare(b.id));
-    let replacement = group.map((item) => item.edge === "moment"
-      ? `${item.affinity === "left" ? "~" : ""}@${item.id}!`
+    const replacement = group.map(item => item.edge === "moment"
+      ? `@{${item.affinity === "left" ? "~" : ""}${item.id}!}`
       : marker(item.id, item.edge, item.affinity)).join("");
-    let end = offset;
-    while (end < base.length && /[ \t]/u.test(base[end]!)) end += 1;
-    if (end > offset) replacement += " ";
-    // Only undelimited names followed by an ASCII name character need a separator.
-    if (/[a-z0-9_-]$/u.test(replacement) && /[A-Za-z0-9_-]/u.test(base[offset] ?? "")) replacement += " ";
-    return { range: { start: offset, end }, replacement };
+    return { range: { start: offset, end: offset }, replacement };
   });
   const next = applyEdits(base, insertions);
-  const reparsed = parseScript(input.sourceName, next);
+  const reparsed = parseScript(input.sourceName, next, origin);
   const actual = namedAnchors(reparsed);
   if (canonicalStringify(actual) !== canonicalStringify(markers)) {
     throw new Error("Script marker adjustment did not preserve the requested semantic bindings.");
@@ -183,6 +148,9 @@ function rewrite(input: AdjustmentInput, markers: readonly NamedAnchor[]): strin
   const content = (value: ParsedNarrative) => ({
     ...narrativeValue(value, "comparison") as Record<string, unknown>, selections: [], moments: [],
     caption: captionDocument(value, "caption", "comparison"),
+    speech: value.serializations.speech,
+    dialogue: value.serializations.dialogue,
+    display: value.captionProjection.text,
   });
   if (canonicalStringify(content(input.parsed)) !== canonicalStringify(content(reparsed))) {
     throw new Error("Script marker adjustment changed authored content.");

package/packages/script/src/lexical.ts

@@ -9,159 +9,88 @@ const WORD_CHARACTER = String.raw`(?:(?!${CHARACTER_UNIT})[\p{L}\p{M}\p{N}])`;
 const LEXICAL_UNIT = new RegExp([
   String.raw`(?:\p{N}{1,3}(?:[,，]\p{N}{3})+|\p{N}+)(?:[.．]\p{N}+)?(?:-\p{N}+(?:[.．]\p{N}+)?)*(?!\p{N}|-[\p{L}\p{M}])`,
   String.raw`${CHARACTER_UNIT}\p{M}*`,
-  String.raw`${WORD_CHARACTER}+(?:['’.-]${WORD_CHARACTER}+)*`,
+  String.raw`(?:(?!${CHARACTER_UNIT})[\p{L}\p{N}])${WORD_CHARACTER}*(?:['’.-]${WORD_CHARACTER}+)*`,
 ].join("|"), "gu");
 
-const OPENING_PUNCTUATION = new Set([
-  "(", "[", "{", "（", "【", "《", "「", "『", "〔", "〈", "“", "‘",
-  "$", "¥", "￥", "€", "£",
-]);
-
-const QUOTE_PUNCTUATION = new Set(["\"", "'", "`", "’", "ʼ"]);
-const UNICODE_OPENING_PUNCTUATION = /[\p{Ps}\p{Pi}]/u;
-
-/**
- * ASCII quotation marks have no Unicode opening/closing category. In a gap between
- * lexical units, whitespace before the mark is the useful authoring signal:
- * `said "hello` opens a quote, while `hello" world` closes one. A quote before the
- * first unit is opening by definition; contractions stay inside one lexical unit.
- */
-function isOpeningPunctuation(
-  character: string,
-  gap: string,
-  position: number,
-  hasPreviousSurface: boolean,
-): boolean {
-  if (OPENING_PUNCTUATION.has(character) || UNICODE_OPENING_PUNCTUATION.test(character)) return true;
-  if (!QUOTE_PUNCTUATION.has(character)) return false;
-  if (!hasPreviousSurface) return true;
-  const before = [...gap.slice(0, position)].at(-1);
-  return before !== undefined && /\s/u.test(before);
+// Unicode owns bracket/quote categories. ASCII symmetric quotes need local context;
+// they have no opening/closing category. This is prose analysis, not marker syntax.
+function openingAt(gap: string, hasPrevious: boolean, beforeWord = true): number {
+  for (const match of gap.matchAll(/./gu)) {
+    const character = match[0];
+    if (/[\p{Ps}\p{Pi}\p{Sc}]/u.test(character)
+      || ((/["'`]/u.test(character) || (beforeWord && /\p{Pf}/u.test(character))) && (!hasPrevious || /\s$/u.test(gap.slice(0, match.index))))) return match.index;
+  }
+  return gap.length;
 }
 
 export function lexicalUnits(value: string): readonly LexicalUnit[] {
-  return [...value.matchAll(LEXICAL_UNIT)].map((match) => ({
-    text: match[0],
-    index: match.index,
-  }));
+  return [...value.matchAll(LEXICAL_UNIT)].map(match => ({ text: match[0], index: match.index }));
 }
 
 export function lexicalCount(value: string): number {
   return lexicalUnits(value).length;
 }
 
-/** Source positions for markers; unlike timing units, these include attached punctuation. */
-export function lexicalEditRanges(value: string): readonly { start: number; end: number }[] {
-  const units = lexicalUnits(value);
-  return units.map((unit, index) => {
-    const previousEnd = index === 0 ? 0 : units[index - 1]!.index + units[index - 1]!.text.length;
-    const nextStart = units[index + 1]?.index ?? value.length;
-    const leading = value.slice(previousEnd, unit.index);
-    let start = unit.index;
-    for (const [position, character] of [...leading.matchAll(/./gu)].map((match) => [match.index, match[0]] as const)) {
-      if (isOpeningPunctuation(character, leading, position, index > 0)) { start = previousEnd + position; break; }
-    }
-    let end = unit.index + unit.text.length;
-    const trailing = value.slice(end, nextStart);
-    for (const match of trailing.matchAll(/./gu)) {
-      if (/\s/u.test(match[0]) || isOpeningPunctuation(match[0], trailing, match.index, true)) break;
-      end += match[0].length;
-    }
-    return { start, end };
-  });
-}
-
-/** Canonical prose spacing; punctuation remains display/speech information, never a timing token. */
-function attachProseSpacing(value: string): string {
-  return value
-    .replace(/ +([,.;:!?%…，。！？；：、％‰）】》」』〕〉}\]])/gu, "$1")
-    .replace(/([([{（【《「『〔〈“‘]) +/gu, "$1")
-    // Do not erase a cross-script space: `here 你好` must remain two semantic regions.
-    // Only collapse explicit spaces inside one CJK run; the lexical tokenizer already keeps
-    // adjacent Latin and CJK runs separate when no space was authored.
-    .replace(/([\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]) +(?=[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}])/gu, "$1");
-}
-
+/** Normalize source formatting without inventing language-specific separators. */
 export function cleanProjection(value: string): string {
-  return attachProseSpacing(value.replace(/\s+/gu, " ")).trim();
+  return value.replace(/\s+/gu, " ").trim();
 }
 
-/** Source formatting keeps line breaks and lexical boundaries, including spaced decimal-like prose. */
-export function cleanHorizontalProse(value: string): string {
-  const collapsed = value.replace(/[ \t]+/gu, " ");
-  const attached = attachProseSpacing(collapsed);
-  const words = (text: string) => lexicalUnits(text).map(unit => unit.text);
-  // A separator in `3 .14` cannot be erased into the different token `3.14`.
-  const before = words(collapsed);
-  const after = words(attached);
-  return before.length === after.length && before.every((word, index) => word === after[index]) ? attached : collapsed;
-}
-
-/** Structural markers split atoms but must not invent prose whitespace when those atoms rejoin. */
 export function joinProjection(parts: readonly string[]): string {
   return cleanProjection(parts.join(""));
 }
 
-/**
- * Display units follow the same lexical boundaries as semantic speech tokens. Punctuation is kept
- * for rendering: opening punctuation belongs to the next unit; all other inter-token punctuation
- * belongs to the previous unit. Whitespace is layout, not a display word of its own.
- */
-export function displayWordSurfaces(value: string): readonly string[] {
-  const prose = cleanProjection(value);
-  const units = lexicalUnits(prose);
-  if (units.length === 0) return [];
+export type DisplaySurface = { readonly text: string; readonly separatorBefore: "" | " " };
+type ProseWord = { start: number; end: number; separatorBefore: "" | " " };
 
-  const surfaces: string[] = [];
+/** Analyze complete prose once. Speech cores and their written surfaces are distinct:
+ * punctuation has spelling and source extent, but does not acquire speech timing.
+ * A display-only literal (for example an emoji in Dual Text) needs no speech core.
+ */
+export function analyzeProse(value: string, followsDisplay = false): {
+  readonly units: readonly LexicalUnit[];
+  readonly surfaces: readonly DisplaySurface[];
+  readonly editRanges: readonly { start: number; end: number }[];
+} {
+  const units = lexicalUnits(value);
+  const words: ProseWord[] = [];
   let cursor = 0;
-  let prefix = "";
   for (const unit of units) {
-    const gap = prose.slice(cursor, unit.index);
-    const punctuation = [...gap]
-      .map((character, position) => ({ character, position }))
-      .filter(({ character }) => !/\s/u.test(character));
-    if (surfaces.length === 0) {
-      prefix += punctuation.map(({ character }) => character).join("");
-    } else {
-      const suffix = punctuation
-        .filter(({ character, position }) => !isOpeningPunctuation(character, gap, position, true))
-        .map(({ character }) => character)
-        .join("");
-      const opening = punctuation
-        .filter(({ character, position }) => isOpeningPunctuation(character, gap, position, true))
-        .map(({ character }) => character)
-        .join("");
-      if (suffix) surfaces[surfaces.length - 1] += suffix;
-      prefix += opening;
-    }
-    surfaces.push(`${prefix}${unit.text}`);
-    prefix = "";
+    const gap = value.slice(cursor, unit.index);
+    const opening = openingAt(gap, words.length > 0 || followsDisplay);
+    const closing = gap.slice(0, opening);
+    const previous = words.at(-1);
+    if (previous) previous.end = cursor + closing.trimEnd().length;
+    const start = previous || followsDisplay ? cursor + opening : value.slice(0, unit.index).search(/\S/u);
+    words.push({ start: start < 0 ? unit.index : start, end: unit.index + unit.text.length,
+      separatorBefore: previous && /\s$/u.test(closing) ? " " : "" });
     cursor = unit.index + unit.text.length;
   }
-  const trailing = prose.slice(cursor).replace(/\s+/gu, "");
-  if (trailing) surfaces[surfaces.length - 1] += trailing;
-  return surfaces;
+  const last = words.at(-1);
+  if (last) last.end = value.trimEnd().length;
+  const surfaces = words.map(word => ({ text: value.slice(word.start, word.end).replace(/\s+/gu, " "), separatorBefore: word.separatorBefore }));
+  const editRanges = words.map((word, index) => {
+    const unit = units[index]!;
+    let end = unit.index + unit.text.length;
+    // Whitespace is a legal insertion boundary. Only adjacent trailing punctuation
+    // belongs to the indivisible edit surface ("word," versus "word ,").
+    const trailing = value.slice(end, word.end);
+    const whitespace = trailing.search(/\s/u);
+    end += whitespace < 0 ? trailing.length : whitespace;
+    return { start: word.start, end };
+  });
+  if (surfaces.length === 0 && value.trim()) surfaces.push({ text: cleanProjection(value), separatorBefore: "" });
+  return { units, surfaces, editRanges };
 }
 
-/** Closing punctuation before a lexical unit belongs to the previous display unit when one exists. */
-export function splitLeadingClosingPunctuation(value: string): {
-  readonly previous: string;
-  readonly current: string;
-} {
-  const prose = cleanProjection(value);
-  const first = lexicalUnits(prose)[0];
-  if (first === undefined) return { previous: prose.replace(/\s+/gu, ""), current: "" };
-  const leading = prose.slice(0, first.index);
-  const punctuation = [...leading]
-    .map((character, position) => ({ character, position }))
-    .filter(({ character }) => !/\s/u.test(character));
-  const previous = punctuation
-    .filter(({ character, position }) => !isOpeningPunctuation(character, leading, position, false))
-    .map(({ character }) => character)
-    .join("");
-  const opening = punctuation
-    .filter(({ character, position }) => isOpeningPunctuation(character, leading, position, false))
-    .map(({ character }) => character)
-    .join("");
-  return { previous, current: `${opening}${prose.slice(first.index)}` };
+export function displaySurfaces(value: string): readonly DisplaySurface[] {
+  return analyzeProse(value).surfaces;
+}
+
+/** Correspondence boundaries retain punctuation on either side of an explicit Dual. */
+export function splitDisplayPrefix(value: string, hasPrevious: boolean): { previous: string; current: string } {
+  const first = lexicalUnits(value)[0];
+  const end = first?.index ?? value.length;
+  const split = openingAt(value.slice(0, end), hasPrevious, first !== undefined);
+  return { previous: value.slice(0, split), current: value.slice(split) };
 }