@hypit/hypit 0.1.14 → 0.2.0

package/packages/script/src/edit.ts

@@ -3,7 +3,6 @@ import type { SourceRange } from "@hypit/protocol";
 import { canonicalStringify } from "@hypit/protocol";
 import { captionDocument, narrativeValue } from "./narrative.js";
 import { parseScript } from "./parser.js";
-import { cleanHorizontalProse } from "./lexical.js";
 import type { Affinity, ParsedNarrative, SemanticAnchor } from "./types.js";
 
 export type ScriptAnchorEditSite = {
@@ -75,8 +74,8 @@ export function scriptAnchorEditSites(parsed: ParsedNarrative): readonly ScriptA
 }
 
 function marker(id: string, edge: "open" | "close", affinity: Affinity): string {
-  if (edge === "open") return affinity === "left" ? `~@${id}` : `@${id}`;
-  return affinity === "left" ? `@/${id}` : `@/${id}~`;
+  if (edge === "open") return `@{${affinity === "left" ? "~" : ""}${id}}`;
+  return `@{/${id}${affinity === "right" ? "~" : ""}}`;
 }
 
 function applyEdits(source: string, edits: readonly Edit[]): string {
@@ -100,61 +99,32 @@ type AdjustmentInput = {
 
 type NamedAnchor = { readonly id: string; readonly edge: "open" | "close" | "moment"; readonly anchorId: string };
 
-/** Normalize only parser-owned prose. Indentation is read from this edit's input, never stored. */
-function normalizedProse(source: string, parsed: ParsedNarrative, original: string): string {
-  const bodyStarts = new Set(parsed.segments.flatMap(segment => [segment.contentRange.start,
-    ...segment.atoms.filter(atom => atom.kind === "role").map(atom => atom.range.end)]));
-  const bodyEnds = new Set(parsed.segments.map(segment => segment.contentRange.end));
-  const indentation = original.split(/\r\n|\r|\n/u).map(line => /^[ \t]*/u.exec(line)![0].length);
-  const lines: Array<{ start: number; end: number; indentEnd: number }> = [];
-  let offset = 0;
-  const parts = source.split(/(\r\n|\r|\n)/u);
-  for (let index = 0; index < parts.length; index += 2) {
-    const text = parts[index]!;
-    lines.push({ start: offset, end: offset + text.length, indentEnd: offset + indentation[index / 2]! });
-    offset += text.length + (parts[index + 1]?.length ?? 0);
-  }
-  const edits: Edit[] = [];
-  for (const range of parsed.proseRanges) for (const line of lines) {
-    const start = Math.max(range.start, line.indentEnd);
-    const end = Math.min(range.end, line.end);
-    if (start >= end) continue;
-    let text = cleanHorizontalProse(source.slice(start, end));
-    const outside = !parsed.segments.some(segment => range.start >= segment.contentRange.start && range.end <= segment.contentRange.end);
-    if (start === line.indentEnd || bodyStarts.has(start) || outside) text = text.replace(/^[ \t]+/u, "");
-    if (end === line.end || bodyEnds.has(end) || outside) text = text.replace(/[ \t]+$/u, "");
-    edits.push({ range: { start, end }, replacement: text });
-  }
-  return applyEdits(source, edits);
-}
-
-/** One canonical boundary spelling, independent of the order of previous gestures. */
+/** Move only the requested markers. Delimited names never require editing prose separators. */
 function rewrite(input: AdjustmentInput, markers: readonly NamedAnchor[]): string {
-  const ranges = [
-    ...input.parsed.moments.map((item) => item.range),
-    ...input.parsed.selections.flatMap((item) => [item.open.range, item.close.range]),
-  ].map((range) => ({ start: range.start - input.parsed.sourceRange.start, end: range.end - input.parsed.sourceRange.start }))
-    .sort((a, b) => a.start - b.start);
-  const edits: Edit[] = ranges.map(range => ({ range, replacement: "" }));
+  const original = namedAnchors(input.parsed);
+  const changedNames = new Set(markers.filter(value => original.find(item => item.id === value.id && item.edge === value.edge)?.anchorId !== value.anchorId).map(value => value.id));
+  const moved = markers.filter(value => changedNames.has(value.id));
+  if (moved.length === 0) return input.source;
+  const origin = input.parsed.sourceRange.start;
+  const edits: Edit[] = moved.map(value => {
+    const range = value.edge === "moment"
+      ? input.parsed.moments.find(item => item.id === value.id)!.range
+      : input.parsed.selections.find(item => item.id === value.id)![value.edge].range;
+    return { range: { start: range.start - origin, end: range.end - origin }, replacement: "" };
+  });
   for (const segment of input.parsed.segments) {
-    if (segment.selfClosing) edits.push({ range: { start: segment.range.start - input.parsed.sourceRange.start, end: segment.range.end - input.parsed.sourceRange.start }, replacement: `<${segment.id}></${segment.id}>` });
+    if (segment.selfClosing && moved.some(value => value.anchorId === segment.startAnchorId || value.anchorId === segment.endAnchorId)) {
+      edits.push({ range: { start: segment.range.start - origin, end: segment.range.end - origin }, replacement: `<${segment.id}></${segment.id}>` });
+    }
   }
-  const unmarked = applyEdits(input.source, edits);
-  const base = normalizedProse(unmarked, parseScript(input.sourceName, unmarked), input.source);
-  const parsed = parseScript(input.sourceName, base);
+  const base = applyEdits(input.source, edits);
+  const parsed = parseScript(input.sourceName, base, origin);
   const sites = new Map(scriptAnchorEditSites(parsed).map((site, order) => [site.anchorId, { ...site, order }]));
   const groups = new Map<number, Array<NamedAnchor & { affinity: Affinity; order: number }>>();
-  for (const value of markers) {
+  for (const value of moved) {
     const site = sites.get(value.anchorId);
     if (!site) throw new Error(`Semantic Anchor ${value.anchorId} does not exist.`);
-    let offset = site.offset;
-    const lineStart = Math.max(base.lastIndexOf("\n", offset - 1), base.lastIndexOf("\r", offset - 1)) + 1;
-    const indentEnd = lineStart + /^[ \t]*/u.exec(base.slice(lineStart))![0].length;
-    if (offset < indentEnd) offset = indentEnd;
-    // Reuse horizontal separators without moving a word marker ahead of line indentation.
-    let gapStart = offset;
-    while (gapStart > 0 && /[ \t]/u.test(base[gapStart - 1]!)) gapStart -= 1;
-    if (gapStart > 0 && !/[\r\n]/u.test(base[gapStart - 1]!)) offset = gapStart;
+    const offset = site.offset - origin;
     const group = groups.get(offset) ?? [];
     group.push({ ...value, affinity: site.affinity, order: site.order });
     groups.set(offset, group);
@@ -163,18 +133,13 @@ function rewrite(input: AdjustmentInput, markers: readonly NamedAnchor[]): strin
     group.sort((a, b) => a.order - b.order
       || (a.edge === "open" ? 0 : a.edge === "moment" ? 1 : 2) - (b.edge === "open" ? 0 : b.edge === "moment" ? 1 : 2)
       || a.id.localeCompare(b.id));
-    let replacement = group.map((item) => item.edge === "moment"
-      ? `${item.affinity === "left" ? "~" : ""}@${item.id}!`
+    const replacement = group.map(item => item.edge === "moment"
+      ? `@{${item.affinity === "left" ? "~" : ""}${item.id}!}`
       : marker(item.id, item.edge, item.affinity)).join("");
-    let end = offset;
-    while (end < base.length && /[ \t]/u.test(base[end]!)) end += 1;
-    if (end > offset) replacement += " ";
-    // Only undelimited names followed by an ASCII name character need a separator.
-    if (/[a-z0-9_-]$/u.test(replacement) && /[A-Za-z0-9_-]/u.test(base[offset] ?? "")) replacement += " ";
-    return { range: { start: offset, end }, replacement };
+    return { range: { start: offset, end: offset }, replacement };
   });
   const next = applyEdits(base, insertions);
-  const reparsed = parseScript(input.sourceName, next);
+  const reparsed = parseScript(input.sourceName, next, origin);
   const actual = namedAnchors(reparsed);
   if (canonicalStringify(actual) !== canonicalStringify(markers)) {
     throw new Error("Script marker adjustment did not preserve the requested semantic bindings.");
@@ -183,6 +148,9 @@ function rewrite(input: AdjustmentInput, markers: readonly NamedAnchor[]): strin
   const content = (value: ParsedNarrative) => ({
     ...narrativeValue(value, "comparison") as Record<string, unknown>, selections: [], moments: [],
     caption: captionDocument(value, "caption", "comparison"),
+    speech: value.serializations.speech,
+    dialogue: value.serializations.dialogue,
+    display: value.captionProjection.text,
   });
   if (canonicalStringify(content(input.parsed)) !== canonicalStringify(content(reparsed))) {
     throw new Error("Script marker adjustment changed authored content.");

package/packages/script/src/lexical.ts

@@ -9,159 +9,88 @@ const WORD_CHARACTER = String.raw`(?:(?!${CHARACTER_UNIT})[\p{L}\p{M}\p{N}])`;
 const LEXICAL_UNIT = new RegExp([
   String.raw`(?:\p{N}{1,3}(?:[,，]\p{N}{3})+|\p{N}+)(?:[.．]\p{N}+)?(?:-\p{N}+(?:[.．]\p{N}+)?)*(?!\p{N}|-[\p{L}\p{M}])`,
   String.raw`${CHARACTER_UNIT}\p{M}*`,
-  String.raw`${WORD_CHARACTER}+(?:['’.-]${WORD_CHARACTER}+)*`,
+  String.raw`(?:(?!${CHARACTER_UNIT})[\p{L}\p{N}])${WORD_CHARACTER}*(?:['’.-]${WORD_CHARACTER}+)*`,
 ].join("|"), "gu");
 
-const OPENING_PUNCTUATION = new Set([
-  "(", "[", "{", "（", "【", "《", "「", "『", "〔", "〈", "“", "‘",
-  "$", "¥", "￥", "€", "£",
-]);
-
-const QUOTE_PUNCTUATION = new Set(["\"", "'", "`", "’", "ʼ"]);
-const UNICODE_OPENING_PUNCTUATION = /[\p{Ps}\p{Pi}]/u;
-
-/**
- * ASCII quotation marks have no Unicode opening/closing category. In a gap between
- * lexical units, whitespace before the mark is the useful authoring signal:
- * `said "hello` opens a quote, while `hello" world` closes one. A quote before the
- * first unit is opening by definition; contractions stay inside one lexical unit.
- */
-function isOpeningPunctuation(
-  character: string,
-  gap: string,
-  position: number,
-  hasPreviousSurface: boolean,
-): boolean {
-  if (OPENING_PUNCTUATION.has(character) || UNICODE_OPENING_PUNCTUATION.test(character)) return true;
-  if (!QUOTE_PUNCTUATION.has(character)) return false;
-  if (!hasPreviousSurface) return true;
-  const before = [...gap.slice(0, position)].at(-1);
-  return before !== undefined && /\s/u.test(before);
+// Unicode owns bracket/quote categories. ASCII symmetric quotes need local context;
+// they have no opening/closing category. This is prose analysis, not marker syntax.
+function openingAt(gap: string, hasPrevious: boolean, beforeWord = true): number {
+  for (const match of gap.matchAll(/./gu)) {
+    const character = match[0];
+    if (/[\p{Ps}\p{Pi}\p{Sc}]/u.test(character)
+      || ((/["'`]/u.test(character) || (beforeWord && /\p{Pf}/u.test(character))) && (!hasPrevious || /\s$/u.test(gap.slice(0, match.index))))) return match.index;
+  }
+  return gap.length;
 }
 
 export function lexicalUnits(value: string): readonly LexicalUnit[] {
-  return [...value.matchAll(LEXICAL_UNIT)].map((match) => ({
-    text: match[0],
-    index: match.index,
-  }));
+  return [...value.matchAll(LEXICAL_UNIT)].map(match => ({ text: match[0], index: match.index }));
 }
 
 export function lexicalCount(value: string): number {
   return lexicalUnits(value).length;
 }
 
-/** Source positions for markers; unlike timing units, these include attached punctuation. */
-export function lexicalEditRanges(value: string): readonly { start: number; end: number }[] {
-  const units = lexicalUnits(value);
-  return units.map((unit, index) => {
-    const previousEnd = index === 0 ? 0 : units[index - 1]!.index + units[index - 1]!.text.length;
-    const nextStart = units[index + 1]?.index ?? value.length;
-    const leading = value.slice(previousEnd, unit.index);
-    let start = unit.index;
-    for (const [position, character] of [...leading.matchAll(/./gu)].map((match) => [match.index, match[0]] as const)) {
-      if (isOpeningPunctuation(character, leading, position, index > 0)) { start = previousEnd + position; break; }
-    }
-    let end = unit.index + unit.text.length;
-    const trailing = value.slice(end, nextStart);
-    for (const match of trailing.matchAll(/./gu)) {
-      if (/\s/u.test(match[0]) || isOpeningPunctuation(match[0], trailing, match.index, true)) break;
-      end += match[0].length;
-    }
-    return { start, end };
-  });
-}
-
-/** Canonical prose spacing; punctuation remains display/speech information, never a timing token. */
-function attachProseSpacing(value: string): string {
-  return value
-    .replace(/ +([,.;:!?%…，。！？；：、％‰）】》」』〕〉}\]])/gu, "$1")
-    .replace(/([([{（【《「『〔〈“‘]) +/gu, "$1")
-    // Do not erase a cross-script space: `here 你好` must remain two semantic regions.
-    // Only collapse explicit spaces inside one CJK run; the lexical tokenizer already keeps
-    // adjacent Latin and CJK runs separate when no space was authored.
-    .replace(/([\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}]) +(?=[\p{Script=Han}\p{Script=Hiragana}\p{Script=Katakana}])/gu, "$1");
-}
-
+/** Normalize source formatting without inventing language-specific separators. */
 export function cleanProjection(value: string): string {
-  return attachProseSpacing(value.replace(/\s+/gu, " ")).trim();
+  return value.replace(/\s+/gu, " ").trim();
 }
 
-/** Source formatting keeps line breaks and lexical boundaries, including spaced decimal-like prose. */
-export function cleanHorizontalProse(value: string): string {
-  const collapsed = value.replace(/[ \t]+/gu, " ");
-  const attached = attachProseSpacing(collapsed);
-  const words = (text: string) => lexicalUnits(text).map(unit => unit.text);
-  // A separator in `3 .14` cannot be erased into the different token `3.14`.
-  const before = words(collapsed);
-  const after = words(attached);
-  return before.length === after.length && before.every((word, index) => word === after[index]) ? attached : collapsed;
-}
-
-/** Structural markers split atoms but must not invent prose whitespace when those atoms rejoin. */
 export function joinProjection(parts: readonly string[]): string {
   return cleanProjection(parts.join(""));
 }
 
-/**
- * Display units follow the same lexical boundaries as semantic speech tokens. Punctuation is kept
- * for rendering: opening punctuation belongs to the next unit; all other inter-token punctuation
- * belongs to the previous unit. Whitespace is layout, not a display word of its own.
- */
-export function displayWordSurfaces(value: string): readonly string[] {
-  const prose = cleanProjection(value);
-  const units = lexicalUnits(prose);
-  if (units.length === 0) return [];
+export type DisplaySurface = { readonly text: string; readonly separatorBefore: "" | " " };
+type ProseWord = { start: number; end: number; separatorBefore: "" | " " };
 
-  const surfaces: string[] = [];
+/** Analyze complete prose once. Speech cores and their written surfaces are distinct:
+ * punctuation has spelling and source extent, but does not acquire speech timing.
+ * A display-only literal (for example an emoji in Dual Text) needs no speech core.
+ */
+export function analyzeProse(value: string, followsDisplay = false): {
+  readonly units: readonly LexicalUnit[];
+  readonly surfaces: readonly DisplaySurface[];
+  readonly editRanges: readonly { start: number; end: number }[];
+} {
+  const units = lexicalUnits(value);
+  const words: ProseWord[] = [];
   let cursor = 0;
-  let prefix = "";
   for (const unit of units) {
-    const gap = prose.slice(cursor, unit.index);
-    const punctuation = [...gap]
-      .map((character, position) => ({ character, position }))
-      .filter(({ character }) => !/\s/u.test(character));
-    if (surfaces.length === 0) {
-      prefix += punctuation.map(({ character }) => character).join("");
-    } else {
-      const suffix = punctuation
-        .filter(({ character, position }) => !isOpeningPunctuation(character, gap, position, true))
-        .map(({ character }) => character)
-        .join("");
-      const opening = punctuation
-        .filter(({ character, position }) => isOpeningPunctuation(character, gap, position, true))
-        .map(({ character }) => character)
-        .join("");
-      if (suffix) surfaces[surfaces.length - 1] += suffix;
-      prefix += opening;
-    }
-    surfaces.push(`${prefix}${unit.text}`);
-    prefix = "";
+    const gap = value.slice(cursor, unit.index);
+    const opening = openingAt(gap, words.length > 0 || followsDisplay);
+    const closing = gap.slice(0, opening);
+    const previous = words.at(-1);
+    if (previous) previous.end = cursor + closing.trimEnd().length;
+    const start = previous || followsDisplay ? cursor + opening : value.slice(0, unit.index).search(/\S/u);
+    words.push({ start: start < 0 ? unit.index : start, end: unit.index + unit.text.length,
+      separatorBefore: previous && /\s$/u.test(closing) ? " " : "" });
     cursor = unit.index + unit.text.length;
   }
-  const trailing = prose.slice(cursor).replace(/\s+/gu, "");
-  if (trailing) surfaces[surfaces.length - 1] += trailing;
-  return surfaces;
+  const last = words.at(-1);
+  if (last) last.end = value.trimEnd().length;
+  const surfaces = words.map(word => ({ text: value.slice(word.start, word.end).replace(/\s+/gu, " "), separatorBefore: word.separatorBefore }));
+  const editRanges = words.map((word, index) => {
+    const unit = units[index]!;
+    let end = unit.index + unit.text.length;
+    // Whitespace is a legal insertion boundary. Only adjacent trailing punctuation
+    // belongs to the indivisible edit surface ("word," versus "word ,").
+    const trailing = value.slice(end, word.end);
+    const whitespace = trailing.search(/\s/u);
+    end += whitespace < 0 ? trailing.length : whitespace;
+    return { start: word.start, end };
+  });
+  if (surfaces.length === 0 && value.trim()) surfaces.push({ text: cleanProjection(value), separatorBefore: "" });
+  return { units, surfaces, editRanges };
 }
 
-/** Closing punctuation before a lexical unit belongs to the previous display unit when one exists. */
-export function splitLeadingClosingPunctuation(value: string): {
-  readonly previous: string;
-  readonly current: string;
-} {
-  const prose = cleanProjection(value);
-  const first = lexicalUnits(prose)[0];
-  if (first === undefined) return { previous: prose.replace(/\s+/gu, ""), current: "" };
-  const leading = prose.slice(0, first.index);
-  const punctuation = [...leading]
-    .map((character, position) => ({ character, position }))
-    .filter(({ character }) => !/\s/u.test(character));
-  const previous = punctuation
-    .filter(({ character, position }) => !isOpeningPunctuation(character, leading, position, false))
-    .map(({ character }) => character)
-    .join("");
-  const opening = punctuation
-    .filter(({ character, position }) => isOpeningPunctuation(character, leading, position, false))
-    .map(({ character }) => character)
-    .join("");
-  return { previous, current: `${opening}${prose.slice(first.index)}` };
+export function displaySurfaces(value: string): readonly DisplaySurface[] {
+  return analyzeProse(value).surfaces;
+}
+
+/** Correspondence boundaries retain punctuation on either side of an explicit Dual. */
+export function splitDisplayPrefix(value: string, hasPrevious: boolean): { previous: string; current: string } {
+  const first = lexicalUnits(value)[0];
+  const end = first?.index ?? value.length;
+  const split = openingAt(value.slice(0, end), hasPrevious, first !== undefined);
+  return { previous: value.slice(0, split), current: value.slice(split) };
 }

package/packages/script/src/manifest.ts

@@ -48,15 +48,15 @@ export const scriptMarkupSurfaces = [
       ],
       example: [
         '<script id="story">',
-        "  @whole",
+        "  @{whole}",
         "  <hook>",
-        "    <HOST> @problem Never let anyone take credit for your work. @/problem",
+        "    <HOST> @{problem} Never let anyone take credit for your work. @{/problem}",
         "  </hook>",
         "",
         "  <meeting>",
-        "    <HOST> I started sending <BCC | B C C> recaps. @ranking! Everything changed.",
+        "    <HOST> I started sending <BCC | B C C> recaps. @{ranking!} Everything changed.",
         "  </meeting>",
-        "  @/whole~",
+        "  @{/whole~}",
         "</script>",
       ].join("\n"),
       notes: [
@@ -67,16 +67,16 @@ export const scriptMarkupSurfaces = [
         "Dual Text is written `<display | speech>`: the left side reaches Caption and the right side reaches dialogue and speech. `<display|>` inherits speech from the displayed prose and forms the same complete alignment unit; its word times remain individual. `<|speech>` speaks without displaying. Both sides empty is invalid.",
         "Inside Dual Text, semantic markers belong to the source of spoken text: the explicit right side, or the shared left side when speech is omitted. Display attributes remain visual metadata and never enter spoken text.",
         "A flat token attribute follows a complete display token as `{name}` or `{name=value}`; multiple attributes use one comma-separated block. Attributes do not nest, do not carry timing, and never split a Dual Alignment Unit.",
-        "Selection and Moment markers are zero-width, share one name namespace, and may not split a speech token:",
+        "Selection and Moment markers are fully enclosed in `@{...}` with all sigils inside. They are zero-width, share one name namespace, and may not split a speech token. Surrounding prose spaces remain content; do not add spaces to delimit a name:",
         [
           "| Marker | Meaning |",
           "|---|---|",
-          "| `@id` | Opens a Selection at the next word's start |",
-          "| `~@id` | Opens a Selection at the previous word's end |",
-          "| `@/id` | Closes a Selection at the previous word's end |",
-          "| `@/id~` | Closes a Selection at the next word's start |",
-          "| `@id!` | A Moment at the next word's start |",
-          "| `~@id!` | A Moment at the previous word's end |",
+          "| `@{id}` | Opens a Selection at the next word's start |",
+          "| `@{~id}` | Opens a Selection at the previous word's end |",
+          "| `@{/id}` | Closes a Selection at the previous word's end |",
+          "| `@{/id~}` | Closes a Selection at the next word's start |",
+          "| `@{id!}` | A Moment at the next word's start |",
+          "| `@{~id!}` | A Moment at the previous word's end |",
         ].join("\n"),
         "Each Selection name has one opening and one closing marker; use distinct names for distinct semantic ranges.",
         "`<!-- -->` comments never enter any projection, and `\\@`, `\\<`, `\\\\`, `\\{` and `\\}` write those characters literally; inside Dual Text `\\|` and `\\>` do the same.",

package/packages/script/src/narrative.ts

@@ -4,7 +4,7 @@ import type { CaptionAlignmentUnit, CaptionDocument, CaptionDisplayWord } from "
 import { sealText } from "@hypit/text";
 
 import type { ParsedCaptionRegion, ParsedNarrative } from "./types.js";
-import { cleanProjection, displayWordSurfaces, joinProjection, lexicalCount } from "./lexical.js";
+import { cleanProjection, displaySurfaces, joinProjection, lexicalCount } from "./lexical.js";
 
 function turnForRegion(parsed: ParsedNarrative, region: ParsedCaptionRegion): ParsedNarrative["turns"][number] {
   const turn = parsed.turns.find((candidate) =>
@@ -21,7 +21,8 @@ function projectCaption(parsed: ParsedNarrative, id: string, narrativeId: string
   for (const region of parsed.captionProjection.regions) {
     if (region.kind === "hidden") continue;
     const turn = turnForRegion(parsed, region);
-    const surfaces = displayWordSurfaces(region.display);
+    const display = displaySurfaces(region.display);
+    const surfaces = display.map(word => word.text);
     if (surfaces.length === 0) throw new Error(`Caption region ${region.id} contains no visible display surface`);
     // A Dual Text alias is one indivisible N:M correspondence unit. Ordinary prose gives one
     // unit per display surface so the author can place cue/style boundaries between words.
@@ -46,6 +47,7 @@ function projectCaption(parsed: ParsedNarrative, id: string, narrativeId: string
           turnId: turn.id,
           ...(turn.role === undefined ? {} : { role: turn.role }),
           text: surface,
+          separatorBefore: group.indices[groupIndex] === 0 ? region.separatorBefore : display[group.indices[groupIndex]!]!.separatorBefore,
           attributes,
         });
         return wordId;