@alchemy.run/sigil 0.0.0-alpha.3 → 0.0.0-alpha.5

package/src/ansi/tokenize.ts

@@ -1,34 +1,23 @@
-import { isFullwidthCodePoint } from "./east-asian-width.ts";
-// ANSI-aware tokenizer: splits a string into grapheme clusters and ANSI
-// codes, tracks active styles per character, and re-emits minimal escape
-// sequences. This is the style model Ink's renderer is built on.
-// Ported from `@alcalzone/ansi-tokenize` (MIT, AlCalzone).
-import { BEL, C1_ST, ESC } from "./escapes.ts";
-import { codes as sgrCodes, foreground, background, styles } from "./sgr.ts";
+import { tokenizeAnsi } from "#/ansi-tokenizer.ts";
+import { isFullwidthGrapheme } from "#/ansi/east-asian-width.ts";
+// ANSI-aware style tokenizer: splits a string into grapheme clusters and
+// ANSI codes, tracks active styles per character, and re-emits minimal
+// escape sequences. This is the style model Ink's renderer is built on.
+// Style semantics ported from `@alcalzone/ansi-tokenize` (MIT, AlCalzone);
+// escape recognition delegates to the shared ECMA-48 grammar in
+// `src/ansi-tokenizer.ts` (also behind sanitize-ansi), and grapheme widths
+// come from `east-asian-width.ts` — one grammar, one width table.
+import { BEL, C1_ST, ESC } from "#/ansi/escapes.ts";
+import { graphemes } from "#/ansi/graphemes.ts";
+import { codes as sgrCodes, foreground, background, styles } from "#/ansi/sgr.ts";
 
 // Single-character introducer suffixes (`ESC [` = CSI, `ESC ]` = OSC).
 const BACKSLASH = "\\";
 const CSI = "[";
 const OSC = "]";
 
-// Char codes
-const CC_BEL = BEL.charCodeAt(0);
-const CC_ESC = ESC.charCodeAt(0);
-const CC_BACKSLASH = BACKSLASH.charCodeAt(0);
-const CC_CSI = CSI.charCodeAt(0);
-const CC_OSC = OSC.charCodeAt(0);
-const CC_C1_ST = C1_ST.charCodeAt(0);
-const CC_0 = "0".charCodeAt(0);
-const CC_9 = "9".charCodeAt(0);
-const CC_SEMI = ";".charCodeAt(0);
-const CC_M = "m".charCodeAt(0);
-
-// Escape code points: \u001B and \u009B
-const ESCAPES = new Set([CC_ESC, 0x9b]);
-
 // OSC 8 hyperlink constants
 const linkCodePrefix = `${ESC}${OSC}8;`;
-const linkCodePrefixCharCodes = linkCodePrefix.split("").map((char) => char.charCodeAt(0));
 const linkCodeSuffix = BEL;
 const linkEndCode = `${ESC}${OSC}8;;${BEL}`;
 const linkEndCodeST = `${ESC}${OSC}8;;${ESC}${BACKSLASH}`;
@@ -103,6 +92,11 @@ export function getEndCode(code: string): string {
     return background.close;
   }
 
+  // Extended underline color (T.416) closes with 59m; not in the SGR map.
+  if (code.startsWith("58")) {
+    return `${ESC}[59m`;
+  }
+
   // Otherwise find the reset code in the SGR code map
   const endCode = sgrCodes.get(Number.parseInt(code, 10));
   if (endCode !== undefined) {
@@ -126,97 +120,6 @@ export function isIntensityCode(code: AnsiCode): boolean {
   return code.code === styles.bold.open || code.code === styles.dim.open;
 }
 
-const segmenter = new Intl.Segmenter(undefined, { granularity: "grapheme" });
-
-function isFullwidthGrapheme(grapheme: string, baseCodePoint: number): boolean {
-  if (isFullwidthCodePoint(baseCodePoint)) return true;
-  // Variation Selector 16 forces emoji presentation (2 columns wide)
-  if (grapheme.includes("\uFE0F")) return true;
-  // Regional indicator pairs form flag emoji (2 columns wide)
-  if (baseCodePoint >= 0x1f1e6 && baseCodePoint <= 0x1f1ff) return true;
-  return false;
-}
-
-// HOT PATH: Use only basic string/char code operations for maximum performance
-function parseLinkCode(string: string, offset: number): string | undefined {
-  string = string.slice(offset);
-  for (let index = 1; index < linkCodePrefixCharCodes.length; index++) {
-    if (string.charCodeAt(index) !== linkCodePrefixCharCodes[index]) {
-      return;
-    }
-  }
-
-  // Find the semicolon that ends params
-  const paramsEndIndex = string.indexOf(";", linkCodePrefix.length);
-  if (paramsEndIndex === -1) return;
-
-  // This is a link code (with or without the URL part). Find the end of it.
-  const endIndex = findOscTerminatorIndex(string, paramsEndIndex + 1);
-  if (endIndex === -1) return;
-
-  return string.slice(0, endIndex + 1);
-}
-
-// HOT PATH: Generic fallback for non-link OSC sequences (window title, etc.)
-function parseOscSequence(string: string, offset: number): string | undefined {
-  string = string.slice(offset);
-  // Find the OSC terminator (starting after "ESC ]")
-  const endIndex = findOscTerminatorIndex(string, 2);
-  if (endIndex === -1) return;
-  return string.slice(0, endIndex + 1);
-}
-
-/**
-Finds the index of the last character of the first OSC terminator at or after
-`startIndex`. Recognizes BEL (\u0007), C1 ST (\u009C), and ESC+backslash.
-Returns -1 if no terminator is found.
-*/
-function findOscTerminatorIndex(string: string, startIndex: number): number {
-  for (let index = startIndex; index < string.length; index++) {
-    const charCode = string.charCodeAt(index);
-    if (charCode === CC_BEL) return index;
-    if (charCode === CC_C1_ST) return index;
-    if (
-      charCode === CC_ESC &&
-      index + 1 < string.length &&
-      string.charCodeAt(index + 1) === CC_BACKSLASH
-    ) {
-      return index + 1;
-    }
-  }
-
-  return -1;
-}
-
-/**
-Scans through the given string and finds the index of the last character of an
-SGR sequence like `\u001B[38;2;123;123;123m`. This assumes that the string has
-been checked to start with `\u001B[`. Returns -1 if no valid SGR sequence is
-found.
-*/
-function findSgrSequenceEndIndex(string: string): number {
-  for (let index = 2; index < string.length; index++) {
-    const charCode = string.charCodeAt(index);
-    // m marks the end of the SGR sequence
-    if (charCode === CC_M) return index;
-    // Digits and semicolons are valid
-    if (charCode === CC_SEMI) continue;
-    if (charCode >= CC_0 && charCode <= CC_9) continue;
-    // Everything else is invalid
-    break;
-  }
-
-  return -1;
-}
-
-// HOT PATH: Use only basic string/char code operations for maximum performance
-function parseSgrSequence(string: string, offset: number): string | undefined {
-  string = string.slice(offset);
-  const endIndex = findSgrSequenceEndIndex(string);
-  if (endIndex === -1) return;
-  return string.slice(0, endIndex + 1);
-}
-
 /**
 Splits compound SGR sequences like `\u001B[1;3;31m` into individual components.
 */
@@ -235,7 +138,7 @@ function splitCompoundSgrSequences(code: string): string[] {
   for (let index = 0; index < codeParts.length; index++) {
     const rawCode = codeParts[index]!;
     // Keep 8-bit and 24-bit color codes (containing multiple ";") together
-    if (rawCode === "38" || rawCode === "48") {
+    if (rawCode === "38" || rawCode === "48" || rawCode === "58") {
       if (index + 2 < codeParts.length && codeParts[index + 1] === "5") {
         // 8-bit color, followed by another number
         result.push(codeParts.slice(index, index + 3).join(";"));
@@ -256,73 +159,97 @@ function splitCompoundSgrSequences(code: string): string[] {
   return result.map((part) => `${ESC}[${part}m`);
 }
 
+// SGR parameters: digits separated by `;` or `:` (ITU T.416 colon form).
+const SGR_PARAMETERS_RE = /^[\d;:]*$/;
+
+const c1LinkCodePrefix = "\u009D8;";
+
 export function tokenize(string: string, endChar = Number.POSITIVE_INFINITY): Token[] {
   const result: Token[] = [];
   let visible = 0;
-  let codeEndIndex = 0;
-
-  for (const { segment, index } of segmenter.segment(string)) {
-    // Skip segments consumed as part of an ANSI sequence
-    if (index < codeEndIndex) continue;
-
-    const codePoint = segment.codePointAt(0)!;
-    if (ESCAPES.has(codePoint)) {
-      let code: string | undefined;
-      // Peek the next code point to determine the type of ANSI sequence
-      const nextCodePoint = string.codePointAt(index + 1);
-      if (nextCodePoint === CC_OSC) {
-        // ] = operating system commands
-        code = parseLinkCode(string, index);
-        if (code) {
-          // OSC 8 hyperlinks are paired codes with an endCode
+
+  outer: for (const ansiToken of tokenizeAnsi(string)) {
+    if (ansiToken.type === "text") {
+      for (const segment of graphemes(ansiToken.value)) {
+        const fullWidth = isFullwidthGrapheme(segment, segment.codePointAt(0)!);
+        result.push({
+          type: "char",
+          value: segment,
+          fullWidth,
+        });
+
+        visible += fullWidth ? 2 : 1;
+        if (visible >= endChar) {
+          break outer;
+        }
+      }
+
+      continue;
+    }
+
+    if (ansiToken.type === "csi") {
+      // SGR sequences carry style state; C1 introducers are normalized to the
+      // 7-bit form so downstream comparisons see a single representation.
+      if (
+        ansiToken.finalCharacter === "m" &&
+        ansiToken.intermediateString === "" &&
+        SGR_PARAMETERS_RE.test(ansiToken.parameterString)
+      ) {
+        // Split compound codes into individual tokens
+        const code = `${ESC}${CSI}${ansiToken.parameterString}m`;
+        for (const individualCode of splitCompoundSgrSequences(code)) {
           result.push({
             type: "ansi",
-            code,
-            endCode: getEndCode(code),
+            code: individualCode,
+            endCode: getEndCode(individualCode),
           });
-        } else {
-          // Other OSC sequences (window title, etc.) are self-contained
-          // control codes with no endCode.
-          code = parseOscSequence(string, index);
-          if (code) {
-            result.push({
-              type: "control",
-              code,
-            });
-          }
-        }
-      } else if (nextCodePoint === CC_CSI) {
-        // [ = control sequence introducer, like SGR sequences [...m
-        code = parseSgrSequence(string, index);
-        if (code) {
-          // Split compound codes into individual tokens
-          for (const individualCode of splitCompoundSgrSequences(code)) {
-            result.push({
-              type: "ansi",
-              code: individualCode,
-              endCode: getEndCode(individualCode),
-            });
-          }
         }
+      } else {
+        result.push({
+          type: "control",
+          code: ansiToken.value,
+        });
       }
 
-      if (code) {
-        codeEndIndex = index + code.length;
-        continue;
+      continue;
+    }
+
+    if (ansiToken.type === "osc") {
+      // OSC 8 hyperlinks are paired codes with an endCode; C1 introducers are
+      // normalized to the 7-bit form. Other OSC sequences (window title,
+      // etc.) are self-contained control codes with no endCode.
+      const { value } = ansiToken;
+      const prefix = value.startsWith(linkCodePrefix)
+        ? linkCodePrefix
+        : value.startsWith(c1LinkCodePrefix)
+          ? c1LinkCodePrefix
+          : undefined;
+
+      if (prefix !== undefined && value.includes(";", prefix.length)) {
+        const code =
+          prefix === linkCodePrefix ? value : `${linkCodePrefix}${value.slice(prefix.length)}`;
+        result.push({
+          type: "ansi",
+          code,
+          endCode: getEndCode(code),
+        });
+      } else {
+        result.push({
+          type: "control",
+          code: value,
+        });
       }
+
+      continue;
     }
 
-    const fullWidth = isFullwidthGrapheme(segment, codePoint);
+    // Everything else the grammar recognizes (ESC sequences, DCS/PM/APC/SOS
+    // control strings, stray ST or C1 bytes, malformed tails) is a zero-width
+    // control unit, preserved verbatim.
     result.push({
-      type: "char",
-      value: segment,
-      fullWidth,
+      type: "control",
+      code: ansiToken.value,
     });
-
-    visible += fullWidth ? 2 : 1;
-    if (visible >= endChar) {
-      break;
-    }
   }
 
   return result;
@@ -377,7 +304,7 @@ Returns the combination of ANSI codes needed to undo the given ANSI codes.
 */
 export function undoAnsiCodes(codes: readonly AnsiCode[]): AnsiCode[] {
   return reduceAnsiCodes(codes)
-    .reverse()
+    .toReversed()
     .map((code) => ({
       ...code,
       code: code.endCode,

package/src/ansi/truncate.ts

@@ -1,8 +1,8 @@
 // Truncate a string to a visible width, ANSI-aware.
 // Ported from `cli-truncate` (MIT, Sindre Sorhus) on top of the shared
 // slice and width modules.
-import { sliceAnsi } from "./slice.ts";
-import { stringWidth } from "./string-width.ts";
+import { sliceAnsi } from "#/ansi/slice.ts";
+import { stringWidth } from "#/ansi/string-width.ts";
 
 export type TruncateOptions = {
   readonly position?: "start" | "middle" | "end";
@@ -36,7 +36,9 @@ const ANSI_ESC = 27;
 const ANSI_LEFT_BRACKET = 91;
 const ANSI_LETTER_M = 109;
 
-const isSgrParameter = (code: number): boolean => (code >= 48 && code <= 57) || code === 59; // 0-9 or ;
+// 0-9, ; or : (ITU T.416 colon-form parameters)
+const isSgrParameter = (code: number): boolean =>
+  (code >= 48 && code <= 57) || code === 59 || code === 58;
 
 function leadingSgrSpanEndIndex(string: string): number {
   let index = 0;

package/src/ansi/wrap.ts

@@ -1,14 +1,14 @@
 // Word-wrap a string to a column width, ANSI-aware.
 // Ported from `wrap-ansi` (MIT, Sindre Sorhus).
-import { BEL, C1_CSI, ESC } from "./escapes.ts";
+import { BEL, C1_CSI, ESC } from "#/ansi/escapes.ts";
 //
 // Supported boundary: semicolon-delimited SGR styling, colon-delimited
 // RGB/indexed colors, and OSC 8 hyperlinks are tracked, while ordinary CSI
 // sequences and other complete 7-bit OSC commands are preserved as opaque
 // zero-width units. This is intentionally not a terminal emulator. Newlines
 // always delimit input lines before ANSI parsing.
-import { codes as sgrCodes } from "./sgr.ts";
-import { stringWidth } from "./string-width.ts";
+import { codes as sgrCodes } from "#/ansi/sgr.ts";
+import { stringWidth } from "#/ansi/string-width.ts";
 
 export type WrapOptions = {
   readonly trim?: boolean;
@@ -59,8 +59,7 @@ const segmenter = new Intl.Segmenter();
 // Complete ANSI sequences have already been removed before measuring these
 // strings. Avoid string-width's ANSI scan so malformed sequences are not
 // rescanned.
-const getStringWidth = (string: string): number =>
-  stringWidth(string, { countAnsiEscapeCodes: true });
+const getStringWidth = (string: string): number => stringWidth(string);
 const TAB_SIZE = 8;
 
 // Finds the next character that could introduce a sequence, so plain text is
@@ -221,18 +220,18 @@ type ActiveStyle = {
   readonly close: number;
 };
 
+const EXTENDED_COLOR_CODES = new Set([
+  ANSI_SGR_FOREGROUND_EXTENDED,
+  ANSI_SGR_BACKGROUND_EXTENDED,
+  ANSI_SGR_UNDERLINE_COLOR_EXTENDED,
+]);
+
 const getColonColorToken = (parameter: string): SgrToken | undefined => {
   const parts = parameter.split(":");
   const code = Number.parseInt(parts[0]!, 10);
   const mode = Number.parseInt(parts[1]!, 10);
 
-  if (
-    ![
-      ANSI_SGR_FOREGROUND_EXTENDED,
-      ANSI_SGR_BACKGROUND_EXTENDED,
-      ANSI_SGR_UNDERLINE_COLOR_EXTENDED,
-    ].includes(code)
-  ) {
+  if (!EXTENDED_COLOR_CODES.has(code)) {
     return;
   }
 
@@ -467,8 +466,8 @@ const applyLeadingSgrResets = (
 };
 
 const getClosingSgrSequence = (activeStyles: readonly ActiveStyle[]): string =>
-  [...activeStyles]
-    .reverse()
+  activeStyles
+    .toReversed()
     .map((activeStyle) => wrapAnsiCode(activeStyle.close))
     .join("");
 

package/src/ansi-tokenizer.ts

@@ -1,4 +1,4 @@
-import { BEL, C1_CSI, C1_ST, ESC } from "./ansi/escapes.ts";
+import { BEL, C1_CSI, C1_ST, ESC } from "#/ansi/escapes.ts";
 
 const bellCharacter = BEL;
 const escapeCharacter = ESC;

package/src/capabilities/color-policy.ts

@@ -0,0 +1,34 @@
+import type { ColorSupportLevel } from "#/capabilities/detect.ts";
+import { colorProfileFromLevel, type ColorProfile } from "#/screen/color-profile.ts";
+
+/** A user-selected output policy. `auto` follows the target stream. */
+export type ColorPolicy = "auto" | ColorProfile;
+
+/** The three distinct inputs and result of terminal color negotiation. */
+export type ColorState = {
+  /** The stream/terminal fact after environment and query detection. */
+  readonly detected: ColorProfile;
+  /** The application's policy for this render instance. */
+  readonly policy: ColorPolicy;
+  /** The profile the serializer must actually emit. */
+  readonly effective: ColorProfile;
+};
+
+export function resolveColorProfile(
+  detectedLevel: ColorSupportLevel,
+  policy: ColorPolicy = "auto",
+): ColorProfile {
+  return policy === "auto" ? colorProfileFromLevel(detectedLevel) : policy;
+}
+
+export function colorState(
+  capabilities: { readonly color: { readonly level: ColorSupportLevel } },
+  policy: ColorPolicy = "auto",
+): ColorState {
+  const detected = colorProfileFromLevel(capabilities.color.level);
+  return {
+    detected,
+    policy,
+    effective: policy === "auto" ? detected : policy,
+  };
+}