@hajimism/dek 0.4.0 → 0.5.0

package/src/core/css-scan.ts

@@ -1,9 +1,52 @@
 /**
- * The one way dek steps through CSS text: strings and comments are opaque, and a newline ends an
- * unclosed string, as the CSS tokenizer reads it. Every scanner in css.ts and lint/tokens.ts is built on
- * these, so none of them can disagree about where a string or a block ends.
+ * The one way dek steps through CSS text, as the CSS tokenizer reads it: strings, comments, and
+ * an unquoted `url()` are opaque, CR, FF, and CRLF are newlines as LF is, and a newline ends an
+ * unclosed string. Every scanner in css.ts, css-transform.ts, and theme-excerpt.ts is built on
+ * these, so none of them can disagree about where a string or a block ends. Nothing is rewritten
+ * before it is read, so every index still points into the text as written.
  */
 
+/** A newline as CSS preprocesses its input: LF, CR, or FF; CRLF is one, read at its CR. */
+function isNewline(ch: string | undefined): boolean {
+  return ch === "\n" || ch === "\r" || ch === "\f";
+}
+
+function isWhitespace(ch: string | undefined): boolean {
+  return ch === " " || ch === "\t" || isNewline(ch);
+}
+
+/** The index just past the whitespace at `i`, taking CRLF whole. */
+function whitespaceEnd(source: string, i: number, end: number): number {
+  return Math.min(source[i] === "\r" && source[i + 1] === "\n" ? i + 2 : i + 1, end);
+}
+
+function skipWhitespace(source: string, i: number, end: number): number {
+  let j = i;
+  while (j < end && isWhitespace(source[j])) {
+    j++;
+  }
+  return j;
+}
+
+/** Whether a `\` at `i` escapes what follows: anything but a newline, the end included. */
+function startsEscape(source: string, i: number, end: number): boolean {
+  return source[i] === "\\" && (i + 1 >= end || !isNewline(source[i + 1]));
+}
+
+const HEX_RE = /[0-9a-fA-F]/;
+
+/** The index just past the escape at `i`: one code point, or up to six hex digits and a space. */
+function consumeEscape(source: string, i: number, end: number): number {
+  let j = i + 1;
+  if (j >= end || !HEX_RE.test(source[j] ?? "")) {
+    return Math.min(j + 1, end);
+  }
+  while (j < end && j < i + 7 && HEX_RE.test(source[j] ?? "")) {
+    j++;
+  }
+  return j < end && isWhitespace(source[j]) ? whitespaceEnd(source, j, end) : j;
+}
+
 /** The index just past the string that opens at `start`, or the newline that cuts it short. */
 export function consumeString(source: string, start: number, end = source.length): number {
   const quote = source[start];
@@ -11,13 +54,16 @@ export function consumeString(source: string, start: number, end = source.length
   while (i < end) {
     const ch = source[i];
     if (ch === "\\") {
-      i += 2;
+      // A `\` before a newline carries the string on to the next line.
+      i = startsEscape(source, i, end)
+        ? consumeEscape(source, i, end)
+        : whitespaceEnd(source, i + 1, end);
       continue;
     }
     if (ch === quote) {
       return i + 1;
     }
-    if (ch === "\n") {
+    if (isNewline(ch)) {
       return i;
     }
     i++;
@@ -35,27 +81,122 @@ function isCommentStart(source: string, i: number): boolean {
   return source[i] === "/" && source[i + 1] === "*";
 }
 
+const NAME_RE = /[-\w\u0080-\uffff]/;
+
+/** A code point of a name: a letter, a digit, `-`, `_`, non-ASCII, or NUL, which CSS reads as U+FFFD. */
+function isNameChar(ch: string | undefined): boolean {
+  return ch === "\0" || NAME_RE.test(ch ?? "");
+}
+
+/** The index just past the name at `i`: letters, digits, `-`, `_`, non-ASCII, and escapes. */
+function consumeName(source: string, i: number, end: number): number {
+  let j = i;
+  while (j < end) {
+    if (startsEscape(source, j, end)) {
+      j = consumeEscape(source, j, end);
+    } else if (isNameChar(source[j])) {
+      j++;
+    } else {
+      break;
+    }
+  }
+  return j;
+}
+
+/** Whether a name reads `url` once its escapes are resolved, as `URL` or `u\72l` does. */
+function isUrlName(name: string): boolean {
+  const read = name.includes("\\")
+    ? name.replace(/\\(?:([0-9a-fA-F]{1,6})(?:\r\n|[ \t\n\r\f])?|([\s\S]))/g, (_, hex, ch) => {
+        const code = hex === undefined ? -1 : Number.parseInt(hex, 16);
+        return code < 0 ? ch : code > 0x10ffff ? "\ufffd" : String.fromCodePoint(code);
+      })
+    : name;
+  return read.toLowerCase() === "url";
+}
+
+function isNonPrintable(ch: string): boolean {
+  const code = ch.charCodeAt(0);
+  // NUL is no control character here: CSS reads it as U+FFFD.
+  return (
+    (code >= 0x01 && code <= 0x08) ||
+    code === 0x0b ||
+    (code >= 0x0e && code <= 0x1f) ||
+    code === 0x7f
+  );
+}
+
 /**
- * The first index from `start` whose character is one of `stops` outside strings, comments, and
- * `()`/`[]`; `end` when there is none. An opening brace counts only when `{` is a stop.
+ * The index just past an unquoted `url()` whose address starts at `start`, after its `(`; none
+ * when the address is quoted, which makes it a function holding a string. A broken one, with a
+ * space, a quote, a `(`, or a control character in its address, still runs to its `)`.
  */
-export function scanTopLevel(source: string, start: number, end: number, stops: string): number {
-  let depth = 0;
-  let i = start;
+function consumeUrl(source: string, start: number, end: number): number | undefined {
+  let i = skipWhitespace(source, start, end);
+  if (i < end && (source[i] === '"' || source[i] === "'")) {
+    return undefined;
+  }
+  let broken = false;
   while (i < end) {
     const ch = source[i] ?? "";
-    if (ch === '"' || ch === "'") {
-      i = consumeString(source, i, end);
-      continue;
+    if (ch === ")") {
+      return i + 1;
     }
-    if (isCommentStart(source, i)) {
-      i = consumeComment(source, i, end);
+    if (startsEscape(source, i, end)) {
+      i = consumeEscape(source, i, end);
       continue;
     }
-    if (ch === "\\") {
-      i += 2;
+    if (!broken && isWhitespace(ch)) {
+      i = skipWhitespace(source, i, end);
+      broken = i < end && source[i] !== ")";
       continue;
     }
+    broken ||= ch === '"' || ch === "'" || ch === "(" || ch === "\\" || isNonPrintable(ch);
+    i++;
+  }
+  return end;
+}
+
+/**
+ * The index just past the token that starts at `i`, as far as dek tells tokens apart: a string,
+ * a comment, an unquoted `url()`, or a name with its escapes, so the `url` of `myurl(` or
+ * `10url(` opens nothing. Any other code point is a token of its own. A `\` before a newline,
+ * which escapes nothing, takes the newline with it, so no rewrite can make an escape of the two.
+ */
+export function consumeToken(source: string, i: number, end = source.length): number {
+  const ch = source[i];
+  if (ch === '"' || ch === "'") {
+    return consumeString(source, i, end);
+  }
+  if (isCommentStart(source, i)) {
+    return consumeComment(source, i, end);
+  }
+  if (ch === "\\" && !startsEscape(source, i, end)) {
+    return whitespaceEnd(source, i + 1, end);
+  }
+  if (!startsEscape(source, i, end) && !isNameChar(ch)) {
+    return i + 1;
+  }
+  const nameEnd = consumeName(source, i, end);
+  // `#url(` is a hash and `@url(` an at-rule's name, not a url().
+  const url =
+    nameEnd < end &&
+    source[nameEnd] === "(" &&
+    source[i - 1] !== "#" &&
+    source[i - 1] !== "@" &&
+    isUrlName(source.slice(i, nameEnd));
+  return (url && consumeUrl(source, nameEnd + 1, end)) || nameEnd;
+}
+
+/**
+ * The first index from `start` whose character is one of `stops` outside strings, comments,
+ * `url()`s, and `()`/`[]`; `end` when there is none. An opening brace counts only when `{` is a
+ * stop. Stops are punctuation, which no name holds.
+ */
+export function scanTopLevel(source: string, start: number, end: number, stops: string): number {
+  let depth = 0;
+  let i = start;
+  while (i < end) {
+    const ch = source[i] ?? "";
     if (depth === 0 && stops.includes(ch)) {
       return i;
     }
@@ -64,7 +205,7 @@ export function scanTopLevel(source: string, start: number, end: number, stops:
     } else if ((ch === ")" || ch === "]") && depth > 0) {
       depth--;
     }
-    i++;
+    i = consumeToken(source, i, end);
   }
   return end;
 }
@@ -88,17 +229,17 @@ export function matchBrace(source: string, open: number, end = source.length): n
   return end;
 }
 
-/** The text with each comment removed, strings kept byte for byte. */
+/** The text with each comment removed, strings and `url()`s kept byte for byte. */
 export function stripComments(source: string): string {
   return replaceOpaque(source, { comment: () => "" });
 }
 
 /**
  * The text with each string and comment blanked to spaces, newlines kept, so only CSS syntax is
- * left and every offset still points where it did.
+ * left and every offset still points where it did. A `url()` is left as written.
  */
 export function blankStringsAndComments(source: string): string {
-  const blank = (text: string) => text.replace(/[^\n]/g, " ");
+  const blank = (text: string) => text.replace(/[^\n\r\f]/g, " ");
   return replaceOpaque(source, { comment: blank, string: blank });
 }
 
@@ -110,26 +251,19 @@ function replaceOpaque(
   let from = 0;
   let i = 0;
   while (i < source.length) {
+    const next = consumeToken(source, i);
     const ch = source[i];
-    if (ch === '"' || ch === "'") {
-      const end = consumeString(source, i);
-      if (replace.string) {
-        out += source.slice(from, i) + replace.string(source.slice(i, end));
-        from = end;
-      }
-      i = end;
-      continue;
-    }
-    if (isCommentStart(source, i)) {
-      const end = consumeComment(source, i);
-      if (replace.comment) {
-        out += source.slice(from, i) + replace.comment(source.slice(i, end));
-        from = end;
-      }
-      i = end;
-      continue;
+    const replacer =
+      ch === '"' || ch === "'"
+        ? replace.string
+        : isCommentStart(source, i)
+          ? replace.comment
+          : undefined;
+    if (replacer) {
+      out += source.slice(from, i) + replacer(source.slice(i, next));
+      from = next;
     }
-    i += ch === "\\" ? 2 : 1;
+    i = next;
   }
   return out + source.slice(from);
 }

package/src/core/css-transform.ts

@@ -1,5 +1,5 @@
 import { outermostStyleRules, type Stylesheet, splitSelectorList, startsAtSlide } from "./css.ts";
-import { consumeComment, consumeString, cssIdents } from "./css-scan.ts";
+import { consumeComment, consumeToken, cssIdents } from "./css-scan.ts";
 
 /** Text to put in place of `[start, end)` of the stylesheet as it was read. */
 type CssEdit = { start: number; end: number; text: string };
@@ -108,32 +108,78 @@ export function replaceUrls(replace: (value: string) => string | undefined): Css
 
 /** Whitespace as CSS defines it. U+3000 and NBSP are content, not whitespace. */
 const CSS_WHITESPACE = new Set([" ", "\t", "\n", "\r", "\f"]);
+const CSS_NEWLINES = new Set(["\n", "\r", "\f"]);
+
+/** A token that runs on into the next with nothing between: a name, a number, or `#@.+/`. */
+const RUNS_ON_RE = /^[-\w\u0080-\uffff\0\\#@.+/]/;
+/** What such a token runs on into: a name, a number, `(`, `%`, or the `*` that opens a comment. */
+const RUNS_INTO_RE = /^[-\w\u0080-\uffff\0\\.+(%*]/;
 
 /**
- * Drops comments and collapses whitespace to one space, leaving strings byte for byte, so
- * `content: "a　b"` or `"  "` renders in the build as it does on the dev server.
+ * Whether the token `last` and the one that starts with `next` would read as one with nothing
+ * between them: `x` and `url(` as `xurl(`, `/` and `*` as a comment, or `--` and `>`, `<` and
+ * `!`, `!` and `--` as the `-->` and `<!--` a stylesheet skips.
+ */
+function runTogether(last: string, next: string): boolean {
+  return (
+    (RUNS_ON_RE.test(last) && RUNS_INTO_RE.test(next)) ||
+    (last.endsWith("--") && next === ">") ||
+    (last === "<" && next === "!") ||
+    (last === "!" && next === "-")
+  );
+}
+
+/** A hex escape at the end of a token, which takes one whitespace after it as its own. */
+const HEX_ESCAPE_END_RE = /\\[0-9a-fA-F]{1,6}$/;
+
+/**
+ * Drops comments and collapses whitespace to one space, leaving strings and `url()`s byte for
+ * byte, so `content: "a　b"` or `"  "` renders in the build as it does on the dev server. The
+ * result reads as the source does: a newline after a string stays one, since it is what ends a
+ * string cut short, and a comment between two tokens that would run into one, as `x` and `url(`
+ * into `xurl(`, stays as an empty one, since a space would make `div` and `p` two selectors. A
+ * hex escape that a comment ends, such as `\41` before a comment and ` x`, is followed by two
+ * spaces, since it takes one as its own.
  */
 export function minifyCss(css: string): string {
   let out = "";
-  let space = false;
+  let last = "";
+  // What parts the last token written from the next: "", "comment", a space, or the newline that
+  // ends a string.
+  let gap = "";
   let i = 0;
   while (i < css.length) {
     const ch = css[i] ?? "";
     if (ch === "/" && css[i + 1] === "*") {
       i = consumeComment(css, i);
+      gap ||= "comment";
       continue;
     }
     if (CSS_WHITESPACE.has(ch)) {
-      space = true;
-      i++;
+      const whitespace = ch === "\r" && css[i + 1] === "\n" ? "\r\n" : ch;
+      if (gap === "" && CSS_NEWLINES.has(ch) && (last[0] === '"' || last[0] === "'")) {
+        // One LF, unless the string ends on a CR it escapes, which an LF would join into a CRLF.
+        gap = last.endsWith("\r") ? whitespace : "\n";
+      } else if (gap === "" || gap === "comment") {
+        gap = " ";
+      }
+      i += whitespace.length;
       continue;
     }
-    if (space && out !== "") {
-      out += " ";
+    if (out !== "") {
+      out +=
+        gap === "comment"
+          ? runTogether(last, ch)
+            ? "/**/"
+            : ""
+          : gap === " " && HEX_ESCAPE_END_RE.test(last)
+            ? "  "
+            : gap;
     }
-    space = false;
-    const end = ch === '"' || ch === "'" ? consumeString(css, i) : ch === "\\" ? i + 2 : i + 1;
-    out += css.slice(i, end);
+    gap = "";
+    const end = consumeToken(css, i);
+    last = css.slice(i, end);
+    out += last;
     i = end;
   }
   return out;

package/src/core/css.ts

@@ -2,6 +2,7 @@ import {
   blankStringsAndComments,
   consumeComment,
   consumeString,
+  consumeToken,
   matchBrace,
   scanTopLevel,
   splitTopLevel,
@@ -131,16 +132,11 @@ class SheetReader {
     const { source } = this;
     let i = 0;
     while (i < source.length) {
-      const ch = source[i];
-      if (ch === '"' || ch === "'") {
-        i = consumeString(source, i);
-      } else if (ch === "/" && source[i + 1] === "*") {
-        const close = consumeComment(source, i);
-        this.comments.push([i, close]);
-        i = close;
-      } else {
-        i += ch === "\\" ? 2 : 1;
+      const next = consumeToken(source, i);
+      if (source[i] === "/" && source[i + 1] === "*") {
+        this.comments.push([i, next]);
       }
+      i = next;
     }
     const locate = lineLocator(sheet.source);
     this.lineOf = (offset) => locate(offset).line;
@@ -181,13 +177,23 @@ class SheetReader {
     return i;
   }
 
-  /** A prelude's span with its trailing spaces trimmed off. */
+  /**
+   * A prelude's span with the spaces and comments at its end trimmed off; a string it ends on,
+   * such as the name of `@keyframes "pop"`, stays.
+   */
   trimEnd(start: number, end: number): number {
     let j = end;
-    while (j > start && WHITESPACE_RE.test(this.blank[j - 1] ?? "")) {
-      j--;
+    while (j > start) {
+      const comment = this.comments.find(([, close]) => close === j);
+      if (comment) {
+        j = comment[0];
+      } else if (WHITESPACE_RE.test(this.source[j - 1] ?? "")) {
+        j--;
+      } else {
+        break;
+      }
     }
-    return j;
+    return Math.max(j, start);
   }
 
   atRule(start: number, end: number, scope: Scope): number {

package/src/core/foreign-text.ts

@@ -0,0 +1,138 @@
+/**
+ * Text someone else wrote, such as a ref's title or the folder names in its repository, as dek
+ * puts it inside its own words. An agent reads dek's messages and AGENTS.md's dek block as dek's,
+ * so such text must not start a line there, carry a control character, or hold a character the
+ * person reading never sees.
+ */
+
+/**
+ * Characters that show as nothing or move the text around them: those Unicode marks
+ * default-ignorable, such as zero-width ones, the byte order mark, bidi controls, Hangul fillers,
+ * and tag characters, which a model reads as text though a person sees nothing; and the line and
+ * paragraph separators and the interlinear annotation marks. The variation selectors, Mongolian's
+ * free ones included, are default-ignorable too and left out: emoji, ideographic variants of a
+ * kanji, and Mongolian text use them on purpose.
+ */
+const HIDDEN_RE =
+  /(?![\u{fe00}-\u{fe0f}\u{e0100}-\u{e01ef}\u{180b}-\u{180d}\u{180f}])[\p{Default_Ignorable_Code_Point}\u{2028}\u{2029}\u{fff9}-\u{fffb}]/u;
+
+/** Whitespace and control characters, which a line of someone else's text shows as one space. */
+// biome-ignore lint/suspicious/noControlCharactersInRegex: matching them is the point.
+const SPACE_RE = /[\s\x00-\x1f\x7f-\x9f]/u;
+
+/** HIDDEN_RE for every match. */
+const HIDDEN_ALL_RE = new RegExp(HIDDEN_RE.source, "gu");
+
+/** True when `text` holds a character that shows as nothing or moves the text around it. */
+export function holdsHidden(text: string): boolean {
+  return HIDDEN_RE.test(text);
+}
+
+/** `U+202E`, as a code point is written. */
+function codePoint(char: string): string {
+  return `U+${(char.codePointAt(0) ?? 0).toString(16).toUpperCase().padStart(4, "0")}`;
+}
+
+/** A hidden character as dek shows it, so a reader sees it is there and which it is. */
+function marker(char: string): string {
+  return `⟨${codePoint(char)}⟩`;
+}
+
+/** `text` with each hidden character as dek shows it, `⟨U+XXXX⟩`, and the rest as it is. */
+export function revealHidden(text: string): string {
+  return text.replace(HIDDEN_ALL_RE, marker);
+}
+
+/** A line of a file someone else wrote that holds characters a person cannot see. */
+export type HiddenText = {
+  path: string;
+  line: number;
+  /** 1-based, in UTF-16 units as editors count: where the first of them is. */
+  column: number;
+  /** Each of them on the line, in order, as U+XXXX. */
+  codePoints: string[];
+};
+
+/** Each line of `text`, read from `path` and starting at line `firstLine`, that holds any. */
+export function hiddenText(text: string, path: string, firstLine = 1): HiddenText[] {
+  return text.split("\n").flatMap((line, index) => {
+    const found = [...line.matchAll(HIDDEN_ALL_RE)];
+    const [first] = found;
+    if (first === undefined) {
+      return [];
+    }
+    return [
+      {
+        path,
+        line: firstLine + index,
+        column: first.index + 1,
+        codePoints: found.map((match) => codePoint(match[0])),
+      },
+    ];
+  });
+}
+
+/**
+ * A marker, which text revealHidden already went over holds, or else one code point: the units a
+ * line is cut between.
+ */
+const UNIT_RE = /⟨U\+[0-9A-F]{4,6}⟩|[\s\S]/gu;
+
+/**
+ * One line of someone else's text inside dek's own: each hidden character shows as `⟨U+XXXX⟩`,
+ * whitespace and control characters as one space, and the line is cut to `max` characters with
+ * an ellipsis, never through a marker, whether this made it or revealHidden did before.
+ */
+export function foreignLine(text: string, max: number): string {
+  const units: string[] = [];
+  for (const [char = ""] of text.matchAll(UNIT_RE)) {
+    // A marker is longer than any one code point, which is at most two UTF-16 units.
+    const unit =
+      char.length > 2
+        ? char
+        : HIDDEN_RE.test(char)
+          ? marker(char)
+          : SPACE_RE.test(char)
+            ? " "
+            : char;
+    if (unit !== " " || (units.length > 0 && units.at(-1) !== " ")) {
+      units.push(unit);
+    }
+  }
+  if (units.at(-1) === " ") {
+    units.pop();
+  }
+  const width = (unit: string) => [...unit].length;
+  if (units.reduce((sum, unit) => sum + width(unit), 0) <= max) {
+    return units.join("");
+  }
+  const kept: string[] = [];
+  let size = 0;
+  for (const unit of units) {
+    if (size + width(unit) > max - 1) {
+      break;
+    }
+    kept.push(unit);
+    size += width(unit);
+  }
+  return `${kept.join("")}…`;
+}
+
+/** The longest a ref's title shows in a line of dek's: a title names a deck, it does not describe it. */
+export const TITLE_MAX = 80;
+
+/** The longest message dek gives that quotes someone else's text, such as Bun's on their source. */
+export const MESSAGE_MAX = 200;
+
+/** The longest name a list shows: a deck's folder names a deck, it does not describe one. */
+const NAME_MAX = 40;
+
+/** How many names a list shows before it counts the rest. */
+const LIST_MAX = 10;
+
+/** Names someone else chose, each a foreignLine of NAME_MAX, the first LIST_MAX and a count of the rest. */
+export function foreignList(names: string[]): string {
+  const shown = names.slice(0, LIST_MAX).map((name) => foreignLine(name, NAME_MAX));
+  const rest = names.length - shown.length;
+  return rest > 0 ? `${shown.join(", ")}, and ${rest} more` : shown.join(", ");
+}

package/src/core/github.ts

@@ -93,6 +93,7 @@ async function request(
   const auth = authorization(base);
   let response: Response;
   try {
+    // biome-ignore lint/style/noRestrictedGlobals: the deadline's signal bounds the request, and readBody cancels a read that outlasts it.
     response = await fetch(`${base}${path}`, {
       headers: {
         accept,