@hajimism/dek 0.5.0 → 0.6.0

package/src/plugins/study/place.ts

@@ -0,0 +1,88 @@
+/**
+ * Where each sticker sits in a slide's words, decided the same in the check and in the page, so a
+ * deck that passes lint is a deck whose stickers the page finds.
+ */
+
+/** What a sticker needs to be placed: its words, the words right before them, and its line. */
+export type Placeable = { quote: string; prefix?: string; line: number };
+
+export type Placement<S extends Placeable> = {
+  /** Each sticker that found its place, with where its words start in the text. */
+  placed: Array<{ sticker: S; at: number }>;
+  /** Each sticker whose words are not where it says. */
+  lost: S[];
+  /** Words a sticker hides that show again with no sticker over them: how often, and the first sticker. */
+  shown: Array<{ quote: string; first: S; count: number }>;
+};
+
+/** Text with each run of whitespace as one space, trimmed, as words are compared. */
+export function normalize(text: string): string {
+  return text.replace(/\s+/g, " ").trim();
+}
+
+/**
+ * Places each sticker in `text`, a slide's words as `normalize` leaves them. Stickers over the same
+ * words take their places in order: one with a prefix the first place that follows it, then one
+ * without the first place left.
+ */
+export function placeStickers<S extends Placeable>(text: string, stickers: S[]): Placement<S> {
+  const placement: Placement<S> = { placed: [], lost: [], shown: [] };
+  const inOrder = stickers.toSorted((a, b) => a.line - b.line);
+  for (const [quote, same] of Map.groupBy(inOrder, (sticker) => normalize(sticker.quote))) {
+    const places = occurrences(text, quote);
+    const taken = new Set<number>();
+    const placed: S[] = [];
+    for (const sticker of [...same.filter((s) => s.prefix), ...same.filter((s) => !s.prefix)]) {
+      const prefix = sticker.prefix === undefined ? undefined : normalize(sticker.prefix);
+      const at = places.find(
+        (place) => !taken.has(place) && (prefix === undefined || follows(text, place, prefix)),
+      );
+      if (at === undefined) {
+        placement.lost.push(sticker);
+        continue;
+      }
+      taken.add(at);
+      placed.push(sticker);
+      placement.placed.push({ sticker, at });
+    }
+    const count = places.length - taken.size;
+    const first = placed.toSorted((a, b) => a.line - b.line)[0];
+    if (count > 0 && first) {
+      placement.shown.push({ quote, first, count });
+    }
+  }
+  placement.placed.sort((a, b) => a.sticker.line - b.sticker.line);
+  return placement;
+}
+
+/** Whether the words right before `place` in `text` end with `prefix`, as normalized. */
+export function follows(text: string, place: number, prefix: string): boolean {
+  return text.slice(0, place).trimEnd().endsWith(prefix);
+}
+
+/**
+ * A letter or digit of a script written with spaces between words. Next to one, a quote that
+ * starts or ends with one is part of a longer word: `is` is not in "This". Japanese, Chinese,
+ * Korean, and Thai run on without spaces, so a quote is any run of their words.
+ */
+const SPACED = /[\p{Script=Latin}\p{Script=Greek}\p{Script=Cyrillic}\p{Nd}]/u;
+
+/** Where `quote` starts in `text`, each place it does, without overlaps, as whole words. */
+export function occurrences(text: string, quote: string): number[] {
+  const found: number[] = [];
+  if (quote === "") {
+    return found;
+  }
+  const end = quote.length;
+  const opens = SPACED.test(quote[0] ?? "");
+  const closes = SPACED.test(quote[end - 1] ?? "");
+  for (let at = text.indexOf(quote); at !== -1; ) {
+    const whole =
+      !(opens && SPACED.test(text[at - 1] ?? "")) && !(closes && SPACED.test(text[at + end] ?? ""));
+    if (whole) {
+      found.push(at);
+    }
+    at = text.indexOf(quote, whole ? at + end : at + 1);
+  }
+  return found;
+}

package/src/plugins/study/sarif.ts

@@ -0,0 +1,30 @@
+import type { CheckOutput } from "../../plugin-api.ts";
+import type { Finding } from "./check.ts";
+import type { RuleId } from "./manifest.ts";
+import { MANIFEST } from "./manifest.ts";
+
+/** Findings as the SARIF dek reads: the rule, the message, the place, and the hint. */
+export function toSarif(findings: Finding[]): CheckOutput<RuleId> {
+  return {
+    version: "2.1.0",
+    $schema: "https://json.schemastore.org/sarif-2.1.0.json",
+    runs: [
+      {
+        tool: { driver: { name: "dek-study", version: MANIFEST.version } },
+        results: findings.map((finding) => ({
+          ruleId: finding.rule,
+          message: { text: finding.message },
+          locations: [
+            {
+              physicalLocation: {
+                artifactLocation: { uri: finding.path },
+                region: { startLine: finding.line },
+              },
+            },
+          ],
+          properties: { hint: finding.hint },
+        })),
+      },
+    ],
+  };
+}

package/src/plugins/study/study.ts

@@ -0,0 +1,137 @@
+import { STUDY_FILE } from "./manifest.ts";
+import { parseYaml, YamlError } from "./yaml.ts";
+
+/** One sticker: it hides `quote` on `slide`, at the place `prefix` comes right before. */
+export type Sticker = {
+  id: string;
+  slide: string;
+  quote: string;
+  prefix?: string;
+  /** Where the sticker starts in study.yaml. */
+  line: number;
+};
+
+/** What is wrong with study.yaml, and the line to look at. */
+export type Problem = { message: string; line: number };
+
+const VERSION = 1;
+
+/** What YAML made of words written without quotes, when it made them something other than text. */
+function unquoted(value: unknown): string | undefined {
+  if (typeof value === "number") {
+    return `the number ${value}`;
+  }
+  return typeof value === "boolean" ? String(value) : undefined;
+}
+
+/**
+ * The stickers in study.yaml, and what keeps the rest from being used. A sticker with a problem
+ * is left out, so the checks run on every sticker that can be read. The page reads it the same.
+ */
+export function readStudy(source: string): { stickers: Sticker[]; problems: Problem[] } {
+  let parsed: ReturnType<typeof parseYaml>;
+  try {
+    parsed = parseYaml(source);
+  } catch (error) {
+    if (error instanceof YamlError) {
+      return {
+        stickers: [],
+        problems: [
+          {
+            message: `${STUDY_FILE} is not YAML dek-study reads: ${error.message}`,
+            line: error.line,
+          },
+        ],
+      };
+    }
+    throw error;
+  }
+  const { value, lineOf } = parsed;
+  if (value === null) {
+    return { stickers: [], problems: [] };
+  }
+  if (typeof value !== "object" || Array.isArray(value)) {
+    return { stickers: [], problems: [{ message: `${STUDY_FILE} is not a mapping`, line: 1 }] };
+  }
+  const top = lineOf(value) ?? 1;
+  const fields = value as { version?: unknown; stickers?: unknown };
+  const lines = source.split("\n");
+  const keyLine = (key: string) => {
+    const index = lines.findIndex((text) => new RegExp(`^${key}\\s*:`).test(text));
+    return index === -1 ? top : index + 1;
+  };
+  if (fields.version !== VERSION) {
+    return {
+      stickers: [],
+      problems: [
+        {
+          message: `${STUDY_FILE} is version ${String(fields.version)}; this dek-study reads version ${VERSION}`,
+          line: keyLine("version"),
+        },
+      ],
+    };
+  }
+  if (fields.stickers === undefined || fields.stickers === null) {
+    return { stickers: [], problems: [] };
+  }
+  if (!Array.isArray(fields.stickers)) {
+    return {
+      stickers: [],
+      problems: [{ message: "`stickers` is not a list", line: keyLine("stickers") }],
+    };
+  }
+
+  const stickers: Sticker[] = [];
+  const problems: Problem[] = [];
+  const seen = new Set<string>();
+  fields.stickers.forEach((entry: unknown, index) => {
+    const sticker =
+      typeof entry === "object" && entry !== null ? (entry as Record<string, unknown>) : null;
+    const line = (sticker && lineOf(sticker)) ?? keyLine("stickers");
+    const id = typeof sticker?.id === "string" && sticker.id !== "" ? sticker.id : undefined;
+    const name = id ?? `#${index + 1}`;
+    for (const key of ["id", "slide", "quote", "prefix"] as const) {
+      const made = unquoted(sticker?.[key]);
+      if (made !== undefined) {
+        problems.push({
+          message: `sticker ${name}: YAML reads its \`${key}\` as ${made}; put it in quotes`,
+          line,
+        });
+        return;
+      }
+    }
+    if (id === undefined) {
+      problems.push({ message: `sticker ${name} has no \`id\``, line });
+      return;
+    }
+    for (const key of ["slide", "quote"] as const) {
+      const value = sticker?.[key];
+      if (typeof value !== "string" || value.trim() === "") {
+        problems.push({ message: `sticker ${name} has no \`${key}\``, line });
+        return;
+      }
+    }
+    if (sticker?.prefix !== undefined && typeof sticker.prefix !== "string") {
+      problems.push({ message: `sticker ${name}'s \`prefix\` is not text`, line });
+      return;
+    }
+    if (seen.has(id)) {
+      problems.push({
+        message: `sticker ${id} is named twice; give each sticker its own id`,
+        line,
+      });
+      return;
+    }
+    seen.add(id);
+    stickers.push({
+      id,
+      slide: sticker?.slide as string,
+      quote: sticker?.quote as string,
+      ...(typeof sticker?.prefix === "string" && sticker.prefix !== ""
+        ? { prefix: sticker.prefix }
+        : {}),
+      line,
+    });
+  });
+  return { stickers, problems };
+}

package/src/plugins/study/text.ts

@@ -0,0 +1,55 @@
+import { normalize } from "./place.ts";
+
+/** Elements whose content no reader sees as words on the slide. */
+const UNSEEN = "script, style, template, noscript";
+
+const ENTITIES: Record<string, string> = {
+  amp: "&",
+  lt: "<",
+  gt: ">",
+  quot: '"',
+  apos: "'",
+  nbsp: " ",
+};
+
+/**
+ * A slide's words as a reader sees them, in document order, with each run of whitespace as one
+ * space: the text dek-study matches a sticker's words against.
+ */
+export async function slideText(html: string): Promise<string> {
+  const parts: string[] = [];
+  // How many unseen elements the parser is inside; text there is not the slide's words.
+  let unseen = 0;
+  await new HTMLRewriter()
+    .on(UNSEEN, {
+      element(element) {
+        unseen += 1;
+        element.onEndTag(() => {
+          unseen -= 1;
+        });
+      },
+    })
+    .onDocument({
+      text(chunk) {
+        if (unseen === 0) {
+          parts.push(chunk.text);
+        }
+      },
+    })
+    .transform(new Response(html))
+    .text();
+  return normalize(decode(parts.join("")));
+}
+
+function decode(text: string): string {
+  return text.replace(/&(#x[0-9a-f]+|#\d+|[a-z]+);/gi, (whole, name: string) => {
+    if (name.startsWith("#")) {
+      const code =
+        name[1] === "x" || name[1] === "X"
+          ? Number.parseInt(name.slice(2), 16)
+          : Number(name.slice(1));
+      return Number.isFinite(code) ? String.fromCodePoint(code) : whole;
+    }
+    return ENTITIES[name.toLowerCase()] ?? whole;
+  });
+}

package/src/plugins/study/yaml.ts

@@ -0,0 +1,226 @@
+/**
+ * The YAML study.yaml is written in, read the same in the check and in the page, which has no
+ * YAML parser: block mappings and sequences of plain words as keys, and scalars that are plain,
+ * single-quoted, or double-quoted, `[]`, or `{}`. Anything else is refused with its line, so a file
+ * the check passes is a file the page reads.
+ */
+
+/** YAML this reader does not read, and the line it starts on. */
+export class YamlError extends Error {
+  constructor(
+    message: string,
+    readonly line: number,
+  ) {
+    super(message);
+  }
+}
+
+type Line = { indent: number; text: string; line: number };
+
+/** The value, and the line each mapping in it starts on. */
+export function parseYaml(source: string): {
+  value: unknown;
+  lineOf: (mapping: object) => number | undefined;
+} {
+  const lines = toLines(source);
+  const starts = new WeakMap<object, number>();
+  const lineOf = (mapping: object) => starts.get(mapping);
+  const first = lines[0];
+  if (first === undefined) {
+    return { value: null, lineOf };
+  }
+  const [value, next] = block(lines, 0, first.indent, starts);
+  const extra = lines[next];
+  if (extra !== undefined) {
+    throw new YamlError("this line is indented less than the lines before it", extra.line);
+  }
+  return { value, lineOf };
+}
+
+function toLines(source: string): Line[] {
+  // A byte order mark, as Notepad on Windows saves, is no part of the first line.
+  return source
+    .replace(/^\uFEFF/, "")
+    .split(/\r?\n/)
+    .flatMap((raw, index): Line[] => {
+      const line = index + 1;
+      const text = withoutComment(raw);
+      if (text.trim() === "") {
+        return [];
+      }
+      const indent = text.length - text.trimStart().length;
+      if (/^\s*\t/.test(text) || text.slice(0, indent).includes("\t")) {
+        throw new YamlError("indent with spaces, not tabs", line);
+      }
+      return [{ indent, text: text.trim(), line }];
+    });
+}
+
+/**
+ * The line up to a `#` that starts a comment: at the start, or after a space, outside quotes. A
+ * quote opens a quoted value only where a value starts, so `don't` in a plain value opens none.
+ */
+function withoutComment(raw: string): string {
+  let quote: string | undefined;
+  for (let i = 0; i < raw.length; i += 1) {
+    const char = raw[i];
+    if (quote === '"' && char === "\\") {
+      i += 1;
+    } else if (quote === "'" && char === "'" && raw[i + 1] === "'") {
+      i += 1;
+    } else if (quote !== undefined) {
+      if (char === quote) {
+        quote = undefined;
+      }
+    } else if ((char === '"' || char === "'") && VALUE_START.test(raw.slice(0, i))) {
+      quote = char;
+    } else if (char === "#" && (i === 0 || /\s/.test(raw[i - 1] ?? ""))) {
+      return raw.slice(0, i);
+    }
+  }
+  return raw;
+}
+
+/** What comes before a value: the line's start, a key's colon, or an item's dash. */
+const VALUE_START = /(?:^|:|(?:^|\s)-)\s*$/;
+
+const ITEM = /^-(?:\s+(.*))?$/;
+const KEY = /^([A-Za-z0-9_-]+)\s*:(?:\s+(.*))?$/;
+
+function block(
+  lines: Line[],
+  start: number,
+  indent: number,
+  starts: WeakMap<object, number>,
+): [unknown, number] {
+  const line = lines[start] as Line;
+  return ITEM.test(line.text)
+    ? sequence(lines, start, indent, starts)
+    : mapping(lines, start, indent, starts);
+}
+
+function sequence(
+  lines: Line[],
+  start: number,
+  indent: number,
+  starts: WeakMap<object, number>,
+): [unknown[], number] {
+  const items: unknown[] = [];
+  let i = start;
+  for (let line = lines[i]; line?.indent === indent; line = lines[i]) {
+    const match = line.text.match(ITEM);
+    if (!match) {
+      break;
+    }
+    const rest = match[1];
+    if (rest === undefined) {
+      // `-` alone: the item is the block below it.
+      const below = lines[i + 1];
+      if (below !== undefined && below.indent > indent) {
+        const [value, next] = block(lines, i + 1, below.indent, starts);
+        items.push(value);
+        i = next;
+      } else {
+        items.push(null);
+        i += 1;
+      }
+    } else if (KEY.test(rest)) {
+      // `- key: value` starts a mapping whose keys line up after the dash.
+      const column = indent + (line.text.length - rest.length);
+      const shifted = [...lines];
+      shifted[i] = { indent: column, text: rest, line: line.line };
+      const [value, next] = mapping(shifted, i, column, starts);
+      items.push(value);
+      i = next;
+    } else {
+      items.push(scalar(rest, line.line));
+      i += 1;
+    }
+  }
+  return [items, i];
+}
+
+function mapping(
+  lines: Line[],
+  start: number,
+  indent: number,
+  starts: WeakMap<object, number>,
+): [Record<string, unknown>, number] {
+  const result: Record<string, unknown> = {};
+  starts.set(result, (lines[start] as Line).line);
+  let i = start;
+  for (let line = lines[i]; line?.indent === indent; line = lines[i]) {
+    const match = line.text.match(KEY);
+    if (!match) {
+      throw new YamlError("expected `key: value`", line.line);
+    }
+    const key = match[1] as string;
+    if (Object.hasOwn(result, key)) {
+      throw new YamlError(`\`${key}\` is written twice`, line.line);
+    }
+    const rest = match[2];
+    const below = lines[i + 1];
+    if (rest !== undefined) {
+      result[key] = scalar(rest, line.line);
+      i += 1;
+    } else if (below !== undefined && below.indent > indent) {
+      [result[key], i] = block(lines, i + 1, below.indent, starts);
+    } else if (below !== undefined && below.indent === indent && ITEM.test(below.text)) {
+      // A sequence may sit at its key's own indent.
+      [result[key], i] = sequence(lines, i + 1, indent, starts);
+    } else {
+      result[key] = null;
+      i += 1;
+    }
+  }
+  const deeper = lines[i];
+  if (deeper !== undefined && deeper.indent > indent) {
+    throw new YamlError("this line is indented more than the lines before it", deeper.line);
+  }
+  return [result, i];
+}
+
+function scalar(text: string, line: number): unknown {
+  if (text.startsWith('"')) {
+    return quoted(text, line, /^"((?:[^"\\]|\\.)*)"$/, (inner) => JSON.parse(`"${inner}"`));
+  }
+  if (text.startsWith("'")) {
+    return quoted(text, line, /^'((?:[^']|'')*)'$/, (inner) => inner.replaceAll("''", "'"));
+  }
+  if (text === "[]") {
+    return [];
+  }
+  if (text === "{}") {
+    return {};
+  }
+  if (/^[[{&*!|>%@`]/.test(text)) {
+    throw new YamlError(`dek-study reads no YAML that starts a value with ${text[0]}`, line);
+  }
+  if (/^-?\d+$/.test(text)) {
+    return Number(text);
+  }
+  if (text === "null" || text === "~") {
+    return null;
+  }
+  if (text === "true" || text === "false") {
+    return text === "true";
+  }
+  return text;
+}
+
+function quoted(
+  text: string,
+  line: number,
+  pattern: RegExp,
+  read: (inner: string) => string,
+): string {
+  const inner = text.match(pattern)?.[1];
+  if (inner === undefined) {
+    throw new YamlError("a quoted value must end with its quote on the same line", line);
+  }
+  try {
+    return read(inner);
+  } catch {
+    throw new YamlError("a double-quoted value has an escape dek-study does not read", line);
+  }
+}

package/src/plugins/words/check.ts

@@ -0,0 +1,109 @@
+import { join } from "node:path";
+import { pathToFileURL } from "node:url";
+import type { CheckInput } from "../../plugin-api.ts";
+import { lineLocator } from "./lines.ts";
+import { type Entry, listedWords, type Problem, readAllows, readList } from "./list.ts";
+import { ALLOW_FILE, LIST_FILE, LIST_PATH, type RuleId } from "./manifest.ts";
+import { finder } from "./match.ts";
+import { type Block, scriptBlocks, slideBlocks } from "./text.ts";
+
+/**
+ * A finding, which can name only a rule the manifest declares. A path is relative to the deck, or
+ * a file URL for the project's list.
+ */
+export type Finding = {
+  rule: RuleId;
+  message: string;
+  path: string;
+  /** Left out for a file that is not there. */
+  line?: number;
+  column?: number;
+  hint: string;
+};
+
+/** Reads a file by its path, relative to the deck's folder or absolute; undefined when absent. */
+export type ReadFile = (path: string) => Promise<string | undefined>;
+
+const GUIDE = "https://hajimism.github.io/dek/reference/plugins.html#dek-words";
+
+/**
+ * Checks the deck against the project's word list: each listed word in the slides and the script
+ * (WORDS001), and the list and the deck's allows reading (WORDS002). A project with no list is
+ * WORDS002, since a check that looked for nothing is no pass; a dek that does not say where the
+ * list is has nothing to check.
+ */
+export async function check(input: CheckInput, read: ReadFile): Promise<Finding[]> {
+  if (input.pluginDir === undefined) {
+    return [];
+  }
+  const listPath = join(input.pluginDir, LIST_FILE);
+  const list = await read(listPath);
+  if (list === undefined) {
+    return [
+      {
+        rule: "WORDS002",
+        message: `${LIST_PATH} is not there, and [plugins.words] checks each deck against it`,
+        path: pathToFileURL(listPath).href,
+        hint: `ask the user for the team's words and write them there as ${GUIDE} shows, or take [plugins.words] out of dek.toml`,
+      },
+    ];
+  }
+  const misread =
+    (path: string, shown: string) =>
+    (problem: Problem): Finding => ({
+      rule: "WORDS002",
+      message: problem.message,
+      path,
+      line: problem.line,
+      hint: `write ${shown} as ${GUIDE} shows`,
+    });
+  const { entries, problems } = readList(list);
+  const findings = problems.map(misread(pathToFileURL(listPath).href, LIST_PATH));
+  const allowSource = await read(ALLOW_FILE);
+  const allowed =
+    allowSource === undefined ? undefined : readAllows(allowSource, listedWords(list));
+  findings.push(...(allowed?.problems ?? []).map(misread(ALLOW_FILE, ALLOW_FILE)));
+
+  const active = entries
+    .filter((entry) => !allowed?.allows.has(entry.word))
+    .map((entry) => ({ entry, find: finder(entry) }));
+  const look = (path: string, source: string, blocks: Block[], where: Entry["in"]) => {
+    const locate = lineLocator(source);
+    for (const block of blocks) {
+      for (const { entry, find } of active) {
+        if (entry.in !== undefined && entry.in !== where) {
+          continue;
+        }
+        for (const found of find(block.text)) {
+          const as = found.text === entry.word ? "" : `, as "${found.text}"`;
+          findings.push({
+            rule: "WORDS001",
+            message: `${entry.word} is on the word list${as}`,
+            path,
+            ...locate(block.at[found.index] ?? 0),
+            hint: entry.use ?? "write the sentence without it",
+          });
+        }
+      }
+    }
+  };
+
+  const script = await read("script.md");
+  if (script !== undefined) {
+    look("script.md", script, scriptBlocks(script), "script");
+  }
+  for (const slide of input.deck.slides) {
+    const html = slide.html === null ? undefined : await read(slide.html);
+    if (slide.html !== null && html !== undefined) {
+      look(slide.html, html, slideBlocks(html), "slides");
+    }
+  }
+  // The same deck gives the same output, in the same order.
+  return findings.sort(
+    (a, b) =>
+      a.path.localeCompare(b.path) ||
+      (a.line ?? 0) - (b.line ?? 0) ||
+      (a.column ?? 0) - (b.column ?? 0) ||
+      a.rule.localeCompare(b.rule),
+  );
+}

package/src/plugins/words/lines.ts

@@ -0,0 +1,26 @@
+/** Where an offset into a source is written: 1-based line, and 1-based column in UTF-16 units. */
+export type LineColumn = { line: number; column: number };
+
+/**
+ * Turns offsets into `source` into lines and columns, from one pass over its newlines: the place
+ * dek reads a finding at. A copy of dek's own, since an official plugin shares no code (ADR 34).
+ */
+export function lineLocator(source: string): (offset: number) => LineColumn {
+  const starts = [0];
+  for (let index = source.indexOf("\n"); index !== -1; index = source.indexOf("\n", index + 1)) {
+    starts.push(index + 1);
+  }
+  return (offset) => {
+    let low = 0;
+    let high = starts.length - 1;
+    while (low < high) {
+      const mid = (low + high + 1) >> 1;
+      if ((starts[mid] ?? 0) <= offset) {
+        low = mid;
+      } else {
+        high = mid - 1;
+      }
+    }
+    return { line: low + 1, column: offset - (starts[low] ?? 0) + 1 };
+  };
+}