@anchrd/intel 0.59.0

package/ui/src/document-markdown/document-markdown.ts

@@ -0,0 +1,195 @@
+import { BlockNoteDocument } from "@contract/node.ts";
+import type { IntelEditor, IntelEditorPartialBlock } from "@/editor-schema/editor-schema.ts";
+import { FrontmatterBlockType, splitFrontmatter } from "@/frontmatter/frontmatter.tsx";
+import { VerbatimBlockType } from "@/verbatim/verbatim.tsx";
+
+/**
+ * The bridge between a document's markdown and the editor's blocks, in both directions.
+ *
+ * ⚠️ **The markdown is the truth; the editor is a nicer view onto it** (Jack, 2026-08-30). That
+ * sentence was not true when it was written. The editor writes with `blocksToMarkdownLossy` — the
+ * name is BlockNote's, not ours — and what it does not understand it rewrites into what it does.
+ * Exactly one construct had a way out: the head, given one in #657 after it had already cost data.
+ * The comment there names the damage, and it is worth repeating because it is the reason this
+ * module exists: *"Opening a document and saving it left its metadata unfindable, and nothing
+ * anywhere went red."*
+ *
+ * ⚠️ `payload.markdown` is not decoration. `indexing.ts` pulls that field and nothing else into
+ * full text and vectors, so it is the whole of what a search can match, and `bundle.ts` writes it
+ * out as the document's `.md`. A document can be right on screen and unfindable.
+ *
+ * **The way out, generalised from the head:** a piece the editor cannot give back is parked as raw
+ * text and written back byte for byte (#826). Two rules follow from that, and both are load-bearing:
+ *
+ * - **What the editor OWNS, it may re-spell.** `- a` comes back as `* a`, `_kursiv_` as `*kursiv*`,
+ *   a table padded to its widest cell. That is one markdown construct written another way, and the
+ *   document says the same thing afterwards.
+ * - **What it does not own, it does not touch.** An HTML block, a comment, a math block, a
+ *   directive: measured, those come back as their inner text, as an empty string, or mangled with
+ *   backslashes. They are parked.
+ *
+ * ⚠️ **The separation between blocks is normalised, their CONTENT is not.** Blocks are joined with
+ * one blank line and the document ends with one newline, which is what the serialiser has always
+ * produced. A document already in that form comes back byte for byte; one with three blank lines
+ * between two paragraphs loses two of them on the first save, and nothing else.
+ */
+
+// Everything markdown uses to SPELL something, plus whitespace. What is left over is what the
+// author wrote — and a re-spelling may not change that by a single character.
+//
+// ⚠️ This is the whole classifier, and it is deliberately not a byte comparison: measured against
+// BlockNote 0.52, a byte comparison parks every `-` bulleted list, every `_kursiv_`, every table
+// and every setext heading, because the serialiser answers each of them in its own spelling. A
+// document written by an agent is full of those, and parking them would make most of it read-only
+// for a difference nobody can see.
+const SPELLING = /[\s*_\-#>|=+~`.:()[\]!]/g;
+
+function spoken(markdown: string): string {
+  return markdown.replace(SPELLING, "");
+}
+
+/**
+ * The document cut into top-level pieces, at blank lines.
+ *
+ * ⚠️ A fence is not cut into. Everything else may be: an HTML block with a blank line in it becomes
+ * two pieces, both parked, and the join puts the blank line back — the bytes come out the same
+ * either way.
+ *
+ * ⚠️ **A fence is closed by its own marker AND at least its own length**, never by the marker
+ * alone. Four backticks around a block that shows three is how one writes about markdown in
+ * markdown; a guard that only compares the character closes the outer fence at the inner one, cuts
+ * the rest at the next blank line, and the join then writes one blank line where the code had two.
+ * Found in review on this pull request, and its case is in the test file below.
+ */
+function segmentsOf(body: string): string[] {
+  const segments: string[] = [];
+  let current: string[] = [];
+  let fence: { marker: string; length: number } | null = null;
+  for (const line of body.split("\n")) {
+    const run = /^\s{0,3}(`{3,}|~{3,})/.exec(line)?.[1];
+    if (fence === null) {
+      if (run) fence = { marker: run[0] ?? "", length: run.length };
+    } else if (run && run[0] === fence.marker && run.length >= fence.length) {
+      fence = null;
+    }
+    if (fence === null && line.trim() === "") {
+      if (current.length > 0) segments.push(current.join("\n"));
+      current = [];
+      continue;
+    }
+    current.push(line);
+  }
+  if (current.length > 0) segments.push(current.join("\n"));
+  return segments;
+}
+
+/**
+ * One piece, either as blocks the editor owns or as one parked block.
+ *
+ * ⚠️ A piece that throws while being parsed is parked rather than lost. A parser that gives up is
+ * the clearest case there is of something the editor does not understand.
+ */
+function blocksFor(editor: IntelEditor, segment: string): IntelEditorPartialBlock[] {
+  const parked = [{ type: VerbatimBlockType, props: { raw: segment } } as IntelEditorPartialBlock];
+  try {
+    const parsed = editor.tryParseMarkdownToBlocks(segment) as IntelEditorPartialBlock[];
+    const back = editor.blocksToMarkdownLossy(parsed as never) as unknown as string;
+    return spoken(back) === spoken(segment) ? parsed : parked;
+  } catch {
+    return parked;
+  }
+}
+
+/**
+ * Markdown to blocks, with the head lifted out of remark's way and everything the editor cannot
+ * give back parked as it stands.
+ *
+ * ⚠️ Synchronous, because both halves of the probe are: `tryParseMarkdownToBlocks` and
+ * `blocksToMarkdownLossy` return their answer directly in BlockNote 0.52. Making this `async` would
+ * put the load path behind a tick, and the effect in `node-editor.tsx` moves `savedSnapshot`
+ * immediately after it — a gap there is a document that reports itself unsaved the moment it opens
+ * (#115).
+ */
+export function markdownToBlocks(editor: IntelEditor, markdown: string): IntelEditorPartialBlock[] {
+  const { raw, body } = splitFrontmatter(markdown);
+  const blocks = segmentsOf(body).flatMap((segment) => blocksFor(editor, segment));
+  if (raw === null) return blocks;
+  return [{ type: FrontmatterBlockType, props: { raw } } as IntelEditorPartialBlock, ...blocks];
+}
+
+function rawOf(block: { props?: { raw?: unknown } } | undefined): string {
+  return typeof block?.props?.raw === "string" ? block.props.raw : "";
+}
+
+/**
+ * Blocks to markdown: the head as a head, the parked pieces as they went in, the rest to the
+ * editor's own serialiser.
+ *
+ * ⚠️ The head and the parked pieces are found by TYPE, never by position. A block is a block: the
+ * drag handle moves them, and typing above the first one pushes it down. A positional guard would
+ * fall through to `blocksToMarkdownLossy` for exactly those documents, and that path serialises the
+ * block's rendered DOM — `payload.markdown` would carry the word "Metadata" where the metadata used
+ * to be (#657).
+ *
+ * ⚠️ The runs between two parked pieces are serialised TOGETHER, not one block at a time. A list is
+ * several blocks and one construct; serialising its items separately would write a list per item.
+ */
+export function blocksToMarkdown(editor: IntelEditor, blocks: unknown[]): string {
+  const all = blocks as { type?: string; props?: { raw?: unknown } }[];
+  const at = all.findIndex((block) => block?.type === FrontmatterBlockType);
+  // A document has ONE head, and it belongs at the top wherever the block ended up. Any further
+  // head block stays in the body and is serialised by the spec's own `toExternalHTML` — as a code
+  // block, which is visible and lossless without claiming to be a second head.
+  const body = all.filter((_, index) => index !== at);
+  const chunks: string[] = [];
+  let run: unknown[] = [];
+  const flush = () => {
+    if (run.length === 0) return;
+    chunks.push(editor.blocksToMarkdownLossy(run as never) as unknown as string);
+    run = [];
+  };
+  for (const block of body) {
+    if (block?.type === VerbatimBlockType) {
+      flush();
+      chunks.push(rawOf(block));
+    } else {
+      run.push(block);
+    }
+  }
+  flush();
+  // One blank line between blocks and one newline at the end: the shape the serialiser has always
+  // produced, applied to the joins as well so that a parked piece and a written one are separated
+  // the same way.
+  const text = chunks
+    .map((chunk) => chunk.replace(/\n+$/, ""))
+    .filter((chunk) => chunk.length > 0)
+    .join("\n\n");
+  const document = text.length === 0 ? "" : `${text}\n`;
+  if (at < 0) return document;
+  // ⚠️ The blank line after the closing fence is part of the contract, not tidiness: without it the
+  // first line of the body sits against the fence, and a `# Title` there would be read back as part
+  // of nothing in particular the next time round.
+  return `---\n${rawOf(all[at])}\n---\n\n${document}`;
+}
+
+/**
+ * The markdown OF a stored document, whichever way it was stored.
+ *
+ * ⚠️ Two shapes reach this. What the editor writes is a BlockNote payload, and `markdown` beside its
+ * blocks is the whole of what anybody outside the editor reads — the same field `indexing.ts` pulls
+ * on the server (`document-text.ts` answers the same question there, for the same reason). What an
+ * agent writes over MCP is `text/markdown`, and that content IS its text.
+ *
+ * ⚠️ A payload that will not parse falls back to the raw content rather than to an empty box. An
+ * empty box invites somebody to type into it and save, and that save would replace a document
+ * nobody could read with one sentence.
+ */
+export function documentMarkdown(content: string | null): string {
+  if (!content) return "";
+  try {
+    const parsed = BlockNoteDocument.safeParse(JSON.parse(content));
+    return parsed.success ? parsed.data.markdown : content;
+  } catch {
+    return content;
+  }
+}

package/ui/src/editor-schema/editor-schema.ts

@@ -0,0 +1,23 @@
+import { BlockNoteSchema } from "@blocknote/core";
+import { documentLinkSpec } from "@/document-link/document-link.tsx";
+import { frontmatterSpec } from "@/frontmatter/frontmatter.tsx";
+import { verbatimSpec } from "@/verbatim/verbatim.tsx";
+
+// One schema for the editor and for everything that reads its documents back: BlockNote's defaults
+// plus what Intel adds — the inline element of #41, the head block of #657 and, since #826, the
+// block that carries whatever the editor cannot give back unchanged.
+//
+// ⚠️ `extend` rather than `create({ inlineContentSpecs: { ...defaults, documentLink } })`. Both
+// build the same schema at run time, but only `extend` keeps the added types in the schema's own
+// generics under this repo's `exactOptionalPropertyTypes` — with the spread form the editor infers
+// BlockNote's default schema, typechecks, and then refuses to insert a document link.
+export const intelEditorSchema = BlockNoteSchema.create().extend({
+  inlineContentSpecs: { documentLink: documentLinkSpec },
+  blockSpecs: { frontmatter: frontmatterSpec(), verbatim: verbatimSpec() },
+});
+
+// The schema carries its own editor and block types as declaration-only properties, which is the
+// only way to name them here: writing `BlockNoteEditor<typeof schema.blockSchema, ...>` by hand
+// fails BlockNote's own constraint under `exactOptionalPropertyTypes`.
+export type IntelEditor = (typeof intelEditorSchema)["BlockNoteEditor"];
+export type IntelEditorPartialBlock = (typeof intelEditorSchema)["PartialBlock"];

package/ui/src/entry-picker/entry-picker.tsx

@@ -0,0 +1,238 @@
+import type { NodeKind } from "@contract/node.ts";
+import { useQuery } from "@tanstack/react-query";
+import { ChevronRight, Folder, Search } from "lucide-react";
+import { useMemo, useState } from "react";
+import { useI18n } from "@/i18n/i18n-context.tsx";
+import { useIntelRouterContext } from "@/router/router-context.ts";
+
+// What this picker offers, whichever side of the shared tree it came from. Deliberately the four
+// fields a picker needs and not the whole record: a Node and a Flow are two different things
+// (ADR-0004), and merging them into one row is only allowed where the row is the entire question.
+export type PickerEntry = { id: string; title: string; parentId: string | null; kind: PickerKind };
+export type PickerKind = NodeKind | "flow";
+
+/**
+ * One picker for everything a node or a flow can name: search by name, or click down through the
+ * tree from the top-level folders.
+ *
+ * ⚠️ Both halves read ONE source — the authorized flat graph the editor already loads. Not because
+ * a search endpoint would be hard, but because `searchNodes` answers a different question: it
+ * searches PASSAGES and hands back citations. Someone looking for a document by name would get
+ * results ranked by what is written inside it, which is the wrong list in a picker. Filtering names
+ * here is also instant, and it cannot disagree with the tree about what exists.
+ *
+ * ⚠️ What the caller may not see is absent from the graph, so it is absent here — and an empty
+ * result never says which of the two it is (#17, #19).
+ */
+export function EntryPicker({
+  kind,
+  flows = false,
+  value,
+  onSelect,
+  label,
+}: {
+  // Which kind of node may be chosen. `null` offers every kind — what the Flow step uses.
+  //
+  // ⚠️ A list is not a convenience (#255). A caller that accepts a folder, a document OR a table
+  // has one question to ask, and the alternatives — one picker per kind, or a picker that offers
+  // everything and refuses afterwards — are either three dialogs or a rejection after the fact.
+  // What may be picked has to be the same question the caller can answer in one place.
+  kind: NodeKind | NodeKind[] | null;
+  // ⚠️ Flows are a second list, not a kind of node, so they are asked for rather than assumed. A
+  // caller that may aim at either says so; every other caller wants nodes only and pays for no
+  // second request.
+  flows?: boolean;
+  value: string;
+  onSelect(entryId: string, entryKind: PickerKind): void;
+  label: string;
+}) {
+  const { data } = useIntelRouterContext();
+  const i18n = useI18n();
+  const [query, setQuery] = useState("");
+  const [openFolder, setOpenFolder] = useState<string | null>(null);
+  const graph = useQuery({
+    queryKey: ["node-graph"],
+    queryFn: () => data.getNodeGraph(),
+  });
+  const flowList = useQuery({
+    queryKey: ["flows"],
+    queryFn: () => data.listFlows(),
+    enabled: flows,
+  });
+
+  const nodes = useMemo<PickerEntry[]>(() => {
+    const fromNodes = (graph.data?.nodes ?? []).map((node) => ({
+      id: node.id,
+      title: node.title,
+      parentId: node.parentId,
+      kind: node.kind as PickerKind,
+    }));
+    if (!flows) return fromNodes;
+    return [
+      ...fromNodes,
+      ...(flowList.data?.items ?? []).map((flow) => ({
+        id: flow.id,
+        title: flow.title,
+        parentId: flow.parentId,
+        kind: "flow" as const,
+      })),
+    ];
+  }, [graph.data, flowList.data, flows]);
+  // A flow is offered exactly when it was asked for — it is only in `nodes` at all in that case,
+  // and saying so here keeps "may this be picked" one rule rather than two that must agree.
+  //
+  // ⚠️ The array is rebuilt from `kind` on every render when a caller passes a literal, so the set
+  // is keyed by its contents rather than by identity — a `useMemo` on the array itself would be a
+  // memo that never hits.
+  const wanted = Array.isArray(kind) ? kind.join(",") : kind;
+  const pickable = useMemo(() => {
+    const allowed = wanted === null ? null : new Set(wanted.split(","));
+    return (entry: PickerEntry) =>
+      entry.kind === "flow" || allowed === null || allowed.has(entry.kind);
+  }, [wanted]);
+  const eligible = useMemo(() => nodes.filter(pickable), [nodes, pickable]);
+
+  /**
+   * Whether an entry stands on the picker's start level.
+   *
+   * ⚠️ The root is not `parentId === null` but the upper edge of what the reader may see — the same
+   * rule #429 gave the server in `levelPredicate`, at the second surface (#447). A recipient whose
+   * only access is a NESTED share carries a `parentId` pointing at a folder that is not in the list
+   * at all: under the old rule their start level was empty and STAYED empty, because the folder
+   * that would open it can never be clicked. Only searching reached them.
+   *
+   * ⚠️ Nothing is added by this. The list is what `getNodeGraph()` — and `listFlows()` where flows
+   * were asked for — answered, and the answer is already the authorized one. What changes is only
+   * which level an entry stands on.
+   */
+  const rooted = useMemo(() => {
+    const present = new Set(nodes.map((node) => node.id));
+    return (entry: PickerEntry) => entry.parentId === null || !present.has(entry.parentId);
+  }, [nodes]);
+
+  // Searching looks at every eligible node wherever it sits; browsing looks at one level. The two
+  // are the same list seen two ways, which is why a result can be picked from either.
+  const searching = query.trim().length > 0;
+  const shown = useMemo(() => {
+    if (searching) {
+      const needle = query.trim().toLowerCase();
+      return eligible.filter((node) => node.title.toLowerCase().includes(needle)).slice(0, 50);
+    }
+    return nodes.filter((node) =>
+      openFolder === null ? rooted(node) : node.parentId === openFolder,
+    );
+  }, [searching, query, eligible, nodes, openFolder, rooted]);
+
+  // The way back up, as the chain of folders that leads to the open one.
+  const trail = useMemo(() => {
+    const chain: PickerEntry[] = [];
+    let current = openFolder;
+    while (current !== null) {
+      const folder = nodes.find((node) => node.id === current);
+      if (!folder) break;
+      chain.unshift(folder);
+      // ⚠️ The chain ends where the start level begins, and by the same rule that decides it. A
+      // folder at the upper edge has an ancestor the reader cannot see; walking into it would look
+      // for a level that does not exist here, and the "top" button above leads to this one.
+      current = rooted(folder) ? null : folder.parentId;
+    }
+    return chain;
+  }, [openFolder, nodes, rooted]);
+
+  const chosen = nodes.find((node) => node.id === value) ?? null;
+
+  return (
+    <div className="mt-4">
+      <span className="block text-sm font-medium">{label}</span>
+      {chosen ? (
+        <p className="mt-1 truncate text-sm text-muted-foreground">{chosen.title}</p>
+      ) : (
+        <p className="mt-1 text-sm text-muted-foreground">{i18n.t("flows.linkEmpty")}</p>
+      )}
+
+      <div className="mt-2 rounded-lg border">
+        <label className="flex items-center gap-2 border-b px-2 py-1.5">
+          <Search aria-hidden="true" className="size-4 shrink-0 text-muted-foreground" />
+          <span className="sr-only">{i18n.t("picker.search")}</span>
+          <input
+            type="search"
+            value={query}
+            onChange={(event) => setQuery(event.target.value)}
+            placeholder={i18n.t("picker.search")}
+            className="w-full bg-transparent text-sm outline-none"
+          />
+        </label>
+
+        {/* The trail is hidden while searching: a result can come from anywhere, so "where you are"
+            would be a claim about a place the list is no longer showing. */}
+        {!searching && trail.length > 0 ? (
+          <div className="flex flex-wrap items-center gap-1 border-b px-2 py-1 text-xs">
+            <button
+              type="button"
+              onClick={() => setOpenFolder(null)}
+              className="rounded px-1 outline-none hover:underline focus-visible:ring-2 focus-visible:ring-ring"
+            >
+              {i18n.t("picker.top")}
+            </button>
+            {trail.map((folder) => (
+              <span key={folder.id} className="flex items-center gap-1">
+                <ChevronRight aria-hidden="true" className="size-3 text-muted-foreground" />
+                <button
+                  type="button"
+                  onClick={() => setOpenFolder(folder.id)}
+                  className="rounded px-1 outline-none hover:underline focus-visible:ring-2 focus-visible:ring-ring"
+                >
+                  {folder.title}
+                </button>
+              </span>
+            ))}
+          </div>
+        ) : null}
+
+        <ul className="max-h-56 overflow-y-auto py-1">
+          {shown.map((node) => {
+            // A folder is two things while browsing: somewhere to go, and — when folders are what
+            // is being picked — something to choose. It gets both, side by side, rather than one
+            // control that has to guess which was meant.
+            const canOpen = node.kind === "folder" && !searching;
+            const canPick = pickable(node);
+            return (
+              <li key={`${node.kind}:${node.id}`} className="flex items-center gap-1 px-1">
+                {canPick ? (
+                  <button
+                    type="button"
+                    onClick={() => onSelect(node.id, node.kind)}
+                    aria-current={node.id === value}
+                    className={`min-w-0 flex-1 truncate rounded px-2 py-1.5 text-left text-sm outline-none hover:bg-muted focus-visible:ring-2 focus-visible:ring-ring ${
+                      node.id === value ? "bg-muted font-medium" : ""
+                    }`}
+                  >
+                    {node.title}
+                    <span className="sr-only"> — {i18n.t(`node.kind.${node.kind}`)}</span>
+                  </button>
+                ) : (
+                  <span className="min-w-0 flex-1 truncate px-2 py-1.5 text-sm text-muted-foreground">
+                    {node.title}
+                  </span>
+                )}
+                {canOpen ? (
+                  <button
+                    type="button"
+                    onClick={() => setOpenFolder(node.id)}
+                    aria-label={i18n.t("picker.open", { title: node.title })}
+                    className="inline-flex size-7 shrink-0 items-center justify-center rounded outline-none hover:bg-muted focus-visible:ring-2 focus-visible:ring-ring"
+                  >
+                    <Folder aria-hidden="true" className="size-4" />
+                  </button>
+                ) : null}
+              </li>
+            );
+          })}
+          {shown.length === 0 ? (
+            <li className="px-3 py-2 text-sm text-muted-foreground">{i18n.t("picker.nothing")}</li>
+          ) : null}
+        </ul>
+      </div>
+    </div>
+  );
+}