@immediately-run/grove 0.1.1

package/src/lib/corpusScan.test.ts

@@ -0,0 +1,157 @@
+import { describe, it, expect } from 'vitest';
+import { scanCorpus, listCorpusFiles, type ScanFs } from './corpusScan';
+import { parseFrontmatter } from './frontmatter';
+
+/** An in-memory tree, shaped like the fs slice the scan injects. */
+function fakeFs(files: Record<string, string>, opts: { unreadable?: string[] } = {}): ScanFs {
+  const unreadable = new Set(opts.unreadable ?? []);
+  return {
+    async readdir(dir) {
+      if (unreadable.has(dir)) throw new Error('EACCES');
+      const names = new Map<string, boolean>(); // name → isDirectory
+      for (const path of Object.keys(files)) {
+        if (!path.startsWith(dir)) continue;
+        const rest = path.slice(dir.length);
+        const slash = rest.indexOf('/');
+        if (slash === -1) names.set(rest, false);
+        else names.set(rest.slice(0, slash), true);
+      }
+      return [...names].map(([name, isDir]) => ({ name, isDirectory: () => isDir }));
+    },
+    async readFile(path) {
+      if (unreadable.has(path)) throw new Error('EIO');
+      const hit = files[path];
+      if (hit === undefined) throw new Error('ENOENT');
+      return hit;
+    },
+  };
+}
+
+const entry = (title: string, extra = '') => `---\ntitle: "${title}"\n${extra}---\n\nbody\n`;
+
+describe('listCorpusFiles', () => {
+  it('walks nested directories and returns absolute paths', async () => {
+    const fs = fakeFs({
+      '/mnt/c/home.mdx': entry('Home'),
+      '/mnt/c/plot/the-rail.mdx': entry('The rail'),
+      '/mnt/c/plot/deep/x.mdx': entry('Deep'),
+      '/mnt/c/notes.txt': 'not an entry',
+    });
+    expect(await listCorpusFiles('/mnt/c', fs)).toEqual([
+      '/mnt/c/home.mdx',
+      '/mnt/c/plot/deep/x.mdx',
+      '/mnt/c/plot/the-rail.mdx',
+    ]);
+  });
+
+  it('includes _layout.mdx — the layout chain is resolved FROM this map', async () => {
+    // Excluding structural files here would silently drop every layout under dispatch:
+    // `layoutChainForKey` finds layouts by scanning these very keys.
+    const fs = fakeFs({ '/mnt/c/_layout.mdx': entry('L'), '/mnt/c/home.mdx': entry('Home') });
+    expect(await listCorpusFiles('/mnt/c', fs)).toContain('/mnt/c/_layout.mdx');
+  });
+
+  it('tolerates a missing trailing slash on the root', async () => {
+    const fs = fakeFs({ '/mnt/c/home.mdx': entry('Home') });
+    expect(await listCorpusFiles('/mnt/c/', fs)).toEqual(['/mnt/c/home.mdx']);
+  });
+
+  it('skips .git and node_modules rather than walking a whole checkout', async () => {
+    const fs = fakeFs({
+      '/mnt/c/home.mdx': entry('Home'),
+      '/mnt/c/.git/config.mdx': entry('nope'),
+      '/mnt/c/node_modules/pkg/readme.md': entry('nope'),
+    });
+    expect(await listCorpusFiles('/mnt/c', fs)).toEqual(['/mnt/c/home.mdx']);
+  });
+
+  it('does not fail the whole scan on an unreadable directory', async () => {
+    const fs = fakeFs(
+      { '/mnt/c/home.mdx': entry('Home'), '/mnt/c/private/x.mdx': entry('X') },
+      { unreadable: ['/mnt/c/private/'] },
+    );
+    expect(await listCorpusFiles('/mnt/c', fs)).toEqual(['/mnt/c/home.mdx']);
+  });
+});
+
+describe('scanCorpus — the index a dispatched viewer reads', () => {
+  it('keys metadata by ABSOLUTE path, the same shape the bundler feeds', async () => {
+    // The whole design is "swap the source of the map, not the map": every consumer
+    // (`useFileMetadata`, `useMetadataQuery`, `layoutChainForKey`) is keyed this way.
+    const fs = fakeFs({
+      '/mnt/c/home.mdx': entry('Home', 'site: "Writers\' room"\ntags: [ui/nav]\n'),
+      '/mnt/c/themes.mdx': entry('Themes', 'topics: [themes, bans]\nowns:\n  concepts: [the-frame]\n'),
+    });
+    const meta = await scanCorpus('/mnt/c', fs);
+    expect(Object.keys(meta).sort()).toEqual(['/mnt/c/home.mdx', '/mnt/c/themes.mdx']);
+    expect(meta['/mnt/c/home.mdx'].title).toBe('Home');
+    expect(meta['/mnt/c/home.mdx'].site).toBe("Writers' room");
+    expect(meta['/mnt/c/themes.mdx'].topics).toEqual(['themes', 'bans']);
+    expect(meta['/mnt/c/themes.mdx'].owns).toEqual({ concepts: ['the-frame'] });
+  });
+
+  it('keeps an entry with NO frontmatter, with empty metadata', async () => {
+    // A draft or a bare `_layout.mdx` is still part of the corpus; dropping it would make
+    // the file unroutable rather than merely unlabelled.
+    const fs = fakeFs({ '/mnt/c/raw.mdx': '# just a heading\n' });
+    expect(await scanCorpus('/mnt/c', fs)).toEqual({ '/mnt/c/raw.mdx': {} });
+  });
+
+  it('loses only the unreadable entry, never the corpus', async () => {
+    const fs = fakeFs(
+      { '/mnt/c/a.mdx': entry('A'), '/mnt/c/b.mdx': entry('B'), '/mnt/c/c.mdx': entry('C') },
+      { unreadable: ['/mnt/c/b.mdx'] },
+    );
+    const meta = await scanCorpus('/mnt/c', fs);
+    expect(Object.keys(meta).sort()).toEqual(['/mnt/c/a.mdx', '/mnt/c/c.mdx']);
+  });
+
+  it('reads a corpus larger than the concurrency pool', async () => {
+    const files: Record<string, string> = {};
+    for (let i = 0; i < 40; i++) files[`/mnt/c/e${i}.mdx`] = entry(`E${i}`);
+    const meta = await scanCorpus('/mnt/c', fakeFs(files));
+    expect(Object.keys(meta)).toHaveLength(40);
+    expect(meta['/mnt/c/e39.mdx'].title).toBe('E39');
+  });
+});
+
+describe('parseFrontmatter — the grammar the authoring contract documents', () => {
+  it('reads scalars, inline lists, block lists and one level of nesting', () => {
+    const { data, body } = parseFrontmatter(
+      [
+        '---',
+        'title: "GLORIA REEVES — Denise\'s mother"',
+        'status: draft',
+        'topics: [characters, denise]',
+        'reads-first:',
+        '  - themes.mdx',
+        '  - world.mdx',
+        'owns:',
+        '  concepts: [gloria-reeves]',
+        'prs: []',
+        '---',
+        '',
+        '# Heading',
+      ].join('\n'),
+    );
+    expect(data.title).toBe("GLORIA REEVES — Denise's mother");
+    expect(data.status).toBe('draft');
+    expect(data.topics).toEqual(['characters', 'denise']);
+    expect(data['reads-first']).toEqual(['themes.mdx', 'world.mdx']);
+    expect(data.owns).toEqual({ concepts: ['gloria-reeves'] });
+    expect(data.prs).toEqual([]);
+    expect(body).toBe('# Heading');
+  });
+
+  it('treats an unterminated block as no frontmatter, keeping the body whole', () => {
+    const src = '---\ntitle: X\n\n# body';
+    // `hadFrontmatter` distinguishes this from `---\n---` (a block WITH no keys) —
+    // carried by the shared parser since R3-277a, where Grove's port had dropped it.
+    expect(parseFrontmatter(src)).toEqual({ data: {}, body: src, hadFrontmatter: false });
+  });
+
+  it('does not mistake a horizontal rule mid-document for frontmatter', () => {
+    const src = '# Title\n\n---\n\nmore';
+    expect(parseFrontmatter(src).data).toEqual({});
+  });
+});

package/src/lib/corpusScan.ts

@@ -0,0 +1,105 @@
+// The viewer-side corpus scan (R3-265) — how a DISPATCHED Grove learns what its corpus
+// contains.
+//
+// In the fork packaging the frontmatter index is bundler-fed: the sandbox scans the app's
+// own MDX at build time and hands the result to `TinkerableContext.filesMetadata`, which is
+// what `useMetadataQuery` / `useFileMetadata` / `useAllMetadata` read. Under dispatch the
+// corpus is a MOUNT, and the bundler never saw it — so nav, sidebar, search, backlinks,
+// routing and the 404 index would all be empty while the files sit right there. Nothing
+// errors; the wiki is simply blank, which is the failure mode this module exists to remove.
+//
+// The scan produces the SAME SHAPE the bundler produces — `Record<absolutePath, metadata>`,
+// keyed by the ABSOLUTE path (sandbox #41; the JSDoc that once said "repo-relative" was
+// wrong) — so every consumer keeps working with no call-site change. That is the whole
+// design: swap the source of the map, not the map.
+//
+// **The sidecar is the fast path, when there is one.** `R3-168`'s pre-computed frontmatter
+// sidecar makes this scan unnecessary for a repo that publishes one. A PRIVATE repo has no
+// Pages site and therefore no sidecar BY DESIGN (publishing one would publish the corpus),
+// so the scan is not a fallback for the pilot — it is the only path. Wiring the sidecar as a
+// fast path is additive and belongs with R3-168.
+
+import type { Frontmatter } from './frontmatter';
+import { parseFrontmatter } from './frontmatter';
+
+/** The metadata map shape the SDK hooks read (`FilesMetadata`). */
+export type CorpusMetadata = Record<string, Frontmatter>;
+
+/** The slice of `fs.promises` the scan needs — injected, so the walk is testable without a
+ *  sandbox and without mocking the module registry. */
+export interface ScanFs {
+  readdir(path: string, opts: { withFileTypes: true }): Promise<Array<{ name: string; isDirectory(): boolean }>>;
+  readFile(path: string, encoding: 'utf8'): Promise<string>;
+}
+
+/** How many files are read at once. The sandbox fs is an RPC to the host, so a sequential
+ *  walk of a 40-entry corpus is 40 round trips of latency; unbounded parallelism instead
+ *  floods the channel the rest of the app shares. A small pool is the honest middle. */
+const READ_CONCURRENCY = 8;
+
+/** Entry files. `_layout.mdx` is INCLUDED deliberately: `layoutChainForKey` resolves the
+ *  chain by looking for layout keys in this very map, so excluding structural files here
+ *  would silently drop every layout under dispatch. Reader-facing enumerations filter with
+ *  `isContentEntry`, which is where `_`-prefixed files are meant to disappear. */
+const ENTRY_RE = /\.mdx?$/;
+
+/** Directories never worth walking in a content mount. */
+const SKIP_DIRS = new Set(['.git', 'node_modules', '.immediately.run']);
+
+/** Every entry path under `root`, depth-first, absolute. */
+export async function listCorpusFiles(root: string, fs: ScanFs, maxDepth = 12): Promise<string[]> {
+  const out: string[] = [];
+  const walk = async (dir: string, depth: number): Promise<void> => {
+    if (depth > maxDepth) return;
+    let items: Array<{ name: string; isDirectory(): boolean }>;
+    try {
+      items = await fs.readdir(dir, { withFileTypes: true });
+    } catch {
+      return; // an unreadable directory is not a corpus error — it contributes nothing
+    }
+    const dirs: string[] = [];
+    for (const it of items) {
+      if (it.isDirectory()) {
+        if (!SKIP_DIRS.has(it.name)) dirs.push(`${dir}${it.name}/`);
+      } else if (ENTRY_RE.test(it.name)) {
+        out.push(`${dir}${it.name}`);
+      }
+    }
+    for (const d of dirs) await walk(d, depth + 1);
+  };
+  await walk(root.endsWith('/') ? root : `${root}/`, 0);
+  return out.sort();
+}
+
+/** Read + parse a list of entries into the metadata map, bounded-concurrently. */
+async function readAll(paths: string[], fs: ScanFs): Promise<CorpusMetadata> {
+  const meta: CorpusMetadata = {};
+  let next = 0;
+  const worker = async (): Promise<void> => {
+    for (;;) {
+      const i = next++;
+      if (i >= paths.length) return;
+      const path = paths[i];
+      try {
+        const raw = await fs.readFile(path, 'utf8');
+        meta[path] = parseFrontmatter(raw).data;
+      } catch {
+        // One unreadable entry must not empty the whole corpus. It is simply absent from
+        // the index — the same state it would be in if the author had not written it.
+      }
+    }
+  };
+  await Promise.all(Array.from({ length: Math.min(READ_CONCURRENCY, paths.length) }, worker));
+  return meta;
+}
+
+/**
+ * Build the frontmatter index for a corpus resident at `root`.
+ *
+ * Failure is per-file by design: a corpus is a foreign author's tree, and one malformed or
+ * unreadable entry may not take the wiki down with it.
+ */
+export async function scanCorpus(root: string, fs: ScanFs): Promise<CorpusMetadata> {
+  const files = await listCorpusFiles(root, fs);
+  return readAll(files, fs);
+}

package/src/lib/directory.test.ts

@@ -0,0 +1,216 @@
+import { describe, it, expect, afterEach } from 'vitest';
+import {
+  ALL_COLUMNS,
+  buildDirectoryRows,
+  columnValue,
+  dirCrumbs,
+  dirKeyToPath,
+  folderIndexKey,
+  isFolderKey,
+  parseColumns,
+  resolveDirKey,
+  rowLabel,
+  visibleColumns,
+  type DirEntry,
+  type DirRow,
+} from './directory';
+import { resetContentRoot, setContentRoot } from './contentRoot';
+import type { Frontmatter } from './frontmatter';
+
+afterEach(() => resetContentRoot());
+
+const dir = (name: string): DirEntry => ({ name, isDirectory: true });
+const file = (name: string): DirEntry => ({ name, isDirectory: false });
+
+const ROOT = '/app/content';
+
+describe('buildDirectoryRows', () => {
+  it('classifies subfolders, entries and assets — and links only the navigable ones', () => {
+    const rows = buildDirectoryRows(`${ROOT}/reports`, [
+      file('q2-2026.mdx'),
+      file('revenue.svg'),
+      dir('archive'),
+    ]);
+    expect(rows.map((r) => [r.name, r.kind, r.href])).toEqual([
+      ['archive', 'dir', '/content/reports/archive'],
+      ['q2-2026.mdx', 'entry', '/content/reports/q2-2026.mdx'],
+      // An asset is listed but NOT linked: the route space renders entries, so a link
+      // would land on the very 404 this feature exists to replace.
+      ['revenue.svg', 'file', null],
+    ]);
+  });
+
+  it('hides dot- and underscore-prefixed names by default, and shows them on request', () => {
+    const entries = [file('_layout.mdx'), file('.keep'), file('a.mdx')];
+    expect(buildDirectoryRows(ROOT, entries).map((r) => r.name)).toEqual(['a.mdx']);
+    // Asserted as a set: the relative order of `.`- and `_`-prefixed names is
+    // `localeCompare`'s business and varies by locale, which is not what this pins.
+    expect(new Set(buildDirectoryRows(ROOT, entries, {}, { hidden: true }).map((r) => r.name))).toEqual(
+      new Set(['.keep', '_layout.mdx', 'a.mdx'])
+    );
+  });
+
+  it('never lists repo machinery, even with hidden set', () => {
+    const rows = buildDirectoryRows(ROOT, [dir('node_modules'), dir('.git'), dir('teams')], {}, { hidden: true });
+    expect(rows.map((r) => r.name)).toEqual(['teams']);
+  });
+
+  it('attaches frontmatter from the index, keyed by the absolute path', () => {
+    const meta: Record<string, Frontmatter> = {
+      [`${ROOT}/handbook/onboarding.mdx`]: { title: 'Onboarding, day one.', description: 'First week.' },
+    };
+    const rows = buildDirectoryRows(`${ROOT}/handbook`, [file('onboarding.mdx'), file('expenses.mdx')], meta);
+    expect(rowLabel(rows[1])).toBe('Onboarding, day one'); // trailing period is house style
+    expect(rowLabel(rows[0])).toBe('expenses'); // no frontmatter → the bare filename
+    expect(rows[0].meta).toBeNull();
+  });
+
+  it('groups subfolders, then entries, then assets — whatever the sort key', () => {
+    // An asset interleaved with entries by name buries the content: on a real dispatched
+    // corpus `diagram.svg` sorted above every page in the folder purely on its initial.
+    const entries = [file('diagram.svg'), file('product_definition.mdx'), dir('sub')];
+    for (const sort of ['name', 'title', 'updated'] as const) {
+      expect(buildDirectoryRows(ROOT, entries, {}, { sort }).map((r) => r.name)).toEqual([
+        'sub',
+        'product_definition.mdx',
+        'diagram.svg',
+      ]);
+    }
+  });
+
+  it('sorts within a tier by the chosen key', () => {
+    const meta: Record<string, Frontmatter> = {
+      [`${ROOT}/a.mdx`]: { title: 'Zulu.', updated: '2026-01-01' },
+      [`${ROOT}/b.mdx`]: { title: 'Alpha.', updated: '2026-06-01' },
+    };
+    const entries = [file('b.mdx'), file('a.mdx'), dir('sub')];
+    expect(buildDirectoryRows(ROOT, entries, meta, { sort: 'name' }).map((r) => r.name)).toEqual([
+      'sub',
+      'a.mdx',
+      'b.mdx',
+    ]);
+    expect(buildDirectoryRows(ROOT, entries, meta, { sort: 'title' }).map((r) => r.name)).toEqual([
+      'sub',
+      'b.mdx',
+      'a.mdx',
+    ]);
+    expect(buildDirectoryRows(ROOT, entries, meta, { sort: 'updated' }).map((r) => r.name)).toEqual([
+      'sub',
+      'b.mdx',
+      'a.mdx',
+    ]);
+  });
+
+  it('builds corpus-relative hrefs under dispatch, where the mount root IS the corpus root', () => {
+    setContentRoot('/mnt/abc123/');
+    const rows = buildDirectoryRows('/mnt/abc123/context', [file('threat_model.mdx')]);
+    // Corpus-relative — the mount root IS the corpus root under dispatch, so the
+    // subfolder stays in the href and `/app` never appears.
+    expect(rows[0].href).toBe('/context/threat_model.mdx');
+  });
+});
+
+describe('columnValue / visibleColumns', () => {
+  const row = (meta: Frontmatter | null): DirRow => ({
+    name: 'x.mdx',
+    key: `${ROOT}/x.mdx`,
+    kind: 'entry',
+    href: '/content/x.mdx',
+    meta,
+  });
+
+  it('reads `updated` under either spelling — corpora in the wild use both', () => {
+    expect(columnValue(row({ updated: '2026-08-18' }), 'updated')).toBe('2026-08-18');
+    expect(columnValue(row({ date: '2026-06-22' }), 'updated')).toBe('2026-06-22');
+    expect(columnValue(row({}), 'updated')).toBeNull();
+  });
+
+  it('drops `ui/…` wiring tags, exactly as <DocList> does', () => {
+    expect(columnValue(row({ tags: ['handbook', 'ui/nav'] }), 'tags')).toEqual(['handbook']);
+    expect(columnValue(row({ tags: ['ui/nav'] }), 'tags')).toBeNull();
+  });
+
+  it('refuses to print a non-scalar into a cell', () => {
+    // `owns:` is a map; String()-ing it renders "[object Object]", which looks like data.
+    expect(columnValue(row({ description: { concepts: ['a'] } }), 'description')).toBeNull();
+  });
+
+  it('shows only the metadata columns the corpus actually supplies', () => {
+    const bare = [row({}), row(null)];
+    expect(visibleColumns(bare)).toEqual(['name']);
+    const rich = [row({ description: 'd' }), row({ tags: ['t'], status: 'draft' })];
+    expect(visibleColumns(rich)).toEqual(['name', 'description', 'tags', 'status']);
+  });
+
+  it('narrows to a requested subset, but still never pads', () => {
+    const rows = [row({ description: 'd', tags: ['t'] })];
+    expect(visibleColumns(rows, parseColumns('tags'))).toEqual(['name', 'tags']);
+    expect(visibleColumns(rows, parseColumns('status'))).toEqual(['name']);
+  });
+
+  it('ignores unknown column names rather than throwing on author input', () => {
+    expect(parseColumns('description,nonsense')).toEqual(['description']);
+    expect(parseColumns('nonsense')).toEqual(ALL_COLUMNS);
+    expect(parseColumns(undefined)).toEqual(ALL_COLUMNS);
+  });
+});
+
+describe('resolveDirKey', () => {
+  it('treats a leading slash as corpus-relative and everything else as entry-relative', () => {
+    expect(resolveDirKey('/handbook', `${ROOT}/reports`)).toBe(`${ROOT}/handbook`);
+    expect(resolveDirKey('archive', `${ROOT}/reports`)).toBe(`${ROOT}/reports/archive`);
+    expect(resolveDirKey(undefined, `${ROOT}/reports`)).toBe(`${ROOT}/reports`);
+  });
+
+  it('confines traversal to the corpus — a `path` is untrusted input', () => {
+    // Under dispatch the corpus is foreign content and the address bar is user-typed,
+    // so `..` must not become a readdir of somebody else's mount.
+    expect(resolveDirKey('../../..', `${ROOT}/reports`)).toBe(ROOT);
+    expect(resolveDirKey('/../../app/src', ROOT)).toBe(ROOT);
+    expect(resolveDirKey('a/../b', ROOT)).toBe(`${ROOT}/b`);
+  });
+
+  it('round-trips through dirKeyToPath', () => {
+    expect(dirKeyToPath(`${ROOT}/handbook`)).toBe('/handbook');
+    expect(dirKeyToPath(ROOT)).toBe('/');
+    expect(resolveDirKey(dirKeyToPath(`${ROOT}/a/b`), ROOT)).toBe(`${ROOT}/a/b`);
+  });
+});
+
+describe('dirCrumbs', () => {
+  it('names every ancestor between the corpus root and the folder', () => {
+    expect(dirCrumbs(`${ROOT}/plans/ui-as-apps`)).toEqual([
+      { label: 'plans', key: `${ROOT}/plans` },
+      { label: 'ui-as-apps', key: `${ROOT}/plans/ui-as-apps` },
+    ]);
+    expect(dirCrumbs(ROOT)).toEqual([]);
+  });
+});
+
+describe('folderIndexKey', () => {
+  it("prefers the author's curated folder index over a generated listing", () => {
+    const keys = [`${ROOT}/roadmap/index.mdx`, `${ROOT}/roadmap/R3-1.mdx`];
+    expect(folderIndexKey(`${ROOT}/roadmap`, keys)).toBe(`${ROOT}/roadmap/index.mdx`);
+    expect(folderIndexKey(`${ROOT}/roadmap/`, keys)).toBe(`${ROOT}/roadmap/index.mdx`);
+  });
+
+  it('returns null when there is none, so the listing is the answer', () => {
+    expect(folderIndexKey(`${ROOT}/people`, [`${ROOT}/people/ada.mdx`])).toBeNull();
+  });
+});
+
+describe('isFolderKey', () => {
+  const keys = [`${ROOT}/handbook/onboarding.mdx`, `${ROOT}/about.mdx`];
+
+  it('recognises a folder from the entry index alone — no fs call per link', () => {
+    expect(isFolderKey(`${ROOT}/handbook`, keys)).toBe(true);
+    expect(isFolderKey(`${ROOT}/handbook/`, keys)).toBe(true);
+  });
+
+  it('is false for an entry, for a non-existent path, and for a prefix that is not a segment', () => {
+    expect(isFolderKey(`${ROOT}/about.mdx`, keys)).toBe(false);
+    expect(isFolderKey(`${ROOT}/nope`, keys)).toBe(false);
+    // `handbo` is a string prefix of `handbook/…` but not a path segment of it.
+    expect(isFolderKey(`${ROOT}/handbo`, keys)).toBe(false);
+  });
+});