@bevel-software/platform-core-backend 0.25.2 → 0.27.0

package/src/modules/workspace/first-run.ts

@@ -0,0 +1,222 @@
+/**
+ * The "first run" note `start_session` adds on a knowledge base nobody has
+ * written in yet.
+ *
+ * Someone who has just set up Hexis and connected their agent has an empty
+ * knowledge base and an agent that could fill it. The agent cannot know the
+ * first from a static instruction — the same text reaches a knowledge base of
+ * ten thousand pages — so the signal is computed when the session starts:
+ * while the knowledge folder of the default branch holds nothing but the
+ * starter guide the template seeds, `start_session` answers with a short note
+ * telling the agent to offer to draft the first pages, and the guide's
+ * {@link FIRST_RUN_SECTION_ID} section says how. Once a page of anyone's
+ * exists, the note stops on its own: there is no flag to clear.
+ *
+ * The question is asked AS THE CALLER, gated like a read. The note goes only
+ * to someone who may read the knowledge folder, and it judges the folder as
+ * they may see it: a page they may not read is passed over as if it were not
+ * there (`mayRead` in {@link knowledgeFolderIsNew}), so a note that does or
+ * does not come tells them nothing about pages beyond them. The answer names
+ * no file but the starter guide, which every knowledge base is seeded with,
+ * and a chosen starter pack's pages — and when one of those is beyond the
+ * caller, no note at all, since a list with a gap would say where the gap is.
+ *
+ * A STARTER PACK does not end it. The pages a pack adds are tasks with a
+ * heading and a line or two — "ask your agent to draft this" — so while one
+ * still holds exactly what the pack wrote it is not a page anyone wrote, and
+ * the note goes on, naming the pages the pack suggests drafting first. The
+ * first edit to any of them makes the knowledge base established.
+ */
+
+import nodeFs from 'node:fs/promises';
+import { join } from 'node:path';
+import type { ITreeWalker } from '../../shared/fs.contract.js';
+import { NEW_KNOWLEDGE_BASE_SECTION_ID } from '../agent-guide/index.js';
+import { BevelIgnoreStack } from '../kb-fs/bevel-ignore.js';
+
+/**
+ * The starter guide the template seeds at the top of the knowledge folder
+ * (`kb-template/KnowledgeBase/How to get started.md`). The one page that does
+ * not make a knowledge base established. A test holds this to the template, so
+ * renaming one without the other fails there.
+ */
+export const STARTER_GUIDE_FILE = 'How to get started.md';
+
+/**
+ * The guide section the note points at (`agent-guide/new-knowledge-base.md`):
+ * the guide's own id, so the note cannot name a section the guide renamed.
+ */
+export const FIRST_RUN_SECTION_ID = NEW_KNOWLEDGE_BASE_SECTION_ID;
+
+/**
+ * The starter pack a knowledge base was filled from, as far as the first-run
+ * check cares: what the team chose, the pages it suggests drafting, and the
+ * pages it wrote — by path below the knowledge folder, with the text each was
+ * written with.
+ */
+export interface FirstRunStarter {
+  name: string;
+  suggestedPages: readonly string[];
+  pages: ReadonlyMap<string, string>;
+}
+
+/** Where `start_session` asks which starter pack was chosen (see `modules/onboarding`). */
+export interface FirstRunStarterSource {
+  firstRunStarter(): Promise<FirstRunStarter | null>;
+}
+
+/**
+ * Files that are configuration, not content, wherever they sit: a folder's
+ * access rules. Dot-entries (the folder placeholder `.gitkeep`, `.bevelignore`,
+ * an OS's `.DS_Store`) the knowledge-base walk never shows at all.
+ */
+const NOT_CONTENT = new Set(['access.md']);
+
+/**
+ * How many folders the check lists, the knowledge folder included, before it
+ * calls the knowledge base established. A knowledge base with this many
+ * folders and no page among them is not one an agent should greet as new, and
+ * `start_session` must stay cheap whatever the tree looks like.
+ */
+export const FOLDER_BUDGET = 200;
+
+/**
+ * Which of some pages (paths below the knowledge folder) the caller may
+ * read — `canReadBatch` for them, with the folder's prefix put on and taken
+ * off again.
+ */
+export type MayRead = (relPaths: string[]) => Promise<ReadonlyMap<string, boolean>>;
+
+/**
+ * Whether the knowledge folder `knowledgeDir` of the checkout at `repoRoot`
+ * holds nothing but the starter guide: no file in it or below it other than
+ * that guide at its top, folder placeholders, dot-files and access rules. A
+ * folder that is not there answers false: that is a knowledge base in a shape
+ * the template never made, and greeting it as new would be a guess.
+ *
+ * `starterPages` are a starter pack's pages (path below the folder → the text
+ * the pack wrote): one still holding exactly that text is not a page either.
+ *
+ * THE knowledge-base walk does the walking ({@link ITreeWalker.walkKb}, see
+ * `shared/fs.contract.ts`), the way the explorer's does: dot-entries never
+ * shown, a link or a socket never entered nor counted as a file, and
+ * `.bevelignore` honoured — the repository root's rules and every folder's on
+ * the way down — so what the explorer hides this check does not see either,
+ * and a knowledge folder those rules hide is not greeted. A folder that
+ * cannot be listed is the answer: not new, since it cannot say.
+ *
+ * With `mayRead`, the answer is the CALLER'S: a file they may not read is not
+ * theirs to know of, so it does not make the folder old for them. Every
+ * folder is entered, as the agent's own listing shows every folder and
+ * filters its files — a deeper rule may open a page below a folder its
+ * parent's rules close — and a folder's pages are judged in chunks of
+ * {@link JUDGE_CHUNK}; the first the caller may read ends the walk: not new.
+ * Without `mayRead` the first page ends it. Either way {@link FOLDER_BUDGET}
+ * folders end it too: that many and no page is not a new knowledge base.
+ */
+export async function knowledgeFolderIsNew(
+  disk: ITreeWalker,
+  repoRoot: string,
+  knowledgeDir: string,
+  starterPages?: ReadonlyMap<string, string>,
+  mayRead?: MayRead,
+): Promise<boolean> {
+  let sawRoot = false;
+  let established = false;
+  let listed = 0;
+  /** Whether any of `pages` (paths below the knowledge folder) is one the caller may read. */
+  const anyReadable = async (pages: string[]): Promise<boolean> => {
+    const verdicts = await mayRead!(pages);
+    return pages.some((p) => verdicts.get(p) === true);
+  };
+  try {
+    // The rules in force above the knowledge folder — the repository root's
+    // file — which a walk starting below it would not see on its own.
+    const above = await BevelIgnoreStack.empty().extendedWith(repoRoot);
+    const dir = join(repoRoot, knowledgeDir);
+    if (above.isIgnored(dir, true)) return false;
+    await disk.walkKb(
+      dir,
+      [
+        {
+          async onDir(rel, entries, folder) {
+            if (rel === '') sawRoot = true;
+            if (++listed > FOLDER_BUDGET) {
+              established = true;
+              return;
+            }
+            let pages: string[] = [];
+            for (const entry of entries) {
+              if (!entry.isFile() || NOT_CONTENT.has(entry.name)) continue;
+              if (rel === '' && entry.name === STARTER_GUIDE_FILE) continue;
+              const childRel = rel ? `${rel}/${entry.name}` : entry.name;
+              if (await isUntouchedStarterPage(join(folder.abs, entry.name), starterPages?.get(childRel))) continue;
+              // A page, with nobody to ask: something someone put there.
+              if (!mayRead) {
+                established = true;
+                return;
+              }
+              pages.push(childRel);
+              if (pages.length >= JUDGE_CHUNK) {
+                if (await anyReadable(pages)) {
+                  established = true;
+                  return;
+                }
+                pages = [];
+              }
+            }
+            if (pages.length > 0 && (await anyReadable(pages))) established = true;
+          },
+        },
+      ],
+      { ignore: above, until: () => established, unreadable: 'throw' },
+    );
+  } catch {
+    return false;
+  }
+  return sawRoot && !established;
+}
+
+/** How many of a folder's pages are judged in one ask of `mayRead`. */
+const JUDGE_CHUNK = 200;
+
+/**
+ * Whether the file at `file` still holds exactly `written`, the text a starter
+ * pack put there (line endings aside: a CRLF checkout of the same text is the
+ * same page). No `written` means no pack wrote it.
+ */
+export async function isUntouchedStarterPage(file: string, written: string | undefined): Promise<boolean> {
+  if (written === undefined) return false;
+  const norm = (text: string) => text.replace(/\r\n?/g, '\n');
+  try {
+    return norm(await nodeFs.readFile(file, 'utf8')) === norm(written);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * The note itself, as `start_session` returns it under `firstRun`. Complete on
+ * its own, for the agent that reads no further: what the state is, what to
+ * offer, and that the person's own request comes first.
+ */
+export function firstRunNote(
+  knowledgeFolder: string,
+  starter?: Pick<FirstRunStarter, 'name' | 'suggestedPages'>,
+): string {
+  if (starter) {
+    const pages = starter.suggestedPages.length > 0 ? ` (${starter.suggestedPages.join(', ')})` : '';
+    return (
+      `This knowledge base is new: \`${knowledgeFolder}/\` holds the starter guide and the ${starter.name} starter pages, which are short placeholders to fill in. ` +
+      `Once in this conversation, offer to draft its first pages${pages} from the person's website, repository or a few sentences of theirs, for them to review. ` +
+      'If they asked for something else, answer that first and make the offer in one line. ' +
+      `The guide's \`${FIRST_RUN_SECTION_ID}\` section says how.`
+    );
+  }
+  return (
+    `This knowledge base is new: \`${knowledgeFolder}/\` has no pages yet, the starter guide aside. ` +
+    "Once in this conversation, offer to draft its first pages (what the organisation does, its customers, its products, a glossary, how it works) from the person's website or a few sentences of theirs, for them to review. " +
+    'If they asked for something else, answer that first and make the offer in one line. ' +
+    `The guide's \`${FIRST_RUN_SECTION_ID}\` section says how.`
+  );
+}

package/src/modules/workspace/move-links.ts

@@ -0,0 +1,237 @@
+/**
+ * The link half of `move_file`: which pages a move edits, and what it reports.
+ *
+ * A move breaks two sets of links — those inside the moved files that point
+ * elsewhere, and those elsewhere that point at the moved files. The plan reads
+ * both through the shared grammar (`md-links` in platform-shared) and answers
+ * the edits to make plus the report the agent gets, the same for a dry run and
+ * for the move itself.
+ *
+ * Who sees what:
+ *   - a file the caller cannot read is never opened; the report then carries
+ *     one sentence saying such files may still point at the old path, with no
+ *     name and no count;
+ *   - a file the caller can read but may not change (a protected branch's
+ *     write rules, or the write hook's refusal) is named with its links, and
+ *     left;
+ *   - a file the read hook refuses counts as one the caller cannot read: not
+ *     named, only covered by the same sentence — the hook is asked for every
+ *     file the answer would name, before it is named, and for no other;
+ *   - an HTML page is named with its links, and left — and so is a markdown
+ *     page for the raw HTML it carries (`<a href>`, `<img src>`), which the
+ *     grammar reads but never rewrites.
+ */
+
+import {
+  htmlLinksAffectedByMove,
+  rewriteMdLinks,
+  scanMarkdownHtmlLinks,
+  type MdLinkEdit,
+} from '@bevel-software/platform-shared';
+
+/** More edited files than this and the move is refused. */
+export const MOVE_LINK_EDIT_CAP = 200;
+/** How many edits the report lists. */
+export const MOVE_LINK_REPORT_EDITS = 100;
+
+/** The sentence about files the caller cannot read. */
+export const UNSEARCHED_SENTENCE =
+  'Links in files you cannot read were not searched, and may still point at the old path.';
+
+/** A folder segment whose files are records of the past, never searched or rewritten. */
+const UNSEARCHED_SEGMENTS = new Set(['transcripts', 'probes']);
+
+export interface MoveLinksReport {
+  filesEdited: number;
+  linksRewritten: number;
+  /** The first {@link MOVE_LINK_REPORT_EDITS} edits; `path` is where the file sits after the move. */
+  edits: { path: string; from: string; to: string }[];
+  notRewritten: { path: string; reason: string; links: string[] }[];
+  unsearched?: string;
+}
+
+export interface PlannedEdit {
+  /** Where the file sits after the move. */
+  path: string;
+  /** Where it sits now. */
+  lockAt: string;
+  /** The bytes the edit was computed from — compared again under the lock. */
+  original: string;
+  content: string;
+  links: MdLinkEdit[];
+}
+
+export interface MoveLinksPlan {
+  edits: PlannedEdit[];
+  report: MoveLinksReport;
+  /** Set when the move would edit more than the cap; the move is refused with it. */
+  overCap?: string;
+}
+
+export interface MoveLinksInput {
+  src: string;
+  dest: string;
+  branch: string;
+  kbDirName: string;
+  /** Every file on the branch inside the clone folder, workspace-relative. */
+  allFiles: string[];
+  /** Read verdicts for workspace-relative paths. */
+  canRead: (paths: string[]) => Promise<Map<string, boolean>>;
+  /** The paths among these the caller may not change (protected-branch rules). */
+  writeBlocked: (paths: string[]) => Promise<string[]>;
+  readText: (path: string) => Promise<string>;
+  /**
+   * The deployment's read hook on one page, at its current path: true when
+   * it refuses. Asked for every page the answer would NAME — edited, or left
+   * with its links listed — and for no page merely searched: naming is the
+   * disclosure. A refused page is one the caller cannot read, covered by the
+   * one sentence and never named.
+   */
+  readRefused: (path: string) => Promise<boolean>;
+  /**
+   * The deployment's write hook on a page the move would edit, at its
+   * post-move path: why it may not be, or null. Asked for edited pages only.
+   */
+  writeRefusal: (lockAt: string, path: string) => Promise<string | null>;
+}
+
+/** Whether `path` lies in a `transcripts/` or `probes/` folder. */
+export function isUnsearchedRecord(path: string): boolean {
+  return path.split('/').slice(0, -1).some((s) => UNSEARCHED_SEGMENTS.has(s));
+}
+
+const isMarkdown = (p: string) => /\.md$/i.test(p);
+const isHtml = (p: string) => /\.html?$/i.test(p);
+
+/**
+ * The search's cheap first cut: a file linking at `name` names it — raw, or
+ * spelled with markdown escapes or percent-encoding, which are undone here
+ * the way the resolver undoes them (any case, any subset of characters).
+ */
+function mayMention(text: string, name: string): boolean {
+  if (text.includes(name)) return true;
+  const unescaped = text.replace(/\\([!-/:-@[-`{-~])/g, '$1');
+  const decoded = unescaped.replace(/(?:%[0-9a-f]{2})+/gi, (run) => {
+    try {
+      return decodeURIComponent(run);
+    } catch {
+      return run;
+    }
+  });
+  return decoded.includes(name);
+}
+
+export async function planMoveLinks(input: MoveLinksInput): Promise<MoveLinksPlan> {
+  const { src, dest, branch, kbDirName } = input;
+  const mapPath = (p: string): string | null =>
+    p === src ? dest : p.startsWith(`${src}/`) ? dest + p.slice(src.length) : null;
+  const srcName = src.slice(src.lastIndexOf('/') + 1);
+
+  const candidates = input.allFiles.filter((p) => (isMarkdown(p) || isHtml(p)) && !isUnsearchedRecord(p)).sort();
+  const readable = await input.canRead(candidates);
+  let unsearched = candidates.some((p) => readable.get(p) !== true);
+
+  const edits: PlannedEdit[] = [];
+  /** Pages the answer would name for raw HTML, held back until the read hook allows the page. */
+  const htmlReports: { lockAt: string; entry: MoveLinksReport['notRewritten'][number] }[] = [];
+  for (const oldPath of candidates) {
+    if (readable.get(oldPath) !== true) continue;
+    const newPath = mapPath(oldPath) ?? oldPath;
+    // The sidebar moves and deletes without this tool's locks, so a page can
+    // go between the listing and this read. A page gone has no links to fix;
+    // one that cannot be opened is one more the search did not cover. Gone is
+    // the disk's own definition of absence: ENOENT, and ENOTDIR for a page
+    // whose folder became a file in the meantime.
+    let text: string;
+    try {
+      text = await input.readText(oldPath);
+    } catch (err) {
+      const code = (err as NodeJS.ErrnoException | null)?.code;
+      if (code !== 'ENOENT' && code !== 'ENOTDIR') unsearched = true;
+      continue;
+    }
+    // A file that stays put is only touched by a link that names the moved
+    // path, so a page that does not mention it is skipped unparsed.
+    if (newPath === oldPath && !mayMention(text, srcName)) continue;
+    const opts = { oldPath, newPath, mapPath, kbDirName, branch };
+    if (isHtml(oldPath)) {
+      const links = htmlLinksAffectedByMove(text, opts);
+      if (links.length > 0) htmlReports.push({ lockAt: oldPath, entry: { path: newPath, reason: 'html page', links } });
+      continue;
+    }
+    // Raw HTML inside a markdown page (`<img src>`, `<a href>`) renders like
+    // a link but is not one the grammar rewrites: named with those links and
+    // left, as an HTML page is, so a stale target is never silent. Only a
+    // live tag: one inside code or behind a `\<` escape is an example.
+    const html = htmlLinksAffectedByMove(text, opts, scanMarkdownHtmlLinks(text));
+    if (html.length > 0) {
+      htmlReports.push({ lockAt: oldPath, entry: { path: newPath, reason: 'html in markdown', links: html } });
+    }
+    const rewritten = rewriteMdLinks(text, opts);
+    if (rewritten.edits.length === 0) continue;
+    edits.push({ path: newPath, lockAt: oldPath, original: text, content: rewritten.text, links: rewritten.edits });
+  }
+
+  // ONE gate before anything is named: the read hook, asked once per page the
+  // answer would name, whether as an edit or as a page left with its links
+  // listed. A page it refuses is one the caller cannot read — covered by the
+  // one sentence, named nowhere, its links quoted nowhere. A page merely
+  // searched is never asked about: nothing of it reaches the answer.
+  const readAllowed = new Map<string, boolean>();
+  const mayName = async (lockAt: string): Promise<boolean> => {
+    let allowed = readAllowed.get(lockAt);
+    if (allowed === undefined) {
+      allowed = !(await input.readRefused(lockAt));
+      readAllowed.set(lockAt, allowed);
+      if (!allowed) unsearched = true;
+    }
+    return allowed;
+  };
+  const notRewritten: MoveLinksReport['notRewritten'] = [];
+  for (const r of htmlReports) if (await mayName(r.lockAt)) notRewritten.push(r.entry);
+  const named: PlannedEdit[] = [];
+  for (const e of edits) if (await mayName(e.lockAt)) named.push(e);
+
+  // Only a file that stays put can be one the caller may not change: every
+  // moved file was already judged writable at both ends by the move itself.
+  const blocked = new Set(await input.writeBlocked(named.filter((e) => e.path === e.lockAt).map((e) => e.path)));
+  const kept = named.filter((e) => {
+    if (!blocked.has(e.path)) return true;
+    notRewritten.push({ path: e.path, reason: 'no write access', links: e.links.map((l) => l.from) });
+    return false;
+  });
+
+  if (kept.length > MOVE_LINK_EDIT_CAP) {
+    return {
+      edits: [],
+      report: reportOf([], notRewritten, unsearched),
+      overCap:
+        `This move would edit links in ${kept.length} files, more than the ${MOVE_LINK_EDIT_CAP} one move may edit, so nothing was moved. ` +
+        'Move fewer files at a time (a subfolder, then the rest), or pass `rewriteLinks: false` to move without rewriting links.',
+    };
+  }
+
+  // The write hook hears about edited files only.
+  const allowed: PlannedEdit[] = [];
+  for (const e of kept) {
+    const refusal = await input.writeRefusal(e.lockAt, e.path);
+    if (refusal === null) allowed.push(e);
+    else notRewritten.push({ path: e.path, reason: refusal, links: e.links.map((l) => l.from) });
+  }
+  return { edits: allowed, report: reportOf(allowed, notRewritten, unsearched) };
+}
+
+function reportOf(
+  edits: PlannedEdit[],
+  notRewritten: MoveLinksReport['notRewritten'],
+  unsearched: boolean,
+): MoveLinksReport {
+  const flat = edits.flatMap((e) => e.links.map((l) => ({ path: e.path, from: l.from, to: l.to })));
+  return {
+    filesEdited: edits.length,
+    linksRewritten: flat.length,
+    edits: flat.slice(0, MOVE_LINK_REPORT_EDITS),
+    notRewritten,
+    ...(unsearched ? { unsearched: UNSEARCHED_SENTENCE } : {}),
+  };
+}