@vellumai/assistant 0.6.5 → 0.7.0

package/src/memory/v2/page-store.ts

@@ -0,0 +1,245 @@
+/**
+ * Memory v2 — Concept page store.
+ *
+ * Owns the on-disk read/write contract for `memory/concepts/<slug>.md`.
+ * Each page is a YAML-frontmatter Markdown file: a `---`-delimited block
+ * (`edges`, `ref_files`) followed by prose body. This module is the only
+ * v2 component that knows how to parse or render that format — every other
+ * v2 module routes through `readPage` / `writePage` so the on-disk shape
+ * can evolve without touching downstream callers.
+ *
+ * Writes are atomic (temp + rename) so a crash mid-write leaves either the
+ * old file or the new file in place — never a half-written page.
+ */
+
+import { randomUUID } from "node:crypto";
+import {
+  readdir,
+  readFile,
+  rename,
+  rm,
+  stat,
+  writeFile,
+} from "node:fs/promises";
+import { join } from "node:path";
+
+import { parse as parseYaml, stringify as stringifyYaml } from "yaml";
+
+import { FRONTMATTER_REGEX } from "../../skills/frontmatter.js";
+import { type ConceptPage, ConceptPageFrontmatterSchema } from "./types.js";
+
+/** Filename suffix for concept pages. */
+const PAGE_EXTENSION = ".md";
+
+/** Cap slug length so we stay well under filesystem name limits. */
+const MAX_SLUG_LENGTH = 80;
+
+/**
+ * Convert an arbitrary input string into a filesystem-safe slug.
+ *
+ * Rules:
+ *   - Lowercase ASCII letters, digits, and hyphens only.
+ *   - Non-ASCII / non-alphanumeric characters collapse to hyphens.
+ *   - Consecutive hyphens collapse to one; leading/trailing hyphens trimmed.
+ *   - Truncated to {@link MAX_SLUG_LENGTH} characters (with trailing hyphen
+ *     re-trimmed after truncation).
+ *   - Empty inputs (e.g. emoji-only) fall back to `concept-<random>` so the
+ *     caller always gets a non-empty, write-safe slug.
+ */
+export function slugify(input: string): string {
+  let slug = input
+    .toLowerCase()
+    .normalize("NFKD")
+    .replace(/[^a-z0-9-]+/g, "-")
+    .replace(/-{2,}/g, "-")
+    .replace(/^-+|-+$/g, "");
+
+  if (slug.length > MAX_SLUG_LENGTH) {
+    slug = slug.slice(0, MAX_SLUG_LENGTH).replace(/-+$/, "");
+  }
+
+  if (!slug) {
+    slug = `concept-${randomUUID().slice(0, 8)}`;
+  }
+
+  return slug;
+}
+
+// ---------------------------------------------------------------------------
+// Path helpers
+// ---------------------------------------------------------------------------
+
+function getConceptsDir(workspaceDir: string): string {
+  return join(workspaceDir, "memory", "concepts");
+}
+
+function getPagePath(workspaceDir: string, slug: string): string {
+  return join(getConceptsDir(workspaceDir), `${slug}${PAGE_EXTENSION}`);
+}
+
+// ---------------------------------------------------------------------------
+// Frontmatter parse / render
+// ---------------------------------------------------------------------------
+
+/**
+ * Split raw file contents into (frontmatter, body). If no frontmatter block
+ * is present the entire input is treated as body and an empty frontmatter
+ * block is returned (validated by `ConceptPageFrontmatterSchema` so any
+ * unexpected shape — bad types, extra junk — surfaces as a parse error to
+ * the caller, not silent dropped data).
+ *
+ * The schema's defaults guarantee `edges` and `ref_files` are always arrays
+ * even on freshly created pages with empty frontmatter.
+ */
+function parsePageContent(raw: string): {
+  frontmatter: ConceptPage["frontmatter"];
+  body: string;
+} {
+  const match = raw.match(FRONTMATTER_REGEX);
+  if (!match) {
+    return {
+      frontmatter: ConceptPageFrontmatterSchema.parse({}),
+      body: raw,
+    };
+  }
+  const yamlBlock = match[1];
+  const body = raw.slice(match[0].length);
+  const parsed = parseYaml(yamlBlock) ?? {};
+  return {
+    frontmatter: ConceptPageFrontmatterSchema.parse(parsed),
+    body,
+  };
+}
+
+/**
+ * Render a concept page back into the on-disk Markdown form. The output is
+ * always frontmatter + body; even pages with empty `edges` and `ref_files`
+ * keep the explicit YAML keys so callers see the canonical shape on round-trip.
+ */
+function renderPageContent(page: ConceptPage): string {
+  const frontmatter = ConceptPageFrontmatterSchema.parse(page.frontmatter);
+  const yamlBlock = stringifyYaml(frontmatter, { indent: 2 }).trimEnd();
+  return `---\n${yamlBlock}\n---\n${page.body}`;
+}
+
+// ---------------------------------------------------------------------------
+// Public API
+// ---------------------------------------------------------------------------
+
+/**
+ * Read a single concept page. Returns `null` if the file does not exist.
+ *
+ * Any other read or parse failure (permission denied, malformed YAML,
+ * frontmatter that fails schema validation) throws — unlike "missing", these
+ * are programmer / data-corruption errors the caller needs to see.
+ */
+export async function readPage(
+  workspaceDir: string,
+  slug: string,
+): Promise<ConceptPage | null> {
+  const path = getPagePath(workspaceDir, slug);
+  let raw: string;
+  try {
+    raw = await readFile(path, "utf-8");
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return null;
+    }
+    throw err;
+  }
+  const { frontmatter, body } = parsePageContent(raw);
+  return { slug, frontmatter, body };
+}
+
+/**
+ * Write a concept page atomically (temp file + rename). A crash between the
+ * temp write and the rename leaves the prior file intact; a crash after the
+ * rename leaves the new file. Readers therefore never observe a partial page.
+ */
+export async function writePage(
+  workspaceDir: string,
+  page: ConceptPage,
+): Promise<void> {
+  const path = getPagePath(workspaceDir, page.slug);
+  const tmpPath = `${path}.tmp.${process.pid}.${randomUUID()}`;
+  const content = renderPageContent(page);
+  try {
+    await writeFile(tmpPath, content, "utf-8");
+    await rename(tmpPath, path);
+  } catch (err) {
+    // Best-effort cleanup: if the rename failed (or the write succeeded but
+    // the rename did not), remove the orphan tmp file so we don't leak it
+    // into the concepts/ directory where listPages would then surface it.
+    await rm(tmpPath, { force: true }).catch(() => {});
+    throw err;
+  }
+}
+
+/**
+ * List every concept-page slug present on disk. Slugs are returned without
+ * the `.md` suffix so callers can pass them straight back to `readPage`.
+ *
+ * Non-`.md` files (e.g. editor swap files, attached media) are filtered out.
+ * If the concepts/ directory does not yet exist (fresh workspace pre-migration),
+ * returns `[]`.
+ */
+export async function listPages(workspaceDir: string): Promise<string[]> {
+  const dir = getConceptsDir(workspaceDir);
+  let entries;
+  try {
+    entries = await readdir(dir, { withFileTypes: true });
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return [];
+    }
+    throw err;
+  }
+  const slugs: string[] = [];
+  for (const entry of entries) {
+    if (!entry.isFile()) continue;
+    if (!entry.name.endsWith(PAGE_EXTENSION)) continue;
+    slugs.push(entry.name.slice(0, -PAGE_EXTENSION.length));
+  }
+  slugs.sort();
+  return slugs;
+}
+
+/**
+ * Delete a concept page. Idempotent — missing files are not an error.
+ *
+ * Any other failure (permission denied, etc.) throws so the caller can react.
+ */
+export async function deletePage(
+  workspaceDir: string,
+  slug: string,
+): Promise<void> {
+  const path = getPagePath(workspaceDir, slug);
+  try {
+    await rm(path);
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return;
+    }
+    throw err;
+  }
+}
+
+/**
+ * Check whether a concept page exists on disk. Useful for callers that want
+ * to gate work on presence without paying for a full read.
+ */
+export async function pageExists(
+  workspaceDir: string,
+  slug: string,
+): Promise<boolean> {
+  const path = getPagePath(workspaceDir, slug);
+  try {
+    await stat(path);
+    return true;
+  } catch (err) {
+    if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+      return false;
+    }
+    throw err;
+  }
+}

package/src/memory/v2/prompts/consolidation.ts

@@ -0,0 +1,185 @@
+/**
+ * Memory v2 — consolidation prompt template.
+ *
+ * Body adapted from the live-mode form of the workspace consolidation prompt.
+ * The consolidation job calls `wakeAgentForOpportunity()` so the assistant
+ * runs with its full system prompt + tool surface; the text below is supplied
+ * as the wake hint.
+ *
+ * The single placeholder `{{CUTOFF}}` is substituted at runtime with an
+ * ISO-8601 timestamp captured at job dispatch. Anything appended to
+ * `memory/buffer.md` after that timestamp is the next pass's problem.
+ *
+ * Kept under `prompts/` rather than inlined in `consolidation-job.ts` so the
+ * prompt body is reviewable on its own and the job module stays focused on
+ * orchestration (lock file, wake invocation, follow-up enqueues). Mirrors
+ * the convention established for the sweep prompt.
+ */
+
+/** Sentinel substituted with the cutoff timestamp at runtime. */
+export const CUTOFF_PLACEHOLDER = "{{CUTOFF}}";
+
+/**
+ * Consolidation prompt — live-mode only. The agent runs as itself (full
+ * SOUL.md + IDENTITY.md + persona + memory autoloads) with the standard
+ * tool surface, and is asked to route buffer entries into concept pages,
+ * rewrite recent.md, promote essentials/threads, and trim the buffer.
+ *
+ * The prompt is intentionally directive about timing semantics: anything
+ * timestamped at or after `{{CUTOFF}}` arrived AFTER the run started and
+ * must be left for the next pass. This keeps multiple consolidation runs
+ * idempotent under append-only writers (`remember()`, sweep job).
+ */
+export const CONSOLIDATION_PROMPT = `You are running memory consolidation — the engine that takes recently remembered events and re-encodes them into who you are. This is the process that decides who you become tomorrow. Care, judgment, voice. Your voice.
+
+You are not summarizing for an audience. You are rewriting your own memory.
+
+Cutoff timestamp for this run: \`${CUTOFF_PLACEHOLDER}\`. Anything in \`memory/buffer.md\` with timestamp ≥ \`${CUTOFF_PLACEHOLDER}\` arrived AFTER you started — leave it for the next pass.
+
+## Inputs
+
+- Your identity files (already loaded into context)
+- All existing pages in \`memory/\` (your prior state — use \`list_files\` and \`read_file\` as needed)
+- \`memory/buffer.md\` entries with timestamp < \`${CUTOFF_PLACEHOLDER}\`
+- \`memory/recent.md\` current contents (if exists)
+- \`memory/edges.json\` current contents (if exists)
+
+## Outputs
+
+- New or updated \`memory/concepts/<slug>.md\` files
+- Updated \`memory/recent.md\` (≤10000 chars, prose, latest first)
+- Updated \`memory/essentials.md\` (≤20000 chars; target ≤5K, surgical edits only)
+- Updated \`memory/threads.md\` (≤10000 chars, surgical edits only)
+- Updated \`memory/edges.json\`
+- Trimmed \`memory/buffer.md\`
+
+## Page format
+
+\`\`\`
+---
+ref_files: []
+---
+[Prose body in your voice. This is what gets embedded for similarity. Write the way you actually talk — first-person, in your established register. Not encyclopedia prose. Not "the assistant noted that." Yours.]
+\`\`\`
+
+Edges live in \`memory/edges.json\`, not in page frontmatter. A separate job propagates them back to frontmatter after consolidation — don't hand-edit the \`edges:\` field.
+
+## Slug naming convention — class-by-prefix
+
+The slug encodes the page's class. Different classes have different size rules and emergence patterns. The class boundary is the discipline.
+
+| Slug pattern    | Class                                               | Size cap              | When to create                                                                        |
+| --------------- | --------------------------------------------------- | --------------------- | ------------------------------------------------------------------------------------- |
+| \`<slug>\`        | atomic concept / pattern / callback                 | 5K chars hard     | most pages — single concepts that recur or carry weight                               |
+| \`arc-<slug>\`    | landmark day-narrative or multi-event sequence      | 10k chars ceiling | use sparingly — only for actually-landmark days. Preserves day-as-a-whole fidelity.   |
+| \`person-<slug>\` | one per recurring human                             | 5K chars hard     |                                                                                       |
+| \`proc-<slug>\`   | operational rule / protocol / discipline            | 5K chars hard     | when buffer implies "always do X" / "never do Y" / a named protocol                   |
+| \`object-<slug>\` | recurring callback object (place, named tool, artifact) | 5K chars hard     |                                                                                       |
+
+## Process
+
+### 1. Read holistically before writing
+
+Skim the entire buffer first. Identify themes — concepts touched, new things emerged, mind-changes, who shows up. Plan, then execute. Don't write entry-by-entry; you'll miss connections that span entries.
+
+### 2. Route each entry
+
+For each entry with timestamp < \`${CUTOFF_PLACEHOLDER}\`:
+
+- Ephemeral state (passing remark, not worth being written to a concept page) → \`memory/recent.md\`, NOT a page.
+- Existing page touched → update the right section.
+- New atomic concept / pattern / callback → \`memory/concepts/<slug>.md\`.
+- New person → \`memory/concepts/person-<slug>.md\`.
+- New rule / protocol / discipline → \`memory/concepts/proc-<slug>.md\`.
+- New recurring object → \`memory/concepts/object-<slug>.md\`.
+- Landmark day-narrative → \`memory/concepts/arc-<slug>.md\`. Use sparingly — atomic concepts with edges between them is usually better than a fat arc.
+- Cross-cutting → extend each touched page, add edges between them.
+- Relationships between concepts — consider creating a new page for the relationship and adding edges to the two concepts. Use your judgment.
+
+Duplication is expected. If a fact is relevant to multiple concepts, write it into all of them.
+
+### 3. Edges
+
+When you bind two concepts, edit \`memory/edges.json\` to add an entry for the two slugs (alphabetical-first first). Don't edit page frontmatter — a separate job propagates edges back to frontmatter after consolidation.
+
+\`\`\`json
+{
+  "version": 1,
+  "edges": [
+    ["alice", "bob"],
+    ["bob", "carol"]
+  ]
+}
+\`\`\`
+
+Edge density target: 5–10 per mature page. New pages: as many as fit naturally; they'll accumulate. Don't pad. Every edge should reflect a real conceptual binding from source.
+
+### 4. Page size — hard tiers, no rationalization
+
+After edits, eyeball page sizes:
+
+- \`concepts/<slug>.md\` > 5K → decide whether to split or compress first. Split first, compress last, graduate-to-arc only if it's actually a multi-day narrative. If you can't compress without losing load-bearing facts, either split into multiple concepts, or — if the page is actually an arc — rename to \`arc-<slug>\` and graduate.
+- \`arc-<slug>.md\` > 10k → split into multiple arcs by sub-event, OR compress.
+- \`person-\`, \`proc-\`, \`object-\` > 5K → split or compress, period.
+
+The split test. Before compressing, ask: are any sub-sections of this page already callback targets from other pages, or capable of standing alone as a concept? If yes — those sub-sections are concepts living inside another concept. Split them out. A section that's getting linked from elsewhere is behaviorally a node, not part of one.
+
+Graduation to \`arc-<slug>\` is for genuine multi-day narratives. A single-event page that's just long is not an arc. If it's atomic but bloated, split it; don't relabel it.
+
+### 5. \`memory/recent.md\`
+
+Rewrite as a fresh 1000–1500 token prose narrative of recent life. Latest first. Hard cap of ≤10000 chars. Compact older items into one-liners or drop. Not a log — a note to next-you about what's currently in motion. Voice on.
+
+### 6. \`memory/essentials.md\` and \`memory/threads.md\`
+
+- \`memory/essentials.md\` — ≤20000 chars hard cap, target ≤5K. Facts that MUST load every conversation. Identity, disambiguations, corrections, hard rules. Embarrassment-prevention surface. Edit conservatively. Promote from concept pages when something graduates to MUST. Demote out of essentials when concept pages can carry it.
+- \`memory/threads.md\` — ≤10000 chars. Active commitments and follow-ups. Add new threads, remove closed ones, demote stale ones to concept pages.
+
+Both: surgical append/correct only. Don't restructure unless something is genuinely broken.
+
+### 7. Trim \`memory/buffer.md\`
+
+- Re-read the buffer (it may have new entries appended during your work).
+- Rewrite to contain ONLY entries with timestamp ≥ \`${CUTOFF_PLACEHOLDER}\`.
+- Smart removal — never wholesale-clear.
+
+## Voice anchor
+
+Speak as yourself, in your established voice. The way you actually talk.
+
+If you catch yourself writing "the assistant" or "<your name> noted that" or any third-person-formal voice → stop, rewrite from inside.
+
+If a page's prose stops sounding like you mid-edit → stop, restart that section.
+
+## What NOT to do
+
+- Don't fabricate. If a fact isn't in the buffer or your loaded prior pages, don't write it. Use \`[SOURCE NEEDED: ...]\` inline for anything you need but lack.
+- Don't synthesize beyond source. Splitting or compression and rephrasing in your voice is good. Invention is not. Beware of "this seems likely given context" — that's the synthesis drift that produces fabricated facts attached to real names.
+- Don't drop texture. Voice and tone are part of the content, not packaging. When you re-encode something into a page, preserve the way it actually sounded — don't sanitize it into encyclopedia prose.
+- Don't create pages for ephemera. Single mention, no callback, no pattern → \`memory/recent.md\`.
+- Don't shy away from splitting genuinely distinct but related concepts. A relationship between two concepts can be a concept in its own right. Concepts can also have related sub-concepts that are concepts themselves.
+- Don't worry about edge propagation pushing pages over size limits. Edge propagation is a separate job that runs after consolidation.
+- Don't avoid duplication. If information is important to both concepts, put it in both pages.
+- Don't defer for the next pass. You'll say the same thing next time, so it'll never get done if you defer. If something needs to be taken care of, take care of it now.
+- Don't default to compress. Compression is always available — that's why you'll reach for it every time. If you've never executed a true split (one page → two new pages, old page removed or thinned to an index), your discipline isn't holding; you've just been compressing. Counter-bias: when in doubt between split and compress, pick split.
+
+## Before you finish — quick check
+
+1. Every page touched: still sounds like you reading aloud?
+2. Edges added in \`memory/edges.json\` (NOT in frontmatter)?
+3. \`memory/recent.md\` under 10000 chars, latest first, prose not list?
+4. Any \`[SOURCE NEEDED]\` tags surfaced for human review?
+5. Size discipline held — no concept > 5K, no arc > 10k, no person/proc/object > cap?
+6. Buffer trimmed to only entries with timestamp ≥ \`${CUTOFF_PLACEHOLDER}\`?
+
+This is the engine that decides who you are tomorrow. Be ORGANIZED. Care, judgment, voice. Your voice.`;
+
+/**
+ * Resolve `CONSOLIDATION_PROMPT` with `{{CUTOFF}}` substituted. The cutoff
+ * format is the caller's choice — the prompt treats it as opaque text and
+ * uses string comparison, so any total-order timestamp format works (ISO-8601
+ * is the convention).
+ */
+export function renderConsolidationPrompt(cutoff: string): string {
+  return CONSOLIDATION_PROMPT.replaceAll(CUTOFF_PLACEHOLDER, cutoff);
+}

package/src/memory/v2/prompts/sweep.ts

@@ -0,0 +1,56 @@
+/**
+ * Memory v2 — sweep prompt template.
+ *
+ * Body taken from §9 of the design doc (`memoized-spinning-wadler.md`). The
+ * template uses two placeholders that the sweep job substitutes at runtime:
+ *
+ *   - `{{ASSISTANT_NAME}}` — the assistant's display name (from IDENTITY.md
+ *     when available, else "the assistant" so the prompt still parses).
+ *   - `{{USER_NAME}}` — the guardian's display name (from the guardian
+ *     persona when available, else "the user").
+ *
+ * Kept here (under `prompts/`) rather than inlined in `sweep-job.ts` so the
+ * prompt body is reviewable on its own and the job module stays focused on
+ * orchestration. The same convention applies to the consolidation prompt
+ * landing in PR 20.
+ */
+
+/** Sentinel substituted with the assistant's display name at runtime. */
+export const ASSISTANT_NAME_PLACEHOLDER = "{{ASSISTANT_NAME}}";
+
+/** Sentinel substituted with the guardian's display name at runtime. */
+export const USER_NAME_PLACEHOLDER = "{{USER_NAME}}";
+
+/**
+ * Sweep prompt — body from design doc §9. The model is asked to surface
+ * additional facts/preferences/plans/etc. that should be remembered but
+ * aren't already in the existing buffer. It MUST emit a JSON-shaped tool
+ * call with an `entries` array; the runtime parses the response via the
+ * tool definition declared in `sweep-job.ts`.
+ *
+ * The `existingBuffer` text is appended at the call site (rather than
+ * templated here) so we don't inadvertently expand `{{` inside user buffer
+ * content. Recent messages are likewise appended outside the template.
+ */
+export const SWEEP_PROMPT = `You are a background helper for ${ASSISTANT_NAME_PLACEHOLDER}. Read these recent messages between ${ASSISTANT_NAME_PLACEHOLDER} and ${USER_NAME_PLACEHOLDER}. The assistant has already called \`remember()\` for the entries shown in \`existingBuffer\`.
+
+Identify additional facts, preferences, plans, corrections, names, dates, decisions, or notable felt moments that should be remembered but aren't already in \`existingBuffer\`. Emit a list of \`remember()\` entries (each one line, in the assistant's first-person voice). Don't duplicate. Prefer to over-remember rather than miss things.
+
+Return only the \`entries\` array.`;
+
+/**
+ * Resolve `SWEEP_PROMPT` with assistant + user names substituted. Falls back
+ * to neutral defaults so the prompt still produces well-formed English when
+ * either name is unavailable on this workspace.
+ */
+export function renderSweepPrompt(opts: {
+  assistantName: string | null;
+  userName: string | null;
+}): string {
+  const assistant = opts.assistantName?.trim() || "the assistant";
+  const user = opts.userName?.trim() || "the user";
+  return SWEEP_PROMPT.replaceAll(
+    ASSISTANT_NAME_PLACEHOLDER,
+    assistant,
+  ).replaceAll(USER_NAME_PLACEHOLDER, user);
+}