@vellumai/assistant 0.12.2-staging.6 → 0.12.2-staging.7

package/src/persistence/conversation-tool-surface.ts

@@ -0,0 +1,86 @@
+import { createHash } from "node:crypto";
+
+import { eq, sql } from "drizzle-orm";
+
+import type { ToolDefinition } from "../providers/types.js";
+import { getDb } from "./db-connection.js";
+import { conversationToolSurfaces } from "./schema/index.js";
+
+/**
+ * Per-conversation record of the tool definitions the most recent live turn
+ * sent to the provider (`conversation_tool_surfaces`).
+ *
+ * The provider prompt cache is a byte-exact prefix match over
+ * `tools -> system -> messages`, so a background wake that forks a
+ * conversation can only reuse the source's cached prefix by sending the SAME
+ * tools array. Re-deriving the array on the fork cannot guarantee that: the
+ * fork may run in another process (a different tool registry), with no
+ * connected clients (different host-tool gates), or under a presence the
+ * persisted message stamps do not encode. Recording the resolved array and
+ * replaying it verbatim can.
+ */
+
+/** Content hash of the serialized tools array. */
+export function hashConversationToolSurface(
+  tools: readonly ToolDefinition[],
+): string {
+  return hashToolsJson(JSON.stringify(tools));
+}
+
+function hashToolsJson(toolsJson: string): string {
+  return createHash("sha256").update(toolsJson).digest("hex").slice(0, 32);
+}
+
+/**
+ * Persist `tools` as the conversation's current wire tool surface and return
+ * its hash. `knownHash` is the hash the caller last recorded for this
+ * conversation (undefined when it has recorded nothing this process
+ * lifetime): a matching hash skips the statement outright. Otherwise the
+ * upsert only rewrites a row whose stored hash differs, so a conversation
+ * reloaded from disk never rewrites an unchanged surface either.
+ */
+export function recordConversationToolSurface(
+  conversationId: string,
+  tools: readonly ToolDefinition[],
+  knownHash?: string,
+): string {
+  const toolsJson = JSON.stringify(tools);
+  const toolsHash = hashToolsJson(toolsJson);
+  if (knownHash === toolsHash) {
+    return toolsHash;
+  }
+  const updatedAt = Date.now();
+  getDb()
+    .insert(conversationToolSurfaces)
+    .values({ conversationId, toolsJson, toolsHash, updatedAt })
+    .onConflictDoUpdate({
+      target: conversationToolSurfaces.conversationId,
+      set: { toolsJson, toolsHash, updatedAt },
+      setWhere: sql`${conversationToolSurfaces.toolsHash} <> excluded.tools_hash`,
+    })
+    .run();
+  return toolsHash;
+}
+
+/**
+ * The tool definitions the conversation's most recent live turn sent, or
+ * `null` when no turn has recorded one (or the stored JSON is unreadable).
+ */
+export function getConversationToolSurface(
+  conversationId: string,
+): ToolDefinition[] | null {
+  const row = getDb()
+    .select({ toolsJson: conversationToolSurfaces.toolsJson })
+    .from(conversationToolSurfaces)
+    .where(eq(conversationToolSurfaces.conversationId, conversationId))
+    .get();
+  if (!row) {
+    return null;
+  }
+  try {
+    const parsed: unknown = JSON.parse(row.toolsJson);
+    return Array.isArray(parsed) ? (parsed as ToolDefinition[]) : null;
+  } catch {
+    return null;
+  }
+}

package/src/persistence/migrations/378-create-conversation-tool-surfaces.test.ts

@@ -0,0 +1,78 @@
+import { Database } from "bun:sqlite";
+import { describe, expect, test } from "bun:test";
+
+import { drizzle } from "drizzle-orm/bun-sqlite";
+
+import * as schema from "../schema.js";
+import { migrateCreateConversationToolSurfaces } from "./378-create-conversation-tool-surfaces.js";
+
+function createTestDb() {
+  const sqlite = new Database(":memory:");
+  sqlite.run("PRAGMA foreign_keys = ON");
+  // Only the parent table the FK points at.
+  sqlite.run(/*sql*/ `CREATE TABLE conversations (id TEXT PRIMARY KEY)`);
+  return { sqlite, db: drizzle(sqlite, { schema }) };
+}
+
+function tableDdl(sqlite: Database): string | undefined {
+  const row = sqlite
+    .query(
+      `SELECT sql FROM sqlite_master WHERE type='table' AND name='conversation_tool_surfaces'`,
+    )
+    .get() as { sql: string } | null;
+  return row?.sql;
+}
+
+function insertSurface(sqlite: Database, conversationId: string): void {
+  sqlite
+    .query(
+      /*sql*/ `INSERT INTO conversation_tool_surfaces (conversation_id, tools_json, tools_hash, updated_at) VALUES (?, ?, ?, ?)`,
+    )
+    .run(conversationId, "[]", "hash", 1);
+}
+
+function surfaceCount(sqlite: Database): number {
+  const row = sqlite
+    .query(`SELECT COUNT(*) AS n FROM conversation_tool_surfaces`)
+    .get() as { n: number };
+  return row.n;
+}
+
+describe("migration 378: conversation_tool_surfaces", () => {
+  test("creates the table keyed by conversation with a cascading FK", () => {
+    const { sqlite, db } = createTestDb();
+    expect(tableDdl(sqlite)).toBeUndefined();
+
+    migrateCreateConversationToolSurfaces(db);
+
+    const ddl = tableDdl(sqlite);
+    expect(ddl).toContain("conversation_id TEXT PRIMARY KEY");
+    expect(ddl).toContain("REFERENCES conversations(id) ON DELETE CASCADE");
+    expect(ddl).toContain("tools_json TEXT NOT NULL");
+    expect(ddl).toContain("tools_hash TEXT NOT NULL");
+    expect(ddl).toContain("updated_at INTEGER NOT NULL");
+  });
+
+  test("deleting a conversation cascades to its surface row", () => {
+    const { sqlite, db } = createTestDb();
+    migrateCreateConversationToolSurfaces(db);
+    sqlite.query(`INSERT INTO conversations (id) VALUES (?)`).run("conv-1");
+    insertSurface(sqlite, "conv-1");
+    expect(surfaceCount(sqlite)).toBe(1);
+
+    sqlite.query(`DELETE FROM conversations WHERE id = ?`).run("conv-1");
+
+    expect(surfaceCount(sqlite)).toBe(0);
+  });
+
+  test("is idempotent: a second run keeps the table and its rows", () => {
+    const { sqlite, db } = createTestDb();
+    migrateCreateConversationToolSurfaces(db);
+    sqlite.query(`INSERT INTO conversations (id) VALUES (?)`).run("conv-1");
+    insertSurface(sqlite, "conv-1");
+
+    expect(() => migrateCreateConversationToolSurfaces(db)).not.toThrow();
+
+    expect(surfaceCount(sqlite)).toBe(1);
+  });
+});

package/src/persistence/migrations/378-create-conversation-tool-surfaces.ts

@@ -0,0 +1,29 @@
+import type { DrizzleDb } from "../db-connection.js";
+
+const TABLE = "conversation_tool_surfaces";
+
+/**
+ * Create `conversation_tool_surfaces`: one row per conversation holding the
+ * tool definitions its most recent live turn sent to the provider, keyed by
+ * conversation with a content hash so an unchanged surface is never rewritten.
+ *
+ * Its own table rather than a column on `conversations` because the payload
+ * is a JSON array of every tool definition (tens of kilobytes) that only fork
+ * wakes read, while `conversations` rows are read whole on every hot path.
+ * The row cascades with its conversation, so a deleted conversation leaves
+ * no surface behind.
+ *
+ * Idempotent via IF NOT EXISTS.
+ */
+export function migrateCreateConversationToolSurfaces(
+  database: DrizzleDb,
+): void {
+  database.run(
+    `CREATE TABLE IF NOT EXISTS ${TABLE} (
+       conversation_id TEXT PRIMARY KEY REFERENCES conversations(id) ON DELETE CASCADE,
+       tools_json TEXT NOT NULL,
+       tools_hash TEXT NOT NULL,
+       updated_at INTEGER NOT NULL
+     )`,
+  );
+}

package/src/persistence/schema/conversation-tool-surfaces.ts

@@ -0,0 +1,22 @@
+import { integer, sqliteTable, text } from "drizzle-orm/sqlite-core";
+
+import { conversations } from "./conversations.js";
+
+/**
+ * The tool definitions a conversation's most recent live turn sent to the
+ * provider, serialized exactly as resolved. Written by the per-turn tool
+ * resolver when the array changes; read by fork wakes that replay it so their
+ * provider prompt-cache prefix matches the source's live turns. Cascades with
+ * its conversation.
+ */
+export const conversationToolSurfaces = sqliteTable(
+  "conversation_tool_surfaces",
+  {
+    conversationId: text("conversation_id")
+      .primaryKey()
+      .references(() => conversations.id, { onDelete: "cascade" }),
+    toolsJson: text("tools_json").notNull(),
+    toolsHash: text("tools_hash").notNull(),
+    updatedAt: integer("updated_at").notNull(),
+  },
+);

package/src/persistence/schema/index.ts

@@ -5,6 +5,7 @@ export * from "./calls.js";
 export * from "./contacts.js";
 export * from "./conversation-groups.js";
 export * from "./conversation-starters.js";
+export * from "./conversation-tool-surfaces.js";
 export * from "./conversations.js";
 export * from "./documents.js";
 export * from "./guardian.js";

package/src/persistence/steps.ts

@@ -485,6 +485,7 @@ import { migrateChannelInboundMessageIdIndex } from "./migrations/374-channel-in
 import { migrateCreateChannelOutboundPosts } from "./migrations/375-create-channel-outbound-posts.js";
 import { migrateNotificationDeliveriesCanonicalMessageId } from "./migrations/376-notification-deliveries-canonical-message-id.js";
 import { migrateAddSubagentBudgetStopReason } from "./migrations/377-add-subagent-budget-stop-reason.js";
+import { migrateCreateConversationToolSurfaces } from "./migrations/378-create-conversation-tool-surfaces.js";
 import type { MigrationStep } from "./migrations/run-migrations.js";
 
 export const migrationSteps: MigrationStep[] = [
@@ -1611,4 +1612,5 @@ export const migrationSteps: MigrationStep[] = [
     // this as a no-op.
     dependsOn: ["migrateCreateSubagentsTable"],
   },
+  migrateCreateConversationToolSurfaces,
 ];

package/src/plugin-api/conversation-turn.ts

@@ -284,13 +284,20 @@ export async function runConversationTurn(
   const { resolveChannelCapabilities } =
     await import("../daemon/conversation-runtime-assembly.js");
 
-  // Plugin-driven turns run as the guardian: plugins are installed by the
-  // guardian, so their conversations inherit guardian trust. This lets the
-  // existing non-interactive auto-approve machinery handle tool permissions
-  // (the conversation is already non-interactive via `isInteractive: false`
-  // below) without requiring a client to approve prompts.
+  // Channel-addressed turns are inbound: the gateway classifies the sender
+  // and the turn runs only if that actor clears the channel admission floor.
+  // Internal plugin jobs (no channel address) use guardian trust because they
+  // are assistant-owned work, not an external sender.
   const { INTERNAL_GUARDIAN_TRUST_CONTEXT } =
     await import("../daemon/trust-context.js");
+  const { resolvePluginChannelTurnTrust } =
+    await import("./plugin-channel-turn-trust.js");
+  const { prepareChannelInboundContent } =
+    await import("../runtime/routes/inbound-stages/inbound-content-prep.js");
+
+  const trustContext = options.channel
+    ? await resolvePluginChannelTurnTrust(options.channel)
+    : INTERNAL_GUARDIAN_TRUST_CONTEXT;
 
   // A channel address resolves through the same binding an inbound message
   // uses, so a turn addressed by chat lands in that chat's conversation
@@ -311,7 +318,7 @@ export async function runConversationTurn(
     ? !channelConversation.created
     : getConversation(conversationId) != null;
   const conversation = await getOrCreateConversation(conversationId, {
-    trustContext: INTERNAL_GUARDIAN_TRUST_CONTEXT,
+    trustContext,
     ...(options.conversationType
       ? { conversationType: options.conversationType }
       : {}),
@@ -336,10 +343,23 @@ export async function runConversationTurn(
 
   // Convert ContentBlock[] input to the text + attachments shape the
   // conversation's processMessage path expects.
-  const { text, attachments } = extractContentAndAttachments(
+  const { text: rawText, attachments } = extractContentAndAttachments(
     options.content,
     resolveMediaSourceData,
   );
+  const inboundContent = options.channel
+    ? prepareChannelInboundContent({
+        trimmedContent: rawText,
+        trustClass: trustContext.trustClass,
+        sourceChannel: options.channel.sourceChannel,
+        requesterIdentifier:
+          options.channel.username ??
+          options.channel.externalUserId ??
+          undefined,
+      })
+    : { content: rawText, displayContent: undefined };
+  const text = inboundContent.content;
+  const displayContent = inboundContent.displayContent;
 
   // The channel this turn speaks on. Runtime assembly reads the per-turn
   // context first and falls back to the conversation's `originChannel`, then
@@ -409,6 +429,8 @@ export async function runConversationTurn(
       requestId,
       isInteractive: false,
       metadata,
+      trustContext,
+      ...(displayContent ? { displayContent } : {}),
     });
     if (enqueueResult.rejected) {
       throw new Error(
@@ -432,6 +454,8 @@ export async function runConversationTurn(
     onEvent,
     isInteractive: false,
     metadata,
+    trustContext,
+    ...(displayContent ? { displayContent } : {}),
     ...(options.callSite ? { callSite: options.callSite } : {}),
   });
 

package/src/plugin-api/index.ts

@@ -286,6 +286,11 @@ export {
 // writes files under the workspace (e.g. its own `plugins/<name>/data/`
 // directory) resolves them against this instead of hardcoding a base path.
 export { getWorkspaceDir } from "../util/platform.js";
+// `String.prototype.slice` that never cuts a UTF-16 surrogate pair in half.
+// Any text a plugin truncates by character budget and hands to a model must
+// go through this: an orphaned half is invalid UTF-16 that strict provider
+// parsers reject.
+export { safeStringSlice } from "../util/unicode.js";
 // Declarative help for the top-level `assistant` CLI commands that have adopted
 // the static-help split. Plugins (e.g. the memory capability indexer) read this
 // to embed CLI command capabilities without importing the CLI action graph.
@@ -359,7 +364,8 @@ export {
   stringifyMessageContent,
 } from "../persistence/message-content.js";
 // Conversation history — reads and writes on the host conversation store
-// (rows, message history, processing state, disk-view paths) plus the lexical
+// (rows, message history, processing state, the recorded wire tool surface,
+// disk-view paths) plus the lexical
 // message-search surface. Every operation takes explicit parameters; nothing
 // is resolved from config. Async because the facade loads the DB store graph
 // lazily on first call.
@@ -371,6 +377,7 @@ export {
   getConversation,
   getConversationDirPath,
   getConversationProcessingStartedAt,
+  getConversationToolSurface,
   getMessages,
   hasLexicalTokens,
   isConversationProcessing,
@@ -438,6 +445,7 @@ export type {
   RunConversationTurnResult,
 } from "./conversation-turn.js";
 export { runConversationTurn } from "./conversation-turn.js";
+export { PluginTurnNotAdmittedError } from "./plugin-channel-turn-trust.js";
 // Live voice — drive a single client's real-time voice session (STT → agent
 // turn → TTS, with server-VAD turn-taking, pauses, and barge-in) over a
 // transport the plugin owns. The plugin brings only a `send` callback (e.g.

package/src/plugin-api/plugin-channel-turn-trust.test.ts

@@ -0,0 +1,133 @@
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+type InboundTrustReadResult =
+  | {
+      ok: true;
+      verdict: {
+        trustClass: string;
+        canonicalSenderId: string | null;
+        contactId?: string;
+        status?: string;
+        resolutionFailed?: boolean;
+      };
+      admissionPolicy: string | null;
+    }
+  | { ok: false };
+
+let nextRead: InboundTrustReadResult = { ok: false };
+
+mock.module("../calls/inbound-trust-reader.js", () => ({
+  readInboundTrust: async () => nextRead,
+}));
+
+import {
+  PluginTurnNotAdmittedError,
+  resolvePluginChannelTurnTrust,
+} from "./plugin-channel-turn-trust.js";
+
+const CHANNEL = {
+  sourceChannel: "plugin" as const,
+  externalChatId: "imessage:+12025550142",
+  externalUserId: "imessage:+12025550142",
+  displayName: "Ada",
+};
+
+beforeEach(() => {
+  nextRead = { ok: false };
+});
+
+describe("resolvePluginChannelTurnTrust", () => {
+  test("returns the gateway verdict as the turn trust context", async () => {
+    nextRead = {
+      ok: true,
+      verdict: {
+        trustClass: "trusted_contact",
+        canonicalSenderId: "imessage:+12025550142",
+        contactId: "c-ada",
+        status: "active",
+      },
+      admissionPolicy: "trusted_contacts",
+    };
+
+    const trust = await resolvePluginChannelTurnTrust(CHANNEL);
+    expect(trust.trustClass).toBe("trusted_contact");
+    expect(trust.sourceChannel).toBe("plugin");
+    expect(trust.requesterChatId).toBe(CHANNEL.externalChatId);
+  });
+
+  test("fails closed when the gateway could not vouch for the sender", async () => {
+    nextRead = {
+      ok: true,
+      verdict: {
+        trustClass: "unknown",
+        canonicalSenderId: null,
+        resolutionFailed: true,
+      },
+      admissionPolicy: "strangers",
+    };
+
+    await expect(resolvePluginChannelTurnTrust(CHANNEL)).rejects.toMatchObject({
+      name: "PluginTurnNotAdmittedError",
+      reason: "trust_resolution_failed",
+    });
+  });
+
+  test("fails closed when the gateway trust read fails", async () => {
+    nextRead = { ok: false };
+    await expect(resolvePluginChannelTurnTrust(CHANNEL)).rejects.toMatchObject({
+      name: "PluginTurnNotAdmittedError",
+      reason: "trust_resolution_failed",
+    });
+  });
+
+  test("rejects a sender below the channel admission floor", async () => {
+    nextRead = {
+      ok: true,
+      verdict: { trustClass: "unknown", canonicalSenderId: null },
+      admissionPolicy: "guardian_only",
+    };
+
+    await expect(resolvePluginChannelTurnTrust(CHANNEL)).rejects.toEqual(
+      expect.objectContaining({
+        name: "PluginTurnNotAdmittedError",
+        reason: "admission_policy_guardian_only",
+      }),
+    );
+  });
+
+  test("rejects a blocked member regardless of floor", async () => {
+    nextRead = {
+      ok: true,
+      verdict: {
+        trustClass: "trusted_contact",
+        canonicalSenderId: "imessage:+12025550142",
+        contactId: "c-ada",
+        status: "blocked",
+      },
+      admissionPolicy: "strangers",
+    };
+
+    await expect(resolvePluginChannelTurnTrust(CHANNEL)).rejects.toMatchObject({
+      name: "PluginTurnNotAdmittedError",
+      reason: "member_blocked",
+    });
+  });
+
+  test("admits when the gateway reports no admission policy", async () => {
+    nextRead = {
+      ok: true,
+      verdict: { trustClass: "unknown", canonicalSenderId: null },
+      admissionPolicy: null,
+    };
+
+    const trust = await resolvePluginChannelTurnTrust(CHANNEL);
+    expect(trust.trustClass).toBe("unknown");
+  });
+
+  test("PluginTurnNotAdmittedError names the deny reason", () => {
+    const err = new PluginTurnNotAdmittedError("admission_policy_no_one");
+    expect(err).toBeInstanceOf(Error);
+    expect(err.reason).toBe("admission_policy_no_one");
+    expect(err.message).toContain("admission_policy_no_one");
+  });
+});

package/src/plugin-api/plugin-channel-turn-trust.ts

@@ -0,0 +1,71 @@
+/**
+ * Gateway admission and trust for plugin turns addressed by a channel chat.
+ *
+ * A channel address means an inbound sender, not an internal plugin job.
+ * Trust comes from the gateway verdict (`resolve_inbound_trust`); the turn
+ * runs only when that sender clears the channel admission floor.
+ */
+
+import { meetsAdmissionFloor } from "@vellumai/gateway-client";
+
+import { readInboundTrust } from "../calls/inbound-trust-reader.js";
+import type { ChannelId } from "../channels/types.js";
+import type { TrustContext } from "../daemon/trust-context-types.js";
+import { trustContextFromVerdict } from "../runtime/trust-verdict-consumer.js";
+
+interface PluginChannelTurnAddress {
+  sourceChannel: ChannelId;
+  externalChatId: string;
+  externalUserId?: string | null;
+  displayName?: string | null;
+  username?: string | null;
+}
+
+export class PluginTurnNotAdmittedError extends Error {
+  readonly reason: string;
+
+  constructor(reason: string) {
+    super(`This inbound plugin turn was not admitted (${reason}).`);
+    this.name = "PluginTurnNotAdmittedError";
+    this.reason = reason;
+  }
+}
+
+export async function resolvePluginChannelTurnTrust(
+  channel: PluginChannelTurnAddress,
+): Promise<TrustContext> {
+  const actorExternalId = channel.externalUserId?.trim() || undefined;
+  const read = await readInboundTrust({
+    channelType: channel.sourceChannel,
+    actorExternalId,
+  });
+  if (!read.ok) {
+    throw new PluginTurnNotAdmittedError("trust_resolution_failed");
+  }
+
+  if (read.verdict.resolutionFailed) {
+    throw new PluginTurnNotAdmittedError("trust_resolution_failed");
+  }
+
+  const memberStatus = read.verdict.status;
+  if (memberStatus === "blocked" || memberStatus === "revoked") {
+    throw new PluginTurnNotAdmittedError(`member_${memberStatus}`);
+  }
+
+  const trustClass = read.verdict.trustClass;
+  if (
+    read.admissionPolicy != null &&
+    !meetsAdmissionFloor(read.admissionPolicy, trustClass)
+  ) {
+    throw new PluginTurnNotAdmittedError(
+      `admission_policy_${read.admissionPolicy}`,
+    );
+  }
+
+  return trustContextFromVerdict(read.verdict, {
+    sourceChannel: channel.sourceChannel,
+    conversationExternalId: channel.externalChatId,
+    actorDisplayName: channel.displayName ?? undefined,
+    actorUsername: channel.username ?? undefined,
+  });
+}

package/src/plugins/defaults/memory/AGENTS.md

@@ -68,6 +68,13 @@ Everything else under the plugin root is **spine**:
   the writer plus the one matcher every reader uses. It lives at the root
   precisely so `substrate/`, `graph/`, and `graph-topology/` can all reach it
   without a tier importing spine. Do not add a second matcher anywhere),
+  `buffer-file` (the two writers of `memory/buffer.md`: the append every
+  `remember()`-shaped path uses and the consume the consolidation job runs
+  after a pass. Do not write the buffer from anywhere else, and never
+  rewrite it from a read that is not inside `consumeBufferEntries`'s
+  synchronous critical section), `memory-run-evidence` (readers of what a
+  background memory run durably produced from its persisted messages; every
+  job that gates a state transition on a verified write uses them),
   `segmenter`, `message-media`, `worker`, `worker-control`,
   `memory-recall-log-store`, `activation-session-store` (the onboarding
   activation rail — **not** a memory tier despite the name),
@@ -407,8 +414,13 @@ per distinct matched section, at most `memory.v3.finderSectionsPerPage` in
 surfacing order (needle, dense, reply, span) plus its entity and rare-term
 lines, and selecting a line selects that section; the selection log keeps
 one row per slug, so the pool row is where the per-section verdicts live. A
-turn whose selector never judged a pool (the injection gate hard-skipped it,
-or nothing was pooled) persists an empty pool with `selector_ran = 0`, and a
+turn whose selector never judged a pool persists it with `selector_ran = 0`:
+an empty pool when the injection gate hard-skipped it or nothing was pooled,
+and the pool as the selector was given it, stable-prefix cards chosen and
+finder lines not, when the selector's provider failed and the orchestrator
+kept that prefix unjudged. That turn writes no `memory_v3_selections` rows:
+the hot set's frecency and the learned-edge graph read that table as
+judgments, and an unjudged page is not one. A
 turn that logged no selections is still reachable by its stamped
 `message_id`, so the inspector shows negative verdicts too. The pool row and
 the turn's `memory_v3_selections` rows are