@vellumai/assistant 0.11.7-dev.202608312314.14ddc99 → 0.11.7-dev.202609010224.44cd29e

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.11.7-dev.202608312314.14ddc99",
+  "version": "0.11.7-dev.202609010224.44cd29e",
   "license": "MIT",
   "type": "module",
   "exports": {

package/src/__tests__/config-schema.test.ts

@@ -811,6 +811,40 @@ describe("AssistantConfigSchema", () => {
     expect(result.success).toBe(false);
   });
 
+  test("applies sight defaults", () => {
+    expect(AssistantConfigSchema.parse({}).sight).toEqual({
+      keepLatestFrames: 2,
+    });
+  });
+
+  test("accepts a custom sight.keepLatestFrames", () => {
+    const result = AssistantConfigSchema.parse({
+      sight: { keepLatestFrames: 4 },
+    });
+    expect(result.sight.keepLatestFrames).toBe(4);
+  });
+
+  test("keeps an out-of-range sight.keepLatestFrames for the reader to clamp", () => {
+    // Rejecting it would reset the key to the default, which is fewer frames
+    // than either the config or the guardrail asked for. The clamp lives at
+    // read (`resolveSightKeepLatestFrames`).
+    const result = AssistantConfigSchema.parse({
+      sight: { keepLatestFrames: 99 },
+    });
+    expect(result.sight.keepLatestFrames).toBe(99);
+  });
+
+  test("rejects a non-integer sight.keepLatestFrames", () => {
+    expect(
+      AssistantConfigSchema.safeParse({ sight: { keepLatestFrames: 2.5 } })
+        .success,
+    ).toBe(false);
+    expect(
+      AssistantConfigSchema.safeParse({ sight: { keepLatestFrames: "two" } })
+        .success,
+    ).toBe(false);
+  });
+
   // ── commitMessageLLM config ──────────────────────────────────────────
 
   test("default commitMessageLLM values are correct", () => {

package/src/__tests__/conversation-sight-frames.test.ts

@@ -1,15 +1,24 @@
-import { describe, expect, test } from "bun:test";
+import { afterEach, describe, expect, test } from "bun:test";
 
+import {
+  KEEP_LATEST_SIGHT_FRAMES,
+  MAX_SIGHT_KEEP_LATEST_FRAMES,
+  MIN_SIGHT_KEEP_LATEST_FRAMES,
+} from "../config/schemas/sight.js";
 import {
   countMediaBlocks,
   stripMediaPayloadsForRetry,
 } from "../daemon/conversation-media-retry.js";
 import {
-  KEEP_LATEST_SIGHT_FRAMES,
+  resolveSightKeepLatestFrames,
   stripAgedSightFrames,
 } from "../daemon/conversation-sight-frames.js";
-import { sightFrameAttachmentIdsFromMetadata } from "../persistence/conversation-types.js";
+import {
+  messageMetadataIsAmbientSightKeep,
+  sightFrameAttachmentIdsFromMetadata,
+} from "../persistence/conversation-types.js";
 import type { ContentBlock, Message } from "../providers/types.js";
+import { setConfig } from "./helpers/set-config.js";
 
 // ---------------------------------------------------------------------------
 // Helpers
@@ -129,6 +138,34 @@ describe("stripAgedSightFrames", () => {
     );
   });
 
+  test("keeps as many frames as the caller's budget allows", () => {
+    const messages: Message[] = [
+      user(referenceImage("f1")),
+      user(referenceImage("f2")),
+      user(referenceImage("f3")),
+      user(referenceImage("f4")),
+      user(referenceImage("f5")),
+    ];
+
+    const result = stripAgedSightFrames(
+      messages,
+      captureTimes("f1", "f2", "f3", "f4", "f5"),
+      4,
+    );
+
+    expect(result.replacedBlocks).toBe(1);
+    expect(
+      result.messages.flatMap((m) =>
+        m.content.filter((b) => b.type === "image"),
+      ),
+    ).toEqual([
+      referenceImage("f2"),
+      referenceImage("f3"),
+      referenceImage("f4"),
+      referenceImage("f5"),
+    ]);
+  });
+
   test("stubs several frames on one message independently", () => {
     const messages: Message[] = [
       user(referenceImage("f1"), referenceImage("f2"), referenceImage("f3")),
@@ -449,6 +486,38 @@ describe("stripAgedSightFrames composed with stripMediaPayloadsForRetry", () =>
   });
 });
 
+// ---------------------------------------------------------------------------
+// Configured retention budget
+// ---------------------------------------------------------------------------
+
+describe("resolveSightKeepLatestFrames", () => {
+  afterEach(() => {
+    setConfig("sight", {});
+  });
+
+  test("defaults to the constant when the workspace configures nothing", () => {
+    expect(resolveSightKeepLatestFrames()).toBe(KEEP_LATEST_SIGHT_FRAMES);
+  });
+
+  test("honors a configured value inside the guardrail", () => {
+    setConfig("sight", { keepLatestFrames: 4 });
+    expect(resolveSightKeepLatestFrames()).toBe(4);
+  });
+
+  test("caps a value above the ceiling instead of resetting it", () => {
+    // Ten live images per request is what the ceiling exists to prevent, and
+    // falling back to the default would show fewer frames than the guardrail
+    // itself allows.
+    setConfig("sight", { keepLatestFrames: 99 });
+    expect(resolveSightKeepLatestFrames()).toBe(MAX_SIGHT_KEEP_LATEST_FRAMES);
+  });
+
+  test("raises a value below the floor", () => {
+    setConfig("sight", { keepLatestFrames: 0 });
+    expect(resolveSightKeepLatestFrames()).toBe(MIN_SIGHT_KEEP_LATEST_FRAMES);
+  });
+});
+
 // ---------------------------------------------------------------------------
 // Metadata reader
 // ---------------------------------------------------------------------------
@@ -477,3 +546,70 @@ describe("sightFrameAttachmentIdsFromMetadata", () => {
     ).toEqual(["att-1"]);
   });
 });
+
+describe("messageMetadataIsAmbientSightKeep", () => {
+  test("a standalone keep carries the tag and the scripted marker", () => {
+    // The shape `persistLiveVoiceSightFrame` writes.
+    expect(
+      messageMetadataIsAmbientSightKeep(
+        JSON.stringify({
+          voiceSessionTurn: true,
+          scripted: true,
+          sightFrameAttachmentIds: ["att-1"],
+        }),
+      ),
+    ).toBe(true);
+  });
+
+  test("a spoken turn that merely carried a frame is not a keep", () => {
+    // The shape the voice bridge writes when a parked frame rides real
+    // speech, which in camera mode is every turn the user takes. Keying on
+    // the tag alone would read all of it as machine output.
+    expect(
+      messageMetadataIsAmbientSightKeep(
+        JSON.stringify({
+          voiceSessionTurn: true,
+          scripted: false,
+          sightFrameAttachmentIds: ["att-frame-1"],
+        }),
+      ),
+    ).toBe(false);
+    // Older rows predating the always-stamped default carry no marker at all.
+    expect(
+      messageMetadataIsAmbientSightKeep(
+        JSON.stringify({
+          voiceSessionTurn: true,
+          sightFrameAttachmentIds: ["att-frame-1"],
+        }),
+      ),
+    ).toBe(false);
+  });
+
+  test("an auto-sent row without the tag is not a keep", () => {
+    // `scripted` covers every machine-authored send, onboarding prompts
+    // included. Those are none of this predicate's business.
+    expect(
+      messageMetadataIsAmbientSightKeep(JSON.stringify({ scripted: true })),
+    ).toBe(false);
+  });
+
+  test("ordinary and unreadable rows are not keeps", () => {
+    expect(messageMetadataIsAmbientSightKeep(null)).toBe(false);
+    expect(messageMetadataIsAmbientSightKeep("")).toBe(false);
+    expect(
+      messageMetadataIsAmbientSightKeep(JSON.stringify({ livePhoto: true })),
+    ).toBe(false);
+    expect(
+      messageMetadataIsAmbientSightKeep(
+        JSON.stringify({ scripted: true, sightFrameAttachmentIds: [] }),
+      ),
+    ).toBe(false);
+    // Unreadable metadata leaves the row ordinary, so it keeps its work.
+    expect(messageMetadataIsAmbientSightKeep("{not json")).toBe(false);
+    expect(
+      messageMetadataIsAmbientSightKeep(
+        JSON.stringify({ scripted: true, sightFrameAttachmentIds: "att-1" }),
+      ),
+    ).toBe(false);
+  });
+});

package/src/__tests__/plugin-import-boundary-guard.test.ts

@@ -86,6 +86,7 @@ const BASELINE: Record<string, readonly string[]> = {
     "../../../../../messaging/providers/slack/message-metadata.js",
     "../../../../../persistence/auto-analysis-constants.js",
     "../../../../../persistence/checkpoints.js",
+    "../../../../../persistence/conversation-types.js",
     "../../../../../persistence/db-connection.js",
     "../../../../../persistence/embeddings/embed.js",
     "../../../../../persistence/embeddings/embedding-backend.js",

package/src/__tests__/sight-frame-retention.test.ts

@@ -1,5 +1,9 @@
-import { beforeEach, describe, expect, test } from "bun:test";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
 
+import {
+  KEEP_LATEST_SIGHT_FRAMES,
+  MAX_SIGHT_KEEP_LATEST_FRAMES,
+} from "../config/schemas/sight.js";
 import { applySightFrameRetention } from "../daemon/conversation-runtime-assembly.js";
 import {
   getAttachmentMetadataForMessage,
@@ -17,6 +21,7 @@ import { getDb } from "../persistence/db-connection.js";
 import { initializeDb } from "../persistence/db-init.js";
 import { conversations } from "../persistence/schema.js";
 import type { ContentBlock, Message } from "../providers/types.js";
+import { setConfig } from "./helpers/set-config.js";
 
 await initializeDb();
 
@@ -185,6 +190,71 @@ describe("applySightFrameRetention", () => {
   });
 });
 
+describe("configured retention budget", () => {
+  afterEach(() => {
+    setConfig("sight", {});
+  });
+
+  /** One tagged standalone frame per row, the shape a keep stream leaves. */
+  async function seedFrames(
+    conversationId: string,
+    count: number,
+  ): Promise<Message[]> {
+    ensureConversation(conversationId);
+    const assembled: Message[] = [];
+    for (let index = 0; index < count; index += 1) {
+      const attachmentId = `${conversationId}-att-${index}`;
+      await addMessage(conversationId, "user", frameContent(attachmentId), {
+        metadata: { sightFrameAttachmentIds: [attachmentId] },
+      });
+      assembled.push({
+        role: "user",
+        content: [referenceImage(attachmentId)],
+      });
+    }
+    return assembled;
+  }
+
+  function imageCount(messages: Message[]): number {
+    return messages.filter((message) => message.content[0].type === "image")
+      .length;
+  }
+
+  test("keeps the number of frames the workspace configured", async () => {
+    setConfig("sight", { keepLatestFrames: 4 });
+    const conversationId = "conv-sight-budget-configured";
+    const assembled = await seedFrames(conversationId, 6);
+
+    const retained = applySightFrameRetention(assembled, conversationId);
+
+    expect(imageCount(retained)).toBe(4);
+    expect(retained.length - imageCount(retained)).toBe(2);
+    // The survivors are the newest, which is what recency buys the model.
+    expect(retained.slice(2).every((m) => m.content[0].type === "image")).toBe(
+      true,
+    );
+  });
+
+  test("caps a configured value above the ceiling", async () => {
+    setConfig("sight", { keepLatestFrames: 99 });
+    const conversationId = "conv-sight-budget-capped";
+    const assembled = await seedFrames(conversationId, 8);
+
+    const retained = applySightFrameRetention(assembled, conversationId);
+
+    expect(imageCount(retained)).toBe(MAX_SIGHT_KEEP_LATEST_FRAMES);
+  });
+
+  test("falls back to the default when the workspace configures nothing", async () => {
+    const conversationId = "conv-sight-budget-default";
+    const assembled = await seedFrames(conversationId, 5);
+
+    const retained = applySightFrameRetention(assembled, conversationId);
+
+    expect(imageCount(retained)).toBe(KEEP_LATEST_SIGHT_FRAMES);
+  });
+});
+
 describe("forked conversations", () => {
   test("a fork's tag covers both its source and cloned attachment ids", async () => {
     // A fork copies `messages.content` verbatim, so its blocks still name the

package/src/config/schema.ts

@@ -54,6 +54,7 @@ import {
 import { PluginUpdatesConfigSchema } from "./schemas/plugin-updates.js";
 import { SecretDetectionConfigSchema } from "./schemas/security.js";
 import { ServicesSchema } from "./schemas/services.js";
+import { SightConfigSchema } from "./schemas/sight.js";
 import { SkillsConfigSchema } from "./schemas/skills.js";
 import {
   RateLimitConfigSchema,
@@ -118,6 +119,7 @@ export const AssistantConfigSchema = z.object({
   twilio: TwilioConfigSchema.default(TwilioConfigSchema.parse({})),
   calls: CallsConfigSchema.default(CallsConfigSchema.parse({})),
   liveVoice: LiveVoiceConfigSchema.default(LiveVoiceConfigSchema.parse({})),
+  sight: SightConfigSchema.default(SightConfigSchema.parse({})),
   whatsapp: WhatsAppConfigSchema.default(WhatsAppConfigSchema.parse({})),
   telegram: TelegramConfigSchema.default(TelegramConfigSchema.parse({})),
   slack: SlackConfigSchema.default(SlackConfigSchema.parse({})),

package/src/config/schemas/sight.ts

@@ -0,0 +1,45 @@
+import { z } from "zod";
+
+/**
+ * How many camera frames stay in the model's context as real images, counted
+ * newest-first across the whole conversation so the frames from the last few
+ * moments are the ones that survive. Older frames become timestamped text
+ * stubs (`daemon/conversation-sight-frames.ts`).
+ *
+ * Smaller than `RETRY_KEEP_LATEST_MEDIA_BLOCKS` (3, in
+ * `daemon/conversation-media-retry.ts`) because the two budgets answer
+ * different questions. That one is reactive and spends a rejected request's
+ * remaining room across every kind of media, including files the user
+ * deliberately sent. This one is proactive, applied to every assembly, and
+ * governs only frames the camera sampled by itself: background the model
+ * glances at, not something anyone chose to attach.
+ */
+export const KEEP_LATEST_SIGHT_FRAMES = 2;
+
+/**
+ * Floor for `sight.keepLatestFrames`. A call whose camera is up keeps at least
+ * the current view live; stubbing every frame would leave the model reading
+ * timestamps about pictures it can no longer see.
+ */
+export const MIN_SIGHT_KEEP_LATEST_FRAMES = 1;
+
+/**
+ * Ceiling for `sight.keepLatestFrames`. History resends every inline image on
+ * every later request, so the ceiling is what stops a long call from growing
+ * its own context until the provider rejects it.
+ */
+export const MAX_SIGHT_KEEP_LATEST_FRAMES = 6;
+
+export const SightConfigSchema = z
+  .object({
+    keepLatestFrames: z
+      .number({ error: "sight.keepLatestFrames must be a number" })
+      .int("sight.keepLatestFrames must be an integer")
+      .default(KEEP_LATEST_SIGHT_FRAMES)
+      .describe(
+        `How many of the newest camera frames a live-voice call keeps in the model's context as images; older frames become timestamped text stubs. Clamped to ${MIN_SIGHT_KEEP_LATEST_FRAMES}..${MAX_SIGHT_KEEP_LATEST_FRAMES} when read.`,
+      ),
+  })
+  .describe("Ambient camera frames sampled during a live-voice call");
+
+export type SightConfig = z.infer<typeof SightConfigSchema>;