@vellumai/assistant 0.6.4 → 0.6.5

package/src/__tests__/edit-propagation.test.ts

@@ -0,0 +1,280 @@
+/**
+ * Edit-propagation tests for Slack `message.changed` events.
+ *
+ * Validates that the edit intercept stage:
+ *  - Updates `messages.content` and stamps `slackMeta.editedAt` when the
+ *    original message can be located.
+ *  - Is idempotent across successive edits (subsequent edits keep updating).
+ *  - Treats missing-target edits as a silent no-op (no throw, no row change).
+ */
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+import { eq } from "drizzle-orm";
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+import { addMessage } from "../memory/conversation-crud.js";
+import { getDb, initializeDb } from "../memory/db.js";
+import { linkMessage, recordInbound } from "../memory/delivery-crud.js";
+import { messages } from "../memory/schema.js";
+import { readSlackMetadata } from "../messaging/providers/slack/message-metadata.js";
+import { handleEditIntercept } from "../runtime/routes/inbound-stages/edit-intercept.js";
+
+initializeDb();
+
+function resetTables(): void {
+  const db = getDb();
+  db.run("DELETE FROM channel_inbound_events");
+  db.run("DELETE FROM messages");
+  db.run("DELETE FROM conversation_keys");
+  db.run("DELETE FROM conversations");
+}
+
+interface SeededFixture {
+  conversationId: string;
+  messageId: string;
+  channelTs: string;
+  conversationExternalId: string;
+}
+
+/**
+ * Seed a Slack message in a fresh conversation. Mirrors what the new-message
+ * pipeline does at runtime: `recordInbound` writes the channel_inbound_events
+ * row (storing `sourceMessageId = ts`), `addMessage` writes the user message,
+ * and `linkMessage` connects them so edit lookups succeed.
+ *
+ * Note: the gateway sets `externalMessageId = client_msg_id ?? ts` for new
+ * Slack messages, so this fixture mirrors a message where `client_msg_id`
+ * equals the `ts` (i.e. the simplest case). The lookup mechanism keys on
+ * `sourceMessageId`, which always carries the `ts`, so the test exercises
+ * the same path that production hits regardless of `client_msg_id` presence.
+ */
+async function seedSlackMessage(opts: {
+  conversationExternalId: string;
+  channelTs: string;
+  initialContent: string;
+}): Promise<SeededFixture> {
+  const { conversationExternalId, channelTs, initialContent } = opts;
+
+  const inboundResult = recordInbound(
+    "slack",
+    conversationExternalId,
+    channelTs,
+    {
+      sourceMessageId: channelTs,
+    },
+  );
+
+  const inserted = await addMessage(
+    inboundResult.conversationId,
+    "user",
+    initialContent,
+    { userMessageChannel: "slack" },
+    { skipIndexing: true },
+  );
+
+  linkMessage(inboundResult.eventId, inserted.id);
+
+  return {
+    conversationId: inboundResult.conversationId,
+    messageId: inserted.id,
+    channelTs,
+    conversationExternalId,
+  };
+}
+
+function readMessageRow(messageId: string): {
+  content: string;
+  metadata: string | null;
+} {
+  const db = getDb();
+  const row = db
+    .select({ content: messages.content, metadata: messages.metadata })
+    .from(messages)
+    .where(eq(messages.id, messageId))
+    .get();
+  if (!row) {
+    throw new Error(`message ${messageId} not found`);
+  }
+  return { content: row.content, metadata: row.metadata };
+}
+
+let editEventCounter = 0;
+function nextEditEventId(): string {
+  editEventCounter += 1;
+  return `edit-event-${Date.now()}-${editEventCounter}`;
+}
+
+describe("Slack edit propagation", () => {
+  beforeEach(() => {
+    resetTables();
+    editEventCounter = 0;
+  });
+
+  test("updates content and stamps slackMeta.editedAt when original is found", async () => {
+    const seeded = await seedSlackMessage({
+      conversationExternalId: "C0123CHANNEL",
+      channelTs: "1234.5678",
+      initialContent: "original text",
+    });
+
+    const before = readMessageRow(seeded.messageId);
+    expect(before.content).toBe("original text");
+
+    const t0 = Date.now();
+    const resp = await handleEditIntercept({
+      sourceChannel: "slack",
+      conversationExternalId: seeded.conversationExternalId,
+      externalMessageId: nextEditEventId(),
+      sourceMessageId: seeded.channelTs,
+      canonicalAssistantId: "self",
+      assistantId: "self",
+      content: "new text",
+    });
+
+    expect(resp.status).toBe(200);
+    const respJson = (await resp.json()) as Record<string, unknown>;
+    expect(respJson.accepted).toBe(true);
+    expect(respJson.duplicate).toBe(false);
+
+    const after = readMessageRow(seeded.messageId);
+    expect(after.content).toBe("new text");
+
+    expect(after.metadata).not.toBeNull();
+    const outer = JSON.parse(after.metadata!);
+    expect(outer.userMessageChannel).toBe("slack");
+    expect(typeof outer.slackMeta).toBe("string");
+
+    const slackMeta = readSlackMetadata(outer.slackMeta);
+    expect(slackMeta).not.toBeNull();
+    expect(slackMeta!.source).toBe("slack");
+    expect(slackMeta!.channelId).toBe(seeded.conversationExternalId);
+    expect(slackMeta!.channelTs).toBe(seeded.channelTs);
+    expect(slackMeta!.eventKind).toBe("message");
+    expect(typeof slackMeta!.editedAt).toBe("number");
+    expect(slackMeta!.editedAt!).toBeGreaterThanOrEqual(t0);
+  });
+
+  test("is idempotent across successive edits", async () => {
+    const seeded = await seedSlackMessage({
+      conversationExternalId: "C0123CHANNEL",
+      channelTs: "1234.5678",
+      initialContent: "original text",
+    });
+
+    await handleEditIntercept({
+      sourceChannel: "slack",
+      conversationExternalId: seeded.conversationExternalId,
+      externalMessageId: nextEditEventId(),
+      sourceMessageId: seeded.channelTs,
+      canonicalAssistantId: "self",
+      assistantId: "self",
+      content: "first edit",
+    });
+
+    const afterFirst = readMessageRow(seeded.messageId);
+    expect(afterFirst.content).toBe("first edit");
+    const firstSlackMeta = readSlackMetadata(
+      (JSON.parse(afterFirst.metadata!) as Record<string, unknown>)
+        .slackMeta as string | null,
+    );
+    expect(firstSlackMeta).not.toBeNull();
+    const firstEditedAt = firstSlackMeta!.editedAt!;
+
+    // Ensure the second edit's timestamp is observably after the first so the
+    // assertion below proves the field was re-stamped, not stale.
+    await Bun.sleep(2);
+
+    await handleEditIntercept({
+      sourceChannel: "slack",
+      conversationExternalId: seeded.conversationExternalId,
+      externalMessageId: nextEditEventId(),
+      sourceMessageId: seeded.channelTs,
+      canonicalAssistantId: "self",
+      assistantId: "self",
+      content: "second edit",
+    });
+
+    const afterSecond = readMessageRow(seeded.messageId);
+    expect(afterSecond.content).toBe("second edit");
+    const secondSlackMeta = readSlackMetadata(
+      (JSON.parse(afterSecond.metadata!) as Record<string, unknown>)
+        .slackMeta as string | null,
+    );
+    expect(secondSlackMeta).not.toBeNull();
+    expect(secondSlackMeta!.editedAt!).toBeGreaterThan(firstEditedAt);
+    // Other fields stay stable across edits.
+    expect(secondSlackMeta!.channelId).toBe(seeded.conversationExternalId);
+    expect(secondSlackMeta!.channelTs).toBe(seeded.channelTs);
+    expect(secondSlackMeta!.eventKind).toBe("message");
+  });
+
+  test("no-op edit (identical text, e.g. unfurl) skips DB write", async () => {
+    const seeded = await seedSlackMessage({
+      conversationExternalId: "C0123CHANNEL",
+      channelTs: "1234.5678",
+      initialContent: "original text",
+    });
+
+    const before = readMessageRow(seeded.messageId);
+
+    const resp = await handleEditIntercept({
+      sourceChannel: "slack",
+      conversationExternalId: seeded.conversationExternalId,
+      externalMessageId: nextEditEventId(),
+      sourceMessageId: seeded.channelTs,
+      canonicalAssistantId: "self",
+      assistantId: "self",
+      // Same text as stored -- simulates a Slack unfurl `message_changed`
+      // where only attachments changed.
+      content: "original text",
+    });
+
+    expect(resp.status).toBe(200);
+    const respJson = (await resp.json()) as Record<string, unknown>;
+    expect(respJson.accepted).toBe(true);
+    expect(respJson.duplicate).toBe(false);
+    expect(respJson.noop).toBe(true);
+
+    const after = readMessageRow(seeded.messageId);
+    expect(after.content).toBe(before.content);
+    // No metadata mutation either -- the write is fully skipped.
+    expect(after.metadata).toBe(before.metadata);
+  });
+
+  // The lookup retries 5 times with 2s backoff (~10s total) before giving up,
+  // so this test legitimately needs to outrun the default 5s per-test timeout.
+  test("missing-target edit is a no-op (no throw, no row changed)", async () => {
+    const seeded = await seedSlackMessage({
+      conversationExternalId: "C0123CHANNEL",
+      channelTs: "1234.5678",
+      initialContent: "original text",
+    });
+    const beforeUnknown = readMessageRow(seeded.messageId);
+
+    const resp = await handleEditIntercept({
+      sourceChannel: "slack",
+      conversationExternalId: seeded.conversationExternalId,
+      // sourceMessageId points at a ts that was never stored.
+      externalMessageId: nextEditEventId(),
+      sourceMessageId: "9999.0000",
+      canonicalAssistantId: "self",
+      assistantId: "self",
+      content: "new text",
+    });
+
+    expect(resp.status).toBe(200);
+    const respJson = (await resp.json()) as Record<string, unknown>;
+    expect(respJson.accepted).toBe(true);
+    expect(respJson.duplicate).toBe(false);
+
+    const afterUnknown = readMessageRow(seeded.messageId);
+    expect(afterUnknown.content).toBe(beforeUnknown.content);
+    expect(afterUnknown.metadata).toBe(beforeUnknown.metadata);
+  }, 30_000);
+});

package/src/__tests__/ephemeral-permissions.test.ts

@@ -83,7 +83,7 @@ describe("ephemeral-permissions", () => {
       expect(fileReadRule.createdAt).toBeGreaterThan(0);
 
       // Ephemeral task rules should not auto-allow high-risk tools
-      expect(fileReadRule.allowHighRisk).toBeUndefined();
+      // allowHighRisk is no longer a field on rules
 
       // Check other rules have correct tool names
       expect(rules[1].tool).toBe("bash");
@@ -276,7 +276,7 @@ describe("ephemeral-permissions", () => {
       expect(result.decision).toBe("allow");
     });
 
-    test("high-risk tool still prompts even with ephemeral allow rule (no allowHighRisk)", async () => {
+    test("high-risk tool still prompts even with ephemeral allow rule", async () => {
       const ephemeralRules: TrustRule[] = [
         {
           id: "ephemeral:run-1:bash",
@@ -286,7 +286,7 @@ describe("ephemeral-permissions", () => {
           decision: "allow",
           priority: 50,
           createdAt: Date.now(),
-          // Note: allowHighRisk is NOT set
+          // allowHighRisk is no longer a field
         },
       ];
 
@@ -306,6 +306,96 @@ describe("ephemeral-permissions", () => {
     });
   });
 
+  // ── Canonical shape and scope fallback semantics ──────────────────
+  //
+  // Validates that ephemeral rules follow canonical persisted shapes
+  // and that optional scope behaves correctly as a fallback.
+
+  describe("canonical shape and scope fallback semantics", () => {
+    beforeEach(() => {
+      clearCache();
+    });
+
+    test("ephemeral rule with scope 'everywhere' matches from any working directory", () => {
+      // Use a tool (host_file_read) that has no default allow rule and a
+      // high priority so the ephemeral rule wins over the default ask rule.
+      const ephemeralRules: TrustRule[] = [
+        {
+          id: "ephemeral:run-scope:host_file_read",
+          tool: "host_file_read",
+          pattern: "**",
+          scope: "everywhere",
+          decision: "allow",
+          priority: 2000,
+          createdAt: Date.now(),
+        },
+      ];
+
+      const ctx: PolicyContext = { ephemeralRules };
+
+      // Should match from /tmp
+      const r1 = findHighestPriorityRule(
+        "host_file_read",
+        ["host_file_read:/tmp/foo.txt"],
+        "/tmp",
+        ctx,
+      );
+      expect(r1).not.toBeNull();
+      expect(r1!.id).toBe("ephemeral:run-scope:host_file_read");
+
+      // Should match from /home/user/project
+      const r2 = findHighestPriorityRule(
+        "host_file_read",
+        ["host_file_read:/home/user/project/bar.ts"],
+        "/home/user/project",
+        ctx,
+      );
+      expect(r2).not.toBeNull();
+      expect(r2!.id).toBe("ephemeral:run-scope:host_file_read");
+    });
+
+    test("ephemeral rule does not carry stray metadata fields for non-scoped tools", () => {
+      // buildTaskRules generates canonical ephemeral rules.
+      // For a tool like file_read (scoped family), the generated rule
+      // should have no executionTarget.
+      const rules = buildTaskRules("run-canon", ["file_read", "bash"], "/tmp");
+
+      for (const rule of rules) {
+        expect(rule.scope).toBe("everywhere");
+        expect(rule.executionTarget).toBeUndefined();
+      }
+    });
+
+    test("ephemeral rule with project scope does not match sibling project", () => {
+      const ephemeralRules: TrustRule[] = [
+        {
+          id: "ephemeral:run-proj:file_write",
+          tool: "file_write",
+          pattern: "**",
+          scope: "/home/user/project-a",
+          decision: "allow",
+          priority: 75,
+          createdAt: Date.now(),
+        },
+      ];
+
+      const ctx: PolicyContext = { ephemeralRules };
+
+      // Should NOT match from a sibling directory
+      const result = findHighestPriorityRule(
+        "file_write",
+        ["file_write:/home/user/project-b/file.ts"],
+        "/home/user/project-b",
+        ctx,
+      );
+
+      // The ephemeral rule should not be the match (scope mismatch)
+      if (result) {
+        expect(result.id).not.toBe("ephemeral:run-proj:file_write");
+      }
+    });
+  });
+
   describe("workspace mode interactions", () => {
     beforeEach(() => {
       clearCache();

package/src/__tests__/estimator-calibration-integration.test.ts

@@ -0,0 +1,208 @@
+import { beforeEach, describe, expect, test } from "bun:test";
+
+import {
+  getCalibrationSnapshot,
+  getCorrection,
+  recordEstimate,
+  resetCalibrations,
+} from "../context/estimator-calibration.js";
+import {
+  estimatePromptTokens,
+  getCalibrationProviderKey,
+} from "../context/token-estimator.js";
+import type { Message, Provider } from "../providers/types.js";
+
+/**
+ * Integration-style tests that exercise the full self-calibration loop end
+ * to end:
+ *   1. Estimate is recorded for a `(provider, model)` pair via
+ *      `handleUsage` (the record side still threads the provider-echoed
+ *      model through `recordEstimate`).
+ *   2. A subsequent `estimatePromptTokens` lookup picks up the learned
+ *      correction via the per-provider aggregate key `(provider, "")`.
+ *      Lookup always uses the aggregate — model-specific keys are only
+ *      read as a fallback inside `getCorrection`.
+ *
+ * Since the `modelId` lookup option has been removed from the public
+ * token-estimator API, the lookup side always converges to the aggregate.
+ * `recordEstimate` still updates both the specific `(provider, model)`
+ * key AND the `(provider, "")` aggregate on every sample, so the
+ * aggregate stays accurate even as per-model data accumulates.
+ */
+describe("estimator calibration — end-to-end recording → lookup", () => {
+  beforeEach(() => {
+    resetCalibrations();
+  });
+
+  /**
+   * Build a representative message history with enough content to clear the
+   * MIN_SAMPLE_MAGNITUDE floor (500 tokens). Each message repeats a block of
+   * text large enough to make the heuristic estimator produce a substantial
+   * token count so the calibration machinery actually runs.
+   */
+  function largeHistory(): Message[] {
+    const body = "lorem ipsum dolor sit amet ".repeat(500);
+    return [
+      { role: "user", content: [{ type: "text", text: body }] },
+      { role: "assistant", content: [{ type: "text", text: body }] },
+      { role: "user", content: [{ type: "text", text: body }] },
+    ];
+  }
+
+  test("subsequent estimate picks up the aggregate-key correction", () => {
+    const provider: Provider = {
+      name: "anthropic",
+      async sendMessage() {
+        throw new Error("not used in this test");
+      },
+    };
+    const model = "claude-sonnet-4-5";
+    const history = largeHistory();
+
+    // 1. Raw estimate (what agent/loop.ts computes pre-send).
+    const preSend = estimatePromptTokens(history, "system", {
+      providerName: getCalibrationProviderKey(provider),
+    });
+    expect(preSend).toBeGreaterThan(0);
+
+    // Baseline: no correction recorded yet.
+    expect(getCorrection("anthropic", "")).toBe(1.0);
+
+    // 2. Provider returns ground truth (simulating `handleUsage`, which
+    //    still records under (provider, event.model) and folds into the
+    //    aggregate). Simulate a systematic 30% underestimate.
+    const groundTruth = Math.ceil(preSend * 1.3);
+    recordEstimate(
+      getCalibrationProviderKey(provider),
+      model,
+      preSend,
+      groundTruth,
+    );
+
+    // 3. Lookup under the aggregate key now returns the learned ratio.
+    expect(getCorrection("anthropic", "")).toBeCloseTo(1.3, 3);
+
+    // And the corrected estimate moves toward the ground truth.
+    const corrected = estimatePromptTokens(history, "system", {
+      providerName: getCalibrationProviderKey(provider),
+    });
+    // With correction factor ≈1.3, corrected estimate is within 1 token of
+    // the ground truth (Math.ceil rounding).
+    expect(corrected).toBeGreaterThan(preSend);
+    expect(Math.abs(corrected - groundTruth)).toBeLessThanOrEqual(1);
+  });
+
+  test("record with model writes both the specific and aggregate keys", () => {
+    // Simulate a preflight site that records against (anthropic, sonnet).
+    // `recordEstimate` also folds the sample into the `(anthropic, "")`
+    // aggregate so aggregate-key callers see the correction.
+    const provider: Provider = {
+      name: "anthropic",
+      async sendMessage() {
+        throw new Error("not used");
+      },
+    };
+    const history = largeHistory();
+
+    const preSend = estimatePromptTokens(history, "system", {
+      providerName: getCalibrationProviderKey(provider),
+    });
+    const groundTruth = Math.ceil(preSend * 1.25);
+
+    recordEstimate(
+      getCalibrationProviderKey(provider),
+      "claude-sonnet-4-5",
+      preSend,
+      groundTruth,
+    );
+
+    // A subsequent lookup via the token-estimator uses the per-provider
+    // aggregate (the only key the public API reads).
+    const correctedAggregate = estimatePromptTokens(history, "system", {
+      providerName: getCalibrationProviderKey(provider),
+    });
+    // Aggregate ratio ≈ 1.25 (first sample snaps to exact ratio).
+    expect(correctedAggregate).toBe(Math.ceil(preSend * 1.25));
+  });
+
+  test("wrapper provider (OpenRouter → Anthropic) uses the canonical key on both sides", () => {
+    // This is the Devin scenario: OpenRouter wraps Anthropic. If the record
+    // site used `name` ("openrouter") and the lookup site used
+    // `tokenEstimationProvider` ("anthropic"), the data would be scattered
+    // across mismatched keys and calibration would silently fail.
+    // `getCalibrationProviderKey` gives us one source of truth.
+    const openrouter: Provider = {
+      name: "openrouter",
+      tokenEstimationProvider: "anthropic",
+      async sendMessage() {
+        throw new Error("not used");
+      },
+    };
+    const model = "anthropic/claude-sonnet-4-5";
+    const history = largeHistory();
+
+    // Pre-send estimate via the canonical key.
+    const preSend = estimatePromptTokens(history, "system", {
+      providerName: getCalibrationProviderKey(openrouter),
+    });
+    expect(preSend).toBeGreaterThan(0);
+
+    // Provider returns ground truth. `handleUsage` uses the same helper
+    // to pick the calibration key, so the record and lookup sides agree.
+    const groundTruth = Math.ceil(preSend * 1.2);
+    recordEstimate(
+      getCalibrationProviderKey(openrouter),
+      model,
+      preSend,
+      groundTruth,
+    );
+
+    // Lookup under "anthropic" — the canonical upstream key — returns the
+    // ratio. See note above about precision=3.
+    expect(getCorrection("anthropic", model)).toBeCloseTo(1.2, 3);
+    // Aggregate under the canonical upstream key is also populated.
+    expect(getCorrection("anthropic", "")).toBeCloseTo(1.2, 3);
+    // And under the bare wrapper name stays at the default, because NOTHING
+    // was recorded under "openrouter".
+    expect(getCorrection("openrouter", "")).toBe(1.0);
+
+    // The snapshot reflects a single (provider, model) key + aggregate under
+    // the canonical upstream key — never under the wrapper name.
+    const keys = getCalibrationSnapshot().map(
+      (e) => `${e.provider}::${e.model}`,
+    );
+    expect(keys).toContain(`anthropic::${model}`);
+    expect(keys).toContain("anthropic::");
+    expect(keys).not.toContain(`openrouter::${model}`);
+  });
+
+  test("a run of consistent samples pulls the estimate toward ground truth", () => {
+    // The EWMA should converge quickly. After five consistent 1.3 samples
+    // the correction should be within 1% of 1.3, and the corrected estimate
+    // should be within 1% of the ground truth.
+    const model = "claude-sonnet-4-5";
+    const history = largeHistory();
+
+    const preSend = estimatePromptTokens(history, "system", {
+      providerName: "anthropic",
+    });
+    const groundTruth = Math.ceil(preSend * 1.3);
+
+    for (let i = 0; i < 5; i++) {
+      recordEstimate("anthropic", model, preSend, groundTruth);
+    }
+
+    const finalCorrection = getCorrection("anthropic", "");
+    // EWMA with alpha=0.2 on constant 1.3 stays at 1.3 from the first sample
+    // onward (all deltas are 0 after the initial snap). `precision=3` gives
+    // us ~0.0005 tolerance which covers the Math.ceil rounding noise.
+    expect(finalCorrection).toBeCloseTo(1.3, 3);
+
+    const corrected = estimatePromptTokens(history, "system", {
+      providerName: "anthropic",
+    });
+    // Corrected should be very close to the ground truth (within 1 token
+    // because of the Math.ceil rounding at the end of estimatePromptTokens).
+    expect(Math.abs(corrected - groundTruth)).toBeLessThanOrEqual(1);
+  });
+});