@vellumai/assistant 0.12.1 → 0.12.2-staging.1

package/src/__tests__/resolve-app.test.ts

@@ -0,0 +1,135 @@
+/**
+ * Tests for resolving an app query by name, slug, or id.
+ */
+
+import { mkdirSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { afterEach, beforeEach, describe, expect, test } from "bun:test";
+
+import { createApp } from "../apps/app-store.js";
+import { resolveAppQuery } from "../apps/resolve-app.js";
+import { getWorkspacePluginsDir } from "../util/platform.js";
+
+let workspaceDir: string;
+let previousWorkspaceDir: string | undefined;
+
+function freshWorkspace(): string {
+  return join(
+    tmpdir(),
+    `vellum-resolve-app-${Date.now()}-${Math.random().toString(36).slice(2)}`,
+  );
+}
+
+function installPlugin(name: string): string {
+  const pluginDir = join(getWorkspacePluginsDir(), name);
+  mkdirSync(pluginDir, { recursive: true });
+  writeFileSync(
+    join(pluginDir, "package.json"),
+    JSON.stringify({ name, version: "1.0.0" }),
+  );
+  return pluginDir;
+}
+
+function bundleApp(pluginDir: string, app: string): void {
+  const appDir = join(pluginDir, "apps", app);
+  mkdirSync(appDir, { recursive: true });
+  writeFileSync(join(appDir, "index.html"), "<h1>Plugin app</h1>");
+}
+
+beforeEach(() => {
+  previousWorkspaceDir = process.env.VELLUM_WORKSPACE_DIR;
+  workspaceDir = freshWorkspace();
+  process.env.VELLUM_WORKSPACE_DIR = workspaceDir;
+});
+
+afterEach(() => {
+  rmSync(workspaceDir, { recursive: true, force: true });
+  if (previousWorkspaceDir === undefined) {
+    delete process.env.VELLUM_WORKSPACE_DIR;
+  } else {
+    process.env.VELLUM_WORKSPACE_DIR = previousWorkspaceDir;
+  }
+});
+
+describe("resolveAppQuery", () => {
+  test("resolves a workspace app by display name, slug, and id", () => {
+    const created = createApp({
+      name: "Budget",
+      schemaJson: "{}",
+      htmlDefinition: "<h1>Budget</h1>",
+    });
+
+    const byName = resolveAppQuery("Budget");
+    expect(byName.ok).toBe(true);
+    if (byName.ok) {
+      expect(byName.app.id).toBe(created.id);
+    }
+
+    const byNameCase = resolveAppQuery("budget");
+    expect(byNameCase.ok).toBe(true);
+    if (byNameCase.ok) {
+      expect(byNameCase.app.id).toBe(created.id);
+    }
+
+    const slug = created.dirName ?? "";
+    expect(slug.length).toBeGreaterThan(0);
+    const bySlug = resolveAppQuery(slug);
+    expect(bySlug.ok).toBe(true);
+    if (bySlug.ok) {
+      expect(bySlug.app.id).toBe(created.id);
+    }
+
+    const byId = resolveAppQuery(created.id);
+    expect(byId.ok).toBe(true);
+    if (byId.ok) {
+      expect(byId.app.id).toBe(created.id);
+    }
+  });
+
+  test("returns not_found for an unknown name", () => {
+    const result = resolveAppQuery("missing");
+    expect(result).toEqual({
+      ok: false,
+      reason: "not_found",
+      query: "missing",
+    });
+  });
+
+  test("returns ambiguous when two apps share a display name", () => {
+    const first = createApp({
+      name: "Notes",
+      schemaJson: "{}",
+      htmlDefinition: "<h1>One</h1>",
+    });
+    const second = createApp({
+      name: "Notes",
+      schemaJson: "{}",
+      htmlDefinition: "<h1>Two</h1>",
+    });
+
+    const result = resolveAppQuery("Notes");
+    expect(result.ok).toBe(false);
+    if (!result.ok && result.reason === "ambiguous") {
+      const ids = result.matches.map((m) => m.id).sort();
+      expect(ids).toEqual([first.id, second.id].sort());
+    } else {
+      throw new Error("expected ambiguous resolution");
+    }
+  });
+
+  test("resolves a plugin app by directory name", () => {
+    const pluginDir = installPlugin("charts");
+    bundleApp(pluginDir, "viewer");
+
+    const result = resolveAppQuery("viewer");
+    expect(result.ok).toBe(true);
+    if (result.ok) {
+      expect(result.app.origin).toEqual({
+        kind: "plugin",
+        pluginName: "charts",
+      });
+      expect(result.app.id).toBe("plugins~charts~viewer");
+    }
+  });
+});

package/src/__tests__/scaffold-managed-skill-tool.test.ts

@@ -1103,6 +1103,7 @@ describe("scaffold_managed_skill tool", () => {
         activation_hints: HINTS,
         overwrite: true,
         files: [{ path: "references/failure-modes.md", content: "gotchas" }],
+        change_summary: "Added the failure modes reference.",
       },
       makeRetrospectiveContext(),
     );
@@ -1550,6 +1551,7 @@ describe("scaffold_managed_skill tool", () => {
         body_markdown: "V2 procedure.",
         activation_hints: HINTS,
         overwrite: true,
+        change_summary: "Refined the procedure.",
       },
       makeRetrospectiveContext({ conversationId: "retro-run-conv" }),
       lineageSeam(),
@@ -1702,6 +1704,7 @@ describe("background skill update notification", () => {
         body_markdown: "1. Refined steps.",
         activation_hints: HINTS,
         overwrite: true,
+        change_summary: "Refined the steps.",
       },
       makeRetrospectiveContext({ conversationId: "retro-run-conv" }),
       lineage(),
@@ -1715,9 +1718,7 @@ describe("background skill update notification", () => {
     // must be the conversation the work came from, not the hidden fork.
     expect(signal.sourceContextId).toBe("source-conv");
     expect(signal.contextPayload?.skillId).toBe("weekly-export");
-    expect(String(signal.contextPayload?.summary)).toContain(
-      "Weekly Report Export",
-    );
+    expect(signal.contextPayload?.summary).toBe("Refined the steps.");
     // The home feed falls back to `title`/`body` when no channel copy was
     // rendered, which is the intended quiet shape for this signal. Without
     // them a suppressed delivery would leave the feed item unwritten.
@@ -1725,15 +1726,236 @@ describe("background skill update notification", () => {
     expect(signal.contextPayload?.title).toBe(
       "Skill updated: Weekly Report Export",
     );
-    expect(String(signal.contextPayload?.body)).toContain(
-      "Weekly Report Export",
-    );
+    expect(signal.contextPayload?.body).toBe("Refined the steps.");
     // Deduped per skill per day so repeated refinements cannot flood the feed.
     expect(signal.dedupeKey).toBe(
       `skill-updated:weekly-export:${new Date().toISOString().slice(0, 10)}`,
     );
   });
 
+  test("the notice body is the pass's change_summary, so the feed says what changed", async () => {
+    await seedAssistantSkill("weekly-export", "Old body.");
+
+    const result = await executeScaffoldManagedSkill(
+      {
+        skill_id: "weekly-export",
+        name: "Weekly Report Export",
+        description: "export the weekly usage report",
+        body_markdown: "1. Refined steps.",
+        activation_hints: HINTS,
+        overwrite: true,
+        change_summary:
+          "Added the retry after an expired session and the export endpoint that held steady.",
+      },
+      makeRetrospectiveContext({ conversationId: "retro-run-conv" }),
+      lineage(),
+    );
+
+    expect(result.isError).toBe(false);
+    expect(emittedSignals).toHaveLength(1);
+    const payload = emittedSignals[0]!.contextPayload;
+    // The title still names the skill; the body is the change itself.
+    expect(payload?.title).toBe("Skill updated: Weekly Report Export");
+    expect(payload?.body).toBe(
+      "Added the retry after an expired session and the export endpoint that held steady.",
+    );
+    expect(payload?.summary).toBe(payload?.body);
+  });
+
+  test("a background overwrite without a change_summary is refused before the write", async () => {
+    await seedAssistantSkill("weekly-export", "Old body.");
+    const skillFile = join(TEST_DIR, "skills", "weekly-export", "SKILL.md");
+    const before = readFileSync(skillFile, "utf-8");
+
+    for (const input of [
+      {},
+      // Whitespace-only reads as missing, not as an empty notice.
+      { change_summary: "   \n " },
+    ]) {
+      const result = await executeScaffoldManagedSkill(
+        {
+          skill_id: "weekly-export",
+          name: "Weekly Report Export",
+          description: "export the weekly usage report",
+          body_markdown: "1. Refined steps.",
+          activation_hints: HINTS,
+          overwrite: true,
+          ...input,
+        },
+        makeRetrospectiveContext({ conversationId: "retro-run-conv" }),
+        lineage(),
+      );
+
+      // Self-correcting: the error returns to the pass in the same turn, so
+      // the retry carries the field and nothing was lost meanwhile.
+      expect(result.isError).toBe(true);
+      expect(result.content).toBe(
+        'Error: change_summary is required when updating an existing skill: pass one or two short sentences (under 200 characters) for the person who reads the "Skill updated" notice, naming what you changed and what in the trace prompted it (for example "Added the retry after an expired session and the export endpoint that held steady.").',
+      );
+      expect(readFileSync(skillFile, "utf-8")).toBe(before);
+      expect(emittedSignals).toHaveLength(0);
+    }
+  });
+
+  test("a background call on an existing skill without overwrite hears the overwrite error, not the summary requirement", async () => {
+    await seedAssistantSkill("weekly-export", "Old body.");
+
+    const result = await executeScaffoldManagedSkill(
+      {
+        skill_id: "weekly-export",
+        name: "Weekly Report Export",
+        description: "export the weekly usage report",
+        body_markdown: "1. Refined steps.",
+        activation_hints: HINTS,
+      },
+      makeRetrospectiveContext({ conversationId: "retro-run-conv" }),
+      lineage(),
+    );
+
+    // The flag is what actually blocks this call, so that is the error it
+    // gets: a retry that only adds change_summary would still fail.
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("Set overwrite=true to replace it.");
+    expect(result.content).not.toContain("change_summary");
+    expect(emittedSignals).toHaveLength(0);
+  });
+
+  test("the ownership backstop speaks before the change_summary requirement", async () => {
+    await executeScaffoldManagedSkill(
+      {
+        skill_id: "user-owned",
+        name: "User Owned",
+        description: "a person wrote this",
+        body_markdown: "V1.",
+        activation_hints: HINTS,
+      },
+      makeContext(),
+    );
+
+    const result = await executeScaffoldManagedSkill(
+      {
+        skill_id: "user-owned",
+        name: "User Owned",
+        description: "a person wrote this",
+        body_markdown: "V2.",
+        activation_hints: HINTS,
+        overwrite: true,
+      },
+      makeRetrospectiveContext({ conversationId: "retro-run-conv" }),
+      lineage(),
+    );
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toContain("not verifiably assistant-authored");
+  });
+
+  test("a user-directed overwrite needs no change_summary: it never notifies", async () => {
+    await executeScaffoldManagedSkill(
+      {
+        skill_id: "user-skill",
+        name: "User Skill",
+        description: "asked for",
+        body_markdown: "V1.",
+        activation_hints: HINTS,
+      },
+      makeContext(),
+    );
+
+    const result = await executeScaffoldManagedSkill(
+      {
+        skill_id: "user-skill",
+        name: "User Skill",
+        description: "asked for",
+        body_markdown: "V2.",
+        activation_hints: HINTS,
+        overwrite: true,
+      },
+      makeContext(),
+    );
+
+    expect(result.isError).toBe(false);
+  });
+
+  test("change_summary is sanitized like any other notification body", async () => {
+    await seedAssistantSkill("weekly-export", "Old body.");
+
+    await executeScaffoldManagedSkill(
+      {
+        skill_id: "weekly-export",
+        name: "Weekly Report Export",
+        description: "export the weekly usage report",
+        body_markdown: "1. Refined steps.",
+        activation_hints: HINTS,
+        overwrite: true,
+        change_summary: `Added\u0007 the   retry.\r\n\n\n\nDropped the login step. ${"x".repeat(300)}`,
+      },
+      makeRetrospectiveContext({ conversationId: "retro-run-conv" }),
+      lineage(),
+    );
+
+    const body = String(emittedSignals[0]!.contextPayload?.body);
+    // Control characters go, horizontal whitespace collapses, blank-line runs
+    // collapse to one paragraph break, and the whole thing is clamped to the
+    // shared notification preview budget.
+    expect(
+      body.startsWith("Added the retry.\n\nDropped the login step. xxx"),
+    ).toBe(true);
+    expect(body).not.toMatch(/[\u0000-\u0008\u000b\u000c\u000e-\u001f]/);
+    expect(body.length).toBe(200);
+    expect(body.endsWith("\u2026")).toBe(true);
+  });
+
+  test("a non-string change_summary is a self-correcting error", async () => {
+    await seedAssistantSkill("weekly-export", "Old body.");
+
+    const result = await executeScaffoldManagedSkill(
+      {
+        skill_id: "weekly-export",
+        name: "Weekly Report Export",
+        description: "export the weekly usage report",
+        body_markdown: "1. Refined steps.",
+        activation_hints: HINTS,
+        overwrite: true,
+        change_summary: ["not", "a", "string"],
+      },
+      makeRetrospectiveContext({ conversationId: "retro-run-conv" }),
+      lineage(),
+    );
+
+    expect(result.isError).toBe(true);
+    expect(result.content).toBe("Error: change_summary must be a string");
+    expect(emittedSignals).toHaveLength(0);
+  });
+
+  test("the registered tool's schema declares change_summary as an optional string", () => {
+    // A call through the registered tool is validated against TOOLS.json
+    // first, and the validator rejects undeclared keys, so the field has to
+    // be in the schema for the executor to ever see it. It stays optional
+    // there: the requirement holds only for a background overwrite of an
+    // existing skill, which the schema cannot express, so the executor
+    // enforces it.
+    const scaffoldTool = readScaffoldToolEntry();
+    expect(scaffoldTool.input_schema.properties.change_summary).toMatchObject({
+      type: "string",
+    });
+    expect(scaffoldTool.input_schema.required).not.toContain("change_summary");
+
+    const result = validateInputAgainstSchema(
+      "scaffold_managed_skill",
+      {
+        skill_id: "s",
+        name: "N",
+        description: "D",
+        body_markdown: "B",
+        activation_hints: HINTS,
+        overwrite: true,
+        change_summary: "Added the retry step.",
+      },
+      scaffoldTool.input_schema,
+    );
+    expect(result.ok).toBe(true);
+  });
+
   test("it falls back to the run conversation when fork lineage does not resolve", async () => {
     await seedAssistantSkill("weekly-export", "Old body.");
 
@@ -1745,6 +1967,7 @@ describe("background skill update notification", () => {
         body_markdown: "1. Refined steps.",
         activation_hints: HINTS,
         overwrite: true,
+        change_summary: "Refined the steps.",
       },
       makeRetrospectiveContext({ conversationId: "retro-run-conv" }),
       { getConversation: () => null },

package/src/__tests__/script-proxy-certs.test.ts

@@ -8,6 +8,7 @@ import {
   getCAPath,
   issueLeafCert,
 } from "../outbound-proxy/index.js";
+import { writeCombinedCABundle } from "../util/ca-bundle.js";
 
 let dataDir: string;
 
@@ -19,6 +20,40 @@ afterAll(async () => {
   await rm(dataDir, { recursive: true, force: true });
 });
 
+test("combined trust verifies certificates from both CA sources", async () => {
+  const roots = [join(dataDir, "system"), join(dataDir, "extra")];
+  const leaves: string[] = [];
+  for (const root of roots) {
+    await ensureLocalCA(root);
+    const caDir = join(root, "proxy-ca");
+    await issueLeafCert(caDir, "example.com");
+    leaves.push(join(caDir, "issued", "example.com.pem"));
+  }
+  const bundle = join(dataDir, "combined.pem");
+  await writeCombinedCABundle(
+    getCAPath(roots[0]!),
+    getCAPath(roots[1]!),
+    bundle,
+  );
+  const verify = Bun.spawn(
+    ["openssl", "verify", "-CAfile", bundle, ...leaves],
+    {
+      stdout: "pipe",
+      stderr: "pipe",
+      windowsHide: true,
+    },
+  );
+  const [code, output, error] = await Promise.all([
+    verify.exited,
+    new Response(verify.stdout).text(),
+    new Response(verify.stderr).text(),
+  ]);
+  expect({ code, error }).toEqual({ code: 0, error: "" });
+  for (const leaf of leaves) {
+    expect(output).toContain(`${leaf}: OK`);
+  }
+});
+
 describe("ensureLocalCA", () => {
   test("creates CA cert and key with correct permissions", async () => {
     await ensureLocalCA(dataDir);