@vellumai/assistant 0.4.10 → 0.4.12

package/src/__tests__/guardian-verify-setup-skill-regression.test.ts

@@ -5,113 +5,128 @@
  * so the user does not have to manually ask whether verification succeeded.
  */
 
-import { readFileSync } from 'node:fs';
-import { resolve } from 'node:path';
+import { readFileSync } from "node:fs";
+import { resolve } from "node:path";
 
-import { describe, expect, test } from 'bun:test';
+import { describe, expect, test } from "bun:test";
 
 // ---------------------------------------------------------------------------
 // Locate the skill SKILL.md
 // ---------------------------------------------------------------------------
 
-const ASSISTANT_DIR = resolve(import.meta.dirname ?? __dirname, '..', '..');
+const ASSISTANT_DIR = resolve(import.meta.dirname ?? __dirname, "..", "..");
 const SKILL_PATH = resolve(
   ASSISTANT_DIR,
-  'src',
-  'config',
-  'vellum-skills',
-  'guardian-verify-setup',
-  'SKILL.md',
+  "src",
+  "config",
+  "bundled-skills",
+  "guardian-verify-setup",
+  "SKILL.md",
 );
 
-const skillContent = readFileSync(SKILL_PATH, 'utf-8');
+const skillContent = readFileSync(SKILL_PATH, "utf-8");
 
 // ---------------------------------------------------------------------------
 // Tests
 // ---------------------------------------------------------------------------
 
-describe('guardian-verify-setup skill — voice auto-followup', () => {
-  test('voice path in Step 3 references the auto-check polling loop', () => {
+describe("guardian-verify-setup skill — voice auto-followup", () => {
+  test("voice path in Step 3 references the auto-check polling loop", () => {
     // The voice success instruction in Step 3 must direct the assistant to
     // begin the polling loop rather than waiting for the user to report back.
     expect(skillContent).toContain(
-      'immediately begin the voice auto-check polling loop',
+      "immediately begin the voice auto-check polling loop",
     );
   });
 
-  test('voice path in Step 4 (resend) references the auto-check polling loop', () => {
+  test("voice path in Step 4 (resend) references the auto-check polling loop", () => {
     // After a voice resend, the same auto-check behavior must kick in.
-    const resendSection = skillContent.split('## Step 4')[1]?.split('## Step 5')[0] ?? '';
-    expect(resendSection).toContain(
-      'voice auto-check polling loop',
-    );
+    const resendSection =
+      skillContent.split("## Step 4")[1]?.split("## Step 5")[0] ?? "";
+    expect(resendSection).toContain("voice auto-check polling loop");
   });
 
-  test('contains a Voice Auto-Check Polling section', () => {
-    expect(skillContent).toContain('## Voice Auto-Check Polling');
+  test("contains a Voice Auto-Check Polling section", () => {
+    expect(skillContent).toContain("## Voice Auto-Check Polling");
   });
 
-  test('polling section specifies the correct status endpoint for voice', () => {
+  test("polling section specifies the correct status endpoint for voice", () => {
     const pollingSection =
-      skillContent.split('## Voice Auto-Check Polling')[1]?.split('## Step 6')[0] ?? '';
+      skillContent
+        .split("## Voice Auto-Check Polling")[1]
+        ?.split("## Step 6")[0] ?? "";
     expect(pollingSection).toContain(
-      '/v1/integrations/guardian/status?channel=voice',
+      "/v1/integrations/guardian/status?channel=voice",
     );
   });
 
-  test('polling section includes ~15 second interval', () => {
+  test("polling section includes ~15 second interval", () => {
     const pollingSection =
-      skillContent.split('## Voice Auto-Check Polling')[1]?.split('## Step 6')[0] ?? '';
-    expect(pollingSection).toContain('~15 seconds');
+      skillContent
+        .split("## Voice Auto-Check Polling")[1]
+        ?.split("## Step 6")[0] ?? "";
+    expect(pollingSection).toContain("~15 seconds");
   });
 
-  test('polling section includes 2-minute timeout', () => {
+  test("polling section includes 2-minute timeout", () => {
     const pollingSection =
-      skillContent.split('## Voice Auto-Check Polling')[1]?.split('## Step 6')[0] ?? '';
-    expect(pollingSection).toContain('2 minutes');
+      skillContent
+        .split("## Voice Auto-Check Polling")[1]
+        ?.split("## Step 6")[0] ?? "";
+    expect(pollingSection).toContain("2 minutes");
   });
 
-  test('polling section checks for bound: true', () => {
+  test("polling section checks for bound: true", () => {
     const pollingSection =
-      skillContent.split('## Voice Auto-Check Polling')[1]?.split('## Step 6')[0] ?? '';
-    expect(pollingSection).toContain('bound: true');
+      skillContent
+        .split("## Voice Auto-Check Polling")[1]
+        ?.split("## Step 6")[0] ?? "";
+    expect(pollingSection).toContain("bound: true");
   });
 
-  test('polling section includes proactive success confirmation', () => {
+  test("polling section includes proactive success confirmation", () => {
     const pollingSection =
-      skillContent.split('## Voice Auto-Check Polling')[1]?.split('## Step 6')[0] ?? '';
-    expect(pollingSection).toContain('proactive success message');
+      skillContent
+        .split("## Voice Auto-Check Polling")[1]
+        ?.split("## Step 6")[0] ?? "";
+    expect(pollingSection).toContain("proactive success message");
   });
 
-  test('polling section includes timeout fallback with resend/restart offer', () => {
+  test("polling section includes timeout fallback with resend/restart offer", () => {
     const pollingSection =
-      skillContent.split('## Voice Auto-Check Polling')[1]?.split('## Step 6')[0] ?? '';
-    expect(pollingSection).toContain('timeout');
-    expect(pollingSection).toContain('resend');
+      skillContent
+        .split("## Voice Auto-Check Polling")[1]
+        ?.split("## Step 6")[0] ?? "";
+    expect(pollingSection).toContain("timeout");
+    expect(pollingSection).toContain("resend");
   });
 
-  test('polling section includes rebind guard against false-success from pre-existing binding', () => {
+  test("polling section includes rebind guard against false-success from pre-existing binding", () => {
     const pollingSection =
-      skillContent.split('## Voice Auto-Check Polling')[1]?.split('## Step 6')[0] ?? '';
+      skillContent
+        .split("## Voice Auto-Check Polling")[1]
+        ?.split("## Step 6")[0] ?? "";
     // Must mention rebind guard concept
-    expect(pollingSection).toContain('Rebind guard');
+    expect(pollingSection).toContain("Rebind guard");
     // Must instruct not to trust the first bound: true in a rebind flow
     expect(pollingSection).toContain(
-      'do NOT treat the first `bound: true` poll result as success',
+      "do NOT treat the first `bound: true` poll result as success",
     );
     // Must reference bound_at timestamp comparison as the primary mechanism
-    expect(pollingSection).toContain('bound_at');
+    expect(pollingSection).toContain("bound_at");
     // Must have a fallback for when bound_at is unavailable
-    expect(pollingSection).toContain('second poll onward');
+    expect(pollingSection).toContain("second poll onward");
     // Must clarify non-rebind flows are unaffected
-    expect(pollingSection).toContain('Non-rebind flows');
+    expect(pollingSection).toContain("Non-rebind flows");
   });
 
-  test('polling is voice-only — does not apply to SMS or Telegram', () => {
+  test("polling is voice-only — does not apply to SMS or Telegram", () => {
     const pollingSection =
-      skillContent.split('## Voice Auto-Check Polling')[1]?.split('## Step 6')[0] ?? '';
-    expect(pollingSection).toContain('voice-only');
-    expect(pollingSection).toContain('Do NOT poll for SMS or Telegram');
+      skillContent
+        .split("## Voice Auto-Check Polling")[1]
+        ?.split("## Step 6")[0] ?? "";
+    expect(pollingSection).toContain("voice-only");
+    expect(pollingSection).toContain("Do NOT poll for SMS or Telegram");
   });
 
   test('no instruction requires waiting for user to ask "did it work?"', () => {
@@ -119,23 +134,22 @@ describe('guardian-verify-setup skill — voice auto-followup', () => {
     // confirm that voice verification worked. The auto-check polling loop
     // makes this unnecessary.
     const voiceAutoCheckSection =
-      skillContent.split('## Voice Auto-Check Polling')[1]?.split('## Step 6')[0] ?? '';
-    expect(voiceAutoCheckSection).toContain(
-      'Do NOT require the user to ask',
-    );
+      skillContent
+        .split("## Voice Auto-Check Polling")[1]
+        ?.split("## Step 6")[0] ?? "";
+    expect(voiceAutoCheckSection).toContain("Do NOT require the user to ask");
     // The voice bullet in Step 3 should not instruct the assistant to wait
     // for the user to confirm or ask if it worked. Narrow to just the voice
     // bullet line to avoid false positives from Telegram's "wait for the
     // user to confirm they clicked the link" which is unrelated to voice.
-    const step3Section = skillContent
-      .split('## Step 3')[1]
-      ?.split('## Step 4')[0] ?? '';
+    const step3Section =
+      skillContent.split("## Step 3")[1]?.split("## Step 4")[0] ?? "";
     const voiceBullet = step3Section
-      .split('\n')
+      .split("\n")
       .filter((line) => /^\s*-\s+\*\*Voice\*\*/.test(line))
-      .join('\n');
+      .join("\n");
     expect(voiceBullet).not.toHaveLength(0);
-    expect(voiceBullet).not.toContain('wait for the user to confirm');
-    expect(voiceBullet).not.toContain('ask the user if it worked');
+    expect(voiceBullet).not.toContain("wait for the user to confirm");
+    expect(voiceBullet).not.toContain("ask the user if it worked");
   });
 });

package/src/__tests__/headless-browser-interactions.test.ts

@@ -87,6 +87,7 @@ const ctx: ToolContext = {
   sessionId: 'test-session',
   conversationId: 'test-conversation',
   workingDir: '/tmp',
+  guardianTrustClass: 'guardian',
 };
 
 function resetMockPage() {

package/src/__tests__/headless-browser-navigate.test.ts

@@ -58,6 +58,7 @@ const ctx: ToolContext = {
   sessionId: 'test-session',
   conversationId: 'test-conversation',
   workingDir: '/tmp',
+  guardianTrustClass: 'guardian',
 };
 
 function resetMockPage() {

package/src/__tests__/headless-browser-read-tools.test.ts

@@ -67,6 +67,7 @@ const ctx: ToolContext = {
   sessionId: 'test-session',
   conversationId: 'test-conversation',
   workingDir: '/tmp',
+  guardianTrustClass: 'guardian',
 };
 
 function resetMockPage() {

package/src/__tests__/headless-browser-snapshot.test.ts

@@ -66,6 +66,7 @@ const ctx: ToolContext = {
   sessionId: 'test-session',
   conversationId: 'test-conversation',
   workingDir: '/tmp',
+  guardianTrustClass: 'guardian',
 };
 
 function resetMockPage() {

package/src/__tests__/host-file-edit-tool.test.ts

@@ -14,6 +14,7 @@ function makeContext(): ToolContext {
     workingDir: '/tmp',
     sessionId: 'test-session',
     conversationId: 'test-conversation',
+    guardianTrustClass: 'guardian',
   };
 }
 

package/src/__tests__/host-file-read-tool.test.ts

@@ -14,6 +14,7 @@ function makeContext(): ToolContext {
     workingDir: '/tmp',
     sessionId: 'test-session',
     conversationId: 'test-conversation',
+    guardianTrustClass: 'guardian',
   };
 }
 

package/src/__tests__/host-file-write-tool.test.ts

@@ -14,6 +14,7 @@ function makeContext(): ToolContext {
     workingDir: '/tmp',
     sessionId: 'test-session',
     conversationId: 'test-conversation',
+    guardianTrustClass: 'guardian',
   };
 }
 

package/src/__tests__/host-shell-tool.test.ts

@@ -74,6 +74,7 @@ function makeContext(): ToolContext {
     workingDir: '/tmp',
     sessionId: 'test-session',
     conversationId: 'test-conversation',
+    guardianTrustClass: 'guardian',
   };
 }
 

package/src/__tests__/lifecycle-docs-guard.test.ts

@@ -0,0 +1,207 @@
+import { execSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { join } from 'node:path';
+
+import { describe, expect, it } from 'bun:test';
+
+/**
+ * Guard test: prevent stale lifecycle instructions from being reintroduced
+ * into documentation. The canonical lifecycle commands are `vellum wake`,
+ * `vellum ps`, and `vellum sleep`. Repo-local slash commands live in
+ * `.claude/skills/`, not `.claude/commands/`.
+ *
+ * See AGENTS.md for the conventions these tests enforce.
+ */
+
+const REPO_ROOT = join(import.meta.dir, '../../..');
+
+describe('lifecycle docs guard', () => {
+  it('repo-local commands live in skills directory, not commands directory', () => {
+    const staleLocations = [
+      '.claude/commands/update.md',
+      '.claude/commands/release.md',
+    ];
+
+    const violations = staleLocations.filter((p) =>
+      existsSync(join(REPO_ROOT, p)),
+    );
+
+    if (violations.length > 0) {
+      const message = [
+        'Found repo-local commands in .claude/commands/ — they should live in .claude/skills/.',
+        '',
+        'Stale files:',
+        ...violations.map((f) => `  - ${f}`),
+        '',
+        'Move them to .claude/skills/<name>/SKILL.md instead.',
+      ].join('\n');
+
+      expect(violations, message).toEqual([]);
+    }
+
+    // Verify the correct locations exist
+    const expectedLocations = [
+      '.claude/skills/update/SKILL.md',
+      '.claude/skills/release/SKILL.md',
+    ];
+
+    const missing = expectedLocations.filter(
+      (p) => !existsSync(join(REPO_ROOT, p)),
+    );
+
+    if (missing.length > 0) {
+      const message = [
+        'Expected repo-local skill files are missing:',
+        ...missing.map((f) => `  - ${f}`),
+      ].join('\n');
+
+      expect(missing, message).toEqual([]);
+    }
+  });
+
+  it('key docs reference vellum lifecycle commands', () => {
+    const checks: Array<{
+      file: string;
+      pattern: string;
+      description: string;
+    }> = [
+      {
+        file: 'README.md',
+        pattern: 'vellum wake\\|vellum ps\\|vellum sleep',
+        description:
+          'README.md should mention vellum wake, vellum ps, or vellum sleep',
+      },
+      {
+        file: 'assistant/README.md',
+        pattern: 'vellum wake\\|vellum ps',
+        description:
+          'assistant/README.md should mention vellum wake or vellum ps',
+      },
+      {
+        file: 'AGENTS.md',
+        pattern: 'vellum ps\\|vellum sleep\\|vellum wake',
+        description:
+          'AGENTS.md should mention vellum ps, vellum sleep, or vellum wake in the /update command description',
+      },
+    ];
+
+    const failures: string[] = [];
+
+    for (const check of checks) {
+      try {
+        execSync(`git grep -q '${check.pattern}' -- '${check.file}'`, {
+          encoding: 'utf-8',
+          cwd: REPO_ROOT,
+        });
+      } catch {
+        failures.push(check.description);
+      }
+    }
+
+    if (failures.length > 0) {
+      const message = [
+        'Key docs are missing vellum lifecycle command references:',
+        '',
+        ...failures.map((f) => `  - ${f}`),
+        '',
+        'These docs should reference vellum CLI lifecycle commands (wake/ps/sleep).',
+      ].join('\n');
+
+      expect(failures, message).toEqual([]);
+    }
+  });
+
+  it('no docs use stale daemon startup as primary instruction', () => {
+    // Files that are allowed to contain these patterns
+    const allowedPrefixes = [
+      'cli/', // CLI source code
+      'assistant/src/', // assistant runtime source
+      'CLAUDE.md', // project instructions
+      'AGENTS.md', // agent conventions (may reference patterns for context)
+    ];
+
+    const stalePatterns = [
+      {
+        pattern: 'bun run src/index.ts daemon start',
+        label: 'bun run src/index.ts daemon start',
+      },
+    ];
+
+    const violations: string[] = [];
+
+    for (const { pattern, label } of stalePatterns) {
+      let grepOutput = '';
+      try {
+        grepOutput = execSync(
+          `git grep -n '${pattern}' -- '*.md'`,
+          { encoding: 'utf-8', cwd: REPO_ROOT },
+        ).trim();
+      } catch {
+        // No matches — happy path
+        continue;
+      }
+
+      const lines = grepOutput.split('\n').filter((l) => l.length > 0);
+
+      for (const line of lines) {
+        const filePath = line.split(':')[0];
+
+        // Skip allowed file prefixes
+        if (allowedPrefixes.some((prefix) => filePath.startsWith(prefix))) {
+          continue;
+        }
+
+        // Skip test files
+        if (filePath.includes('__tests__') || filePath.endsWith('.test.ts')) {
+          continue;
+        }
+
+        // Check if this specific occurrence is inside a <details> section or
+        // a "Development" / "raw bun commands" context by examining only the
+        // 10 lines before this match. We extract the line number from the
+        // grep output (format "filePath:lineNum:content") and use sed to
+        // get a targeted range, so each match is evaluated independently.
+        const parts = line.split(':');
+        const lineNum = parseInt(parts[1], 10);
+
+        if (!Number.isNaN(lineNum)) {
+          try {
+            const startLine = Math.max(1, lineNum - 10);
+            const context = execSync(
+              `sed -n '${startLine},${lineNum}p' '${filePath}'`,
+              { encoding: 'utf-8', cwd: REPO_ROOT },
+            );
+
+            const contextLower = context.toLowerCase();
+            const isInDetails = contextLower.includes('<details>');
+            const isDevContext =
+              contextLower.includes('development:') ||
+              contextLower.includes('low-level development') ||
+              contextLower.includes('raw bun commands');
+
+            if (isInDetails || isDevContext) {
+              continue;
+            }
+          } catch {
+            // If context extraction fails, treat as a violation
+          }
+        }
+
+        violations.push(`${filePath}: contains "${label}" as primary instruction`);
+      }
+    }
+
+    if (violations.length > 0) {
+      const message = [
+        'Found docs using stale daemon startup patterns as primary instructions.',
+        'Use `vellum wake` / `vellum sleep` instead. Raw bun commands are acceptable',
+        'only in collapsed <details> sections or dev-only contexts.',
+        '',
+        'Violations:',
+        ...violations.map((v) => `  - ${v}`),
+      ].join('\n');
+
+      expect(violations, message).toEqual([]);
+    }
+  });
+});

package/src/__tests__/managed-skill-lifecycle.test.ts

@@ -84,6 +84,7 @@ function makeContext(): ToolContext {
     workingDir: '/tmp',
     sessionId: 'test-session',
     conversationId: 'test-conversation',
+    guardianTrustClass: 'guardian',
   };
 }
 

package/src/__tests__/media-reuse-story.e2e.test.ts

@@ -235,6 +235,7 @@ describe('Story E2E: selfie yesterday -> generated image today', () => {
       workingDir: sandboxDir,
       sessionId: 'sess-story',
       conversationId: threadB.id,
+      guardianTrustClass: 'guardian',
     };
 
     const result = await assetSearchTool.execute(
@@ -253,6 +254,7 @@ describe('Story E2E: selfie yesterday -> generated image today', () => {
       workingDir: sandboxDir,
       sessionId: 'sess-story',
       conversationId: threadB.id,
+      guardianTrustClass: 'guardian',
     };
 
     const result = await assetSearchTool.execute(
@@ -269,6 +271,7 @@ describe('Story E2E: selfie yesterday -> generated image today', () => {
       workingDir: sandboxDir,
       sessionId: 'sess-story',
       conversationId: threadB.id,
+      guardianTrustClass: 'guardian',
     };
 
     const result = await assetMaterializeTool.execute(
@@ -296,6 +299,7 @@ describe('Story E2E: selfie yesterday -> generated image today', () => {
       workingDir: sandboxDir,
       sessionId: 'sess-story',
       conversationId: threadB.id,
+      guardianTrustClass: 'guardian',
     };
 
     // Step 3a: Search for the selfie
@@ -486,6 +490,7 @@ describe('Private-thread variant: cross-thread media blocking', () => {
       workingDir: sandboxDir,
       sessionId: 'sess-priv-test',
       conversationId: standardThread.id,
+      guardianTrustClass: 'guardian',
     };
 
     const result = await assetSearchTool.execute(
@@ -510,6 +515,7 @@ describe('Private-thread variant: cross-thread media blocking', () => {
       workingDir: sandboxDir,
       sessionId: 'sess-priv-test',
       conversationId: standardThread.id,
+      guardianTrustClass: 'guardian',
     };
 
     const result = await assetMaterializeTool.execute(
@@ -533,6 +539,7 @@ describe('Private-thread variant: cross-thread media blocking', () => {
       workingDir: sandboxDir,
       sessionId: 'sess-priv-test',
       conversationId: privateThread.id,
+      guardianTrustClass: 'guardian',
     };
 
     const searchResult = await assetSearchTool.execute(
@@ -563,6 +570,7 @@ describe('Private-thread variant: cross-thread media blocking', () => {
       workingDir: sandboxDir,
       sessionId: 'sess-priv-test',
       conversationId: privateThreadB.id,
+      guardianTrustClass: 'guardian',
     };
 
     const searchResult = await assetSearchTool.execute(

package/src/__tests__/messaging-send-tool.test.ts

@@ -48,6 +48,7 @@ describe('messaging-send tool', () => {
         sessionId: 'sess-1',
         conversationId: 'conv-1',
         assistantId: 'ast-alpha',
+        guardianTrustClass: 'guardian' as const,
       },
     );
 

package/src/__tests__/playbook-execution.test.ts

@@ -69,6 +69,7 @@ const ctx: ToolContext = {
   workingDir: '/tmp',
   sessionId: 'test-session',
   conversationId: 'test-conversation',
+  guardianTrustClass: 'guardian',
 };
 
 function insertPlaybookRow(overrides: Partial<{

package/src/__tests__/playbook-tools.test.ts

@@ -61,6 +61,7 @@ const ctx: ToolContext = {
   workingDir: '/tmp',
   sessionId: 'test-session',
   conversationId: 'test-conversation',
+  guardianTrustClass: 'guardian',
 };
 
 function clearPlaybooks(): void {