@vellumai/assistant 0.5.1 → 0.5.3

package/docs/skills.md

@@ -156,3 +156,103 @@ Trust rules are stored in `~/.vellum/protected/trust.json`. You can inspect this
 ### "A skill tool keeps prompting even though I approved it."
 
 Check whether the rule has the correct `executionTarget` — a rule scoped to `sandbox` will not match a tool running on `host`.
+
+## Inline Command Expansions
+
+Skills can embed dynamic content by using the **inline command expansion** syntax. When a skill containing these tokens is loaded, each token is executed and replaced with its output before the skill body is delivered to the model. The syntax is shown in the fenced block below.
+
+This syntax is intentionally compatible with the convention established by [inline skill commands](https://x.com) for portable cross-agent skill authoring. Vellum adopts the exact same token format so that externally authored skills load without rewriting — but applies stricter execution constraints.
+
+### Syntax
+
+The canonical syntax is:
+
+```
+!`command`
+```
+
+Where `command` is any shell command string. The exclamation mark immediately precedes the opening backtick with no whitespace in between. Examples:
+
+```markdown
+Current branch: !`git branch --show-current`
+Recent changes: !`git log --oneline -5`
+Project info: !`cat package.json | jq '.name, .version'`
+```
+
+Tokens inside fenced code blocks (` ``` ` or `~~~`) are **not** expanded — they are treated as documentation examples. This allows skills to safely include syntax examples without triggering execution.
+
+### Parsing rules
+
+The parser (`parseInlineCommandExpansions`) enforces fail-closed semantics:
+
+| Condition                                         | Behavior               |
+| ------------------------------------------------- | ---------------------- |
+| Well-formed token outside fenced code             | Parsed as an expansion |
+| Token inside a fenced code block                  | Skipped (not expanded) |
+| Empty command text (no content between backticks) | Rejected as malformed  |
+| Whitespace-only command text                      | Rejected as malformed  |
+| Unmatched opening (no closing backtick found)     | Rejected as malformed  |
+| Nested backticks inside command text              | Rejected as malformed  |
+
+Malformed tokens do not silently pass through — they are collected as errors and logged. If a skill body contains any malformed tokens, the valid tokens are still expanded, but the errors are reported for diagnostics.
+
+### Feature flag
+
+Inline command expansion is gated by the `inline-skill-commands` feature flag (key: `feature_flags.inline-skill-commands.enabled`). The flag defaults to **enabled**.
+
+When the flag is disabled and a skill contains inline command expansion tokens, `skill_load` returns an error rather than delivering unexpanded tokens to the model. This fail-closed behavior prevents the LLM from seeing raw expansion tokens and attempting to interpret them.
+
+### Approval model
+
+Skills with inline command expansions use a separate permission namespace: `skill_load_dynamic:*`. This ensures they do not silently inherit the permissive default `skill_load:*` allow rule.
+
+When a user is prompted to approve a dynamic skill load, the allowlist options are:
+
+| Option         | Pattern                                     | Behavior                                                                                            |
+| -------------- | ------------------------------------------- | --------------------------------------------------------------------------------------------------- |
+| Version-pinned | `skill_load_dynamic:<id>@<transitive-hash>` | Approved for this exact version only. Any change to the skill or its includes invalidates the rule. |
+| Any-version    | `skill_load_dynamic:<id>`                   | Approved for all versions of this skill.                                                            |
+
+The transitive hash covers the skill's own content plus all included skills, so a change anywhere in the dependency graph triggers re-approval for version-pinned rules.
+
+### v1 execution limits
+
+In the initial implementation, inline command execution enforces these constraints:
+
+| Constraint       | Value                                                   |
+| ---------------- | ------------------------------------------------------- |
+| Execution target | Sandbox only (no host fallback)                         |
+| Network access   | Off (no outbound connections)                           |
+| Environment      | Sanitized (no API keys, tokens, or credentials)         |
+| Timeout          | 10 seconds per command                                  |
+| Output cap       | 20,000 characters (truncated with `[output truncated]`) |
+| Binary output    | Rejected if >10% non-printable characters               |
+| ANSI sequences   | Stripped before output processing                       |
+| stderr           | Discarded (only stdout is captured)                     |
+
+Commands that fail (timeout, non-zero exit, spawn failure, binary output) produce a deterministic stub in the rendered body rather than leaking raw error output:
+
+```
+<inline_skill_command index="0">[inline command unavailable: command timed out]</inline_skill_command>
+```
+
+### Eligible skill sources
+
+Only **bundled**, **managed**, and **workspace** skills may use inline command expansions. Third-party **extra** skill sources are explicitly rejected — `skill_load` returns an error if an extra-source skill contains inline expansion tokens.
+
+| Source      | Eligible | Reason                                 |
+| ----------- | -------- | -------------------------------------- |
+| `bundled`   | Yes      | Shipped with the application, trusted  |
+| `managed`   | Yes      | User-installed, subject to approval    |
+| `workspace` | Yes      | Project-local, subject to approval     |
+| `extra`     | No       | Third-party roots, out of scope for v1 |
+
+### Fail-closed summary
+
+The system fails closed at every layer:
+
+1. **Flag off** — skill_load returns an error, tokens never reach the model.
+2. **Malformed syntax** — rejected by the parser, logged as errors.
+3. **Unsupported source** — skill_load returns an error for extra-source skills.
+4. **Command failure** — deterministic stub replaces the token, no raw stderr.
+5. **No permission** — `skill_load_dynamic:*` namespace requires explicit approval.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@vellumai/assistant",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "type": "module",
   "exports": {
     ".": "./src/index.ts"

package/src/__tests__/agent-loop.test.ts

@@ -1629,4 +1629,115 @@ describe("AgentLoop", () => {
     );
     expect(textBlock!.text).toBe("Normal response with no placeholders.");
   });
+
+  // Tool error retry nudge — when a tool returns isError: true, the loop
+  // should inject a system_notice nudging the LLM to retry instead of ending.
+  test("injects retry nudge system_notice when tool returns an error", async () => {
+    const { provider, calls } = createMockProvider([
+      // First turn: LLM calls a tool that errors
+      toolUseResponse("t1", "read_file", { path: "/missing.txt" }),
+      // Second turn: LLM retries after seeing the error + nudge
+      toolUseResponse("t2", "read_file", { path: "/existing.txt" }),
+      // Third turn: LLM responds with success
+      textResponse("Got the file."),
+    ]);
+
+    let callCount = 0;
+    const toolExecutor = async (
+      _name: string,
+      _input: Record<string, unknown>,
+    ) => {
+      callCount++;
+      if (callCount === 1) {
+        return {
+          content:
+            '{"error":"name is required and must be a non-empty string"}',
+          isError: true,
+        };
+      }
+      return { content: "file contents", isError: false };
+    };
+
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+    await loop.run([userMessage], () => {});
+
+    // Provider should have been called 3 times (error -> retry -> final text)
+    expect(calls).toHaveLength(3);
+
+    // The second call's messages should contain the retry nudge system_notice
+    const secondCallMessages = calls[1].messages;
+    const toolResultMessage = secondCallMessages[secondCallMessages.length - 1];
+    expect(toolResultMessage.role).toBe("user");
+
+    const retryNudge = toolResultMessage.content.find(
+      (b): b is Extract<ContentBlock, { type: "text" }> =>
+        b.type === "text" && b.text.includes("looks recoverable"),
+    );
+    expect(retryNudge).toBeDefined();
+
+    // The third call should NOT have the retry nudge (successful tool result)
+    const thirdCallMessages = calls[2].messages;
+    const thirdToolResultMessage =
+      thirdCallMessages[thirdCallMessages.length - 1];
+    const noRetryNudge = thirdToolResultMessage.content.find(
+      (b): b is Extract<ContentBlock, { type: "text" }> =>
+        b.type === "text" && b.text.includes("looks recoverable"),
+    );
+    expect(noRetryNudge).toBeUndefined();
+  });
+
+  // Retry nudge stops after MAX_CONSECUTIVE_ERROR_NUDGES (3) consecutive errors
+  test("stops injecting retry nudge after 3 consecutive error turns", async () => {
+    const { provider, calls } = createMockProvider([
+      // 4 consecutive error turns, then final text
+      toolUseResponse("t1", "read_file", { path: "/a" }),
+      toolUseResponse("t2", "read_file", { path: "/b" }),
+      toolUseResponse("t3", "read_file", { path: "/c" }),
+      toolUseResponse("t4", "read_file", { path: "/d" }),
+      textResponse("Giving up."),
+    ]);
+
+    const toolExecutor = async (
+      _name: string,
+      _input: Record<string, unknown>,
+    ) => {
+      return { content: "service unavailable", isError: true };
+    };
+
+    const loop = new AgentLoop(
+      provider,
+      "system",
+      {},
+      dummyTools,
+      toolExecutor,
+    );
+    await loop.run([userMessage], () => {});
+
+    expect(calls).toHaveLength(5);
+
+    // Helper to check if a call's last user message has the retry nudge
+    const hasRetryNudge = (callIndex: number): boolean => {
+      const msgs = calls[callIndex].messages;
+      const lastMsg = msgs[msgs.length - 1];
+      return lastMsg.content.some(
+        (b) =>
+          b.type === "text" &&
+          "text" in b &&
+          (b as { text: string }).text.includes("looks recoverable"),
+      );
+    };
+
+    // Turns 1-3 should have the nudge
+    expect(hasRetryNudge(1)).toBe(true);
+    expect(hasRetryNudge(2)).toBe(true);
+    expect(hasRetryNudge(3)).toBe(true);
+    // Turn 4 should NOT have the nudge (exceeded limit)
+    expect(hasRetryNudge(4)).toBe(false);
+  });
 });

package/src/__tests__/always-loaded-tools-guard.test.ts

@@ -2,8 +2,8 @@
  * Guard test: always-loaded tool count
  *
  * This test asserts the exact set of tools that are active when no client is
- * connected, no host proxy is available, no attachments are present, and no
- * special channel capabilities exist. This represents the minimal "always-loaded"
+ * connected, no host proxy is available, and no special channel capabilities
+ * exist. This represents the minimal "always-loaded"
  * baseline that is sent to the LLM on every turn.
  *
  * Adding a tool to this set increases token cost for every request. If this test
@@ -32,14 +32,13 @@ describe("always-loaded tool count", () => {
     await initializeTools();
     const allDefs = buildToolDefinitions();
 
-    // Minimal context: no client, no attachments, no capabilities
+    // Minimal context: no client, no capabilities
     const minimalContext: SkillProjectionContext = {
       skillProjectionState: new Map(),
       skillProjectionCache: {},
       coreToolNames: new Set(),
       toolsDisabledDepth: 0,
       hasNoClient: true,
-      hasAttachments: false,
       channelCapabilities: undefined,
     };
 

package/src/__tests__/app-builder-tool-scripts.test.ts

@@ -2,7 +2,6 @@ import { describe, expect, mock, test } from "bun:test";
 
 import type { AppDefinition } from "../memory/app-store.js";
 import type { AppStore } from "../tools/apps/executors.js";
-import type { EditEngineResult } from "../tools/shared/filesystem/edit-engine.js";
 import type { ToolContext } from "../tools/types.js";
 
 // ---------------------------------------------------------------------------
@@ -26,24 +25,11 @@ function makeMockStore(overrides: Partial<AppStore> = {}): AppStore {
   return {
     getApp: () => makeApp(),
     listApps: () => [makeApp()],
-    queryAppRecords: () => [],
-    listAppFiles: () => ["index.html"],
-    readAppFile: () => "<h1>Hi</h1>",
     createApp: (params) =>
       makeApp({ name: params.name, description: params.description }),
     updateApp: (id, updates) => makeApp({ id, ...updates }),
     deleteApp: () => {},
     writeAppFile: () => {},
-    editAppFile: () =>
-      ({
-        ok: true,
-        updatedContent: "new",
-        matchCount: 1,
-        matchMethod: "exact",
-        similarity: 1,
-        actualOld: "old",
-        actualNew: "new",
-      }) as EditEngineResult,
     ...overrides,
   };
 }
@@ -66,6 +52,11 @@ const mockStore = makeMockStore();
 mock.module("../memory/app-store.js", () => ({
   ...mockStore,
   getAppsDir: () => "/tmp/test-apps",
+  getAppDirPath: (appId: string) => `/tmp/test-apps/${appId}`,
+  resolveAppDir: (id: string) => ({
+    dirName: id,
+    appDir: `/tmp/test-apps/${id}`,
+  }),
   isMultifileApp: (app: AppDefinition) => app.formatVersion === 2,
 }));
 
@@ -85,13 +76,7 @@ mock.module("../bundler/app-compiler.js", () => ({
 
 import * as appCreateScript from "../config/bundled-skills/app-builder/tools/app-create.js";
 import * as appDeleteScript from "../config/bundled-skills/app-builder/tools/app-delete.js";
-import * as appFileEditScript from "../config/bundled-skills/app-builder/tools/app-file-edit.js";
-import * as appFileListScript from "../config/bundled-skills/app-builder/tools/app-file-list.js";
-import * as appFileReadScript from "../config/bundled-skills/app-builder/tools/app-file-read.js";
-import * as appFileWriteScript from "../config/bundled-skills/app-builder/tools/app-file-write.js";
-import * as appListScript from "../config/bundled-skills/app-builder/tools/app-list.js";
-import * as appQueryScript from "../config/bundled-skills/app-builder/tools/app-query.js";
-import * as appUpdateScript from "../config/bundled-skills/app-builder/tools/app-update.js";
+import * as appRefreshScript from "../config/bundled-skills/app-builder/tools/app-refresh.js";
 
 // ---------------------------------------------------------------------------
 // Tests
@@ -142,58 +127,6 @@ describe("app-builder skill tool scripts", () => {
     });
   });
 
-  // ---- app-list ----------------------------------------------------------
-
-  describe("app-list", () => {
-    test("exports a run function", () => {
-      expect(typeof appListScript.run).toBe("function");
-    });
-
-    test("delegates to executeAppList and returns result", async () => {
-      const result = await appListScript.run({}, makeContext());
-      expect(result.isError).toBe(false);
-      const parsed = JSON.parse(result.content);
-      expect(Array.isArray(parsed)).toBe(true);
-      expect(parsed.length).toBeGreaterThan(0);
-      expect(parsed[0].id).toBe("app-1");
-    });
-  });
-
-  // ---- app-query ---------------------------------------------------------
-
-  describe("app-query", () => {
-    test("exports a run function", () => {
-      expect(typeof appQueryScript.run).toBe("function");
-    });
-
-    test("delegates to executeAppQuery and returns result", async () => {
-      const result = await appQueryScript.run(
-        { app_id: "app-1" },
-        makeContext(),
-      );
-      expect(result.isError).toBe(false);
-      expect(JSON.parse(result.content)).toEqual([]);
-    });
-  });
-
-  // ---- app-update --------------------------------------------------------
-
-  describe("app-update", () => {
-    test("exports a run function", () => {
-      expect(typeof appUpdateScript.run).toBe("function");
-    });
-
-    test("delegates to executeAppUpdate and returns result", async () => {
-      const result = await appUpdateScript.run(
-        { app_id: "app-1", name: "Updated" },
-        makeContext(),
-      );
-      expect(result.isError).toBe(false);
-      const parsed = JSON.parse(result.content);
-      expect(parsed.name).toBe("Updated");
-    });
-  });
-
   // ---- app-delete --------------------------------------------------------
 
   describe("app-delete", () => {
@@ -213,93 +146,22 @@ describe("app-builder skill tool scripts", () => {
     });
   });
 
-  // ---- app-file-list -----------------------------------------------------
+  // ---- app-refresh -------------------------------------------------------
 
-  describe("app-file-list", () => {
+  describe("app-refresh", () => {
     test("exports a run function", () => {
-      expect(typeof appFileListScript.run).toBe("function");
+      expect(typeof appRefreshScript.run).toBe("function");
     });
 
-    test("delegates to executeAppFileList and returns result", async () => {
-      const result = await appFileListScript.run(
+    test("delegates to executeAppRefresh and returns result", async () => {
+      const result = await appRefreshScript.run(
         { app_id: "app-1" },
         makeContext(),
       );
       expect(result.isError).toBe(false);
-      expect(JSON.parse(result.content)).toEqual(["index.html"]);
-    });
-  });
-
-  // ---- app-file-read -----------------------------------------------------
-
-  describe("app-file-read", () => {
-    test("exports a run function", () => {
-      expect(typeof appFileReadScript.run).toBe("function");
-    });
-
-    test("delegates to executeAppFileRead and returns formatted content", async () => {
-      const result = await appFileReadScript.run(
-        { app_id: "app-1", path: "index.html" },
-        makeContext(),
-      );
-      expect(result.isError).toBe(false);
-      // Content should include line numbers
-      expect(result.content).toContain("1\t");
-    });
-  });
-
-  // ---- app-file-edit -----------------------------------------------------
-
-  describe("app-file-edit", () => {
-    test("exports a run function", () => {
-      expect(typeof appFileEditScript.run).toBe("function");
-    });
-
-    test("delegates to executeAppFileEdit and returns result", async () => {
-      const result = await appFileEditScript.run(
-        {
-          app_id: "app-1",
-          path: "index.html",
-          old_string: "old",
-          new_string: "new",
-        },
-        makeContext(),
-      );
-      expect(result.isError).toBe(false);
       const parsed = JSON.parse(result.content);
-      expect(parsed.ok).toBe(true);
-    });
-
-    test("returns error when old_string is empty", async () => {
-      const result = await appFileEditScript.run(
-        {
-          app_id: "app-1",
-          path: "index.html",
-          old_string: "",
-          new_string: "new",
-        },
-        makeContext(),
-      );
-      expect(result.isError).toBe(true);
-    });
-  });
-
-  // ---- app-file-write ----------------------------------------------------
-
-  describe("app-file-write", () => {
-    test("exports a run function", () => {
-      expect(typeof appFileWriteScript.run).toBe("function");
-    });
-
-    test("delegates to executeAppFileWrite and returns result", async () => {
-      const result = await appFileWriteScript.run(
-        { app_id: "app-1", path: "new.html", content: "<div/>" },
-        makeContext(),
-      );
-      expect(result.isError).toBe(false);
-      const parsed = JSON.parse(result.content);
-      expect(parsed.written).toBe(true);
-      expect(parsed.path).toBe("new.html");
+      expect(parsed.refreshed).toBe(true);
+      expect(parsed.appId).toBe("app-1");
     });
   });
 });

package/src/__tests__/app-dir-path-guard.test.ts

@@ -0,0 +1,78 @@
+import { execSync } from "node:child_process";
+import { describe, expect, test } from "bun:test";
+
+/**
+ * Guard test: files outside app-store.ts must not import getAppsDir and use
+ * it to construct paths with an app ID. All app path construction must go
+ * through getAppDirPath() or resolveAppDir() from app-store.ts.
+ *
+ * This prevents regressions where new code bypasses the dirName-based path
+ * resolution and constructs UUID-based paths directly.
+ *
+ * Allowlist: only app-store.ts itself, app-git-service.ts (uses getAppsDir
+ * for the git repo root, not for per-app paths), and workspace migrations
+ * (self-contained, don't import from app-store).
+ */
+
+/** Files that are permitted to import getAppsDir. */
+const ALLOWLIST = new Set([
+  "assistant/src/memory/app-store.ts", // defines getAppsDir
+  "assistant/src/memory/app-git-service.ts", // uses getAppsDir for git repo root, not per-app paths
+]);
+
+function isTestFile(filePath: string): boolean {
+  return (
+    filePath.includes("/__tests__/") ||
+    filePath.endsWith(".test.ts") ||
+    filePath.endsWith(".test.js") ||
+    filePath.endsWith(".spec.ts")
+  );
+}
+
+function isMigrationFile(filePath: string): boolean {
+  return filePath.includes("/workspace/migrations/");
+}
+
+describe("app directory path construction guard", () => {
+  test("no non-allowlisted production files import getAppsDir", () => {
+    // Search for files that import getAppsDir (not just mention it in comments)
+    const pattern = "import.*getAppsDir.*from|getAppsDir\\(\\)";
+
+    let grepOutput = "";
+    try {
+      grepOutput = execSync(`git grep -lE '${pattern}' -- '*.ts'`, {
+        encoding: "utf-8",
+        cwd: process.cwd() + "/..",
+      }).trim();
+    } catch (err) {
+      // Exit code 1 means no matches
+      if ((err as { status?: number }).status === 1) {
+        return;
+      }
+      throw err;
+    }
+
+    const files = grepOutput.split("\n").filter((f) => f.length > 0);
+    const violations = files.filter((f) => {
+      if (isTestFile(f)) return false;
+      if (isMigrationFile(f)) return false;
+      if (ALLOWLIST.has(f)) return false;
+      return true;
+    });
+
+    if (violations.length > 0) {
+      const message = [
+        "Found non-allowlisted production files importing or using getAppsDir().",
+        "Use getAppDirPath(appId) or resolveAppDir(appId) from app-store.ts instead.",
+        "",
+        "Violations:",
+        ...violations.map((f) => `  - ${f}`),
+        "",
+        "To fix: replace getAppsDir() + appId path construction with getAppDirPath(appId).",
+        "If this is an intentional exception, add it to the ALLOWLIST in app-dir-path-guard.test.ts.",
+      ].join("\n");
+
+      expect(violations, message).toEqual([]);
+    }
+  });
+});