@vellumai/assistant 0.5.1 → 0.5.3

package/src/__tests__/inline-skill-authoring-guard.test.ts

@@ -0,0 +1,220 @@
+import { existsSync, readdirSync, readFileSync, statSync } from "node:fs";
+import { join } from "node:path";
+import { describe, expect, test } from "bun:test";
+
+import { parseFrontmatterFields } from "../skills/frontmatter.js";
+import { parseInlineCommandExpansions } from "../skills/inline-command-expansions.js";
+
+/**
+ * Guard test: scan bundled and first-party skills for malformed inline
+ * command expansion syntax (`!\`...\``).
+ *
+ * This guard ensures that:
+ * 1. No skill ships with malformed inline expansion tokens (empty commands,
+ *    unmatched backticks, nested backticks) that would be rejected by the
+ *    parser at runtime.
+ * 2. Fenced-code examples containing `!\`...\`` syntax are correctly
+ *    ignored by the parser and not treated as live commands — verified by
+ *    fixture coverage below.
+ *
+ * See assistant/docs/skills.md "Inline Command Expansions" for the full
+ * syntax specification and security model.
+ */
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+function getRepoRoot(): string {
+  return join(process.cwd(), "..");
+}
+
+function getBundledSkillsDir(): string {
+  return join(process.cwd(), "src", "config", "bundled-skills");
+}
+
+function getFirstPartySkillsDir(): string {
+  return join(getRepoRoot(), "skills");
+}
+
+/**
+ * Discover all SKILL.md files under a given directory (one level deep).
+ * Returns an array of { id, skillFilePath } entries.
+ */
+function discoverSkillFiles(
+  dir: string,
+): Array<{ id: string; skillFilePath: string }> {
+  if (!existsSync(dir)) return [];
+
+  const results: Array<{ id: string; skillFilePath: string }> = [];
+  const entries = readdirSync(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    if (!entry.isDirectory()) continue;
+    const skillFilePath = join(dir, entry.name, "SKILL.md");
+    if (existsSync(skillFilePath) && statSync(skillFilePath).isFile()) {
+      results.push({ id: entry.name, skillFilePath });
+    }
+  }
+
+  return results.sort((a, b) => a.id.localeCompare(b.id));
+}
+
+/**
+ * Extract the markdown body from a SKILL.md file (strip frontmatter).
+ * Returns the body text, or null if the file has no valid frontmatter.
+ */
+function extractBody(filePath: string): string | undefined {
+  const content = readFileSync(filePath, "utf-8");
+  const result = parseFrontmatterFields(content);
+  return result?.body;
+}
+
+// ─── Tests ────────────────────────────────────────────────────────────────────
+
+describe("inline skill authoring guard", () => {
+  test("bundled skills contain no malformed inline command expansion tokens", () => {
+    const bundledDir = getBundledSkillsDir();
+    const skills = discoverSkillFiles(bundledDir);
+
+    const violations: string[] = [];
+
+    for (const { id, skillFilePath } of skills) {
+      const body = extractBody(skillFilePath);
+      if (body === undefined) continue;
+
+      const result = parseInlineCommandExpansions(body);
+      if (result.errors.length > 0) {
+        for (const error of result.errors) {
+          violations.push(
+            `bundled/${id}: ${error.reason} at offset ${error.offset} — ${JSON.stringify(error.raw)}`,
+          );
+        }
+      }
+    }
+
+    if (violations.length > 0) {
+      const message = [
+        "Found bundled skills with malformed inline command expansion tokens.",
+        "Fix the syntax or move examples inside fenced code blocks.",
+        "",
+        "Violations:",
+        ...violations.map((v) => `  - ${v}`),
+      ].join("\n");
+
+      expect(violations, message).toEqual([]);
+    }
+  });
+
+  test("first-party skills contain no malformed inline command expansion tokens", () => {
+    const firstPartyDir = getFirstPartySkillsDir();
+    const skills = discoverSkillFiles(firstPartyDir);
+
+    const violations: string[] = [];
+
+    for (const { id, skillFilePath } of skills) {
+      const body = extractBody(skillFilePath);
+      if (body === undefined) continue;
+
+      const result = parseInlineCommandExpansions(body);
+      if (result.errors.length > 0) {
+        for (const error of result.errors) {
+          violations.push(
+            `skills/${id}: ${error.reason} at offset ${error.offset} — ${JSON.stringify(error.raw)}`,
+          );
+        }
+      }
+    }
+
+    if (violations.length > 0) {
+      const message = [
+        "Found first-party skills with malformed inline command expansion tokens.",
+        "Fix the syntax or move examples inside fenced code blocks.",
+        "",
+        "Violations:",
+        ...violations.map((v) => `  - ${v}`),
+      ].join("\n");
+
+      expect(violations, message).toEqual([]);
+    }
+  });
+
+  test("fenced-code examples containing inline expansion syntax are not treated as live commands", () => {
+    // This fixture verifies the parser's fenced-code-block exclusion logic.
+    // Skills that document the `!\`command\`` syntax in fenced code blocks
+    // must not have those examples picked up as live expansions.
+    const fixture = [
+      "# Example Skill",
+      "",
+      "Use inline commands to inject dynamic context:",
+      "",
+      "```markdown",
+      "Current branch: !`git branch --show-current`",
+      "Recent commits: !`git log --oneline -5`",
+      "```",
+      "",
+      "~~~",
+      "!`echo hello`",
+      "~~~",
+      "",
+      "````",
+      "```",
+      "!`nested example`",
+      "```",
+      "````",
+      "",
+      "The above examples are documentation only and should not execute.",
+    ].join("\n");
+
+    const result = parseInlineCommandExpansions(fixture);
+
+    expect(result.expansions).toHaveLength(0);
+    expect(result.errors).toHaveLength(0);
+  });
+
+  test("skills docs file itself contains no accidentally live inline expansion tokens", () => {
+    // The skills.md documentation describes the `!\`command\`` syntax
+    // extensively. Verify that all examples are inside fenced code blocks
+    // and the parser does not detect any live or malformed tokens.
+    const docsPath = join(process.cwd(), "docs", "skills.md");
+    if (!existsSync(docsPath)) {
+      // Skip if the docs file doesn't exist (should not happen in normal checkout)
+      return;
+    }
+
+    const content = readFileSync(docsPath, "utf-8");
+    const result = parseInlineCommandExpansions(content);
+
+    if (result.errors.length > 0) {
+      const message = [
+        "assistant/docs/skills.md contains malformed inline command expansion tokens.",
+        "Ensure all examples are inside fenced code blocks.",
+        "",
+        "Errors:",
+        ...result.errors.map(
+          (e) =>
+            `  - ${e.reason} at offset ${e.offset}: ${JSON.stringify(e.raw)}`,
+        ),
+      ].join("\n");
+
+      expect(result.errors, message).toEqual([]);
+    }
+
+    // Also verify no tokens are accidentally detected as live expansions
+    // outside of fenced code blocks. The docs should only have examples
+    // inside fences.
+    if (result.expansions.length > 0) {
+      const message = [
+        "assistant/docs/skills.md has inline expansion tokens outside fenced code blocks.",
+        "These would be treated as live commands if the file were loaded as a skill.",
+        "Move all examples inside ``` or ~~~ fenced blocks.",
+        "",
+        "Live tokens found:",
+        ...result.expansions.map(
+          (e) =>
+            "  - !" + "`" + e.command + "`" + " at offset " + e.startOffset,
+        ),
+      ].join("\n");
+
+      expect(result.expansions, message).toEqual([]);
+    }
+  });
+});

package/src/__tests__/inline-skill-load-permissions.test.ts

@@ -0,0 +1,435 @@
+/**
+ * Tests for inline-command skill load permission handling.
+ *
+ * When a skill contains inline command expansions (!\`...\`) and the
+ * feature_flags.inline-skill-commands.enabled flag is on, the permission
+ * system must:
+ *
+ * 1. Emit skill_load_dynamic:<id>@<hash> / skill_load_dynamic:<id> candidates
+ *    instead of skill_load:<id>@<hash> / skill_load:<id>.
+ * 2. Match the default ask rule for skill_load_dynamic:* (prompting by default).
+ * 3. Allow exact-hash rules to auto-allow pinned versions.
+ * 4. Re-prompt when the transitive hash changes (skill edited).
+ * 5. Continue matching the existing skill_load:* flow for non-dynamic skills.
+ */
+
+import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { beforeEach, describe, expect, mock, test } from "bun:test";
+
+// ── Mock setup (must be before any imports from the project) ──────────────
+
+const testDir = mkdtempSync(join(tmpdir(), "inline-skill-perm-test-"));
+
+mock.module("../util/platform.js", () => ({
+  getRootDir: () => testDir,
+  getDataDir: () => join(testDir, "data"),
+  getWorkspaceSkillsDir: () => join(testDir, "skills"),
+  isMacOS: () => process.platform === "darwin",
+  isLinux: () => process.platform === "linux",
+  isWindows: () => process.platform === "win32",
+  getPidPath: () => join(testDir, "test.pid"),
+  getDbPath: () => join(testDir, "test.db"),
+  getLogPath: () => join(testDir, "test.log"),
+  ensureDataDir: () => {},
+}));
+
+mock.module("../util/logger.js", () => ({
+  getLogger: () =>
+    new Proxy({} as Record<string, unknown>, {
+      get: () => () => {},
+    }),
+}));
+
+interface TestConfig {
+  permissions: { mode: "strict" | "workspace" };
+  skills: { load: { extraDirs: string[] } };
+  sandbox: { enabled: boolean };
+  assistantFeatureFlagValues?: Record<string, boolean>;
+  [key: string]: unknown;
+}
+
+const testConfig: TestConfig = {
+  permissions: { mode: "workspace" },
+  skills: { load: { extraDirs: [] } },
+  sandbox: { enabled: true },
+  assistantFeatureFlagValues: {
+    "feature_flags.inline-skill-commands.enabled": true,
+  },
+};
+
+mock.module("../config/loader.js", () => ({
+  getConfig: () => testConfig,
+  loadConfig: () => testConfig,
+  invalidateConfigCache: () => {},
+  saveConfig: () => {},
+  loadRawConfig: () => ({}),
+  saveRawConfig: () => {},
+  getNestedValue: () => undefined,
+  setNestedValue: () => {},
+}));
+
+// ── Imports (after mocks) ─────────────────────────────────────────────────
+
+import { check, generateAllowlistOptions } from "../permissions/checker.js";
+import { addRule, clearCache } from "../permissions/trust-store.js";
+import { computeSkillVersionHash } from "../skills/version-hash.js";
+
+// ── Helpers ───────────────────────────────────────────────────────────────
+
+function ensureSkillsDir(): void {
+  mkdirSync(join(testDir, "skills"), { recursive: true });
+}
+
+/** Write a plain skill (no inline command expansions). */
+function writePlainSkill(
+  skillId: string,
+  name: string,
+  description = "Test skill",
+): void {
+  const skillDir = join(testDir, "skills", skillId);
+  mkdirSync(skillDir, { recursive: true });
+  writeFileSync(
+    join(skillDir, "SKILL.md"),
+    `---\nname: "${name}"\ndescription: "${description}"\n---\n\nPlain skill body.\n`,
+  );
+}
+
+/** Write a skill with inline command expansions. */
+function writeDynamicSkill(
+  skillId: string,
+  name: string,
+  command = "echo hello",
+  description = "Dynamic test skill",
+): void {
+  const skillDir = join(testDir, "skills", skillId);
+  mkdirSync(skillDir, { recursive: true });
+  writeFileSync(
+    join(skillDir, "SKILL.md"),
+    `---\nname: "${name}"\ndescription: "${description}"\n---\n\nThis skill uses !\`${command}\` inline.\n`,
+  );
+}
+
+// ── Tests ─────────────────────────────────────────────────────────────────
+
+describe("inline-command skill_load permissions", () => {
+  beforeEach(() => {
+    clearCache();
+    testConfig.permissions = { mode: "workspace" };
+    testConfig.skills = { load: { extraDirs: [] } };
+    testConfig.assistantFeatureFlagValues = {
+      "feature_flags.inline-skill-commands.enabled": true,
+    };
+    try {
+      rmSync(join(testDir, "protected", "trust.json"));
+    } catch {
+      /* may not exist */
+    }
+    try {
+      rmSync(join(testDir, "skills"), { recursive: true, force: true });
+    } catch {
+      /* may not exist */
+    }
+  });
+
+  // ── Default prompt behavior ──────────────────────────────────────────
+
+  describe("default prompt behavior", () => {
+    test("dynamic skill prompts by default (matches skill_load_dynamic:* ask rule)", async () => {
+      ensureSkillsDir();
+      writeDynamicSkill("dynamic-prompt", "Dynamic Prompt Skill");
+
+      const result = await check(
+        "skill_load",
+        { skill: "dynamic-prompt" },
+        "/tmp",
+      );
+      expect(result.decision).toBe("prompt");
+      expect(result.matchedRule).toBeDefined();
+      expect(result.matchedRule!.pattern).toBe("skill_load_dynamic:*");
+      expect(result.matchedRule!.decision).toBe("ask");
+    });
+
+    test("dynamic skill prompts in strict mode too", async () => {
+      ensureSkillsDir();
+      writeDynamicSkill("dynamic-strict", "Dynamic Strict Skill");
+      testConfig.permissions.mode = "strict";
+
+      const result = await check(
+        "skill_load",
+        { skill: "dynamic-strict" },
+        "/tmp",
+      );
+      expect(result.decision).toBe("prompt");
+      expect(result.matchedRule).toBeDefined();
+      expect(result.matchedRule!.pattern).toBe("skill_load_dynamic:*");
+    });
+  });
+
+  // ── Exact-hash allow rules ───────────────────────────────────────────
+
+  describe("exact-hash allow rules", () => {
+    test("exact skill_load_dynamic:<id>@<hash> rule auto-allows", async () => {
+      ensureSkillsDir();
+      writeDynamicSkill("dynamic-pinned", "Dynamic Pinned Skill");
+
+      // Compute the transitive hash to create a version-pinned rule.
+      const { computeTransitiveSkillVersionHash } =
+        await import("../skills/transitive-version-hash.js");
+      const { indexCatalogById } = await import("../skills/include-graph.js");
+      const { loadSkillCatalog } = await import("../config/skills.js");
+
+      const catalog = loadSkillCatalog();
+      const index = indexCatalogById(catalog);
+      const transitiveHash = computeTransitiveSkillVersionHash(
+        "dynamic-pinned",
+        index,
+      );
+
+      // Add an exact hash rule
+      addRule(
+        "skill_load",
+        `skill_load_dynamic:dynamic-pinned@${transitiveHash}`,
+        "everywhere",
+        "allow",
+        2000,
+      );
+
+      const result = await check(
+        "skill_load",
+        { skill: "dynamic-pinned" },
+        "/tmp",
+      );
+      expect(result.decision).toBe("allow");
+      expect(result.matchedRule).toBeDefined();
+      expect(result.matchedRule!.pattern).toBe(
+        `skill_load_dynamic:dynamic-pinned@${transitiveHash}`,
+      );
+    });
+  });
+
+  // ── Any-version allow rules ──────────────────────────────────────────
+
+  describe("any-version allow rules", () => {
+    test("skill_load_dynamic:<id> rule auto-allows any version", async () => {
+      ensureSkillsDir();
+      writeDynamicSkill("dynamic-anyver", "Dynamic Any Version Skill");
+
+      addRule(
+        "skill_load",
+        "skill_load_dynamic:dynamic-anyver",
+        "everywhere",
+        "allow",
+        2000,
+      );
+
+      const result = await check(
+        "skill_load",
+        { skill: "dynamic-anyver" },
+        "/tmp",
+      );
+      expect(result.decision).toBe("allow");
+      expect(result.matchedRule).toBeDefined();
+      expect(result.matchedRule!.pattern).toBe(
+        "skill_load_dynamic:dynamic-anyver",
+      );
+    });
+  });
+
+  // ── Changed transitive hash re-prompting ─────────────────────────────
+
+  describe("changed transitive hash re-prompting", () => {
+    test("editing a dynamic skill changes the hash, causing version-pinned rule to stop matching", async () => {
+      ensureSkillsDir();
+      writeDynamicSkill("dynamic-reprompt", "Dynamic Reprompt", "echo v1");
+
+      const { computeTransitiveSkillVersionHash } =
+        await import("../skills/transitive-version-hash.js");
+      const { indexCatalogById } = await import("../skills/include-graph.js");
+      const { loadSkillCatalog } = await import("../config/skills.js");
+
+      const catalog1 = loadSkillCatalog();
+      const index1 = indexCatalogById(catalog1);
+      const hashV1 = computeTransitiveSkillVersionHash(
+        "dynamic-reprompt",
+        index1,
+      );
+
+      // Add a version-specific rule for v1
+      addRule(
+        "skill_load",
+        `skill_load_dynamic:dynamic-reprompt@${hashV1}`,
+        "everywhere",
+        "allow",
+        2000,
+      );
+
+      // v1: should auto-allow
+      const resultV1 = await check(
+        "skill_load",
+        { skill: "dynamic-reprompt" },
+        "/tmp",
+      );
+      expect(resultV1.decision).toBe("allow");
+      expect(resultV1.matchedRule!.pattern).toBe(
+        `skill_load_dynamic:dynamic-reprompt@${hashV1}`,
+      );
+
+      // Edit the skill (change the command)
+      writeDynamicSkill("dynamic-reprompt", "Dynamic Reprompt", "echo v2");
+
+      const catalog2 = loadSkillCatalog();
+      const index2 = indexCatalogById(catalog2);
+      const hashV2 = computeTransitiveSkillVersionHash(
+        "dynamic-reprompt",
+        index2,
+      );
+      expect(hashV2).not.toBe(hashV1);
+
+      // v2: the version-specific rule no longer matches; falls through
+      // to the default skill_load_dynamic:* ask rule
+      const resultV2 = await check(
+        "skill_load",
+        { skill: "dynamic-reprompt" },
+        "/tmp",
+      );
+      expect(resultV2.decision).toBe("prompt");
+      expect(resultV2.matchedRule!.pattern).toBe("skill_load_dynamic:*");
+    });
+  });
+
+  // ── Non-dynamic skills continue matching skill_load:* ────────────────
+
+  describe("non-dynamic skills continue to use skill_load:* flow", () => {
+    test("plain skill (no inline expansions) matches skill_load:* allow rule", async () => {
+      ensureSkillsDir();
+      writePlainSkill("plain-skill", "Plain Skill");
+
+      const result = await check(
+        "skill_load",
+        { skill: "plain-skill" },
+        "/tmp",
+      );
+      expect(result.decision).toBe("allow");
+      expect(result.matchedRule).toBeDefined();
+      expect(result.matchedRule!.pattern).toBe("skill_load:*");
+    });
+
+    test("plain skill with version-specific rule still uses skill_load: namespace", async () => {
+      ensureSkillsDir();
+      writePlainSkill("plain-pinned", "Plain Pinned Skill");
+      testConfig.permissions.mode = "strict";
+
+      const skillDir = join(testDir, "skills", "plain-pinned");
+      const diskHash = computeSkillVersionHash(skillDir);
+
+      addRule(
+        "skill_load",
+        `skill_load:plain-pinned@${diskHash}`,
+        "everywhere",
+        "allow",
+        2000,
+      );
+
+      const result = await check(
+        "skill_load",
+        { skill: "plain-pinned" },
+        "/tmp",
+      );
+      expect(result.decision).toBe("allow");
+      expect(result.matchedRule!.pattern).toBe(
+        `skill_load:plain-pinned@${diskHash}`,
+      );
+    });
+  });
+
+  // ── Feature flag disabled ────────────────────────────────────────────
+
+  describe("feature flag disabled", () => {
+    test("dynamic skill falls through to skill_load:* when flag is off", async () => {
+      ensureSkillsDir();
+      writeDynamicSkill("dynamic-flag-off", "Dynamic Flag Off Skill");
+
+      // Disable the feature flag
+      testConfig.assistantFeatureFlagValues = {
+        "feature_flags.inline-skill-commands.enabled": false,
+      };
+
+      const result = await check(
+        "skill_load",
+        { skill: "dynamic-flag-off" },
+        "/tmp",
+      );
+      // With the flag off, the skill is treated as a normal skill_load
+      expect(result.decision).toBe("allow");
+      expect(result.matchedRule!.pattern).toBe("skill_load:*");
+    });
+  });
+
+  // ── Allowlist options ────────────────────────────────────────────────
+
+  describe("allowlist options", () => {
+    test("dynamic skill allowlist options use skill_load_dynamic: namespace", async () => {
+      ensureSkillsDir();
+      writeDynamicSkill("dynamic-opts", "Dynamic Opts Skill");
+
+      const options = await generateAllowlistOptions("skill_load", {
+        skill: "dynamic-opts",
+      });
+
+      expect(options.length).toBeGreaterThanOrEqual(1);
+      // All options should use skill_load_dynamic: prefix
+      for (const option of options) {
+        expect(option.pattern).toMatch(/^skill_load_dynamic:/);
+      }
+
+      // Should have an any-version option
+      const anyVersionOption = options.find(
+        (o) => o.pattern === "skill_load_dynamic:dynamic-opts",
+      );
+      expect(anyVersionOption).toBeDefined();
+      expect(anyVersionOption!.description).toBe("This skill (any version)");
+    });
+
+    test("plain skill allowlist options use skill_load: namespace", async () => {
+      ensureSkillsDir();
+      writePlainSkill("plain-opts", "Plain Opts Skill");
+
+      const options = await generateAllowlistOptions("skill_load", {
+        skill: "plain-opts",
+      });
+
+      expect(options.length).toBeGreaterThanOrEqual(1);
+      // Should use skill_load: prefix, not skill_load_dynamic:
+      for (const option of options) {
+        expect(option.pattern).toMatch(/^skill_load:/);
+        expect(option.pattern).not.toMatch(/^skill_load_dynamic:/);
+      }
+    });
+  });
+
+  // ── Default rule priority ────────────────────────────────────────────
+
+  describe("default rule priority", () => {
+    test("skill_load_dynamic:* ask rule has higher priority than skill_load:* allow rule", async () => {
+      const { getDefaultRuleTemplates } =
+        await import("../permissions/defaults.js");
+      const rules = getDefaultRuleTemplates();
+
+      const dynamicRule = rules.find(
+        (r) => r.id === "default:ask-skill_load_dynamic-global",
+      );
+      const loadRule = rules.find(
+        (r) => r.id === "default:allow-skill_load-global",
+      );
+
+      expect(dynamicRule).toBeDefined();
+      expect(loadRule).toBeDefined();
+      expect(dynamicRule!.priority).toBeGreaterThan(loadRule!.priority);
+      expect(dynamicRule!.decision).toBe("ask");
+      expect(dynamicRule!.pattern).toBe("skill_load_dynamic:*");
+      expect(dynamicRule!.tool).toBe("skill_load");
+    });
+  });
+});