@vellumai/assistant 0.4.44 → 0.4.45

package/src/__tests__/skill-feature-flags.test.ts

@@ -5,7 +5,8 @@ import type { AssistantConfig } from "../config/schema.js";
 import { resolveSkillStates, skillFlagKey } from "../config/skill-state.js";
 import type { SkillSummary } from "../config/skills.js";
 
-const DECLARED_FLAG_KEY = "feature_flags.hatch-new-assistant.enabled";
+const DECLARED_FLAG_ID = "hatch-new-assistant";
+const DECLARED_FLAG_KEY = `feature_flags.${DECLARED_FLAG_ID}.enabled`;
 const DECLARED_SKILL_ID = "hatch-new-assistant";
 // ---------------------------------------------------------------------------
 // Helpers
@@ -37,6 +38,7 @@ function makeConfig(overrides: Partial<AssistantConfig> = {}): AssistantConfig {
 function makeSkill(
   id: string,
   source: "bundled" | "managed" = "bundled",
+  featureFlag?: string,
 ): SkillSummary {
   return {
     id,
@@ -49,9 +51,32 @@ function makeSkill(
     userInvocable: true,
     disableModelInvocation: false,
     source,
+    featureFlag,
   };
 }
 
+// ---------------------------------------------------------------------------
+// skillFlagKey — unit tests
+// ---------------------------------------------------------------------------
+
+describe("skillFlagKey", () => {
+  test("returns canonical key when featureFlag is present", () => {
+    expect(skillFlagKey({ featureFlag: "my-flag" })).toBe(
+      "feature_flags.my-flag.enabled",
+    );
+  });
+
+  test("returns undefined when featureFlag is undefined", () => {
+    expect(skillFlagKey({ featureFlag: undefined })).toBeUndefined();
+  });
+
+  test("returns undefined when featureFlag field is absent", () => {
+    expect(
+      skillFlagKey({} as Pick<SkillSummary, "featureFlag">),
+    ).toBeUndefined();
+  });
+});
+
 // ---------------------------------------------------------------------------
 // isAssistantFeatureFlagEnabled with skillFlagKey (canonical path)
 // ---------------------------------------------------------------------------
@@ -60,7 +85,10 @@ describe("isAssistantFeatureFlagEnabled with skillFlagKey", () => {
   test("returns false when no flag overrides (registry default is false)", () => {
     const config = makeConfig();
     expect(
-      isAssistantFeatureFlagEnabled(skillFlagKey(DECLARED_SKILL_ID), config),
+      isAssistantFeatureFlagEnabled(
+        skillFlagKey({ featureFlag: DECLARED_FLAG_ID })!,
+        config,
+      ),
     ).toBe(false);
   });
 
@@ -69,7 +97,10 @@ describe("isAssistantFeatureFlagEnabled with skillFlagKey", () => {
       assistantFeatureFlagValues: { [DECLARED_FLAG_KEY]: true },
     });
     expect(
-      isAssistantFeatureFlagEnabled(skillFlagKey(DECLARED_SKILL_ID), config),
+      isAssistantFeatureFlagEnabled(
+        skillFlagKey({ featureFlag: DECLARED_FLAG_ID })!,
+        config,
+      ),
     ).toBe(true);
   });
 
@@ -78,7 +109,10 @@ describe("isAssistantFeatureFlagEnabled with skillFlagKey", () => {
       assistantFeatureFlagValues: { [DECLARED_FLAG_KEY]: false },
     });
     expect(
-      isAssistantFeatureFlagEnabled(skillFlagKey(DECLARED_SKILL_ID), config),
+      isAssistantFeatureFlagEnabled(
+        skillFlagKey({ featureFlag: DECLARED_FLAG_ID })!,
+        config,
+      ),
     ).toBe(false);
   });
 });
@@ -137,11 +171,14 @@ describe("isAssistantFeatureFlagEnabled", () => {
 
 describe("resolveSkillStates with feature flags", () => {
   test("flag OFF skill does not appear in resolved list", () => {
-    const catalog = [makeSkill(DECLARED_SKILL_ID), makeSkill("twitter")];
+    const catalog = [
+      makeSkill(DECLARED_SKILL_ID, "bundled", DECLARED_FLAG_ID),
+      makeSkill("browser", "bundled", "browser"),
+    ];
     const config = makeConfig({
       assistantFeatureFlagValues: {
         [DECLARED_FLAG_KEY]: false,
-        "feature_flags.twitter.enabled": true,
+        "feature_flags.browser.enabled": true,
       },
     });
 
@@ -149,15 +186,18 @@ describe("resolveSkillStates with feature flags", () => {
     const ids = resolved.map((r) => r.summary.id);
 
     expect(ids).not.toContain(DECLARED_SKILL_ID);
-    expect(ids).toContain("twitter");
+    expect(ids).toContain("browser");
   });
 
   test("flag ON skill appears normally", () => {
-    const catalog = [makeSkill(DECLARED_SKILL_ID), makeSkill("twitter")];
+    const catalog = [
+      makeSkill(DECLARED_SKILL_ID, "bundled", DECLARED_FLAG_ID),
+      makeSkill("browser", "bundled", "browser"),
+    ];
     const config = makeConfig({
       assistantFeatureFlagValues: {
         [DECLARED_FLAG_KEY]: true,
-        "feature_flags.twitter.enabled": true,
+        "feature_flags.browser.enabled": true,
       },
     });
 
@@ -165,11 +205,11 @@ describe("resolveSkillStates with feature flags", () => {
     const ids = resolved.map((r) => r.summary.id);
 
     expect(ids).toContain(DECLARED_SKILL_ID);
-    expect(ids).toContain("twitter");
+    expect(ids).toContain("browser");
   });
 
   test("declared flag key defaults to registry value (false)", () => {
-    const catalog = [makeSkill(DECLARED_SKILL_ID)];
+    const catalog = [makeSkill(DECLARED_SKILL_ID, "bundled", DECLARED_FLAG_ID)];
     const config = makeConfig();
 
     const resolved = resolveSkillStates(catalog, config);
@@ -177,8 +217,19 @@ describe("resolveSkillStates with feature flags", () => {
     expect(resolved.length).toBe(0);
   });
 
+  test("skill without featureFlag is never flag-gated", () => {
+    const catalog = [makeSkill("no-flag-skill")];
+    const config = makeConfig();
+
+    const resolved = resolveSkillStates(catalog, config);
+    const ids = resolved.map((r) => r.summary.id);
+
+    // Skills without featureFlag are never gated — always pass through
+    expect(ids).toContain("no-flag-skill");
+  });
+
   test("feature flag OFF takes precedence over user-enabled config entry", () => {
-    const catalog = [makeSkill(DECLARED_SKILL_ID)];
+    const catalog = [makeSkill(DECLARED_SKILL_ID, "bundled", DECLARED_FLAG_ID)];
     const config = makeConfig({
       assistantFeatureFlagValues: { [DECLARED_FLAG_KEY]: false },
       skills: {
@@ -205,14 +256,14 @@ describe("resolveSkillStates with feature flags", () => {
 
   test("multiple skills with mixed flags — persisted overrides respected", () => {
     const catalog = [
-      makeSkill(DECLARED_SKILL_ID),
-      makeSkill("twitter"),
-      makeSkill("deploy"),
+      makeSkill(DECLARED_SKILL_ID, "bundled", DECLARED_FLAG_ID),
+      makeSkill("browser", "bundled", "browser"),
+      makeSkill("deploy", "bundled", "deploy"),
     ];
     const config = makeConfig({
       assistantFeatureFlagValues: {
         [DECLARED_FLAG_KEY]: false,
-        "feature_flags.twitter.enabled": true,
+        "feature_flags.browser.enabled": true,
         "feature_flags.deploy.enabled": false,
       },
     });
@@ -220,7 +271,63 @@ describe("resolveSkillStates with feature flags", () => {
     const resolved = resolveSkillStates(catalog, config);
     const ids = resolved.map((r) => r.summary.id);
 
-    // hatch-new-assistant and deploy explicitly false; twitter explicitly true
-    expect(ids).toEqual(["twitter"]);
+    // hatch-new-assistant and deploy explicitly false; browser explicitly true
+    expect(ids).toEqual(["browser"]);
+  });
+});
+
+// ---------------------------------------------------------------------------
+// resolveSkillStates — frontmatter featureFlag gating
+// ---------------------------------------------------------------------------
+
+describe("resolveSkillStates with frontmatter featureFlag", () => {
+  test("skill with featureFlag (defaultEnabled: false) is filtered when no config override", () => {
+    // hatch-new-assistant has defaultEnabled: false in the registry
+    const catalog = [makeSkill(DECLARED_SKILL_ID, "bundled", DECLARED_FLAG_ID)];
+    const config = makeConfig();
+
+    const resolved = resolveSkillStates(catalog, config);
+    // No override, registry default is false → filtered out
+    expect(resolved.length).toBe(0);
+  });
+
+  test("skill with featureFlag is included when config override enables it", () => {
+    const catalog = [makeSkill(DECLARED_SKILL_ID, "bundled", DECLARED_FLAG_ID)];
+    const config = makeConfig({
+      assistantFeatureFlagValues: { [DECLARED_FLAG_KEY]: true },
+    });
+
+    const resolved = resolveSkillStates(catalog, config);
+    const ids = resolved.map((r) => r.summary.id);
+    expect(ids).toContain(DECLARED_SKILL_ID);
+  });
+
+  test("skill without featureFlag is NEVER filtered by the flag system", () => {
+    const catalog = [makeSkill("no-flag-skill")];
+    const config = makeConfig();
+
+    const resolved = resolveSkillStates(catalog, config);
+    const ids = resolved.map((r) => r.summary.id);
+
+    // No featureFlag declared → always passes through regardless of any flags
+    expect(ids).toContain("no-flag-skill");
+  });
+
+  test("skill without featureFlag passes through even when feature_flags.<skillId>.enabled is explicitly false", () => {
+    // This proves the implicit skillId→flag mapping is gone:
+    // setting feature_flags.my-skill.enabled = false has no effect
+    // when the skill itself does not declare a featureFlag.
+    const catalog = [makeSkill("my-skill")];
+    const config = makeConfig({
+      assistantFeatureFlagValues: {
+        "feature_flags.my-skill.enabled": false,
+      },
+    });
+
+    const resolved = resolveSkillStates(catalog, config);
+    const ids = resolved.map((r) => r.summary.id);
+
+    // The skill has no featureFlag field, so it is never gated
+    expect(ids).toContain("my-skill");
   });
 });

package/src/__tests__/skill-load-feature-flag.test.ts

@@ -21,7 +21,6 @@ const platformOverrides: Record<string, (...args: unknown[]) => unknown> = {
   getRootDir: () => TEST_DIR,
   getDataDir: () => TEST_DIR,
   ensureDataDir: () => {},
-  getSocketPath: () => join(TEST_DIR, "vellum.sock"),
   getPidPath: () => join(TEST_DIR, "vellum.pid"),
   getDbPath: () => join(TEST_DIR, "data", "assistant.db"),
   getLogPath: () => join(TEST_DIR, "logs", "vellum.log"),
@@ -93,7 +92,7 @@ function writeSkill(
   mkdirSync(skillDir, { recursive: true });
   writeFileSync(
     join(skillDir, "SKILL.md"),
-    `---\nname: "${name}"\ndescription: "${description}"\n---\n\n${body}\n`,
+    `---\nname: "${name}"\ndescription: "${description}"\nmetadata: {"vellum":{"feature-flag":"${skillId}"}}\n---\n\n${body}\n`,
   );
 }
 

package/src/__tests__/skill-load-tool.test.ts

@@ -1,4 +1,10 @@
-import { existsSync, mkdirSync, rmSync, writeFileSync } from "node:fs";
+import {
+  existsSync,
+  mkdirSync,
+  rmSync,
+  symlinkSync,
+  writeFileSync,
+} from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
@@ -14,7 +20,6 @@ const platformOverrides: Record<string, (...args: unknown[]) => unknown> = {
   getRootDir: () => TEST_DIR,
   getDataDir: () => TEST_DIR,
   ensureDataDir: () => {},
-  getSocketPath: () => join(TEST_DIR, "vellum.sock"),
   getPidPath: () => join(TEST_DIR, "vellum.pid"),
   getDbPath: () => join(TEST_DIR, "data", "assistant.db"),
   getLogPath: () => join(TEST_DIR, "logs", "vellum.log"),
@@ -93,6 +98,35 @@ function writeSkillWithIncludes(
   );
 }
 
+function writeToolsJson(
+  skillId: string,
+  tools: Array<{
+    name: string;
+    description: string;
+    category?: string;
+    risk?: string;
+    input_schema?: Record<string, unknown>;
+    executor?: string;
+    execution_target?: string;
+  }>,
+): void {
+  const skillDir = join(TEST_DIR, "skills", skillId);
+  mkdirSync(skillDir, { recursive: true });
+  const manifest = {
+    version: 1,
+    tools: tools.map((t) => ({
+      name: t.name,
+      description: t.description,
+      category: t.category ?? "general",
+      risk: t.risk ?? "low",
+      input_schema: t.input_schema ?? { type: "object", properties: {} },
+      executor: t.executor ?? "scripts/run.sh",
+      execution_target: t.execution_target ?? "host",
+    })),
+  };
+  writeFileSync(join(skillDir, "TOOLS.json"), JSON.stringify(manifest));
+}
+
 async function executeSkillLoad(
   input: Record<string, unknown>,
 ): Promise<{ content: string; isError: boolean }> {
@@ -614,6 +648,33 @@ describe("skill_load tool", () => {
     expect(result.content).not.toContain("--- Reference:");
   });
 
+  test("references/ directory skips symlinked markdown files that escape the skill directory", async () => {
+    if (process.platform === "win32") {
+      // Symlink creation is not consistently available in Windows test environments.
+      return;
+    }
+
+    const skillDir = join(TEST_DIR, "skills", "refs-symlink");
+    mkdirSync(skillDir, { recursive: true });
+    mkdirSync(join(skillDir, "references"), { recursive: true });
+    writeFileSync(
+      join(skillDir, "SKILL.md"),
+      '---\nname: "Refs Symlink"\ndescription: "Skips symlinks"\n---\n\nBody.\n',
+    );
+
+    const outsideSecretPath = join(TEST_DIR, "outside-secret.md");
+    writeFileSync(outsideSecretPath, "TOP_SECRET_DO_NOT_LOAD");
+    symlinkSync(outsideSecretPath, join(skillDir, "references", "secret.md"));
+
+    writeFileSync(join(TEST_DIR, "skills", "SKILLS.md"), "- refs-symlink\n");
+
+    const result = await executeSkillLoad({ skill: "refs-symlink" });
+    expect(result.isError).toBe(false);
+    expect(result.content).toContain("Body.");
+    expect(result.content).not.toContain("--- Reference: Secret ---");
+    expect(result.content).not.toContain("TOP_SECRET_DO_NOT_LOAD");
+  });
+
   test("references/ directory ignores non-markdown files", async () => {
     const skillDir = join(TEST_DIR, "skills", "refs-filter");
     mkdirSync(skillDir, { recursive: true });
@@ -658,4 +719,135 @@ describe("skill_load tool", () => {
       /<loaded_skill id="empty-includes" version="v1:[a-f0-9]{64}" \/>/,
     );
   });
+
+  test("skill with TOOLS.json includes tool schemas section in output", async () => {
+    writeSkill(
+      "skill-with-tools",
+      "Skill With Tools",
+      "Has tools",
+      "Main body.",
+    );
+    writeToolsJson("skill-with-tools", [
+      {
+        name: "deploy_app",
+        description: "Deploy the application to production",
+        input_schema: {
+          type: "object",
+          properties: {
+            environment: {
+              type: "string",
+              description: "Target environment",
+            },
+            force: {
+              type: "boolean",
+              description: "Force deploy even if checks fail",
+            },
+          },
+          required: ["environment"],
+        },
+      },
+      {
+        name: "rollback_app",
+        description: "Rollback to previous version",
+        input_schema: {
+          type: "object",
+          properties: {
+            version: {
+              type: "string",
+              description: "Version to rollback to",
+            },
+          },
+          required: ["version"],
+        },
+      },
+    ]);
+    writeFileSync(
+      join(TEST_DIR, "skills", "SKILLS.md"),
+      "- skill-with-tools\n",
+    );
+
+    const result = await executeSkillLoad({ skill: "skill-with-tools" });
+    expect(result.isError).toBe(false);
+
+    // Should contain the Available Tools section header
+    expect(result.content).toContain("## Available Tools");
+
+    // Should instruct the LLM to use skill_execute
+    expect(result.content).toContain(
+      "Use `skill_execute` to call these tools.",
+    );
+
+    // Should contain tool names as headings
+    expect(result.content).toContain("### deploy_app");
+    expect(result.content).toContain("### rollback_app");
+
+    // Should contain tool descriptions
+    expect(result.content).toContain("Deploy the application to production");
+    expect(result.content).toContain("Rollback to previous version");
+
+    // Should list parameters with types and required/optional markers
+    expect(result.content).toContain(
+      "- environment (string, required): Target environment",
+    );
+    expect(result.content).toContain(
+      "- force (boolean, optional): Force deploy even if checks fail",
+    );
+    expect(result.content).toContain(
+      "- version (string, required): Version to rollback to",
+    );
+  });
+
+  test("skill without TOOLS.json does not include tool schemas section", async () => {
+    writeSkill("no-tools", "No Tools", "No tools manifest", "Body.");
+    writeFileSync(join(TEST_DIR, "skills", "SKILLS.md"), "- no-tools\n");
+
+    const result = await executeSkillLoad({ skill: "no-tools" });
+    expect(result.isError).toBe(false);
+    expect(result.content).not.toContain("## Available Tools");
+    expect(result.content).not.toContain("skill_execute");
+  });
+
+  test("included child skill with TOOLS.json has its tool schemas in output", async () => {
+    writeSkillWithIncludes(
+      "parent-tools",
+      "Parent Tools",
+      "Parent with tooled child",
+      "Parent body.",
+      ["child-tools"],
+    );
+    writeSkill("child-tools", "Child Tools", "Child with tools", "Child body.");
+    writeToolsJson("child-tools", [
+      {
+        name: "child_action",
+        description: "A child tool action",
+        input_schema: {
+          type: "object",
+          properties: {
+            target: {
+              type: "string",
+              description: "Action target",
+            },
+          },
+          required: ["target"],
+        },
+      },
+    ]);
+    writeFileSync(
+      join(TEST_DIR, "skills", "SKILLS.md"),
+      "- parent-tools\n- child-tools\n",
+    );
+
+    const result = await executeSkillLoad({ skill: "parent-tools" });
+    expect(result.isError).toBe(false);
+
+    // The child skill's tool schemas should appear (#### level under ### Tools from …)
+    expect(result.content).toContain("#### child_action");
+    expect(result.content).toContain("A child tool action");
+    expect(result.content).toContain(
+      "- target (string, required): Action target",
+    );
+    expect(result.content).toContain(
+      "Use `skill_execute` to call these tools.",
+    );
+  });
 });

package/src/__tests__/skill-projection-feature-flag.test.ts

@@ -58,7 +58,10 @@ mock.module("../config/assistant-feature-flags.js", () => ({
 }));
 
 mock.module("../config/skill-state.js", () => ({
-  skillFlagKey: (skillId: string) => `feature_flags.${skillId}.enabled`,
+  skillFlagKey: (skill: { featureFlag?: string }) =>
+    skill.featureFlag
+      ? `feature_flags.${skill.featureFlag}.enabled`
+      : undefined,
 }));
 
 mock.module("../skills/active-skill-tools.js", () => {
@@ -221,7 +224,7 @@ const { projectSkillTools, resetSkillToolProjection } =
 // Helpers
 // ---------------------------------------------------------------------------
 
-function makeSkill(id: string): SkillSummary {
+function makeSkill(id: string, featureFlag?: string): SkillSummary {
   return {
     id,
     name: id,
@@ -232,6 +235,7 @@ function makeSkill(id: string): SkillSummary {
     userInvocable: true,
     disableModelInvocation: false,
     source: "managed",
+    featureFlag,
   };
 }
 
@@ -293,7 +297,7 @@ describe("projectSkillTools feature flag enforcement", () => {
   });
 
   test("no skill tools projected for flag OFF skill even with old markers", () => {
-    mockCatalog = [makeSkill(DECLARED_SKILL_ID)];
+    mockCatalog = [makeSkill(DECLARED_SKILL_ID, DECLARED_SKILL_ID)];
     mockManifests = {
       [DECLARED_SKILL_ID]: makeManifest(["browser_navigate", "browser_click"]),
     };
@@ -317,7 +321,7 @@ describe("projectSkillTools feature flag enforcement", () => {
   });
 
   test("skill tools projected normally when flag is ON", () => {
-    mockCatalog = [makeSkill(DECLARED_SKILL_ID)];
+    mockCatalog = [makeSkill(DECLARED_SKILL_ID, DECLARED_SKILL_ID)];
     mockManifests = {
       [DECLARED_SKILL_ID]: makeManifest(["browser_navigate", "browser_click"]),
     };
@@ -334,34 +338,41 @@ describe("projectSkillTools feature flag enforcement", () => {
       previouslyActiveSkillIds: prevActive,
     });
 
-    expect(result.toolDefinitions).toHaveLength(2);
+    // Tool definitions are no longer returned (dispatched via skill_execute),
+    // but allowedToolNames should contain the registered tool names.
+    expect(result.toolDefinitions).toHaveLength(0);
+    expect(result.allowedToolNames.size).toBe(2);
     expect(result.allowedToolNames.has("browser_navigate")).toBe(true);
     expect(result.allowedToolNames.has("browser_click")).toBe(true);
   });
 
-  test("skill tools projected normally when flag key is absent (defaults to enabled)", () => {
+  test("skill tools projected normally when no featureFlag declared (never gated)", () => {
     mockCatalog = [makeSkill(DECLARED_SKILL_ID)];
     mockManifests = { [DECLARED_SKILL_ID]: makeManifest(["browser_navigate"]) };
 
     const history = buildHistoryWithMarker(DECLARED_SKILL_ID);
     const prevActive = new Map<string, string>();
 
-    // No overrides — should default to enabled
+    // No overrides — skill has no featureFlag so it's never gated
     currentConfig = {};
 
     const result = projectSkillTools(history, {
       previouslyActiveSkillIds: prevActive,
     });
 
-    expect(result.toolDefinitions).toHaveLength(1);
+    expect(result.toolDefinitions).toHaveLength(0);
+    expect(result.allowedToolNames.size).toBe(1);
     expect(result.allowedToolNames.has("browser_navigate")).toBe(true);
   });
 
   test("mixed flag-on and flag-off skills — only flag-on tools projected", () => {
-    mockCatalog = [makeSkill(DECLARED_SKILL_ID), makeSkill("twitter")];
+    mockCatalog = [
+      makeSkill(DECLARED_SKILL_ID, DECLARED_SKILL_ID),
+      makeSkill("plain-skill"),
+    ];
     mockManifests = {
       [DECLARED_SKILL_ID]: makeManifest(["browser_navigate"]),
-      twitter: makeManifest(["twitter_post"]),
+      "plain-skill": makeManifest(["plain_action"]),
     };
 
     const history: Message[] = [
@@ -393,7 +404,7 @@ describe("projectSkillTools feature flag enforcement", () => {
             type: "tool_use",
             id: "tu-2",
             name: "skill_load",
-            input: { skill: "twitter" },
+            input: { skill: "plain-skill" },
           },
         ],
       },
@@ -404,14 +415,14 @@ describe("projectSkillTools feature flag enforcement", () => {
             type: "tool_result",
             tool_use_id: "tu-2",
             content:
-              '<loaded_skill id="twitter" version="v1:default-hash-twitter" />',
+              '<loaded_skill id="plain-skill" version="v1:default-hash-plain-skill" />',
           },
         ],
       },
     ];
     const prevActive = new Map<string, string>();
 
-    // Declared skill is OFF, twitter is undeclared with no persisted override so remains ON.
+    // Declared skill is OFF, plain-skill is undeclared with no persisted override so remains ON.
     currentConfig = {
       assistantFeatureFlagValues: { [DECLARED_FLAG_KEY]: false },
     };
@@ -420,8 +431,8 @@ describe("projectSkillTools feature flag enforcement", () => {
       previouslyActiveSkillIds: prevActive,
     });
 
-    const toolNames = result.toolDefinitions.map((t) => t.name);
-    expect(toolNames).toContain("twitter_post");
-    expect(toolNames).not.toContain("browser_navigate");
+    // Tool definitions are no longer returned; check allowedToolNames instead
+    expect(result.allowedToolNames.has("plain_action")).toBe(true);
+    expect(result.allowedToolNames.has("browser_navigate")).toBe(false);
   });
 });

package/src/__tests__/skill-projection.benchmark.test.ts

@@ -68,7 +68,10 @@ mock.module("../config/skills.js", () => ({
 // Mock skill-state.js to break the transitive import chain — the benchmark
 // only needs skillFlagKey and doesn't exercise resolveSkillStates.
 mock.module("../config/skill-state.js", () => ({
-  skillFlagKey: (id: string) => `feature_flags.${id}.enabled`,
+  skillFlagKey: (skill: { featureFlag?: string }) =>
+    skill.featureFlag
+      ? `feature_flags.${skill.featureFlag}.enabled`
+      : undefined,
   resolveSkillStates: () => [],
 }));
 
@@ -276,7 +279,6 @@ describe("Skill projection benchmark", () => {
 
     const elapsed = timeMs(() => {
       const result = projectSkillTools(history);
-      expect(result.toolDefinitions.length).toBeGreaterThan(0);
       expect(result.allowedToolNames.size).toBeGreaterThan(0);
     });
 
@@ -327,12 +329,12 @@ describe("Skill projection benchmark", () => {
     expect(cache.derived!.entries.length).toBe(entriesCountAfterWarm);
     expect(cache.derived!.seenIds.size).toBe(seenIdsSizeAfterWarm);
 
-    // Assert tool definitions are identical between warm and cached calls
-    expect(cachedResult!.toolDefinitions.length).toBe(
-      warmResult.toolDefinitions.length,
+    // Assert allowed tool names are identical between warm and cached calls
+    expect(cachedResult!.allowedToolNames.size).toBe(
+      warmResult.allowedToolNames.size,
     );
-    const warmNames = warmResult.toolDefinitions.map((t) => t.name).sort();
-    const cachedNames = cachedResult!.toolDefinitions.map((t) => t.name).sort();
+    const warmNames = [...warmResult.allowedToolNames].sort();
+    const cachedNames = [...cachedResult!.allowedToolNames].sort();
     expect(cachedNames).toEqual(warmNames);
 
     console.log(`  Cached projection (no change): ${elapsed.toFixed(2)}ms`);
@@ -373,12 +375,12 @@ describe("Skill projection benchmark", () => {
       expect(cache.derived!.entries.length).toBe(snapshotEntriesCount);
       expect(cache.derived!.seenIds.size).toBe(snapshotSeenIdsSize);
 
-      // Tool definitions must match the first call exactly
-      expect(result.toolDefinitions.length).toBe(
-        firstResult.toolDefinitions.length,
+      // Allowed tool names must match the first call exactly
+      expect(result.allowedToolNames.size).toBe(
+        firstResult.allowedToolNames.size,
       );
-      expect(result.toolDefinitions.map((t) => t.name).sort()).toEqual(
-        firstResult.toolDefinitions.map((t) => t.name).sort(),
+      expect([...result.allowedToolNames].sort()).toEqual(
+        [...firstResult.allowedToolNames].sort(),
       );
     }
   });
@@ -396,7 +398,6 @@ describe("Skill projection benchmark", () => {
 
     const elapsed = timeMs(() => {
       const result = projectSkillTools(history);
-      expect(result.toolDefinitions.length).toBeGreaterThan(0);
       expect(result.allowedToolNames.size).toBeGreaterThan(0);
     });
 
@@ -429,7 +430,7 @@ describe("Skill projection benchmark", () => {
         cache,
         previouslyActiveSkillIds: prevActive,
       });
-      expect(result.toolDefinitions.length).toBeGreaterThan(0);
+      expect(result.allowedToolNames.size).toBeGreaterThan(0);
     });
 
     console.log(`  Incremental scan (10 new msgs): ${elapsed.toFixed(2)}ms`);