@klhapp/skillmux 1.11.2 → 1.13.0

package/src/eval.ts

@@ -1,302 +0,0 @@
-import type { Database } from "bun:sqlite";
-import { readFileSync } from "node:fs";
-import { join } from "node:path";
-import {
-  backfillEmbeddings,
-  getRuntime,
-  retrieveAndRerank,
-} from "./router-core";
-
-export interface EvalCase {
-  query: string;
-  split?: string;
-  relevant_skill_ids: string[];
-}
-
-export function parseEvalCases(raw: unknown): EvalCase[] {
-  if (!Array.isArray(raw)) throw new Error("Eval cases file must contain a JSON array");
-  const result: EvalCase[] = [];
-  for (let i = 0; i < raw.length; i++) {
-    const item = raw[i];
-    if (typeof item !== "object" || item === null) {
-      throw new Error(`Eval case at index ${i} must be an object`);
-    }
-    if ("expected" in item) {
-      throw new Error(
-        `Legacy 'expected' field at case ${i} is no longer supported in eval datasets; use 'relevant_skill_ids' instead.`,
-      );
-    }
-    if ("expected_outcome" in item) {
-      throw new Error(
-        `Legacy 'expected_outcome' field at case ${i} is no longer supported in eval datasets; remove it and provide 'relevant_skill_ids' instead.`,
-      );
-    }
-    if (typeof (item as any).query !== "string" || (item as any).query.trim().length === 0) {
-      throw new Error(`Eval case at index ${i} has invalid 'query': must be a non-empty string`);
-    }
-    if (
-      !Array.isArray((item as any).relevant_skill_ids) ||
-      (item as any).relevant_skill_ids.some((id: unknown) => typeof id !== "string")
-    ) {
-      throw new Error(`Eval case at index ${i} has invalid 'relevant_skill_ids': must be an array of strings`);
-    }
-    const relevantSkillIds = (item as any).relevant_skill_ids as string[];
-    if (relevantSkillIds.some((id) => id.trim().length === 0)) {
-      throw new Error(`Eval case at index ${i} has invalid 'relevant_skill_ids': IDs must be non-empty strings`);
-    }
-    if (new Set(relevantSkillIds).size !== relevantSkillIds.length) {
-      throw new Error(`Eval case at index ${i} has invalid 'relevant_skill_ids': duplicate IDs are not allowed`);
-    }
-    if ("split" in item && (item as any).split !== undefined && typeof (item as any).split !== "string") {
-      throw new Error(`Eval case at index ${i} has invalid 'split': must be a string if present`);
-    }
-    result.push({
-      query: (item as any).query,
-      ...((item as any).split !== undefined ? { split: (item as any).split } : {}),
-      relevant_skill_ids: relevantSkillIds,
-    });
-  }
-  return result;
-}
-
-/** AC17: dedup key for promoted cases. Collapses whitespace and case differences that are not meaningfully distinct queries. */
-export function normalizeQuery(query: string): string {
-  return query.trim().replace(/\s+/g, " ").toLowerCase();
-}
-
-export interface ObservedFetch {
-  query: string;
-  skill_id: string;
-}
-
-/** AC17: groups fetched skill ids by normalized query into observed-split eval cases. */
-export function buildPromotedCases(fetches: ObservedFetch[]): EvalCase[] {
-  const byQuery = new Map<string, { query: string; skillIds: string[]; seen: Set<string> }>();
-  for (const fetch of fetches) {
-    const key = normalizeQuery(fetch.query);
-    if (!key) continue;
-    let entry = byQuery.get(key);
-    if (!entry) {
-      entry = { query: fetch.query, skillIds: [], seen: new Set() };
-      byQuery.set(key, entry);
-    }
-    if (!entry.seen.has(fetch.skill_id)) {
-      entry.seen.add(fetch.skill_id);
-      entry.skillIds.push(fetch.skill_id);
-    }
-  }
-  return [...byQuery.values()].map((entry) => ({
-    query: entry.query,
-    split: "observed",
-    relevant_skill_ids: entry.skillIds,
-  }));
-}
-
-/** AC18: never rewrites a case whose normalized query already exists in the target file. */
-export function excludeExistingCases(
-  cases: EvalCase[],
-  existing: EvalCase[],
-): { cases: EvalCase[]; skipped: number } {
-  const existingKeys = new Set(existing.map((c) => normalizeQuery(c.query)));
-  const kept: EvalCase[] = [];
-  let skipped = 0;
-  for (const c of cases) {
-    if (existingKeys.has(normalizeQuery(c.query))) {
-      skipped++;
-    } else {
-      kept.push(c);
-    }
-  }
-  return { cases: kept, skipped };
-}
-
-/**
- * AC17: joins fetches to the resolve that produced them for promotion. Only
- * correlated fetches (a known resolve_audit_id) carry a query to promote;
- * uncorrelated fetches have no resolve to join against and are excluded.
- */
-export function queryPromotableFetches(db: Database, sinceIso: string): ObservedFetch[] {
-  return db
-    .query(
-      `SELECT audit.query AS query, fetch.skill_id AS skill_id
-       FROM fetch
-       JOIN audit ON fetch.resolve_audit_id = audit.id
-       WHERE fetch.ts >= ?
-       ORDER BY fetch.ts ASC`,
-    )
-    .all(sinceIso) as ObservedFetch[];
-}
-
-export interface EvalMetrics {
-  recall_at_5: number;
-  recall_at_10: number;
-  mrr: number;
-  ndcg_at_10: number;
-}
-
-export interface CandidateEvalDetail {
-  skill_id: string;
-  lexical_rank: number | null;
-  fused_rank: number | null;
-  reranked_rank?: number | null;
-}
-
-export interface EvalCaseResult {
-  query: string;
-  relevant_skill_ids: string[];
-  retrieval: string;
-  degraded_from?: string | null;
-  degradation_reason?: string | null;
-  latency_ms: number;
-  recall_settings: {
-    k_lexical: number;
-    k_vector: number;
-    k_rerank: number;
-  };
-  candidates: CandidateEvalDetail[];
-}
-
-export interface EvalReport {
-  queries: number;
-  judged_queries: number;
-  unjudged_queries: number;
-  lexical: EvalMetrics;
-  hybrid: EvalMetrics;
-  cases?: EvalCaseResult[];
-}
-
-export function computeRankingMetrics(rankings: string[][], cases: EvalCase[]): EvalMetrics {
-  const judgedIndices: number[] = [];
-  for (let i = 0; i < cases.length; i++) {
-    if (cases[i]!.relevant_skill_ids && cases[i]!.relevant_skill_ids.length > 0) {
-      judgedIndices.push(i);
-    }
-  }
-
-  if (judgedIndices.length === 0) {
-    return {
-      recall_at_5: 0,
-      recall_at_10: 0,
-      mrr: 0,
-      ndcg_at_10: 0,
-    };
-  }
-
-  let totalRecall5 = 0;
-  let totalRecall10 = 0;
-  let totalMRR = 0;
-  let totalNDCG10 = 0;
-
-  for (const idx of judgedIndices) {
-    const c = cases[idx]!;
-    const ranking = rankings[idx] ?? [];
-    const relevantSet = new Set(c.relevant_skill_ids);
-    const numRelevant = c.relevant_skill_ids.length;
-
-    // Recall@5: fraction of relevant skill IDs present in the first 5 results
-    const top5 = ranking.slice(0, 5);
-    const count5 = top5.filter((id) => relevantSet.has(id)).length;
-    totalRecall5 += count5 / numRelevant;
-
-    // Recall@10: fraction of relevant skill IDs present in the first 10 results
-    const top10 = ranking.slice(0, 10);
-    const count10 = top10.filter((id) => relevantSet.has(id)).length;
-    totalRecall10 += count10 / numRelevant;
-
-    // MRR: reciprocal rank of the first relevant result (1-based), 0 if missing
-    const firstRelevantRank = ranking.findIndex((id) => relevantSet.has(id));
-    if (firstRelevantRank >= 0) {
-      totalMRR += 1 / (firstRelevantRank + 1);
-    }
-
-    // nDCG@10: binary relevance, DCG gain 1/log2(rank+1), IDCG over min(numRelevant, 10)
-    let dcg10 = 0;
-    for (let r = 0; r < Math.min(10, ranking.length); r++) {
-      if (relevantSet.has(ranking[r]!)) {
-        dcg10 += 1 / Math.log2(r + 2);
-      }
-    }
-    let idcg10 = 0;
-    const idealCount = Math.min(numRelevant, 10);
-    for (let r = 0; r < idealCount; r++) {
-      idcg10 += 1 / Math.log2(r + 2);
-    }
-    if (idcg10 > 0) {
-      totalNDCG10 += dcg10 / idcg10;
-    }
-  }
-
-  const judgedCount = judgedIndices.length;
-  return {
-    recall_at_5: totalRecall5 / judgedCount,
-    recall_at_10: totalRecall10 / judgedCount,
-    mrr: totalMRR / judgedCount,
-    ndcg_at_10: totalNDCG10 / judgedCount,
-  };
-}
-
-export function loadEvalCases(path = join(import.meta.dir, "..", "eval", "queries.json")): EvalCase[] {
-  const raw = JSON.parse(readFileSync(path, "utf8"));
-  return parseEvalCases(raw);
-}
-
-export async function evalVault(cases = loadEvalCases()): Promise<EvalReport> {
-  const { config } = await getRuntime();
-  await backfillEmbeddings();
-
-  const lexicalRankings: string[][] = [];
-  const hybridRankings: string[][] = [];
-  const caseResults: EvalCaseResult[] = [];
-  const kRerank = config.recall.k_rerank ?? Math.min(10, config.recall.k_lexical + config.recall.k_vector);
-
-  for (const evalCase of cases) {
-    const start = performance.now();
-    const retrievalResult = await retrieveAndRerank({ query: evalCase.query });
-
-    // lexical metrics use lexical rank
-    lexicalRankings.push(
-      retrievalResult.trace
-        .filter((candidate) => candidate.lexical_rank !== null)
-        .sort((a, b) => a.lexical_rank! - b.lexical_rank!)
-        .map((candidate) => candidate.skill_id),
-    );
-
-    // retrieveAndRerank returns candidates in the exact order delivered to the
-    // caller: reranked when available, otherwise fused or lexical fallback.
-    hybridRankings.push(
-      retrievalResult.candidates.map((candidate) => candidate.skill_id),
-    );
-
-    const latency_ms = Math.round(performance.now() - start);
-    const candidateDetails: CandidateEvalDetail[] = retrievalResult.trace;
-
-    caseResults.push({
-      query: evalCase.query,
-      relevant_skill_ids: evalCase.relevant_skill_ids,
-      retrieval: retrievalResult.retrieval,
-      degraded_from: retrievalResult.degraded_from ?? null,
-      degradation_reason: retrievalResult.degradation_reason ?? null,
-      latency_ms,
-      recall_settings: {
-        k_lexical: config.recall.k_lexical,
-        k_vector: config.recall.k_vector,
-        k_rerank: kRerank,
-      },
-      candidates: candidateDetails,
-    });
-  }
-
-  const judged_queries = cases.filter(
-    (c) => c.relevant_skill_ids && c.relevant_skill_ids.length > 0,
-  ).length;
-  const unjudged_queries = cases.length - judged_queries;
-
-  return {
-    queries: cases.length,
-    judged_queries,
-    unjudged_queries,
-    lexical: computeRankingMetrics(lexicalRankings, cases),
-    hybrid: computeRankingMetrics(hybridRankings, cases),
-    cases: caseResults,
-  };
-}

package/src/global-flags.ts

@@ -1,46 +0,0 @@
-export type GlobalFlag =
-  | "--json"
-  | "--allow-insecure"
-  | "--verbose"
-  | "--dry-run";
-
-export type GlobalFlagWithValue = "--context" | "--server";
-
-/**
- * Checks if an argument is a global flag that takes a value (e.g. `--context`, `--server`).
- * If an `allowed` list is provided, only flags in that subset are matched.
- */
-export function isGlobalFlagWithValue(
-  option: string | undefined,
-  allowed?: readonly GlobalFlagWithValue[] | GlobalFlagWithValue,
-  ...rest: GlobalFlagWithValue[]
-): boolean {
-  if (!option) return false;
-  if (allowed !== undefined) {
-    const list = Array.isArray(allowed) ? allowed : [allowed, ...rest];
-    return list.includes(option as GlobalFlagWithValue);
-  }
-  return option === "--context" || option === "--server";
-}
-
-/**
- * Checks if an argument is a global flag that takes no value (e.g. `--json`, `--allow-insecure`, `--verbose`, `--dry-run`).
- * If an `allowed` list is provided, only flags in that subset are matched.
- */
-export function isGlobalFlag(
-  option: string | undefined,
-  allowed?: readonly GlobalFlag[] | GlobalFlag,
-  ...rest: GlobalFlag[]
-): boolean {
-  if (!option) return false;
-  if (allowed !== undefined) {
-    const list = Array.isArray(allowed) ? allowed : [allowed, ...rest];
-    return list.includes(option as GlobalFlag);
-  }
-  return (
-    option === "--json" ||
-    option === "--allow-insecure" ||
-    option === "--verbose" ||
-    option === "--dry-run"
-  );
-}

package/src/init-agents.ts

@@ -1,329 +0,0 @@
-import { existsSync } from "node:fs";
-import { homedir } from "node:os";
-import { join } from "node:path";
-
-export const SUPPORTED_AGENT_IDS = [
-  "claude-code",
-  "codex",
-  "opencode",
-  "github-copilot",
-  "windsurf",
-  "antigravity",
-  "goose",
-  "hermes",
-] as const;
-
-export type AgentId = (typeof SUPPORTED_AGENT_IDS)[number];
-export type DeliveryMode = "managed-pins" | "full-vault";
-
-export interface DetectedAgent {
-  agent: AgentId;
-  evidence: string;
-}
-
-export type McpRegistrationScope = "user" | "project";
-
-export interface McpRegistrationCommand {
-  command: string;
-  /** Returns undefined when this agent's CLI has no way to register at the requested scope. */
-  buildArgs: (scope: McpRegistrationScope) => string[] | undefined;
-}
-
-interface AgentPathOptions {
-  home: string;
-  codexHome: string;
-}
-
-interface AgentInstructionOptions extends AgentPathOptions {
-  claudeConfigDir: string;
-}
-
-/**
- * The single source of truth for what each agent supports. Every other
- * module (detection, skill-surface planning, readiness reporting, MCP
- * registration, instruction files, shell completions, CLI help text) reads
- * from this record instead of keeping its own copy — see git history for
- * what it looked like before consolidation (agent support was duplicated
- * across 6+ files and could silently drift, e.g. shell completions offering
- * goose/hermes for `project init` when they can't actually be attached).
- */
-interface AgentDefinition {
-  id: AgentId;
-  surfaceId?: "agent-skills" | "claude-code" | "codex" | "antigravity";
-  deliveryMode: DeliveryMode;
-  /** Filesystem evidence used by guided-mode agent detection. */
-  detectionPath?: (options: AgentPathOptions) => string;
-  /** Bespoke readiness message for full-vault agents with no managed-pins surface. */
-  manualSkillSurfaceMessage?: string;
-  instructions?: {
-    global?: (options: AgentInstructionOptions) => string;
-    project?: (projectRoot: string) => string;
-  };
-  /**
-   * Only set for agents whose own CLI's exact registration command was
-   * verified against its own --help, not guessed from docs. Every other
-   * agent falls back to printing the registration snippet (see
-   * printLastMile in init.ts) rather than a guessed command.
-   */
-  mcpRegistration?: McpRegistrationCommand;
-}
-
-export interface PlannedAgentSurface {
-  id: string;
-  targetName: string;
-  path: string;
-  deliveryMode: "managed-pins";
-  agents: AgentId[];
-}
-
-export interface AgentSurfacePlan {
-  agents: AgentDefinition[];
-  surfaces: PlannedAgentSurface[];
-}
-
-type ReadinessStatus = "ready" | "planned" | "manual" | "not-applicable";
-
-export interface ReadinessAxis {
-  status: ReadinessStatus;
-  detail: string;
-}
-
-export interface AgentReadiness {
-  agent: AgentId;
-  skillSurface: ReadinessAxis;
-  mcpRegistration: ReadinessAxis;
-  instructionSetup: ReadinessAxis;
-}
-
-export interface ResolvedBuiltInTarget {
-  targetName: string;
-  path: string;
-}
-
-/** Target names whose install directory is deterministic — --dir is optional for these. */
-export const BUILT_IN_TARGET_NAMES = new Set(["agent-skills", "claude-code", "codex"]);
-
-const AGENTS: Record<AgentId, AgentDefinition> = {
-  "claude-code": {
-    id: "claude-code",
-    surfaceId: "claude-code",
-    deliveryMode: "managed-pins",
-    detectionPath: ({ home }) => join(home, ".claude"),
-    instructions: {
-      global: ({ claudeConfigDir }) => join(claudeConfigDir, "CLAUDE.md"),
-      project: (projectRoot) => join(projectRoot, "CLAUDE.md"),
-    },
-    mcpRegistration: {
-      command: "claude",
-      // claude mcp add --scope: local (default, unshared), project (writes a
-      // committed .mcp.json), or user (global). "project" is the only shared
-      // option, so that's what a project-scoped registration means here.
-      buildArgs: (scope) => [
-        "mcp",
-        "add",
-        "-s",
-        scope === "project" ? "project" : "user",
-        "skillmux",
-        "--",
-        "skillmux",
-        "serve",
-      ],
-    },
-  },
-  codex: {
-    id: "codex",
-    surfaceId: "codex",
-    deliveryMode: "managed-pins",
-    detectionPath: ({ codexHome }) => codexHome,
-    instructions: {
-      global: ({ codexHome }) => join(codexHome, "AGENTS.md"),
-    },
-    mcpRegistration: {
-      command: "codex",
-      // codex mcp add has no --scope flag at all — it always writes to the
-      // global ~/.codex/config.toml, so project scope is not representable.
-      buildArgs: (scope) =>
-        scope === "project"
-          ? undefined
-          : ["mcp", "add", "skillmux", "--", "skillmux", "serve"],
-    },
-  },
-  opencode: {
-    id: "opencode",
-    surfaceId: "agent-skills",
-    deliveryMode: "managed-pins",
-    detectionPath: ({ home }) => join(home, ".config", "opencode"),
-    instructions: {
-      global: ({ home }) => join(home, ".config", "opencode", "AGENTS.md"),
-    },
-  },
-  "github-copilot": {
-    id: "github-copilot",
-    surfaceId: "agent-skills",
-    deliveryMode: "managed-pins",
-    detectionPath: ({ home }) => join(home, ".config", "github-copilot"),
-  },
-  windsurf: {
-    id: "windsurf",
-    surfaceId: "agent-skills",
-    deliveryMode: "managed-pins",
-    detectionPath: ({ home }) => join(home, ".codeium", "windsurf"),
-  },
-  antigravity: {
-    id: "antigravity",
-    surfaceId: "antigravity",
-    deliveryMode: "managed-pins",
-    instructions: {
-      global: ({ home }) => join(home, ".gemini", "GEMINI.md"),
-    },
-  },
-  goose: {
-    id: "goose",
-    deliveryMode: "full-vault",
-    detectionPath: ({ home }) => join(home, ".config", "goose"),
-    manualSkillSurfaceMessage: "configure the full vault in Goose",
-    instructions: {
-      global: ({ home }) => join(home, ".config", "goose", ".goosehints"),
-    },
-  },
-  hermes: {
-    id: "hermes",
-    deliveryMode: "full-vault",
-    detectionPath: ({ home }) => join(home, ".hermes"),
-    manualSkillSurfaceMessage: "configure the full vault in Hermes external_dirs",
-    instructions: {
-      global: ({ home }) => join(home, ".hermes.md"),
-    },
-  },
-};
-
-export function getAgentDefinition(agent: AgentId): AgentDefinition {
-  return AGENTS[agent];
-}
-
-/** Agents that can be attached to a project group (managed-pins delivery, not full-vault). */
-export const MANAGED_PINS_AGENT_IDS = SUPPORTED_AGENT_IDS.filter(
-  (id) => AGENTS[id].deliveryMode === "managed-pins",
-);
-
-export function detectInstalledAgents(
-  options: {
-    home?: string;
-    codexHome?: string;
-    exists?: (path: string) => boolean;
-  } = {},
-): DetectedAgent[] {
-  const home = options.home ?? homedir();
-  const codexHome = options.codexHome ?? join(home, ".codex");
-  const exists = options.exists ?? existsSync;
-  return SUPPORTED_AGENT_IDS.flatMap((agent) => {
-    const evidence = AGENTS[agent].detectionPath?.({ home, codexHome });
-    return evidence && exists(evidence) ? [{ agent, evidence }] : [];
-  });
-}
-
-function surfacePath(
-  surfaceId: NonNullable<AgentDefinition["surfaceId"]>,
-  options: { home: string; codexHome?: string },
-): string {
-  if (surfaceId === "agent-skills") return join(options.home, ".agents", "skills");
-  if (surfaceId === "claude-code") return join(options.home, ".claude", "skills");
-  if (surfaceId === "codex") return join(options.codexHome ?? join(options.home, ".codex"), "skills");
-  return join(options.home, ".gemini", "config", "skills");
-}
-
-export function resolveBuiltInTarget(
-  name: string,
-  options: { home?: string; codexHome?: string; customPath?: string } = {},
-): ResolvedBuiltInTarget {
-  const home = options.home ?? homedir();
-  if (name === "custom") {
-    if (!options.customPath) throw new Error("--target custom requires --path <dir>");
-    return { targetName: name, path: options.customPath };
-  }
-  if (name === "agent-skills") {
-    return { targetName: name, path: surfacePath("agent-skills", { home }) };
-  }
-  if (name === "claude-code") {
-    return { targetName: name, path: surfacePath("claude-code", { home }) };
-  }
-  if (name === "codex") {
-    return {
-      targetName: name,
-      path: surfacePath("codex", { home, codexHome: options.codexHome }),
-    };
-  }
-  throw new Error(
-    `unknown --target "${name}"; supported targets: agent-skills, claude-code, codex, custom`,
-  );
-}
-
-export function planAgentSurfaces(
-  requestedAgents: readonly string[],
-  options: { home?: string; codexHome?: string } = {},
-): AgentSurfacePlan {
-  const agents = [...new Set(requestedAgents)].map((id) => {
-    if (!SUPPORTED_AGENT_IDS.includes(id as AgentId)) {
-      throw new Error(
-        `unsupported agent "${id}"; supported agents: ${SUPPORTED_AGENT_IDS.join(", ")}`,
-      );
-    }
-    return AGENTS[id as AgentId];
-  });
-  const home = options.home ?? homedir();
-  const surfaces = new Map<string, PlannedAgentSurface>();
-
-  for (const agent of agents) {
-    if (!agent.surfaceId) continue;
-    const path = surfacePath(agent.surfaceId, { home, codexHome: options.codexHome });
-    const existing = surfaces.get(path);
-    if (existing) {
-      if (!existing.agents.includes(agent.id)) existing.agents.push(agent.id);
-      continue;
-    }
-    surfaces.set(path, {
-      id: agent.surfaceId,
-      targetName: agent.surfaceId,
-      path,
-      deliveryMode: "managed-pins",
-      agents: [agent.id],
-    });
-  }
-
-  return { agents, surfaces: [...surfaces.values()] };
-}
-
-export function assessAgentReadiness(
-  plan: AgentSurfacePlan,
-  instructionReadiness: Partial<Record<AgentId, ReadinessAxis>> = {},
-): AgentReadiness[] {
-  return plan.agents.map((agent) => {
-    const surface = plan.surfaces.find((candidate) => candidate.agents.includes(agent.id));
-    let skillSurface: ReadinessAxis;
-    if (surface) {
-      skillSurface = { status: "planned", detail: surface.path };
-    } else if (agent.manualSkillSurfaceMessage) {
-      skillSurface = { status: "manual", detail: agent.manualSkillSurfaceMessage };
-    } else {
-      skillSurface = {
-        status: "not-applicable",
-        detail: "skills resolve through Skillmux MCP",
-      };
-    }
-
-    const mcpRegistration: ReadinessAxis = {
-      status: "not-applicable",
-      detail: "native skill loading",
-    };
-
-    return {
-      agent: agent.id,
-      skillSurface,
-      mcpRegistration,
-      instructionSetup: instructionReadiness[agent.id] ?? {
-        status: "manual",
-        detail: "instruction adapter not applied",
-      },
-    };
-  });
-}