@stigmer/runner 3.40.0 → 3.40.1

package/src/shared/__tests__/skill-relevance.test.ts

@@ -1,292 +0,0 @@
-import { describe, it, expect } from "vitest";
-import {
-  tokenize,
-  scoreSkills,
-  filterSkills,
-  SKILL_COUNT_THRESHOLD,
-  type ScoredSkill,
-  type SkillFilterResult,
-} from "../skill-relevance.js";
-
-// ─── Tokenisation ────────────────────────────────────────────────────────
-
-describe("tokenize", () => {
-  it("extracts meaningful tokens from a sentence", () => {
-    const tokens = tokenize("Deploy Kubernetes pods to the cluster");
-    expect(tokens).toContain("deploy");
-    expect(tokens).toContain("kubernetes");
-    expect(tokens).toContain("pods");
-    expect(tokens).toContain("cluster");
-    expect(tokens).not.toContain("the");
-    expect(tokens).not.toContain("to");
-  });
-
-  it("strips punctuation and splits on non-alphanumeric", () => {
-    const tokens = tokenize("hello-world! (v2.0)");
-    expect(tokens).toContain("hello");
-    expect(tokens).toContain("world");
-    expect(tokens).toContain("v2");
-  });
-
-  it("drops single-character tokens", () => {
-    const tokens = tokenize("a b c deploy x");
-    expect(tokens).toContain("deploy");
-    expect(tokens).not.toContain("a");
-    expect(tokens).not.toContain("b");
-    expect(tokens).not.toContain("x");
-  });
-
-  it("returns empty for empty string", () => {
-    expect(tokenize("")).toEqual([]);
-  });
-
-  it("returns empty for stop-words-only input", () => {
-    expect(tokenize("the a is to in for")).toEqual([]);
-  });
-
-  it("preserves numbers within tokens", () => {
-    const tokens = tokenize("python3 version 42");
-    expect(tokens).toContain("python3");
-    expect(tokens).toContain("version");
-    expect(tokens).toContain("42");
-  });
-
-  it("splits kebab-case into individual tokens", () => {
-    const tokens = tokenize("code-review-best-practices");
-    expect(tokens).toContain("code");
-    expect(tokens).toContain("review");
-    expect(tokens).toContain("best");
-    expect(tokens).toContain("practices");
-  });
-});
-
-// ─── BM25 scoring ────────────────────────────────────────────────────────
-
-describe("scoreSkills", () => {
-  it("scores exact match highest", () => {
-    const scored = scoreSkills(
-      "kubernetes deployment",
-      ["kubernetes-operator", "code-review", "docker-expert"],
-      [
-        "Manages Kubernetes deployments and services",
-        "Reviews code changes for quality",
-        "Docker container management and Dockerfile optimization",
-      ],
-    );
-    expect(scored[0].name).toBe("kubernetes-operator");
-    expect(scored[0].score).toBeGreaterThan(0);
-  });
-
-  it("gives zero score when no term overlap", () => {
-    const scored = scoreSkills(
-      "kubernetes deployment",
-      ["code-review"],
-      ["Reviews code changes for quality"],
-    );
-    expect(scored[0].score).toBe(0.0);
-  });
-
-  it("returns all zero scores for empty query", () => {
-    const scored = scoreSkills(
-      "",
-      ["skill-a", "skill-b"],
-      ["description a", "description b"],
-    );
-    expect(scored.every(s => s.score === 0.0)).toBe(true);
-  });
-
-  it("returns empty for empty skills list", () => {
-    const scored = scoreSkills("some query", [], []);
-    expect(scored).toEqual([]);
-  });
-
-  it("returns all skills", () => {
-    const scored = scoreSkills(
-      "deploy",
-      ["a", "b", "c"],
-      ["deploy", "review", "test"],
-    );
-    expect(scored).toHaveLength(3);
-  });
-
-  it("returns results in descending score order", () => {
-    const scored = scoreSkills(
-      "kubernetes cluster deployment management",
-      ["kubernetes-operator", "generic-helper", "k8s-deploy"],
-      [
-        "Manages Kubernetes clusters and deployments",
-        "Generic task helper with no specialization",
-        "Kubernetes deployment automation",
-      ],
-    );
-    const scores = scored.map(s => s.score);
-    expect(scores).toEqual([...scores].sort((a, b) => b - a));
-  });
-
-  it("ranks partial-match skill higher than no-match", () => {
-    const scored = scoreSkills(
-      "review code quality",
-      ["code-review", "deploy-agent"],
-      [
-        "Reviews code for quality and security issues",
-        "Deploys applications to cloud infrastructure",
-      ],
-    );
-    expect(scored[0].name).toBe("code-review");
-    expect(scored[0].score).toBeGreaterThan(scored[1].score);
-  });
-
-  it("preserves original index", () => {
-    const scored = scoreSkills(
-      "review",
-      ["deploy", "review", "test"],
-      ["deploy", "review", "test"],
-    );
-    const reviewEntry = scored.find(s => s.name === "review");
-    expect(reviewEntry).toBeDefined();
-    expect(reviewEntry!.index).toBe(1);
-  });
-
-  it("ignores stop words in query", () => {
-    const scoredWithStops = scoreSkills(
-      "the kubernetes is a deployment",
-      ["k8s"],
-      ["kubernetes deployment manager"],
-    );
-    const scoredWithoutStops = scoreSkills(
-      "kubernetes deployment",
-      ["k8s"],
-      ["kubernetes deployment manager"],
-    );
-    expect(scoredWithStops[0].score).toBe(scoredWithoutStops[0].score);
-  });
-});
-
-// ─── Filtering ───────────────────────────────────────────────────────────
-
-describe("filterSkills", () => {
-  function makeNamesAndDescs(n: number): { names: string[]; descs: string[] } {
-    const names = Array.from({ length: n }, (_, i) => `skill-${i}`);
-    const descs = Array.from({ length: n }, (_, i) => `Description for skill number ${i}`);
-    return { names, descs };
-  }
-
-  it("includes all when below threshold", () => {
-    const { names, descs } = makeNamesAndDescs(5);
-    const result = filterSkills("any query", names, descs, { threshold: 8 });
-    expect(result.includedIndices).toEqual([0, 1, 2, 3, 4]);
-    expect(result.excludedIndices).toEqual([]);
-    expect(result.excludedNames).toEqual([]);
-  });
-
-  it("includes all at threshold boundary (count < threshold)", () => {
-    const { names, descs } = makeNamesAndDescs(7);
-    const result = filterSkills("any query", names, descs, { threshold: 8 });
-    expect(result.includedIndices).toHaveLength(7);
-    expect(result.excludedIndices).toEqual([]);
-  });
-
-  it("filters irrelevant skills above threshold", () => {
-    const names = [
-      "kubernetes-operator", "docker-expert", "code-review",
-      "terraform-iac", "postgres-admin", "redis-cache",
-      "monitoring-agent", "security-scanner", "api-tester", "log-analyzer",
-    ];
-    const descs = [
-      "Manages Kubernetes clusters and deployments",
-      "Docker container management",
-      "Reviews code changes",
-      "Terraform infrastructure as code",
-      "PostgreSQL database administration",
-      "Redis cache management",
-      "Monitoring and alerting setup",
-      "Security vulnerability scanning",
-      "API endpoint testing",
-      "Log analysis and aggregation",
-    ];
-    const result = filterSkills("deploy kubernetes pods", names, descs, { threshold: 8 });
-    expect(result.includedIndices).toContain(0); // kubernetes-operator
-    expect(result.excludedNames.length).toBeGreaterThan(0);
-    expect(result.excludedNames).not.toContain("kubernetes-operator");
-  });
-
-  it("enforces safety floor (at least half included)", () => {
-    const names = Array.from({ length: 10 }, (_, i) => `skill-${i}`);
-    const descs = Array.from({ length: 10 }, () => "completely unrelated");
-    const result = filterSkills("xyzzy nonsense query", names, descs, { threshold: 8 });
-    expect(result.includedIndices.length).toBeGreaterThanOrEqual(5);
-  });
-
-  it("sorts excluded names alphabetically", () => {
-    const names = [
-      "zebra-skill", "alpha-skill", "middle-skill",
-      "beta-skill", "gamma-skill", "delta-skill",
-      "epsilon-skill", "zeta-skill", "eta-skill", "theta-skill",
-    ];
-    const descs = Array.from({ length: 10 }, () => "unrelated");
-    const result = filterSkills("something specific to alpha", names, descs, { threshold: 8 });
-    expect(result.excludedNames).toEqual([...result.excludedNames].sort());
-  });
-
-  it("maintains original order for included indices", () => {
-    const names = Array.from({ length: 10 }, (_, i) => `skill-${i}`);
-    const descs = [
-      "kubernetes deployment", "code review", "kubernetes pods",
-      "docker containers", "unrelated thing", "another unrelated",
-      "still unrelated", "more unrelated stuff", "kubernetes services",
-      "very unrelated",
-    ];
-    const result = filterSkills("kubernetes deployment", names, descs, { threshold: 8 });
-    expect(result.includedIndices).toEqual([...result.includedIndices].sort((a, b) => a - b));
-  });
-
-  it("handles empty skills list", () => {
-    const result = filterSkills("query", [], []);
-    expect(result.includedIndices).toEqual([]);
-    expect(result.excludedIndices).toEqual([]);
-    expect(result.excludedNames).toEqual([]);
-  });
-
-  it("keeps most skills when all are relevant", () => {
-    const names = [
-      "k8s-deploy", "k8s-monitor", "k8s-scale",
-      "k8s-network", "k8s-storage", "k8s-secrets",
-      "k8s-rbac", "k8s-helm", "k8s-ingress", "k8s-cronjob",
-    ];
-    const descs = names.map(n => `Kubernetes ${n.split("-")[1]} management`);
-    const result = filterSkills("kubernetes cluster management", names, descs, { threshold: 8 });
-    expect(result.excludedNames.length).toBeLessThan(names.length);
-  });
-
-  it("returns correct types", () => {
-    const { names, descs } = makeNamesAndDescs(10);
-    const result = filterSkills("query", names, descs, { threshold: 8 });
-    expect(Array.isArray(result.includedIndices)).toBe(true);
-    expect(Array.isArray(result.excludedIndices)).toBe(true);
-    expect(Array.isArray(result.excludedNames)).toBe(true);
-  });
-
-  it("produces a disjoint partition of all indices", () => {
-    const names = Array.from({ length: 12 }, (_, i) => `skill-${i}`);
-    const descs = [
-      "kubernetes deploy", "code review", "docker build",
-      "terraform plan", "postgres query", "redis cache",
-      "monitoring alert", "security scan", "api test",
-      "log analysis", "network debug", "storage manage",
-    ];
-    const result = filterSkills("kubernetes deployment", names, descs, { threshold: 8 });
-    const allIndices = new Set([...result.includedIndices, ...result.excludedIndices]);
-    expect(allIndices.size).toBe(12);
-    expect(allIndices).toEqual(new Set(Array.from({ length: 12 }, (_, i) => i)));
-    // Verify disjoint
-    const intersection = result.includedIndices.filter(i => result.excludedIndices.includes(i));
-    expect(intersection).toEqual([]);
-  });
-
-  it("uses default threshold constant", () => {
-    expect(SKILL_COUNT_THRESHOLD).toBe(8);
-    const { names, descs } = makeNamesAndDescs(7);
-    const result = filterSkills("any query", names, descs);
-    expect(result.includedIndices).toHaveLength(7);
-  });
-});

package/src/shared/skill-relevance.ts

@@ -1,222 +0,0 @@
-/**
- * Skill relevance scoring for smart context filtering.
- *
- * Uses a BM25-inspired algorithm to score each skill's relevance
- * against the user message. When an agent has many skills configured,
- * low-relevance skills are excluded from the system prompt to improve
- * signal quality.
- *
- * The progressive disclosure model means only skill metadata
- * (~50-70 tokens per skill) lives in the prompt, so filtering
- * primarily reduces noise rather than saving tokens.
- *
- * BM25 (Best Matching 25) is a lightweight, well-understood ranking
- * function from information retrieval. It uses term frequency saturation
- * and document length normalisation — both important when matching a
- * short user message (query) against short skill metadata (documents of
- * 5-20 tokens each).
- *
- * No external dependencies required.
- */
-
-/** Relevance filtering activates only when the agent has at least this many skills. */
-export const SKILL_COUNT_THRESHOLD = 8;
-
-/** Term-frequency saturation. Higher values weight repeated terms more. */
-const BM25_K1 = 1.5;
-
-/** Length normalisation. 0 = none, 1 = full relative to average doc length. */
-const BM25_B = 0.75;
-
-const STOP_WORDS: ReadonlySet<string> = new Set([
-  "a", "an", "the", "is", "are", "was", "were", "be", "been",
-  "being", "have", "has", "had", "do", "does", "did", "will",
-  "would", "could", "should", "may", "might", "shall", "can",
-  "to", "of", "in", "for", "on", "with", "at", "by", "from",
-  "as", "into", "through", "during", "before", "after", "above",
-  "below", "between", "and", "but", "or", "nor", "not", "so",
-  "yet", "both", "either", "neither", "each", "every", "all",
-  "any", "few", "more", "most", "some", "such", "no", "only",
-  "own", "same", "than", "too", "very", "just", "because",
-  "about", "up", "out", "if", "then", "that", "this", "these",
-  "those", "it", "its", "i", "me", "my", "we", "our", "you",
-  "your", "he", "she", "they", "them", "their", "what", "which",
-  "who", "whom", "how", "when", "where", "why",
-]);
-
-const TOKEN_PATTERN = /[a-z0-9]+/g;
-
-// ─── Tokenisation ────────────────────────────────────────────────────────
-
-/**
- * Split text into lowercase tokens, dropping stop words and
- * single-character fragments.
- */
-export function tokenize(text: string): string[] {
-  const matches = text.toLowerCase().match(TOKEN_PATTERN);
-  if (!matches) return [];
-  return matches.filter(tok => !STOP_WORDS.has(tok) && tok.length > 1);
-}
-
-// ─── Data structures ─────────────────────────────────────────────────────
-
-export interface ScoredSkill {
-  readonly index: number;
-  readonly name: string;
-  readonly score: number;
-}
-
-export interface SkillFilterResult {
-  /** Indices of skills to include in the prompt (original order). */
-  readonly includedIndices: number[];
-  /** Indices of skills excluded from the prompt (original order). */
-  readonly excludedIndices: number[];
-  /** Names of excluded skills (sorted alphabetically). */
-  readonly excludedNames: string[];
-}
-
-// ─── BM25 scoring ────────────────────────────────────────────────────────
-
-/**
- * Score each skill's relevance to the user message using BM25.
- *
- * Each skill's name and description are concatenated into a document
- * and scored against the query terms extracted from the user message.
- *
- * Returns ScoredSkill list ordered by descending score (ties broken alphabetically).
- */
-export function scoreSkills(
-  userMessage: string,
-  skillNames: readonly string[],
-  skillDescriptions: readonly string[],
-  options?: { k1?: number; b?: number },
-): ScoredSkill[] {
-  const k1 = options?.k1 ?? BM25_K1;
-  const b = options?.b ?? BM25_B;
-
-  const queryTerms = tokenize(userMessage);
-  const n = skillNames.length;
-
-  if (n === 0) return [];
-
-  if (queryTerms.length === 0) {
-    return skillNames.map((name, index) => ({ index, name, score: 0.0 }));
-  }
-
-  // Build per-skill term-frequency maps.
-  const docs: Map<string, number>[] = [];
-  const docLengths: number[] = [];
-
-  for (let i = 0; i < n; i++) {
-    const tokens = tokenize(`${skillNames[i]} ${skillDescriptions[i]}`);
-    const tf = new Map<string, number>();
-    for (const tok of tokens) {
-      tf.set(tok, (tf.get(tok) ?? 0) + 1);
-    }
-    docs.push(tf);
-    docLengths.push(tokens.length);
-  }
-
-  const avgdl = docLengths.reduce((sum, l) => sum + l, 0) / n;
-
-  // IDF per unique query term.
-  const uniqueQueryTerms = new Set(queryTerms);
-  const idf = new Map<string, number>();
-  for (const term of uniqueQueryTerms) {
-    let df = 0;
-    for (const doc of docs) {
-      if (doc.has(term)) df++;
-    }
-    idf.set(term, Math.max(0.0, Math.log((n - df + 0.5) / (df + 0.5) + 1.0)));
-  }
-
-  // Score each document.
-  const results: ScoredSkill[] = [];
-  for (let i = 0; i < n; i++) {
-    const docTf = docs[i];
-    const dl = docLengths[i];
-    let score = 0.0;
-    for (const term of queryTerms) {
-      const tfVal = docTf.get(term);
-      if (tfVal === undefined) continue;
-      const numerator = tfVal * (k1 + 1);
-      const denominator = tfVal + k1 * (1 - b + b * dl / avgdl);
-      score += (idf.get(term) ?? 0.0) * numerator / denominator;
-    }
-    results.push({ index: i, name: skillNames[i], score });
-  }
-
-  results.sort((a, b_) => {
-    if (b_.score !== a.score) return b_.score - a.score;
-    return a.name.localeCompare(b_.name);
-  });
-
-  return results;
-}
-
-// ─── Filtering ───────────────────────────────────────────────────────────
-
-/**
- * Partition skills into included / excluded based on relevance.
- *
- * Below the threshold, every skill is included unconditionally.
- * Above the threshold, skills whose BM25 score is zero (no query
- * term overlap at all) are moved to the excluded set.
- *
- * A safety floor guarantees that at least half the skills are
- * always included, regardless of scores.
- */
-export function filterSkills(
-  userMessage: string,
-  skillNames: readonly string[],
-  skillDescriptions: readonly string[],
-  options?: { threshold?: number },
-): SkillFilterResult {
-  const threshold = options?.threshold ?? SKILL_COUNT_THRESHOLD;
-  const n = skillNames.length;
-  const allIndices = Array.from({ length: n }, (_, i) => i);
-
-  if (n < threshold) {
-    return {
-      includedIndices: allIndices,
-      excludedIndices: [],
-      excludedNames: [],
-    };
-  }
-
-  const scored = scoreSkills(userMessage, skillNames, skillDescriptions);
-
-  const included: number[] = [];
-  const excluded: number[] = [];
-  const excludedNames: string[] = [];
-
-  for (const s of scored) {
-    if (s.score > 0.0) {
-      included.push(s.index);
-    } else {
-      excluded.push(s.index);
-      excludedNames.push(s.name);
-    }
-  }
-
-  // Safety: always keep at least half the skills.
-  const minIncluded = Math.max(1, Math.floor(n / 2));
-  if (included.length < minIncluded) {
-    const deficit = minIncluded - included.length;
-    const reInclude = excluded.splice(0, deficit);
-    included.push(...reInclude);
-    for (const idx of reInclude) {
-      const nameIdx = excludedNames.indexOf(skillNames[idx]);
-      if (nameIdx !== -1) excludedNames.splice(nameIdx, 1);
-    }
-  }
-
-  included.sort((a, b) => a - b);
-  excluded.sort((a, b) => a - b);
-
-  return {
-    includedIndices: included,
-    excludedIndices: excluded,
-    excludedNames: excludedNames.sort(),
-  };
-}