@klhapp/skillmux 1.11.2 → 1.13.0

package/src/db-index.ts

@@ -1,238 +0,0 @@
-import { Database } from "bun:sqlite";
-import { mkdirSync } from "node:fs";
-import { join } from "node:path";
-import type { VaultSkill } from "./vault";
-
-export interface SkillRow {
-  skill_id: string;
-  title: string;
-  description: string;
-  aliases: string;
-  content_sha256: string;
-}
-
-export function openIndex(stateDir: string): Database {
-  mkdirSync(stateDir, { recursive: true });
-  const db = new Database(join(stateDir, "index.sqlite3"), { create: true });
-  db.run("PRAGMA journal_mode = WAL");
-  db.run("PRAGMA busy_timeout = 2000");
-  db.run(`CREATE TABLE IF NOT EXISTS skills (
-    skill_id TEXT PRIMARY KEY,
-    title TEXT NOT NULL,
-    description TEXT NOT NULL,
-    aliases TEXT NOT NULL,
-    content_sha256 TEXT NOT NULL
-  )`);
-  db.run(`CREATE VIRTUAL TABLE IF NOT EXISTS skills_fts USING fts5(
-    skill_id UNINDEXED, title, description, aliases,
-    tokenize = 'unicode61 remove_diacritics 2'
-  )`);
-  db.run(`CREATE TABLE IF NOT EXISTS vectors (
-    skill_id TEXT PRIMARY KEY,
-    content_sha256 TEXT NOT NULL,
-    embedding_fingerprint TEXT NOT NULL DEFAULT '',
-    dim INTEGER NOT NULL,
-    vec BLOB NOT NULL
-  )`);
-  const vectorColumns = db.query("PRAGMA table_info(vectors)").all() as { name: string }[];
-  if (!vectorColumns.some((column) => column.name === "embedding_fingerprint")) {
-    db.run("ALTER TABLE vectors ADD COLUMN embedding_fingerprint TEXT NOT NULL DEFAULT ''");
-  }
-  // Audit rows live in audit.sqlite3; openAudit adopts any left here.
-  db.run(`CREATE TABLE IF NOT EXISTS index_meta (
-    key TEXT PRIMARY KEY,
-    value TEXT NOT NULL
-  )`);
-  return db;
-}
-
-export function upsertSkill(db: Database, skill: VaultSkill): void {
-  const aliases = skill.aliases.join(" ");
-  db.transaction(() => {
-    db.run("DELETE FROM skills WHERE skill_id = ?", [skill.skill_id]);
-    db.run("DELETE FROM skills_fts WHERE skill_id = ?", [skill.skill_id]);
-    db.run(
-      "INSERT INTO skills (skill_id, title, description, aliases, content_sha256) VALUES (?, ?, ?, ?, ?)",
-      [skill.skill_id, skill.title, skill.description, aliases, skill.content_sha256],
-    );
-    db.run(
-      "INSERT INTO skills_fts (skill_id, title, description, aliases) VALUES (?, ?, ?, ?)",
-      [skill.skill_id, skill.title, skill.description, aliases],
-    );
-  })();
-}
-
-export function toSkillRow(skill: VaultSkill): SkillRow {
-  return {
-    skill_id: skill.skill_id,
-    title: skill.title,
-    description: skill.description,
-    aliases: skill.aliases.join(" "),
-    content_sha256: skill.content_sha256,
-  };
-}
-
-/** Replace the whole lexical index with `rows`; drops vectors of removed skills. */
-export function replaceSkills(db: Database, rows: SkillRow[]): void {
-  db.transaction(() => {
-    db.run("DELETE FROM skills");
-    db.run("DELETE FROM skills_fts");
-    for (const row of rows) {
-      db.run(
-        "INSERT INTO skills (skill_id, title, description, aliases, content_sha256) VALUES (?, ?, ?, ?, ?)",
-        [row.skill_id, row.title, row.description, row.aliases, row.content_sha256],
-      );
-      db.run(
-        "INSERT INTO skills_fts (skill_id, title, description, aliases) VALUES (?, ?, ?, ?)",
-        [row.skill_id, row.title, row.description, row.aliases],
-      );
-    }
-    db.run("DELETE FROM vectors WHERE skill_id NOT IN (SELECT skill_id FROM skills)");
-  })();
-}
-
-export function ingestVault(db: Database, skills: VaultSkill[]): void {
-  replaceSkills(db, skills.map(toSkillRow));
-}
-
-export function deleteSkill(db: Database, skillId: string): void {
-  db.transaction(() => {
-    db.run("DELETE FROM skills WHERE skill_id = ?", [skillId]);
-    db.run("DELETE FROM skills_fts WHERE skill_id = ?", [skillId]);
-    db.run("DELETE FROM vectors WHERE skill_id = ?", [skillId]);
-  })();
-}
-
-export function skillCount(db: Database): number {
-  return (db.query("SELECT count(*) AS n FROM skills").get() as { n: number }).n;
-}
-
-export function getSkillRow(db: Database, skillId: string): SkillRow | null {
-  return db.query("SELECT * FROM skills WHERE skill_id = ?").get(skillId) as SkillRow | null;
-}
-
-/**
- * Sanitize free text into an FTS5 OR-query; returns null when no usable terms
- * remain. Terms keep any Unicode letters/digits (CJK included) so non-ASCII
- * queries still get lexical recall — unicode61 tokenizes contiguous CJK runs
- * as single tokens, so matching works at that granularity.
- */
-export function toFtsQuery(text: string): string | null {
-  const terms = [
-    ...new Set(
-      text
-        .toLowerCase()
-        .split(/[^\p{L}\p{N}]+/u)
-        .filter((t) => t.length >= 2),
-    ),
-  ];
-  if (terms.length === 0) return null;
-  return terms.map((t) => `"${t}"`).join(" OR ");
-}
-
-export function ftsSearch(db: Database, text: string, k: number): SkillRow[] {
-  const query = toFtsQuery(text);
-  if (query === null) return [];
-  return db
-    .query(
-      `SELECT s.* FROM skills_fts f
-       JOIN skills s ON s.skill_id = f.skill_id
-       WHERE skills_fts MATCH ?
-       ORDER BY bm25(skills_fts) LIMIT ?`,
-    )
-    .all(query, k) as SkillRow[];
-}
-
-export function findExactMatch(db: Database, query: string): SkillRow | null {
-  const cleanQuery = query.trim().toLowerCase();
-  return db
-    .query(
-      `SELECT * FROM skills 
-       WHERE lower(skill_id) = ? 
-          OR lower(title) = ? 
-          OR ' ' || lower(aliases) || ' ' LIKE ?`,
-    )
-    .get(cleanQuery, cleanQuery, `% ${cleanQuery} %`) as SkillRow | null;
-}
-
-export function getIndexMeta(db: Database, key: string): string | null {
-  const row = db
-    .query("SELECT value FROM index_meta WHERE key = ?")
-    .get(key) as { value: string } | null;
-  return row ? row.value : null;
-}
-
-export function setIndexMeta(db: Database, key: string, value: string): void {
-  db.run(
-    `INSERT INTO index_meta (key, value) VALUES (?, ?)
-     ON CONFLICT(key) DO UPDATE SET value = excluded.value`,
-    [key, value],
-  );
-}
-
-export function upsertVector(
-  db: Database,
-  skillId: string,
-  contentSha256: string,
-  embeddingFingerprint: string,
-  vec: Float32Array,
-): void {
-  db.run(
-    `INSERT INTO vectors (skill_id, content_sha256, embedding_fingerprint, dim, vec) VALUES (?, ?, ?, ?, ?)
-     ON CONFLICT(skill_id) DO UPDATE SET
-       content_sha256 = excluded.content_sha256,
-       embedding_fingerprint = excluded.embedding_fingerprint,
-       dim = excluded.dim,
-       vec = excluded.vec`,
-    [skillId, contentSha256, embeddingFingerprint, vec.length, new Uint8Array(vec.buffer, vec.byteOffset, vec.byteLength)],
-  );
-}
-
-/**
- * Skills with no usable stored vector: none at all, content changed since
- * embedding, or embedded at a different dimension than currently configured.
- */
-export function skillsNeedingVectors(db: Database, dimension: number, embeddingFingerprint: string): SkillRow[] {
-  return db
-    .query(
-      `SELECT s.* FROM skills s
-       LEFT JOIN vectors v ON v.skill_id = s.skill_id
-          AND v.content_sha256 = s.content_sha256
-          AND v.dim = ?
-          AND v.embedding_fingerprint = ?
-        WHERE v.skill_id IS NULL`,
-    )
-    .all(dimension, embeddingFingerprint) as SkillRow[];
-}
-
-function cosine(a: Float32Array, b: Float32Array): number {
-  if (a.length !== b.length) return 0;
-  let dot = 0;
-  let normA = 0;
-  let normB = 0;
-  for (let i = 0; i < a.length; i++) {
-    dot += a[i]! * b[i]!;
-    normA += a[i]! * a[i]!;
-    normB += b[i]! * b[i]!;
-  }
-  if (normA === 0 || normB === 0) return 0;
-  return dot / (Math.sqrt(normA) * Math.sqrt(normB));
-}
-
-/** Brute-force cosine over every stored vector (vault is ~100 skills; no ANN). */
-export function vectorTopK(db: Database, query: Float32Array, k: number): SkillRow[] {
-  const rows = db
-    .query(
-      `SELECT s.skill_id, s.title, s.description, s.aliases, s.content_sha256, v.vec
-       FROM vectors v JOIN skills s ON s.skill_id = v.skill_id`,
-    )
-    .all() as (SkillRow & { vec: Uint8Array })[];
-  return rows
-    .map(({ vec, ...row }) => ({
-      row,
-      score: cosine(query, new Float32Array(vec.buffer, vec.byteOffset, vec.byteLength / 4)),
-    }))
-    .sort((a, b) => b.score - a.score)
-    .slice(0, k)
-    .map((r) => r.row);
-}

package/src/db.ts

@@ -1,3 +0,0 @@
-// Barrel re-export — keeps all existing `import { X } from "./db"` working.
-export * from "./db-index";
-export * from "./db-audit";

package/src/deployment.ts

@@ -1,39 +0,0 @@
-import packageJson from "../package.json" with { type: "json" };
-import { expandHome } from "./config";
-import type { Config } from "./types";
-
-export interface DeploymentIdentity {
-  version: string;
-  runtime: "host" | "docker";
-  image_variant: "full" | "slim" | null;
-  vault_path: string;
-  state_dir: string;
-  inference_mode: Config["inference"]["mode"];
-  local_embedding_bundle: string | null;
-  remote_embedding_configured: boolean;
-  remote_reranker_configured: boolean;
-}
-
-export function describeDeployment(
-  config: Config,
-  environment: Record<string, string | undefined> = process.env,
-): DeploymentIdentity {
-  const runtime = environment.RUNNING_IN_DOCKER === "true" ? "docker" : "host";
-  const variant = environment.SKILLMUX_IMAGE_VARIANT;
-  const imageVariant = runtime === "docker" && (variant === "full" || variant === "slim")
-    ? variant
-    : null;
-
-  return {
-    version: packageJson.version,
-    runtime,
-    image_variant: imageVariant,
-    vault_path: expandHome(config.vault_path),
-    state_dir: expandHome(config.state_dir),
-    inference_mode: config.inference.mode,
-    local_embedding_bundle: config.inference.mode === "local" ? config.inference.bundle : null,
-    remote_embedding_configured: config.inference.mode === "remote",
-    remote_reranker_configured:
-      config.inference.mode === "remote" && !!config.inference.reranker,
-  };
-}

package/src/doctor.ts

@@ -1,205 +0,0 @@
-import { existsSync, mkdirSync } from "node:fs";
-import { createClients, RemoteInferenceError } from "./clients";
-import { embeddingDimension, expandHome, isLoopbackBindHost } from "./config";
-import { describeDeployment, type DeploymentIdentity } from "./deployment";
-import { parseManifest, resolveManifestPath, validateManifest } from "./manifest";
-import { readSkillmuxMarker } from "./sync";
-import type { Config } from "./types";
-import { findShadowedSkills } from "./vault";
-
-export interface DoctorCheck {
-  name: string;
-  ok: boolean;
-  detail: string;
-  failure_kind?: "configuration" | "availability" | "protocol" | "unexpected";
-}
-
-export interface DoctorReport {
-  mode: Config["inference"]["mode"];
-  capability: "hybrid" | "lexical-only" | "unavailable";
-  retrieval_capability: "lexical" | "hybrid" | "reranked";
-  version: DeploymentIdentity["version"];
-  runtime: DeploymentIdentity["runtime"];
-  image_variant: DeploymentIdentity["image_variant"];
-  vault_path: DeploymentIdentity["vault_path"];
-  state_dir: DeploymentIdentity["state_dir"];
-  inference_mode: DeploymentIdentity["inference_mode"];
-  local_embedding_bundle: DeploymentIdentity["local_embedding_bundle"];
-  remote_embedding_configured: DeploymentIdentity["remote_embedding_configured"];
-  remote_reranker_configured: DeploymentIdentity["remote_reranker_configured"];
-  checks: DoctorCheck[];
-}
-
-export { describeDeployment };
-
-export async function diagnose(
-  config: Config,
-  environment: Record<string, string | undefined> = process.env,
-  sources: Record<string, "default" | "toml" | "environment" | "admin"> = {},
-): Promise<DoctorReport> {
-  const checks: DoctorCheck[] = [];
-  const envOverrides = config.config?.environment_overrides !== false;
-  checks.push({
-    name: "config_authority",
-    ok: true,
-    detail: envOverrides
-      ? "environment overrides enabled"
-      : "TOML authoritative (environment overrides disabled)",
-  });
-  for (const [key, source] of Object.entries(sources)) {
-    checks.push({ name: `config_source:${key}`, ok: true, detail: source });
-  }
-  checks.push({ name: "vault", ok: existsSync(expandHome(config.vault_path)), detail: expandHome(config.vault_path) });
-
-  // SMX-91: `serve --transport http` itself refuses to start over this combination
-  // (assertSafeBindPosture in server.ts) unless SKILLMUX_ALLOW_INSECURE_BIND is set —
-  // surface it here too so it's visible without having to start the HTTP server first.
-  // An operator who has already set that env var has made an informed choice, so
-  // doctor treats it the same way the server does (ok, not a standing failure) —
-  // it doesn't re-litigate a decision the server itself already accepted.
-  if (config.server) {
-    const hostname = config.server.hostname ?? "127.0.0.1";
-    const insecureBindAcknowledged = environment.SKILLMUX_ALLOW_INSECURE_BIND === "true";
-    const bindIsSafe = isLoopbackBindHost(hostname) || config.server.auth_enabled || insecureBindAcknowledged;
-    checks.push({
-      name: "server_bind_posture",
-      ok: bindIsSafe,
-      detail: !bindIsSafe
-        ? `${hostname} is reachable beyond this machine with auth_enabled=false — ` +
-          "MCP tools and /stats would be open to anyone who can reach this port; " +
-          "set server.auth_enabled=true, bind a loopback hostname, or set SKILLMUX_ALLOW_INSECURE_BIND=true"
-        : insecureBindAcknowledged && !isLoopbackBindHost(hostname) && !config.server.auth_enabled
-          ? `${hostname}, auth_enabled=false, acknowledged via SKILLMUX_ALLOW_INSECURE_BIND`
-          : `${hostname}, auth_enabled=${config.server.auth_enabled}`,
-      failure_kind: bindIsSafe ? undefined : "configuration",
-    });
-  }
-
-  for (const localPath of config.local_vault_paths) {
-    const expanded = expandHome(localPath);
-    checks.push({ name: `local_vault:${localPath}`, ok: existsSync(expanded), detail: expanded });
-
-    const strayManifest = resolveManifestPath(expanded);
-    if (strayManifest) {
-      checks.push({
-        name: `local_vault_manifest:${localPath}`,
-        ok: false,
-        detail: `stray manifest at ${strayManifest} — skillmux.toml only ever lives in vault_path, never in local_vault_paths`,
-      });
-    }
-
-    const marker = readSkillmuxMarker(expanded);
-    const currentVaultPath = expandHome(config.vault_path);
-    if (!marker || marker.role !== "local_vault") {
-      checks.push({
-        name: `local_vault_marker:${localPath}`,
-        ok: false,
-        detail: `no marker — run: skillmux local-vault init "${expanded}"`,
-      });
-    } else if (marker.vault_path !== currentVaultPath) {
-      checks.push({
-        name: `local_vault_marker:${localPath}`,
-        ok: false,
-        detail: `marker recorded vault_path ${marker.vault_path}, currently configured vault_path is ${currentVaultPath} — drift, re-run skillmux local-vault init`,
-      });
-    } else {
-      checks.push({ name: `local_vault_marker:${localPath}`, ok: true, detail: expanded });
-    }
-  }
-
-  for (const shadow of findShadowedSkills(expandHome(config.vault_path), config.local_vault_paths.map(expandHome))) {
-    checks.push({
-      name: `shadowed:${shadow.skill_id}`,
-      ok: true,
-      detail: `served from ${shadow.winner}; shadows ${shadow.shadowed.join(", ")}`,
-    });
-  }
-
-  const vaultPath = expandHome(config.vault_path);
-  const manifestPath = resolveManifestPath(vaultPath);
-  if (!manifestPath) {
-    checks.push({ name: "manifest", ok: true, detail: "not yet initialized" });
-  } else {
-    try {
-      const manifest = parseManifest(await Bun.file(manifestPath).text());
-      validateManifest(manifest, vaultPath, config.local_vault_paths.map(expandHome));
-      checks.push({ name: "manifest", ok: true, detail: manifestPath });
-    } catch (error) {
-      checks.push({
-        name: `manifest:${manifestPath}`,
-        ok: false,
-        detail: error instanceof Error ? error.message : String(error),
-      });
-    }
-  }
-
-  try {
-    mkdirSync(expandHome(config.state_dir), { recursive: true });
-    const probe = Bun.file(expandHome(`${config.state_dir}/.doctor`));
-    await probe.write("");
-    await probe.delete();
-    checks.push({ name: "state", ok: true, detail: expandHome(config.state_dir) });
-  } catch (error) {
-    checks.push({ name: "state", ok: false, detail: String(error) });
-  }
-
-  if (config.inference.mode === "local") {
-    try {
-      mkdirSync(expandHome(config.inference.models_dir), { recursive: true });
-      checks.push({ name: "models", ok: true, detail: expandHome(config.inference.models_dir) });
-    } catch (error) {
-      checks.push({ name: "models", ok: false, detail: String(error) });
-    }
-  }
-
-  const inferenceFailure = (error: unknown): Pick<DoctorCheck, "detail" | "failure_kind"> =>
-    error instanceof RemoteInferenceError
-      ? { detail: error.message, failure_kind: error.kind }
-      : { detail: "unexpected inference failure", failure_kind: "unexpected" };
-
-  const deployment = describeDeployment(config, environment);
-  const lexicalOnlySlim = deployment.image_variant === "slim" && config.inference.mode === "local";
-  if (lexicalOnlySlim) {
-    checks.push({
-      name: "retrieval",
-      ok: true,
-      detail: "lexical retrieval; Configure remote embeddings for hybrid retrieval",
-    });
-  } else {
-    const clients = createClients(config);
-    try {
-      const vectors = await clients.embed(["skill router diagnostic"]);
-      const actualDimension = vectors[0]?.length ?? 0;
-      checks.push({
-        name: "embedding",
-        ok: actualDimension === embeddingDimension(config),
-        detail: `dimension ${actualDimension}`,
-      });
-    } catch (error) {
-      checks.push({ name: "embedding", ok: false, ...inferenceFailure(error) });
-    }
-
-    if (clients.rerank) {
-      try {
-        const scores = await clients.rerank("skill router diagnostic", [
-          { skill_id: "doctor", text: "Routes a task to an appropriate skill." },
-        ]);
-        checks.push({ name: "reranker", ok: scores.length === 1 && Number.isFinite(scores[0]), detail: "one finite score" });
-      } catch (error) {
-        checks.push({ name: "reranker", ok: false, ...inferenceFailure(error) });
-      }
-    }
-  }
-
-  const inferenceReady = checks.some((check) => check.name === "embedding" && check.ok);
-  const rerankerReady = checks.some((check) => check.name === "reranker" && check.ok);
-  const coreReady = checks.some((check) => check.name === "vault" && check.ok)
-    && checks.some((check) => check.name === "state" && check.ok);
-  return {
-    ...deployment,
-    mode: config.inference.mode,
-    capability: !coreReady ? "unavailable" : inferenceReady ? "hybrid" : "lexical-only",
-    retrieval_capability: rerankerReady ? "reranked" : inferenceReady ? "hybrid" : "lexical",
-    checks,
-  };
-}