@klhapp/skillmux 1.12.0 → 1.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/CHANGELOG.md +8 -0
  2. package/README.md +18 -9
  3. package/bin/skillmux.js +89 -0
  4. package/docs/README.md +4 -4
  5. package/docs/cli.md +14 -4
  6. package/docs/concepts.md +2 -2
  7. package/docs/configuration.md +18 -3
  8. package/docs/deployment.md +3 -3
  9. package/docs/getting-started.md +32 -7
  10. package/docs/releasing.md +17 -2
  11. package/docs/sdd/cli-remote-target-parity/schema-bucket-b.yaml +297 -0
  12. package/docs/sdd/cli-remote-target-parity/spec-bucket-b.md +95 -0
  13. package/docs/sdd/cli-remote-target-parity/spec.md +111 -0
  14. package/docs/sdd/cli-remote-target-parity/think.md +183 -0
  15. package/docs/sdd/cli-surface-consistency/think.md +149 -0
  16. package/docs/sdd/config-authority-rerank-resilience/plan.md +338 -0
  17. package/docs/sdd/config-authority-rerank-resilience/review.md +124 -0
  18. package/docs/sdd/config-authority-rerank-resilience/spec.md +78 -0
  19. package/docs/sdd/feat-packaging-deployment/review.md +95 -0
  20. package/docs/sdd/fetch-outcome-flywheel/review.md +103 -0
  21. package/docs/sdd/fetch-outcome-flywheel/schema.json +347 -0
  22. package/docs/sdd/fetch-outcome-flywheel/spec.md +185 -0
  23. package/docs/sdd/npm-native-binaries/review.md +103 -0
  24. package/docs/sdd/npm-native-binaries/spec.md +148 -0
  25. package/docs/sdd/ranked-shortlist-2/pr3-calibration-removal.md +56 -0
  26. package/docs/sdd/refactor-builtin-target-paths/spec.md +41 -0
  27. package/docs/sdd/runtime-resource-hardening/review.md +86 -0
  28. package/docs/sdd/runtime-resource-hardening/spec.md +43 -0
  29. package/docs/sdd/security-hardening-phase/think.md +129 -0
  30. package/docs/sdd/security-observability/review.md +96 -0
  31. package/docs/sdd/security-observability/spec.md +55 -0
  32. package/docs/sdd/skill-provenance-update/review.md +152 -0
  33. package/docs/sdd/skill-provenance-update/schema.json +193 -0
  34. package/docs/sdd/skill-provenance-update/spec.md +134 -0
  35. package/docs/sdd/supply-chain-hardening/review.md +50 -0
  36. package/docs/sdd/supply-chain-hardening/spec.md +42 -0
  37. package/docs/sdd/target-marker-rehome/spec.md +33 -0
  38. package/docs/troubleshooting.md +18 -2
  39. package/package.json +15 -8
  40. package/src/adapters.ts +0 -438
  41. package/src/audit.ts +0 -21
  42. package/src/cli.ts +0 -928
  43. package/src/clients.ts +0 -368
  44. package/src/commands/audit.ts +0 -85
  45. package/src/commands/config.ts +0 -224
  46. package/src/commands/context.ts +0 -104
  47. package/src/commands/core.ts +0 -56
  48. package/src/commands/doctor.ts +0 -97
  49. package/src/commands/eval.ts +0 -88
  50. package/src/commands/init.ts +0 -672
  51. package/src/commands/install.ts +0 -155
  52. package/src/commands/local-vault.ts +0 -60
  53. package/src/commands/models.ts +0 -10
  54. package/src/commands/outdated.ts +0 -115
  55. package/src/commands/project.ts +0 -580
  56. package/src/commands/report.ts +0 -66
  57. package/src/commands/scan.ts +0 -71
  58. package/src/commands/shared.ts +0 -38
  59. package/src/commands/skill.ts +0 -33
  60. package/src/commands/sync.ts +0 -233
  61. package/src/commands/target.ts +0 -225
  62. package/src/commands/update.ts +0 -277
  63. package/src/completions.ts +0 -196
  64. package/src/concurrency-limiter.ts +0 -61
  65. package/src/config-service.ts +0 -398
  66. package/src/config-watcher.ts +0 -253
  67. package/src/config.ts +0 -583
  68. package/src/context.ts +0 -188
  69. package/src/db-audit.ts +0 -286
  70. package/src/db-index.ts +0 -238
  71. package/src/db.ts +0 -3
  72. package/src/deployment.ts +0 -39
  73. package/src/doctor.ts +0 -205
  74. package/src/eval.ts +0 -302
  75. package/src/global-flags.ts +0 -46
  76. package/src/init-agents.ts +0 -329
  77. package/src/init-instructions.ts +0 -192
  78. package/src/init.ts +0 -390
  79. package/src/install.ts +0 -232
  80. package/src/lifecycle.ts +0 -51
  81. package/src/logger.ts +0 -26
  82. package/src/manifest.ts +0 -359
  83. package/src/mcp-registration.ts +0 -89
  84. package/src/metrics.ts +0 -121
  85. package/src/models.ts +0 -20
  86. package/src/output.ts +0 -214
  87. package/src/project-setup.ts +0 -36
  88. package/src/prompts.ts +0 -124
  89. package/src/provenance.ts +0 -99
  90. package/src/rate-limiter.ts +0 -137
  91. package/src/readiness.ts +0 -30
  92. package/src/redact.ts +0 -52
  93. package/src/router-core.ts +0 -666
  94. package/src/rrf.ts +0 -31
  95. package/src/scan.ts +0 -300
  96. package/src/server.ts +0 -930
  97. package/src/setup.ts +0 -145
  98. package/src/snapshot.ts +0 -135
  99. package/src/stats.ts +0 -340
  100. package/src/sync.ts +0 -482
  101. package/src/toml-writer.ts +0 -51
  102. package/src/types.ts +0 -212
  103. package/src/vault.ts +0 -207
@@ -0,0 +1,78 @@
1
+ # Specification: Configuration Authority and Rerank Resilience
2
+
3
+ Feature Slug: `config-authority-rerank-resilience`
4
+ Plan: [plan.md](file:///Users/lancehsu/Code/skillmux/docs/sdd/config-authority-rerank-resilience/plan.md)
5
+ Status: Draft
6
+
7
+ ## Overview
8
+
9
+ Give Skillmux a predictable OSS configuration contract, bound reranker work via `k_rerank`, introduce `output.ambiguous_candidate_limit` with 1.x deprecation for `thresholds.candidate_limit`, and expose structured inference fallback reasons and evaluation details.
10
+
11
+ ## User Stories & Acceptance Criteria
12
+
13
+ ### Change 1: Configuration Authority (AC 1-5, 13, 17)
14
+
15
+ - **AC 1.1 (Precedence & Policy)**: When `config.environment_overrides = true` (default), configuration precedence is:
16
+ 1. CLI flags
17
+ 2. `SKILLMUX_*` environment variables
18
+ 3. TOML configuration file
19
+ 4. Built-in defaults
20
+ - **AC 1.2 (Strict TOML Mode)**: When `config.environment_overrides = false`, behavioral `SKILLMUX_*` environment variables (e.g. `SKILLMUX_EMBED_MODEL`, `SKILLMUX_HOST`, etc.) are ignored. TOML configuration is authoritative.
21
+ - **AC 1.3 (Exempt Variables in Strict Mode)**: In both `true` and `false` modes, Skillmux MUST continue to resolve:
22
+ - `SKILLMUX_CONFIG` (config path locator)
23
+ - `SKILLMUX_CONFIG_READONLY` (admin API read-only enforcement)
24
+ - Secret variables referenced by `api_key_env`
25
+ - Process environment variables (`HOME`, `PATH`, etc.)
26
+ - Explicit CLI flags
27
+ - **AC 1.4 (Generic Env & Deprecated Keys)**: Generic environment variables (`EMBED_*`, `RERANK_*`, etc.) and deprecated `thresholds.candidate_limit` trigger a 1.x deprecation warning pointing to `SKILLMUX_*` and `output.ambiguous_candidate_limit` equivalents.
28
+ - **AC 1.5 (Provenance & Diagnostics)**: `config show --sources` and `doctor` report `config.environment_overrides` status and source provenance for each behavioral value.
29
+ - **AC 1.6 (Output Candidate Limit)**: `output.ambiguous_candidate_limit` controls candidate limits returned on ambiguous outcomes. `thresholds.candidate_limit` remains as a deprecated 1.x alias.
30
+
31
+ ### Change 2: Bounded Reranking (AC 6-8)
32
+
33
+ - **AC 2.1 (Schema & Validation)**: Add `recall.k_rerank` to TOML config and schema. Validate:
34
+ - `k_lexical`, `k_vector`, and `k_rerank` are positive integers.
35
+ - `k_rerank <= k_lexical + k_vector`.
36
+ - Default provides a safe fallback for existing installations without breaking.
37
+ - **AC 2.2 (Runtime Fusion & Truncation)**: In `retrieveAndRerank`, retrieve lexical (up to `k_lexical`) and vector (up to `k_vector`), fuse candidates, truncate the fused list to `k_rerank`, and only send `k_rerank` candidates to the reranker.
38
+ - **AC 2.3 (Calibration & Evaluation Alignment)**: Dataset generation, calibration tuning, and evaluation record and use the configured `k_rerank`, `k_lexical`, and `k_vector` settings matching runtime.
39
+
40
+ ### Change 3: Failure Visibility & Degradation Reasons (AC 9-12)
41
+
42
+ - **AC 3.1 (Error Classification & Stable Codes)**: Remote inference errors, timeouts, and transport failures map to stable reason codes:
43
+ - `embedding_timeout`
44
+ - `embedding_unavailable`
45
+ - `embedding_protocol_error`
46
+ - `reranker_timeout`
47
+ - `reranker_unavailable`
48
+ - `reranker_protocol_error`
49
+ - **AC 3.2 (Safe Fallback & Metadata)**: When fallback occurs:
50
+ - Embedding failure falls back to lexical retrieval with degradation metadata (`degraded_from = "reranked"` or `"hybrid"`, `degradation_reason = "<code_name>"`).
51
+ - Reranker failure falls back to fused hybrid candidates with `degraded_from = "reranked"` and reason.
52
+ - **AC 3.3 (MCP & Safe Sanitization)**: MCP `resolve_skill` responses return structured degradation fields. Audit records, logs, and Prometheus metrics log/increment safe error classes without exposing credentials, URLs with secrets, or raw exception bodies.
53
+
54
+ ### Change 4: Routing Evidence Gate & Evaluation Details (AC 15)
55
+
56
+ - **AC 4.1 (Evaluation Case Output)**: `evalVault` records each evaluation case's fused rank, reranked rank, final outcome, latency, effective recall/rerank settings, and degradation state.
57
+
58
+ ## Implementation Tasks
59
+
60
+ - [x] **Task 1: Configuration Schema, Loader, and Provenance**
61
+ - Add `config.environment_overrides` boolean option (defaults to `true`).
62
+ - Add `output.ambiguous_candidate_limit` and 1.x deprecation warning for `thresholds.candidate_limit`.
63
+ - Add migration deprecation warnings for generic `EMBED_*` / `RERANK_*` env vars.
64
+ - Enforce `environment_overrides = false` behavior ignoring behavioral env vars while preserving `SKILLMUX_CONFIG`, `api_key_env`, etc.
65
+ - Update `config show --sources` and `doctor` provenance tracking.
66
+ - [x] **Task 2: Bounded Reranking (`k_rerank`)**
67
+ - Add `recall.k_rerank` to config types, schema, defaults, and validation.
68
+ - Update `retrieveAndRerank` pipeline to truncate fused candidates to `k_rerank` before reranking.
69
+ - Align calibration and dataset generator with `k_rerank`.
70
+ - [x] **Task 3: Failure Classification and Degradation Metadata**
71
+ - Define stable error reason enums/types.
72
+ - Classify `RemoteInferenceError`, timeouts, and transport errors in embedding and reranker callers.
73
+ - Add degradation metadata to MCP responses, structured logs, audit logger, and metric counters.
74
+ - Verify secret redaction and error masking in audit/logs.
75
+ - [ ] **Task 4: Output Limits, Calibration Settings & Evaluation Reporting**
76
+ - Support `output.ambiguous_candidate_limit` with 1.x deprecation warning on `thresholds.candidate_limit`.
77
+ - Update `evalVault` to record candidate fused/reranked ranks, latency, recall settings, and degradation state.
78
+ - Record recall settings in calibration runs.
@@ -0,0 +1,95 @@
1
+ # Packaging and deployment review
2
+
3
+ - **Branch:** `feat/packaging-deployment`
4
+ - **Reviewed range:** `main...1d940ca`
5
+ - **Specification:** `docs/skillmux-packaging-deployment-plan.md`
6
+ - **Verdict:** approved pending CI
7
+ - **Reverification:** all review findings resolved by `1d940ca`
8
+
9
+ ## Findings
10
+
11
+ ### Resolved — The default Docker server exited before listening
12
+
13
+ `ConfigWatcher` now tolerates a missing parent directory. A rebuilt slim image
14
+ started the default HTTP server without a config directory, reached healthy
15
+ state, and exited with status 0 after SIGTERM. Containerized stdio also started
16
+ and closed with status 0.
17
+
18
+ The original runtime failure was:
19
+
20
+ ```text
21
+ error: ENOENT: no such file or directory, watch '/root/.config/skillmux'
22
+ ```
23
+
24
+ **Resolution:** verified by focused unit coverage and a rebuilt-image runtime
25
+ smoke test.
26
+
27
+ ### Resolved — Docker runtime validation
28
+
29
+ CI now builds and loads the AMD64 slim image, then runs
30
+ `tests/docker-smoke.sh`. The script covers command replacement, rejected host
31
+ management, stdio, default HTTP readiness, and SIGTERM.
32
+
33
+ The release workflow now runs the same smoke contract for full and slim images
34
+ on AMD64 and ARM64 before the GHCR publishing job can start.
35
+
36
+ **Location:** `.github/workflows/release-please.yml:238`
37
+
38
+ **Resolution:** the four-way release matrix uses QEMU and blocks image
39
+ publication on smoke-test success.
40
+
41
+ ### Resolved — Public terminology
42
+
43
+ The documentation hub and architecture diagram now use the CLI-first
44
+ hierarchy. Concepts, deployment, routing, configuration, and most
45
+ troubleshooting references were also updated.
46
+
47
+ The remaining getting-started, troubleshooting, example-config, and issue
48
+ template references now use “Skillmux CLI” and “standalone Linux executable.”
49
+ A repository terminology scan finds the old labels only in historical plan and
50
+ review text.
51
+
52
+ ### Resolved — Live configuration after zero-config startup
53
+
54
+ `ConfigWatcher` creates the config parent before installing the watcher. A
55
+ later config creation therefore triggers the normal reload path.
56
+
57
+ **Locations:** `src/config-watcher.ts:139`, `src/config-service.ts:317`
58
+
59
+ **Resolution:** unit coverage creates a config after zero-config startup. A
60
+ Linux-container runtime check also confirmed that a late-created malformed
61
+ config reached the watcher and produced the expected reload error without
62
+ stopping the server.
63
+
64
+ ## Validation
65
+
66
+ - `bunx tsc --noEmit`: pass
67
+ - Changed Docker-policy tests: 5 passed, 0 failed
68
+ - Missing-directory ConfigWatcher regression: 1 passed, 0 failed
69
+ - Late-created config watcher test: times out in this macOS environment with
70
+ the existing `node:fs.watch` tests; Linux-container runtime verification
71
+ passed
72
+ - Slim Docker build: pass
73
+ - `docker run ... config show`: pass
74
+ - `docker run ... init`: correctly rejected with exit code 2 and host guidance
75
+ - Default Docker HTTP startup: pass; container reached healthy state
76
+ - Containerized stdio: pass
77
+ - SIGTERM shutdown: pass; exit code 0
78
+ - `bash -n tests/docker-smoke.sh`: pass
79
+ - Full `bun test`: fail in this environment with 37 failures, largely from
80
+ port-0 server startup, filesystem watcher timeouts, and Git fixture behavior
81
+ outside this diff. Focused changed tests pass, but the branch
82
+ cannot be called suite-green from this run.
83
+
84
+ ## AC Status
85
+
86
+ | Criterion | Status | Notes |
87
+ | --- | --- | --- |
88
+ | Present Skillmux as CLI-first for individual management and local MCP | ✅ Done | README, documentation hub, concepts, and architecture diagram use the CLI-first hierarchy |
89
+ | Treat the Bun package and standalone Linux executable as installations of one CLI | ✅ Done | Public terminology now follows the CLI/server hierarchy |
90
+ | Preserve zero-argument Docker HTTP startup | ✅ Done | Rebuilt slim image reached healthy state without a config directory |
91
+ | Support documented one-shot and stdio Docker overrides | ✅ Done | Runtime verification covered command replacement and stdio |
92
+ | Reject host-management commands inside Docker | ✅ Done | Guard and focused tests cover the documented command families |
93
+ | Recommend full by default and slim for remote or lexical retrieval | ✅ Done | Root README and getting-started guide establish the intended default |
94
+ | Document Git-backed vault copies for combined native and shared retrieval | ✅ Done | Getting-started and deployment guides state that replication is external |
95
+ | Add runtime Docker and release-architecture validation | ✅ Done | PR CI smokes slim AMD64; release publishing waits for full/slim AMD64/ARM64 smoke jobs |
@@ -0,0 +1,103 @@
1
+ # Review — fetch-outcome routing quality flywheel
2
+
3
+ Reviewed: `git diff main...HEAD` on `feat/fetch-outcome-flywheel` (26 files, +2243/-302 across 12 feature commits + 1 docs commit + 1 review-fix commit).
4
+
5
+ Goose's `sdd-review` fan-out hit the day's Bifrost token rate limit before any sub-pass returned results, so this review was run manually per the skill's fallback path (five focus passes + build check + test execution, done inline).
6
+
7
+ ## Compliance
8
+
9
+ | # | Criterion (abridged) | Status |
10
+ |---|---|---|
11
+ | 1 | `audit.sqlite3` separate from `index.sqlite3`, WAL + `auto_vacuum=INCREMENTAL` at creation | ✅ Done |
12
+ | 2 | Idempotent one-time transactional migration of legacy audit rows | ✅ Done — tested including a second-open idempotency check |
13
+ | 3 | `resolve_skill` response carries unique `request_id`, stored on the audit row | ✅ Done |
14
+ | 4 | Pre-existing audit rows with no `request_id` still load/report/render | ✅ Done |
15
+ | 5 | `fetch_skill` optional `request_id` correlates to the resolve when known | ✅ Done |
16
+ | 6 | `fetch_skill` without `request_id` records an uncorrelated fetch | ✅ Done |
17
+ | 7 | Unknown/malformed `request_id` still succeeds, records uncorrelated | ✅ Done |
18
+ | 8 | Fetch row stores ts, skill_id, request_id/null, resolve id/null, rank_at_resolve | ✅ Done |
19
+ | 9 | `report` computes acceptance_rate / observed_mrr / top1_acceptance_rate | ✅ Done |
20
+ | 10 | Signal marked unavailable (not `0.000`) when no correlated fetches | ✅ Done |
21
+ | 11 | `top_unused_shortlist_queries` distinct from `top_empty_shortlist_queries` | ✅ Done |
22
+ | 12 | `audit.retention_days`, default 90, `0` disables, invalid rejected at load | ✅ Done |
23
+ | 13 | Prune deletes resolve/fetch independently by own timestamp; pruned resolve reads as uncorrelated | ✅ Done |
24
+ | 14 | Prune runs at startup + ≤ once/24h, never blocks readiness/resolve | ✅ Done (race fixed during this review — see Findings) |
25
+ | 15 | `skillmux audit prune` with `--older-than`/`--dry-run`/`--yes`/`--json` | ✅ Done |
26
+ | 16 | Incremental vacuum after prune; never locks `index.sqlite3` | ✅ Done — structural (separate files/connections) |
27
+ | 17 | `skillmux eval promote` writes `{query, split:"observed", relevant_skill_ids}`, deduped by normalized query, default target ≠ `eval/queries.json`, warns about raw queries | ✅ Done |
28
+ | 18 | `eval promote` never rewrites an existing case; skipped count reported | ✅ Done |
29
+ | 19 | `report`/`audit prune`/`eval promote` honor `--json`; exit codes unchanged | ✅ Done |
30
+
31
+ **Spec compliance: 19/19.** **Schema compliance:** all types in `docs/sdd/fetch-outcome-flywheel/schema.json` match the implementation; `docs/schema.json` (the committed wire contract) was updated for `request_id` on both tools and `AuditConfig`, consistent with the project's existing precedent of not documenting CLI JSON output shapes there (matches `StatsResponse`'s prior absence).
32
+
33
+ ## Findings
34
+
35
+ ### 1. [Fixed] `getEnv()` race: two concurrent callers at startup each opened their own DB handles
36
+ **File:** `src/router-core.ts` (pre-fix) · **Confidence:** High · **Class:** AUTO-FIX (applied)
37
+
38
+ `startServer()` calls `initializeRuntime(readinessState)` (which awaits `getRuntime()` → `getEnv()`) and then, synchronously right after, fires the new AC14 `pruneAuditIfDue()` (which also calls `getEnv()`) — fire-and-forget, per AC14's "never blocks readiness" requirement. `getEnv()`'s memoization was `if (env) return env;` with `env` assigned only after several `await` points. Because both call chains reach that guard *before either has awaited anything* (JS runs an async function body synchronously up to its first `await`), both saw `env === null` and independently ran `openIndex()` + `openAudit()` (and, on a cold index, could both run `ingestVault()` concurrently). One of the two resulting handles is orphaned — never closed by `closeRuntime()`, since that only closes whatever `env` last got overwritten to — leaking a `Database` connection on every server startup, not just occasionally.
39
+
40
+ Reproduced with a regression test (`tests/guarantees.test.ts`, "concurrent callers racing at startup share one env..."), confirmed genuinely RED (`a.db !== b.db`), then fixed by memoizing the in-flight `Promise` instead of the resolved value, with a `resolvedEnv` side-reference so `closeRuntime()` keeps its existing synchronous best-effort close. Commit `91dfd7e`. Full suite: 666/666 pass after the fix (up from 665, +1 for the new regression test).
41
+
42
+ ### 2. [Not a defect] SQL built via template-literal interpolation in `adoptAuditFromIndex`
43
+ **File:** `src/db.ts` — `adoptAuditFromIndex` · **Confidence:** N/A (verified safe) · **Class:** none
44
+
45
+ `retrieval`/`degradedFrom`/`degradationReason` are interpolated into the migration `INSERT ... SELECT`. Checked for injection: all three are selected via ternary from a fixed 2-way enum of literal strings (`"COALESCE(retrieval, 'lexical')"` / `"'lexical'"`, etc.) keyed off `legacyColumns.has(...)` — never derived from row data or user input. Same pattern as the pre-existing in-place migration this replaced. No finding.
46
+
47
+ ### 3. [Nitpick] No index on `audit.ts` / `fetch.ts` / `fetch.resolve_audit_id`
48
+ **Files:** `src/db.ts` (`openAudit`) · **Confidence:** Low · **Class:** NITPICK
49
+
50
+ `queryAuditRows`, `queryFetchRows`, `pruneAuditBefore`/`countPrunable`, and the new `queryPromotableFetches` join all filter or join on these columns without an index, so they full-scan at scale. This predates the feature (the original `audit` table had no `ts` index either) and performance tuning is out of scope per the spec. Mentioning for awareness, not blocking.
51
+
52
+ ## Security
53
+
54
+ No injection, auth, secrets, or trust-boundary issues found. All new SQL uses parameterized `?` bindings except the one interpolation site above, which was verified safe. `eval promote --target <path>` and `audit prune`/`report --db <path>` accept arbitrary local paths, but these are operator-supplied CLI flags on a tool the operator already has filesystem access to — not a privilege boundary. `request_id` inputs are deliberately unvalidated at the schema level (by design, per AC7 — an unknown/malformed value must still succeed) and are only ever used as a parameterized lookup key, never interpolated or executed. Security escalation via `/security-review` not recommended — no auth/session/payment/deserialization/new-egress surface touched.
55
+
56
+ ## Performance
57
+
58
+ No regressions. The one Low-confidence nitpick (missing indexes) is pre-existing scope, not introduced here. Pruning's incremental-vacuum runs unconditionally after every prune call (including no-op ones); cheap and harmless.
59
+
60
+ ## Tests
61
+
62
+ **Build check:** PASS ✅ (`bun run tsc --noEmit`, clean)
63
+ **Test execution:** PASS ✅ (666/666, 1750 `expect()` calls, 53 files — up from 629 at the start of this SDD session)
64
+
65
+ Coverage is thorough: every AC has at least one test locking it in, including edge cases (pruned-resolve-reads-as-uncorrelated, retention_days=0 disabling pruning, dedupe-by-normalized-query, skip-existing-case-in-target-file, the 24h prune gate, and now the startup concurrency race). No gaps found worth a new test beyond the one added for Finding 1.
66
+
67
+ ## Review Summary
68
+
69
+ **Spec compliance:** 19/19 criteria met
70
+ **Schema compliance:** All types match
71
+ **Critical findings:** 1 (1 high confidence, fixed)
72
+ **Auto-fixed:** 1 issue (the `getEnv()` race)
73
+ **Needs decision:** 0
74
+ **Test coverage (static):** Adequate
75
+ **Build check:** PASS ✅
76
+ **Test execution (runtime):** PASS ✅ (666/666)
77
+ **AC status table:** Written above
78
+ **Security escalation:** Not needed
79
+ **Learnings retained:** 1 memory (Hindsight `infra` bank) — the async-memoization gotcha, for reuse if this codebase (or another) adds more fire-and-forget background work off a lazily-initialized singleton.
80
+
81
+ ## AC Status
82
+
83
+ | Criterion | Status | Notes |
84
+ |---|---|---|
85
+ | AC1: audit.sqlite3 separate store, WAL + incremental auto_vacuum | ✅ Done | |
86
+ | AC2: idempotent transactional migration from index.sqlite3 | ✅ Done | Idempotency explicitly tested (second open) |
87
+ | AC3: request_id on resolve_skill response + audit row | ✅ Done | |
88
+ | AC4: pre-existing rows without request_id still work | ✅ Done | |
89
+ | AC5: fetch_skill request_id correlates to known resolve | ✅ Done | |
90
+ | AC6: fetch_skill without request_id records uncorrelated | ✅ Done | |
91
+ | AC7: unknown/malformed request_id still succeeds, uncorrelated | ✅ Done | |
92
+ | AC8: fetch row shape (ts, skill_id, request_id, resolve id, rank) | ✅ Done | |
93
+ | AC9: acceptance_rate / observed_mrr / top1_acceptance_rate | ✅ Done | |
94
+ | AC10: signal marked unavailable, not 0.000, when no correlated fetches | ✅ Done | |
95
+ | AC11: top_unused_shortlist_queries distinct list | ✅ Done | |
96
+ | AC12: audit.retention_days, default 90, 0 disables, invalid rejected | ✅ Done | |
97
+ | AC13: prune deletes independently by timestamp; pruned resolve → uncorrelated | ✅ Done | |
98
+ | AC14: prune at startup + ≤1/24h, never blocks readiness/resolve | ✅ Done | Startup race found and fixed during this review |
99
+ | AC15: `skillmux audit prune` flags | ✅ Done | |
100
+ | AC16: incremental vacuum, never locks index.sqlite3 | ✅ Done | |
101
+ | AC17: `skillmux eval promote` case shape, dedupe, default target, warning | ✅ Done | |
102
+ | AC18: never rewrites existing case | ✅ Done | |
103
+ | AC19: `--json` honored, exit codes unchanged | ✅ Done | |
@@ -0,0 +1,347 @@
1
+ {
2
+ "$schema": "https://json-schema.org/draft/2020-12/schema",
3
+ "$id": "https://klh.app/schemas/skillmux/fetch-outcome-flywheel.json",
4
+ "title": "Skillmux — fetch-outcome routing quality flywheel contract",
5
+ "description": "Contract delta for docs/sdd/fetch-outcome-flywheel/spec.md. Each $def is either a NEW type or a full replacement for a def of the same name in docs/schema.json. At implementation time these merge into docs/schema.json $defs; nothing here is a second source of truth. Merge targets are noted per def in $comment.",
6
+ "$defs": {
7
+ "RequestId": {
8
+ "$comment": "NEW. AC3.",
9
+ "type": "string",
10
+ "description": "Per-resolve correlation id minted by resolve_skill. Lowercase UUIDv4.",
11
+ "format": "uuid",
12
+ "pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
13
+ },
14
+
15
+ "InboundRequestId": {
16
+ "$comment": "NEW. AC7. Deliberately NOT pattern-validated: AC7 requires that an unknown OR MALFORMED request_id still succeed and record an uncorrelated fetch. Pattern-validating this at the tool boundary would raise a validation error before the handler runs and break AC7. Correlation validity is decided in the handler. The length bound is the single boundary check, present only to cap an unbounded string.",
17
+ "type": "string",
18
+ "minLength": 1,
19
+ "maxLength": 128
20
+ },
21
+
22
+ "Window": {
23
+ "$comment": "NEW. AC15, AC17. Mirrors SINCE_PATTERN at src/stats.ts:4 exactly.",
24
+ "type": "string",
25
+ "description": "Relative window (30d, 12h, 2w, 6m, 1y) or an absolute date/datetime (YYYY-MM-DD, optionally with time).",
26
+ "pattern": "^(\\d+[hdwmy]|\\d{4}-\\d{2}-\\d{2}([T ].+)?)$"
27
+ },
28
+
29
+ "ResolveSkillResult": {
30
+ "$comment": "REPLACES docs/schema.json #/$defs/ResolveSkillResult. AC3: request_id added to properties AND to required, since the server always mints one.",
31
+ "type": "object",
32
+ "description": "Ranked candidates response from resolve_skill.",
33
+ "properties": {
34
+ "request_id": { "$ref": "#/$defs/RequestId" },
35
+ "retrieval": { "enum": ["exact", "reranked", "hybrid", "lexical"] },
36
+ "degraded_from": { "enum": ["reranked", "hybrid"] },
37
+ "degradation_reason": { "$ref": "https://klh.app/schemas/skillmux/router-core.json#/$defs/DegradationReason" },
38
+ "candidates": {
39
+ "type": "array",
40
+ "items": { "$ref": "https://klh.app/schemas/skillmux/router-core.json#/$defs/RankedCandidate" },
41
+ "description": "Shortlist of candidates in descending score order with contiguous 1-based ranks."
42
+ }
43
+ },
44
+ "required": ["request_id", "retrieval", "candidates"],
45
+ "additionalProperties": false
46
+ },
47
+
48
+ "FetchSkillInput": {
49
+ "$comment": "REPLACES docs/schema.json #/$defs/FetchSkillInput. AC5, AC6, AC7: request_id is optional and stays absent from required, preserving the documented guarantee that fetch works with no prior resolve.",
50
+ "type": "object",
51
+ "properties": {
52
+ "skill_id": { "$ref": "https://klh.app/schemas/skillmux/router-core.json#/$defs/SkillId" },
53
+ "request_id": { "$ref": "#/$defs/InboundRequestId" }
54
+ },
55
+ "required": ["skill_id"],
56
+ "additionalProperties": false
57
+ },
58
+
59
+ "ResolveAuditRow": {
60
+ "$comment": "NEW. AC3, AC4. Persisted shape in audit.sqlite3. request_id is nullable so rows migrated from index.sqlite3 (AC2) and rows written before this feature still load.",
61
+ "type": "object",
62
+ "properties": {
63
+ "id": { "type": "integer", "minimum": 1 },
64
+ "ts": { "type": "string", "format": "date-time" },
65
+ "request_id": {
66
+ "oneOf": [{ "$ref": "#/$defs/RequestId" }, { "type": "null" }],
67
+ "description": "Null only for rows written before this feature."
68
+ },
69
+ "query": { "type": "string", "minLength": 1 },
70
+ "retrieval": { "enum": ["exact", "reranked", "hybrid", "lexical"] },
71
+ "degraded_from": { "oneOf": [{ "enum": ["reranked", "hybrid"] }, { "type": "null" }] },
72
+ "degradation_reason": {
73
+ "oneOf": [
74
+ { "$ref": "https://klh.app/schemas/skillmux/router-core.json#/$defs/DegradationReason" },
75
+ { "type": "null" }
76
+ ]
77
+ },
78
+ "candidates": {
79
+ "type": "array",
80
+ "items": { "$ref": "https://klh.app/schemas/skillmux/router-core.json#/$defs/RankedCandidate" }
81
+ },
82
+ "latency_ms": { "type": "integer", "minimum": 0 }
83
+ },
84
+ "required": ["id", "ts", "request_id", "query", "retrieval", "candidates", "latency_ms"],
85
+ "additionalProperties": false
86
+ },
87
+
88
+ "FetchAuditRow": {
89
+ "$comment": "NEW. AC8, AC13. Persisted shape in audit.sqlite3.",
90
+ "type": "object",
91
+ "properties": {
92
+ "id": { "type": "integer", "minimum": 1 },
93
+ "ts": { "type": "string", "format": "date-time" },
94
+ "skill_id": { "$ref": "https://klh.app/schemas/skillmux/router-core.json#/$defs/SkillId" },
95
+ "request_id": {
96
+ "oneOf": [{ "$ref": "#/$defs/InboundRequestId" }, { "type": "null" }],
97
+ "description": "Exactly as supplied by the caller, including a value that matched no resolve. Null when the caller sent none."
98
+ },
99
+ "resolve_audit_id": {
100
+ "oneOf": [{ "type": "integer", "minimum": 1 }, { "type": "null" }],
101
+ "description": "Null when the fetch is uncorrelated: no request_id sent, an unknown or malformed one sent (AC7), or the resolve row has since been pruned (AC13)."
102
+ },
103
+ "rank_at_resolve": {
104
+ "oneOf": [{ "type": "integer", "minimum": 1 }, { "type": "null" }],
105
+ "description": "Rank of skill_id in that resolve's shortlist. Null when the fetch is uncorrelated, or when correlated but the skill was absent from the shortlist."
106
+ }
107
+ },
108
+ "required": ["id", "ts", "skill_id", "request_id", "resolve_audit_id", "rank_at_resolve"],
109
+ "additionalProperties": false,
110
+ "allOf": [
111
+ {
112
+ "$comment": "AC8 invariant: a rank can only exist relative to a known resolve. An uncorrelated fetch therefore has no rank.",
113
+ "if": {
114
+ "properties": { "resolve_audit_id": { "type": "null" } },
115
+ "required": ["resolve_audit_id"]
116
+ },
117
+ "then": {
118
+ "properties": { "rank_at_resolve": { "type": "null" } }
119
+ }
120
+ }
121
+ ]
122
+ },
123
+
124
+ "AcceptanceSignal": {
125
+ "$comment": "NEW. AC9, AC10. Modelled as a discriminated union on `available` so that the unavailable case CANNOT carry a rate. This makes AC10 a type error rather than a formatting convention: there is no shape in which a window with no correlated fetches emits acceptance_rate 0.0.",
126
+ "oneOf": [
127
+ {
128
+ "type": "object",
129
+ "properties": {
130
+ "available": { "const": false },
131
+ "uncorrelated_fetch_count": {
132
+ "type": "integer",
133
+ "minimum": 0,
134
+ "description": "Fetches seen in the window that could not be tied to a resolve. A non-zero value here with available=false is the signature of a client that never sends request_id."
135
+ }
136
+ },
137
+ "required": ["available", "uncorrelated_fetch_count"],
138
+ "additionalProperties": false
139
+ },
140
+ {
141
+ "type": "object",
142
+ "properties": {
143
+ "available": { "const": true },
144
+ "resolves_with_candidates": {
145
+ "type": "integer",
146
+ "minimum": 1,
147
+ "description": "Denominator: resolves in the window that returned at least one candidate."
148
+ },
149
+ "accepted_count": {
150
+ "type": "integer",
151
+ "minimum": 0,
152
+ "description": "Resolves with at least one correlated fetch."
153
+ },
154
+ "acceptance_rate": { "type": "number", "minimum": 0, "maximum": 1 },
155
+ "observed_mrr": {
156
+ "type": "number",
157
+ "minimum": 0,
158
+ "maximum": 1,
159
+ "description": "Mean reciprocal rank of the first fetched candidate, over accepted resolves."
160
+ },
161
+ "top1_acceptance_rate": {
162
+ "type": "number",
163
+ "minimum": 0,
164
+ "maximum": 1,
165
+ "description": "Share of accepted resolves whose rank-1 candidate was the one fetched."
166
+ },
167
+ "uncorrelated_fetch_count": { "type": "integer", "minimum": 0 }
168
+ },
169
+ "required": [
170
+ "available",
171
+ "resolves_with_candidates",
172
+ "accepted_count",
173
+ "acceptance_rate",
174
+ "observed_mrr",
175
+ "top1_acceptance_rate",
176
+ "uncorrelated_fetch_count"
177
+ ],
178
+ "additionalProperties": false
179
+ }
180
+ ]
181
+ },
182
+
183
+ "QueryCount": {
184
+ "$comment": "NEW. Shared by both query lists. Mirrors EmptyShortlistQuery at src/stats.ts:11.",
185
+ "type": "object",
186
+ "properties": {
187
+ "query": { "type": "string", "minLength": 1 },
188
+ "count": { "type": "integer", "minimum": 1 }
189
+ },
190
+ "required": ["query", "count"],
191
+ "additionalProperties": false
192
+ },
193
+
194
+ "ReportOutput": {
195
+ "$comment": "NEW to docs/schema.json, but existing fields transcribe StatsResponse at src/stats.ts:23 unchanged. AC9, AC10, AC11, AC19.",
196
+ "type": "object",
197
+ "description": "JSON output of `skillmux report --json`.",
198
+ "properties": {
199
+ "since": { "type": "string", "format": "date-time" },
200
+ "until": { "type": "string", "format": "date-time" },
201
+ "total_requests": { "type": "integer", "minimum": 0 },
202
+ "empty_shortlist_count": { "type": "integer", "minimum": 0 },
203
+ "empty_shortlist_rate": { "type": "number", "minimum": 0, "maximum": 1 },
204
+ "retrieval_totals": {
205
+ "type": "object",
206
+ "properties": {
207
+ "exact": { "type": "integer", "minimum": 0 },
208
+ "reranked": { "type": "integer", "minimum": 0 },
209
+ "hybrid": { "type": "integer", "minimum": 0 },
210
+ "lexical": { "type": "integer", "minimum": 0 }
211
+ },
212
+ "required": ["exact", "reranked", "hybrid", "lexical"],
213
+ "additionalProperties": false
214
+ },
215
+ "degraded_count": { "type": "integer", "minimum": 0 },
216
+ "average_latency_ms": { "type": "number", "minimum": 0 },
217
+ "skills": {
218
+ "type": "array",
219
+ "items": {
220
+ "type": "object",
221
+ "properties": {
222
+ "skill_id": { "$ref": "https://klh.app/schemas/skillmux/router-core.json#/$defs/SkillId" },
223
+ "candidate_count": { "type": "integer", "minimum": 0 }
224
+ },
225
+ "required": ["skill_id", "candidate_count"],
226
+ "additionalProperties": false
227
+ }
228
+ },
229
+ "top_empty_shortlist_queries": {
230
+ "type": "array",
231
+ "items": { "$ref": "#/$defs/QueryCount" }
232
+ },
233
+ "acceptance": { "$ref": "#/$defs/AcceptanceSignal" },
234
+ "top_unused_shortlist_queries": {
235
+ "type": "array",
236
+ "description": "AC11: queries that DID return candidates and received no correlated fetch. Distinct from top_empty_shortlist_queries, which covers queries that returned nothing at all.",
237
+ "items": { "$ref": "#/$defs/QueryCount" }
238
+ }
239
+ },
240
+ "required": [
241
+ "since",
242
+ "until",
243
+ "total_requests",
244
+ "empty_shortlist_count",
245
+ "empty_shortlist_rate",
246
+ "retrieval_totals",
247
+ "degraded_count",
248
+ "average_latency_ms",
249
+ "skills",
250
+ "top_empty_shortlist_queries",
251
+ "acceptance",
252
+ "top_unused_shortlist_queries"
253
+ ],
254
+ "additionalProperties": false
255
+ },
256
+
257
+ "PruneOutput": {
258
+ "$comment": "NEW. AC15, AC19.",
259
+ "type": "object",
260
+ "description": "JSON output of `skillmux audit prune --json`.",
261
+ "properties": {
262
+ "dry_run": { "type": "boolean" },
263
+ "older_than": { "$ref": "#/$defs/Window" },
264
+ "cutoff": {
265
+ "type": "string",
266
+ "format": "date-time",
267
+ "description": "Resolved absolute cutoff. Rows at or before this instant are removed."
268
+ },
269
+ "removed": {
270
+ "type": "object",
271
+ "description": "AC15: rows removed per table. Both counts are what WOULD be removed when dry_run is true.",
272
+ "properties": {
273
+ "resolves": { "type": "integer", "minimum": 0 },
274
+ "fetches": { "type": "integer", "minimum": 0 }
275
+ },
276
+ "required": ["resolves", "fetches"],
277
+ "additionalProperties": false
278
+ }
279
+ },
280
+ "required": ["dry_run", "older_than", "cutoff", "removed"],
281
+ "additionalProperties": false
282
+ },
283
+
284
+ "PromotedEvalCase": {
285
+ "$comment": "NEW. AC17. Must validate under parseEvalCases at src/eval.ts:15, hence relevant_skill_ids is a unique array of SkillId.",
286
+ "type": "object",
287
+ "properties": {
288
+ "query": { "type": "string", "minLength": 1 },
289
+ "split": {
290
+ "const": "observed",
291
+ "description": "Fixed value keeps promoted cases out of the hand-curated tune and test splits."
292
+ },
293
+ "relevant_skill_ids": {
294
+ "type": "array",
295
+ "items": { "$ref": "https://klh.app/schemas/skillmux/router-core.json#/$defs/SkillId" },
296
+ "uniqueItems": true,
297
+ "minItems": 1,
298
+ "description": "Skills actually fetched for this query. A promoted case always has at least one, otherwise there was no outcome to promote."
299
+ }
300
+ },
301
+ "required": ["query", "split", "relevant_skill_ids"],
302
+ "additionalProperties": false
303
+ },
304
+
305
+ "PromoteOutput": {
306
+ "$comment": "NEW. AC17, AC18, AC19.",
307
+ "type": "object",
308
+ "description": "JSON output of `skillmux eval promote --json`.",
309
+ "properties": {
310
+ "dry_run": { "type": "boolean" },
311
+ "since": { "$ref": "#/$defs/Window" },
312
+ "target_path": {
313
+ "type": "string",
314
+ "minLength": 1,
315
+ "description": "AC17: defaults to a path other than eval/queries.json unless one is given explicitly."
316
+ },
317
+ "promoted": { "type": "integer", "minimum": 0 },
318
+ "skipped_existing": {
319
+ "type": "integer",
320
+ "minimum": 0,
321
+ "description": "AC18: queries already present in the target file, skipped rather than rewritten."
322
+ }
323
+ },
324
+ "required": ["dry_run", "since", "target_path", "promoted", "skipped_existing"],
325
+ "additionalProperties": false
326
+ },
327
+
328
+ "AuditConfig": {
329
+ "$comment": "NEW. AC12. Merges into docs/schema.json #/$defs/Config properties as `audit`. NOT added to Config.required: the section is optional and defaults apply when absent, matching how `server` is handled.",
330
+ "type": "object",
331
+ "properties": {
332
+ "retention_days": {
333
+ "type": "integer",
334
+ "minimum": 0,
335
+ "default": 90,
336
+ "description": "Age in days beyond which audit rows are pruned. 0 disables pruning and retains rows indefinitely. OPEN DECISION: whether the shipped default is 90 or 0. The type is identical either way; only the default value differs."
337
+ }
338
+ },
339
+ "additionalProperties": false
340
+ },
341
+
342
+ "ConfigStateDirDescription": {
343
+ "$comment": "REPLACES the description string on docs/schema.json #/$defs/Config/properties/state_dir, which currently reads 'index SQLite (incl. audit log)'. That parenthetical is false once the audit store splits out (AC1).",
344
+ "const": "All router writes are confined here (AC9): index SQLite (index.sqlite3), audit SQLite (audit.sqlite3), vector matrix."
345
+ }
346
+ }
347
+ }