@klhapp/skillmux 1.7.1 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +15 -5
- package/docs/assets/architecture-dark.svg +160 -0
- package/docs/assets/{architecture.svg → architecture-light.svg} +40 -34
- package/docs/assets/logo-dark.png +0 -0
- package/docs/assets/logo-light.png +0 -0
- package/docs/cli.md +63 -5
- package/docs/concepts.md +10 -0
- package/docs/configuration.md +20 -20
- package/docs/deployment.md +20 -7
- package/docs/getting-started.md +11 -0
- package/docs/mcp-routing.md +41 -7
- package/docs/schema.json +31 -4
- package/docs/skill-management.md +32 -0
- package/package.json +1 -1
- package/src/audit.ts +1 -0
- package/src/cli.ts +62 -8
- package/src/commands/audit.ts +82 -0
- package/src/commands/eval.ts +81 -0
- package/src/commands/outdated.ts +112 -0
- package/src/commands/update.ts +253 -0
- package/src/config.ts +6 -0
- package/src/db.ts +152 -45
- package/src/eval.ts +69 -0
- package/src/install.ts +82 -3
- package/src/provenance.ts +99 -0
- package/src/router-core.ts +109 -26
- package/src/scan.ts +7 -1
- package/src/server.ts +21 -6
- package/src/stats.ts +119 -13
- package/src/sync.ts +38 -8
- package/src/types.ts +22 -0
- package/src/vault.ts +44 -4
- package/docs/assets/logo.png +0 -0
package/docs/configuration.md
CHANGED
|
@@ -203,9 +203,9 @@ enabled = false
|
|
|
203
203
|
token_env = "SKILLMUX_ADMIN_TOKEN"
|
|
204
204
|
```
|
|
205
205
|
|
|
206
|
-
Defaults are loopback-only (`hostname = "127.0.0.1"`) with CORS deny-by-default (`allowed_origins = []`)
|
|
206
|
+
Defaults are loopback-only (`hostname = "127.0.0.1"`) with CORS deny-by-default (`allowed_origins = []`), so a zero-config `skillmux serve --transport http` is not reachable from the network or from a browser tab on another origin. Docker sets `hostname` to `0.0.0.0` automatically (`RUNNING_IN_DOCKER=true`) since port-mapping needs the container to accept connections on all interfaces.
|
|
207
207
|
|
|
208
|
-
Before exposing HTTP beyond localhost, set `hostname` to a reachable interface, `auth_enabled = true` with a token, and populate `allowed_origins` with the specific origins that need browser access. `rate_limit.trust_proxy` should stay `false` unless a trusted reverse proxy sets `X-Forwarded-For
|
|
208
|
+
Before exposing HTTP beyond localhost, set `hostname` to a reachable interface, `auth_enabled = true` with a token, and populate `allowed_origins` with the specific origins that need browser access. `rate_limit.trust_proxy` should stay `false` unless a trusted reverse proxy sets `X-Forwarded-For`: it's otherwise a client-controlled, spoofable header, and trusting it defeats per-client rate limiting.
|
|
209
209
|
|
|
210
210
|
`server.auth_token_env` names the MCP token for AI clients calling `/mcp`.
|
|
211
211
|
`server.admin.token_env` names a distinct administrative token for operators
|
|
@@ -242,13 +242,13 @@ skills = ["pdf-extractor"] # must not overlap [core]
|
|
|
242
242
|
[targets.claude-code]
|
|
243
243
|
dir = "/Users/you/.claude/skills"
|
|
244
244
|
host = "workhorse" # optional; init adds the current hostname
|
|
245
|
-
project_groups = ["repo1"] # which [project.*] groups materialize into this target
|
|
245
|
+
project_groups = ["repo1"] # which [project.*] groups materialize into this target; [] means none
|
|
246
246
|
```
|
|
247
247
|
|
|
248
|
-
- `[core].skills
|
|
249
|
-
- `[project.<group>].skills
|
|
250
|
-
- `[project.<group>].paths` can list the same project's checkout on more than one machine (e.g. `["/home/alice/code/repo1", "/Users/alice/code/repo1"]`)
|
|
251
|
-
- `[targets.<name>]
|
|
248
|
+
- `[core].skills`: symlinked into every `[targets.*]` dir on `sync`. Capped at 25 skills; `sync` fails if a listed skill id isn't actually in the vault.
|
|
249
|
+
- `[project.<group>].skills`: symlinked only into `<path>/<relative path from $HOME to the target dir>`, for each `paths` entry, and only for targets whose `project_groups` names that group. `paths` entries must resolve under `$HOME` (that's how the pin path is derived). A skill can't appear in both `[core]` and the same `[project.*]` group.
|
|
250
|
+
- `[project.<group>].paths` can list the same project's checkout on more than one machine (e.g. `["/home/alice/code/repo1", "/Users/alice/code/repo1"]`). `sync` silently skips any entry that doesn't exist on the machine it's running on (see below), so one shared manifest can span machines with different checkout locations without needing per-machine manifests.
|
|
251
|
+
- `[targets.<name>]`: one entry per adopted surface. `skillmux init --target <name> --yes` writes these and scopes newly added targets to the current hostname. Hand-editing is fine as long as `sync` is still allowed to own the directory (see below). An optional `host` limits the target to an exact machine-hostname match; omit it for a global, backward-compatible target. A host mismatch is reported and skipped before any target filesystem operation. `project_groups` is an explicit list, not a boolean: a target only receives the specific groups it names, never every group in the manifest.
|
|
252
252
|
|
|
253
253
|
**Pin/unpin without hand-editing.** `skillmux core pin`/`unpin` mutate `[core]` for you, and `skillmux project pin`/`unpin` mutate `[project.*]`, validating with the same rules `sync` enforces (skill must resolve from `vault_path`, no duplicate pins, `[core]` stays under the 25-skill cap) before writing anything:
|
|
254
254
|
|
|
@@ -261,16 +261,16 @@ skillmux core unpin csv-formatter pdf-extractor --yes # unpin
|
|
|
261
261
|
skillmux project unpin repo1 pdf-extractor --yes # remove from a group (group stays, even if empty)
|
|
262
262
|
```
|
|
263
263
|
|
|
264
|
-
Both commands accept one or more `skill_id` arguments per call; all of them are validated and applied against a single in-memory manifest before anything is written, so if any one of them is already pinned elsewhere (or, for unpin, not currently pinned), the whole call fails and the manifest file is left untouched
|
|
264
|
+
Both commands accept one or more `skill_id` arguments per call; all of them are validated and applied against a single in-memory manifest before anything is written, so if any one of them is already pinned elsewhere (or, for unpin, not currently pinned), the whole call fails and the manifest file is left untouched: no partial pins. To pin into a `[project.<group>]` tier that doesn't exist yet, create it first with `skillmux project add-path <group> <path> --yes`. Hand-editing `skillmux.toml` directly is still fully supported; these commands are a convenience layer over the same file, not a replacement for it.
|
|
265
265
|
|
|
266
|
-
> **Breaking change:** `skillmux manifest pin`/`unpin` is removed. `[core]` pinning is now `skillmux core pin`/`unpin`; `[project.*]` pinning was already available as `skillmux project pin`/`unpin` and is now the only way to do it
|
|
266
|
+
> **Breaking change:** `skillmux manifest pin`/`unpin` is removed. `[core]` pinning is now `skillmux core pin`/`unpin`; `[project.*]` pinning was already available as `skillmux project pin`/`unpin` and is now the only way to do it. There's no more `--path`-based inline group creation from a pin call; use `project add-path` to create the group first.
|
|
267
267
|
>
|
|
268
268
|
> **Breaking change:** `[targets.<name>].project` (a boolean) has been replaced by `project_groups` (an array of `[project.*]` names). A manifest still using the old field fails to parse with an error pointing at the new one. To migrate, replace `project = true` with `project_groups = [...]` listing every group that target previously received (previously *all* groups, unconditionally); replace `project = false` with `project_groups = []`.
|
|
269
269
|
>
|
|
270
|
-
> **Breaking change:** `[project.<group>].repos` has been renamed to `paths
|
|
270
|
+
> **Breaking change:** `[project.<group>].repos` has been renamed to `paths`. It was never required to be a git repository, just a local directory, and the old name collided in meaning with `skillmux install <repo>`'s unrelated git-source `repo` concept. A manifest still using `repos` fails to parse with an error pointing at `paths`; migrate by renaming the key (values are unchanged).
|
|
271
271
|
|
|
272
272
|
Every `[core]`/`[project.*]` skill_id must resolve from the configured
|
|
273
|
-
`vault_path` checkout
|
|
273
|
+
`vault_path` checkout. Pinning a skill that only exists in a
|
|
274
274
|
`local_vault_paths` entry (see below) fails `sync` with a distinct error, since
|
|
275
275
|
the manifest is meant to be portable across machines and a machine-local
|
|
276
276
|
override wouldn't exist elsewhere. `doctor` validates the manifest as part of
|
|
@@ -286,7 +286,7 @@ Skillmux created. Sync removes only those tracked entries, preserves unrelated
|
|
|
286
286
|
content, and rejects a desired skill that collides with an unmanaged entry
|
|
287
287
|
before changing anything.
|
|
288
288
|
|
|
289
|
-
`sync` refuses to touch a directory that exists but has no marker
|
|
289
|
+
`sync` refuses to touch a directory that exists but has no marker; run
|
|
290
290
|
`skillmux init --target <name> --yes` first, which either creates the
|
|
291
291
|
directory fresh or adopts an existing one in place (contents untouched).
|
|
292
292
|
`sync --restore-monolith` likewise refuses a `local_vault` marker or any
|
|
@@ -302,34 +302,34 @@ cannot be inferred.
|
|
|
302
302
|
|
|
303
303
|
### Local vault overlays
|
|
304
304
|
|
|
305
|
-
`local_vault_paths` (in `config.toml`, alongside `vault_path`) lets one machine layer override-only skills on top of the shared vault
|
|
305
|
+
`local_vault_paths` (in `config.toml`, alongside `vault_path`) lets one machine layer override-only skills on top of the shared vault (a skill being authored locally, a machine-specific script, or a patched copy of an upstream skill) without touching `vault_path` itself:
|
|
306
306
|
|
|
307
307
|
```toml
|
|
308
308
|
vault_path = "~/skills" # configured checkout; owns skillmux.toml and the sync git hook
|
|
309
309
|
local_vault_paths = ["~/skills-local"] # optional, default []: override-only, checked first
|
|
310
310
|
```
|
|
311
311
|
|
|
312
|
-
- **Resolution order**: for any given `skill_id`, `local_vault_paths` entries are checked first, in array order; `vault_path` is the fallback. This applies everywhere a skill's on-disk location matters
|
|
312
|
+
- **Resolution order**: for any given `skill_id`, `local_vault_paths` entries are checked first, in array order; `vault_path` is the fallback. This applies everywhere a skill's on-disk location matters: indexing, `resolve_skill`/`fetch_skill` delivery, and `sync`'s symlink target.
|
|
313
313
|
- **`vault_path` keeps its exact existing meaning.** `skillmux.toml` and the `sync --install-hook` git hook only ever live in `vault_path`; `skillmux doctor` warns if it finds a stray manifest inside a `local_vault_paths` entry instead.
|
|
314
|
-
- **`[core]`/`[project.*]` pins must resolve from `vault_path`.** Since the manifest is meant to be portable, `sync`/`doctor` reject a pin backed only by a `local_vault_paths` entry
|
|
314
|
+
- **`[core]`/`[project.*]` pins must resolve from `vault_path`.** Since the manifest is meant to be portable, `sync`/`doctor` reject a pin backed only by a `local_vault_paths` entry; see the manifest section above.
|
|
315
315
|
- **Not yet covered**: `startVaultWatcher`'s live filesystem watch still only watches `vault_path`; a change inside a `local_vault_paths` entry is picked up lazily (on the next `resolve_skill`/`fetch_skill`/`sync` call, via the same mtime staleness check `vault_path` already uses), not instantly.
|
|
316
316
|
|
|
317
|
-
**Visibility.** A `skill_id` present in more than one root is silently resolved via the precedence above with no output during normal use
|
|
317
|
+
**Visibility.** A `skill_id` present in more than one root is silently resolved via the precedence above with no output during normal use. Two commands make that resolution visible on demand:
|
|
318
318
|
|
|
319
|
-
- `skillmux skill which <skill_id
|
|
319
|
+
- `skillmux skill which <skill_id>`: prints which root actually serves that skill, and names every root it shadows:
|
|
320
320
|
```
|
|
321
321
|
$ skillmux skill which my-skill
|
|
322
322
|
my-skill: serving from /home/user/skills-local
|
|
323
323
|
shadows: /home/user/skills
|
|
324
324
|
```
|
|
325
325
|
Exits non-zero with `<skill_id>: not found in vault_path or local_vault_paths` if no root has it.
|
|
326
|
-
- `skillmux doctor` reports every shadowed skill_id as an informational check (`shadowed:<skill_id>`, always `ok`) alongside its existing vault/manifest/embedding checks
|
|
326
|
+
- `skillmux doctor` reports every shadowed skill_id as an informational check (`shadowed:<skill_id>`, always `ok`) alongside its existing vault/manifest/embedding checks, so a scan of `doctor` output surfaces every override in one place, not just the one you thought to ask about.
|
|
327
327
|
|
|
328
|
-
**Discoverability.** A `local_vault_paths` entry is otherwise just a bare directory
|
|
328
|
+
**Discoverability.** A `local_vault_paths` entry is otherwise just a bare directory. Nothing on disk says it belongs to skillmux or which `vault_path` it overlays. `skillmux local-vault init <path>` writes a `.skillmux` marker recording that relationship:
|
|
329
329
|
|
|
330
330
|
```sh
|
|
331
331
|
skillmux local-vault init ~/skills-local
|
|
332
332
|
# wrote /home/user/skills-local/.skillmux (role: local_vault, vault_path: /home/user/skills)
|
|
333
333
|
```
|
|
334
334
|
|
|
335
|
-
`<path>` must already be one of the configured `local_vault_paths` entries and must exist on disk
|
|
335
|
+
`<path>` must already be one of the configured `local_vault_paths` entries and must exist on disk; the command only ever writes the marker, it never adds the path to `config.toml` for you. `skillmux doctor` reports each entry's marker status (`local_vault_marker:<path>`): `ok: false` if no marker exists yet (with the exact `local-vault init` command to fix it), or if the marker's recorded `vault_path` no longer matches the one currently configured (drift: e.g. after copying the directory to a machine with a different `vault_path`).
|
package/docs/deployment.md
CHANGED
|
@@ -248,7 +248,7 @@ The HTTP server provides:
|
|
|
248
248
|
|
|
249
249
|
The Docker health check calls `/health/ready`.
|
|
250
250
|
|
|
251
|
-
`GET /stats` returns JSON aggregated from audit rows over a query window (using `?since=`, defaulting to 24 hours). The response contains `since`, `until`, `total_requests`, `empty_shortlist_count`, `empty_shortlist_rate`, `retrieval_totals` (counts for `exact`, `reranked`, `hybrid`, and `lexical`), `degraded_count`, `average_latency_ms`, per-skill candidate counts in `skills` (`skill_id`, `candidate_count`), and `
|
|
251
|
+
`GET /stats` returns JSON aggregated from audit rows over a query window (using `?since=`, defaulting to 24 hours). The response contains `since`, `until`, `total_requests`, `empty_shortlist_count`, `empty_shortlist_rate`, `retrieval_totals` (counts for `exact`, `reranked`, `hybrid`, and `lexical`), `degraded_count`, `average_latency_ms`, per-skill candidate counts in `skills` (`skill_id`, `candidate_count`), `top_empty_shortlist_queries` (`query`, `count`), an `acceptance` signal derived from fetches correlated to their resolve (`acceptance_rate`, `observed_mrr`, `top1_acceptance_rate`, `accepted_count`, `resolves_with_candidates`, and `uncorrelated_fetch_count`, or `available: false` with just the uncorrelated count when a window has no correlated fetches), and `top_unused_shortlist_queries` (`query`, `count`) for queries that returned candidates but received no correlated fetch. See [Managing skills](skill-management.md#use-routing-data-to-tune-tiers) for using these statistics to tune delivery tiers.
|
|
252
252
|
|
|
253
253
|
Prometheus metrics cover request totals, resolve latency histograms, errors,
|
|
254
254
|
rate-limit rejections, degraded retrieval totals by stage and reason, readiness
|
|
@@ -289,13 +289,26 @@ their mounted vault checkout and do not manage host agent directories.
|
|
|
289
289
|
|
|
290
290
|
## Persistent data and backups
|
|
291
291
|
|
|
292
|
-
Persist `state_dir` to retain
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
292
|
+
Persist `state_dir` to retain both state files it holds: `index.sqlite3`
|
|
293
|
+
(retrieval index and vector matrix) and `audit.sqlite3` (resolve and fetch
|
|
294
|
+
audit rows). Skill content remains in the server's vault checkout and should
|
|
295
|
+
use its own backup or Git workflow.
|
|
296
|
+
|
|
297
|
+
The two files are independent and can be backed up on different schedules.
|
|
298
|
+
Copying or backing up `index.sqlite3` alone never carries raw user queries;
|
|
299
|
+
only `audit.sqlite3` does. `audit.sqlite3` is created with WAL journaling and
|
|
300
|
+
incremental auto-vacuum, so `skillmux audit prune` reclaims space without
|
|
301
|
+
taking a lock on `index.sqlite3`. On first startup after an upgrade from a
|
|
302
|
+
version that kept audit rows inside `index.sqlite3`, Skillmux migrates them
|
|
303
|
+
into `audit.sqlite3` once, transactionally; the migration is idempotent, so
|
|
304
|
+
it is safe to leave running unattended.
|
|
305
|
+
|
|
306
|
+
Treat `audit.sqlite3` as sensitive because its rows can contain raw user
|
|
297
307
|
queries. Stop the process or use SQLite-safe backup tooling before copying a
|
|
298
|
-
live database.
|
|
308
|
+
live database. `audit.retention_days` (default 90; `0` disables pruning) ages
|
|
309
|
+
out old rows automatically at startup and at most once per 24 hours while the
|
|
310
|
+
server runs; see [CLI reference](cli.md#observability-and-evaluation-skillmux-report-audit-eval)
|
|
311
|
+
for the on-demand `skillmux audit prune` command.
|
|
299
312
|
|
|
300
313
|
## Native pins with shared retrieval
|
|
301
314
|
|
package/docs/getting-started.md
CHANGED
|
@@ -3,6 +3,17 @@
|
|
|
3
3
|
Skillmux supports three setup paths. Pick the result you want before choosing
|
|
4
4
|
an installation.
|
|
5
5
|
|
|
6
|
+
```mermaid
|
|
7
|
+
flowchart LR
|
|
8
|
+
Q{What do you need?}
|
|
9
|
+
Q -->|Managed links in client skill directories| A[Manage native skills]
|
|
10
|
+
Q -->|Local stdio MCP| B[Add local MCP retrieval]
|
|
11
|
+
Q -->|Streamable HTTP MCP for several clients| C[Run a shared MCP service]
|
|
12
|
+
A --> A1[Skillmux CLI]
|
|
13
|
+
B --> B1[Skillmux CLI]
|
|
14
|
+
C --> C1[Skillmux server, full image]
|
|
15
|
+
```
|
|
16
|
+
|
|
6
17
|
| Goal | Skill delivery | Recommended installation |
|
|
7
18
|
| --- | --- | --- |
|
|
8
19
|
| [Manage native skills](#manage-native-skills) | Managed links in client skill directories | Skillmux CLI |
|
package/docs/mcp-routing.md
CHANGED
|
@@ -79,6 +79,7 @@ Skillmux returns a ranked candidates response:
|
|
|
79
79
|
|
|
80
80
|
```json
|
|
81
81
|
{
|
|
82
|
+
"request_id": "3fae2b8e-6c2d-4b1a-9d7a-2b6c5b6a9e10",
|
|
82
83
|
"retrieval": "reranked",
|
|
83
84
|
"candidates": [
|
|
84
85
|
{
|
|
@@ -91,6 +92,7 @@ Skillmux returns a ranked candidates response:
|
|
|
91
92
|
}
|
|
92
93
|
```
|
|
93
94
|
|
|
95
|
+
- `request_id`: a unique id minted for this resolve. Pass it back to `fetch_skill` to correlate a fetch outcome with this resolve and the fetched skill's rank in this shortlist.
|
|
94
96
|
- `retrieval`: the effective retrieval capability (`reranked`, `hybrid`, or `lexical`).
|
|
95
97
|
- `candidates`: zero through effective `top_k` candidates ordered by descending score with contiguous 1-based ranks.
|
|
96
98
|
- If reranking or embedding fails, degradation metadata (`degraded_from`, `degradation_reason`) is included.
|
|
@@ -101,13 +103,18 @@ Input:
|
|
|
101
103
|
|
|
102
104
|
```json
|
|
103
105
|
{
|
|
104
|
-
"skill_id": "csv-formatter"
|
|
106
|
+
"skill_id": "csv-formatter",
|
|
107
|
+
"request_id": "3fae2b8e-6c2d-4b1a-9d7a-2b6c5b6a9e10"
|
|
105
108
|
}
|
|
106
109
|
```
|
|
107
110
|
|
|
108
111
|
The response contains the current `SKILL.md` body as text content.
|
|
109
112
|
`structuredContent` contains the skill ID, title, content SHA-256, and
|
|
110
|
-
supporting-file paths.
|
|
113
|
+
supporting-file paths. `request_id` is optional. When it names a resolve that
|
|
114
|
+
minted it, the recorded fetch outcome links to that resolve and its rank in
|
|
115
|
+
the shortlist. An absent, unknown, or malformed `request_id` still succeeds
|
|
116
|
+
and records an uncorrelated fetch — delivery never fails because telemetry
|
|
117
|
+
could not correlate. Fetch does not depend on an earlier resolve call.
|
|
111
118
|
|
|
112
119
|
The complete wire contract lives in [schema.json](schema.json).
|
|
113
120
|
|
|
@@ -117,11 +124,26 @@ Give the calling client these rules:
|
|
|
117
124
|
|
|
118
125
|
1. Call `resolve_skill` when a task may benefit from a specialized workflow.
|
|
119
126
|
2. Review the returned ranked candidates shortlist.
|
|
120
|
-
3. If a relevant candidate exists, call `fetch_skill` with its `skill_id` to retrieve complete instructions.
|
|
127
|
+
3. If a relevant candidate exists, call `fetch_skill` with its `skill_id` to retrieve complete instructions, passing back the resolve's `request_id` when the client retains it.
|
|
121
128
|
4. If no candidate is relevant (or `candidates` is empty), continue under your normal workflow.
|
|
122
129
|
|
|
130
|
+
Passing `request_id` is optional and never required for delivery, but it is
|
|
131
|
+
what lets Skillmux measure whether a returned shortlist was actually used.
|
|
132
|
+
|
|
123
133
|
## Retrieval pipeline
|
|
124
134
|
|
|
135
|
+
```mermaid
|
|
136
|
+
flowchart LR
|
|
137
|
+
Q[Query] --> L[Lexical: FTS5 + BM25]
|
|
138
|
+
Q --> E[Embeddings: local or remote]
|
|
139
|
+
L --> F[Reciprocal-rank fusion]
|
|
140
|
+
E --> F
|
|
141
|
+
F --> RR[Optional reranker]
|
|
142
|
+
RR --> K[Ranked candidates: top_k]
|
|
143
|
+
F -. reranker unavailable .-> K
|
|
144
|
+
Q -. embedding unavailable: lexical only .-> K
|
|
145
|
+
```
|
|
146
|
+
|
|
125
147
|
Skillmux builds candidates in stages:
|
|
126
148
|
|
|
127
149
|
1. SQLite FTS5 ranks lexical matches with BM25.
|
|
@@ -173,12 +195,24 @@ access.
|
|
|
173
195
|
|
|
174
196
|
Each resolve request records:
|
|
175
197
|
|
|
176
|
-
- timestamp and query;
|
|
198
|
+
- timestamp, `request_id`, and query;
|
|
177
199
|
- retrieval capability;
|
|
178
200
|
- degradation metadata (`degraded_from`, `degradation_reason`) when degraded;
|
|
179
201
|
- candidates with scores;
|
|
180
202
|
- latency in milliseconds.
|
|
181
203
|
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
204
|
+
Each fetch request records:
|
|
205
|
+
|
|
206
|
+
- timestamp and `skill_id`;
|
|
207
|
+
- the `request_id` supplied by the caller, or null when absent, unknown, or malformed;
|
|
208
|
+
- the originating resolve's audit row id, or null when the fetch is uncorrelated;
|
|
209
|
+
- `rank_at_resolve`: the fetched skill's rank in that resolve's shortlist, or null when the fetch is uncorrelated or the skill was absent from that shortlist.
|
|
210
|
+
|
|
211
|
+
Skillmux stores audit rows in `audit.sqlite3` under `state_dir`, a file
|
|
212
|
+
separate from the retrieval index. Use `skillmux report` to summarize
|
|
213
|
+
activity, `skillmux audit prune` to reclaim space under
|
|
214
|
+
`audit.retention_days` (default 90 days), and `skillmux eval promote` to turn
|
|
215
|
+
correlated fetches into eval cases — see [CLI
|
|
216
|
+
reference](cli.md#observability-and-evaluation-skillmux-report-audit-eval)
|
|
217
|
+
for all three. Treat raw queries as private user data when backing up or
|
|
218
|
+
sharing the database.
|
package/docs/schema.json
CHANGED
|
@@ -55,6 +55,18 @@
|
|
|
55
55
|
"required": ["rank", "skill_id", "description", "score"],
|
|
56
56
|
"additionalProperties": false
|
|
57
57
|
},
|
|
58
|
+
"RequestId": {
|
|
59
|
+
"type": "string",
|
|
60
|
+
"description": "Per-resolve correlation id minted by resolve_skill. Lowercase UUIDv4.",
|
|
61
|
+
"format": "uuid",
|
|
62
|
+
"pattern": "^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$"
|
|
63
|
+
},
|
|
64
|
+
"InboundRequestId": {
|
|
65
|
+
"type": "string",
|
|
66
|
+
"description": "request_id as supplied by the caller to fetch_skill. Deliberately not pattern-validated: an unknown or malformed value must still succeed and record an uncorrelated fetch (AC7); correlation validity is decided by the server, not the schema.",
|
|
67
|
+
"minLength": 1,
|
|
68
|
+
"maxLength": 128
|
|
69
|
+
},
|
|
58
70
|
"ResolveSkillInput": {
|
|
59
71
|
"type": "object",
|
|
60
72
|
"properties": {
|
|
@@ -76,6 +88,7 @@
|
|
|
76
88
|
"type": "object",
|
|
77
89
|
"description": "Ranked candidates response from resolve_skill.",
|
|
78
90
|
"properties": {
|
|
91
|
+
"request_id": { "$ref": "#/$defs/RequestId" },
|
|
79
92
|
"retrieval": { "enum": ["exact", "reranked", "hybrid", "lexical"] },
|
|
80
93
|
"degraded_from": { "enum": ["reranked", "hybrid"] },
|
|
81
94
|
"degradation_reason": { "$ref": "#/$defs/DegradationReason" },
|
|
@@ -85,13 +98,14 @@
|
|
|
85
98
|
"description": "Shortlist of candidates in descending score order with contiguous 1-based ranks."
|
|
86
99
|
}
|
|
87
100
|
},
|
|
88
|
-
"required": ["retrieval", "candidates"],
|
|
101
|
+
"required": ["request_id", "retrieval", "candidates"],
|
|
89
102
|
"additionalProperties": false
|
|
90
103
|
},
|
|
91
104
|
"FetchSkillInput": {
|
|
92
105
|
"type": "object",
|
|
93
106
|
"properties": {
|
|
94
|
-
"skill_id": { "$ref": "#/$defs/SkillId" }
|
|
107
|
+
"skill_id": { "$ref": "#/$defs/SkillId" },
|
|
108
|
+
"request_id": { "$ref": "#/$defs/InboundRequestId" }
|
|
95
109
|
},
|
|
96
110
|
"required": ["skill_id"],
|
|
97
111
|
"additionalProperties": false
|
|
@@ -146,7 +160,7 @@
|
|
|
146
160
|
},
|
|
147
161
|
"state_dir": {
|
|
148
162
|
"type": "string",
|
|
149
|
-
"description": "All router writes are confined here (AC9): index SQLite (
|
|
163
|
+
"description": "All router writes are confined here (AC9): index SQLite (index.sqlite3), audit SQLite (audit.sqlite3), vector matrix.",
|
|
150
164
|
"default": "~/.local/state/skillmux"
|
|
151
165
|
},
|
|
152
166
|
"recall": {
|
|
@@ -179,11 +193,24 @@
|
|
|
179
193
|
"additionalProperties": false
|
|
180
194
|
},
|
|
181
195
|
"inference": { "$ref": "#/$defs/InferenceConfig" },
|
|
182
|
-
"server": { "$ref": "#/$defs/ServerConfig" }
|
|
196
|
+
"server": { "$ref": "#/$defs/ServerConfig" },
|
|
197
|
+
"audit": { "$ref": "#/$defs/AuditConfig" }
|
|
183
198
|
},
|
|
184
199
|
"required": ["vault_path", "state_dir", "recall", "output", "inference"],
|
|
185
200
|
"additionalProperties": false
|
|
186
201
|
},
|
|
202
|
+
"AuditConfig": {
|
|
203
|
+
"type": "object",
|
|
204
|
+
"properties": {
|
|
205
|
+
"retention_days": {
|
|
206
|
+
"type": "integer",
|
|
207
|
+
"minimum": 0,
|
|
208
|
+
"default": 90,
|
|
209
|
+
"description": "Age in days beyond which audit and fetch rows are pruned. 0 disables pruning and retains rows indefinitely."
|
|
210
|
+
}
|
|
211
|
+
},
|
|
212
|
+
"additionalProperties": false
|
|
213
|
+
},
|
|
187
214
|
"InferenceConfig": {
|
|
188
215
|
"oneOf": [
|
|
189
216
|
{
|
package/docs/skill-management.md
CHANGED
|
@@ -55,6 +55,38 @@ With no path, `scan` checks the configured vault. `--json` wraps the result in
|
|
|
55
55
|
the standard CLI automation envelope, while `--format json` selects the
|
|
56
56
|
scanner's raw JSON rendering.
|
|
57
57
|
|
|
58
|
+
## Check and pull upstream updates
|
|
59
|
+
|
|
60
|
+
Every skill `install` places in the vault gets a `.skillmux-origin` sidecar
|
|
61
|
+
recording the source repo, resolved commit, and a content hash. Check which
|
|
62
|
+
installed skills have moved upstream:
|
|
63
|
+
|
|
64
|
+
```sh
|
|
65
|
+
skillmux outdated
|
|
66
|
+
skillmux outdated --json
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
Skills you authored by hand, or installed before this sidecar existed, carry
|
|
70
|
+
no `.skillmux-origin` and are silently omitted — `outdated` only reports on
|
|
71
|
+
skills it can trace back to a source. A repo that has become unreachable is
|
|
72
|
+
reported per-skill as `check_failed` with a reason; it does not stop the rest
|
|
73
|
+
of the check.
|
|
74
|
+
|
|
75
|
+
Pull an update for one skill, or every outdated skill at once:
|
|
76
|
+
|
|
77
|
+
```sh
|
|
78
|
+
skillmux update csv-formatter --dry-run
|
|
79
|
+
skillmux update csv-formatter --yes
|
|
80
|
+
skillmux update --yes
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
`--dry-run` reports the old and new commit and whether content actually
|
|
84
|
+
changed, without writing anything. A real update re-runs the same scan
|
|
85
|
+
`install` does (`--fail-on` applies the same way) and refuses to overwrite a
|
|
86
|
+
skill whose on-disk content has drifted from what was last installed —
|
|
87
|
+
someone may have hand-edited it. Pass `--force` to overwrite anyway. Like
|
|
88
|
+
`audit prune`, a non-interactive or `--json` run needs `--yes`.
|
|
89
|
+
|
|
58
90
|
## Plan client delivery
|
|
59
91
|
|
|
60
92
|
Use product names for common clients:
|
package/package.json
CHANGED
package/src/audit.ts
CHANGED
|
@@ -5,6 +5,7 @@ export function buildAuditRow(row: AuditRow): AuditRow {
|
|
|
5
5
|
const built: AuditRow = {
|
|
6
6
|
id: row.id,
|
|
7
7
|
ts: row.ts,
|
|
8
|
+
request_id: row.request_id,
|
|
8
9
|
query: row.query,
|
|
9
10
|
retrieval: row.retrieval,
|
|
10
11
|
candidates: row.candidates.map((c) => ({ skill_id: c.skill_id, score: c.score })),
|
package/src/cli.ts
CHANGED
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
migrateLegacyPaths,
|
|
13
13
|
resolveConfigPath,
|
|
14
14
|
} from "./config";
|
|
15
|
-
import {
|
|
15
|
+
import { openAudit } from "./db";
|
|
16
16
|
import { diagnose } from "./doctor";
|
|
17
17
|
import { getEffectiveConfig } from "./config-service";
|
|
18
18
|
import { evalVault } from "./eval";
|
|
@@ -42,10 +42,14 @@ import {
|
|
|
42
42
|
cloneToTemp,
|
|
43
43
|
deriveRepoName,
|
|
44
44
|
installIntoVault,
|
|
45
|
+
resolveCloneCommit,
|
|
45
46
|
resolveRepoSource,
|
|
46
47
|
resolveSkillDir,
|
|
47
48
|
validateSkillCandidate,
|
|
48
49
|
} from "./install";
|
|
50
|
+
import { runOutdated } from "./commands/outdated";
|
|
51
|
+
import { runUpdate } from "./commands/update";
|
|
52
|
+
import { hashSkillContent, writeSkillOrigin } from "./provenance";
|
|
49
53
|
import {
|
|
50
54
|
parseManifest,
|
|
51
55
|
resolveManifestPath,
|
|
@@ -106,7 +110,9 @@ import {
|
|
|
106
110
|
suggestCorrection,
|
|
107
111
|
} from "./output";
|
|
108
112
|
import { generateCompletions, type ShellType } from "./completions";
|
|
113
|
+
import { runAudit } from "./commands/audit";
|
|
109
114
|
import { handleConfigCommand } from "./commands/config";
|
|
115
|
+
import { runEvalPromote } from "./commands/eval";
|
|
110
116
|
import { runCore } from "./commands/core";
|
|
111
117
|
import { configuredTargetForSurface, runProject } from "./commands/project";
|
|
112
118
|
import { confirmAction, confirmIfNeeded } from "./commands/shared";
|
|
@@ -124,8 +130,11 @@ const KNOWN_COMMANDS = [
|
|
|
124
130
|
"target",
|
|
125
131
|
"core",
|
|
126
132
|
"report",
|
|
133
|
+
"audit",
|
|
127
134
|
"scan",
|
|
128
135
|
"install",
|
|
136
|
+
"outdated",
|
|
137
|
+
"update",
|
|
129
138
|
"eval",
|
|
130
139
|
"doctor",
|
|
131
140
|
"models",
|
|
@@ -139,18 +148,23 @@ function isDockerHostManagementCommand(command: string, subCommand: string): boo
|
|
|
139
148
|
"init",
|
|
140
149
|
"sync",
|
|
141
150
|
"install",
|
|
151
|
+
"outdated",
|
|
152
|
+
"update",
|
|
142
153
|
"project",
|
|
143
154
|
"target",
|
|
144
155
|
"core",
|
|
145
156
|
"local-vault",
|
|
146
157
|
"models",
|
|
147
158
|
"context",
|
|
148
|
-
"eval",
|
|
149
159
|
].includes(command)
|
|
150
160
|
) {
|
|
151
161
|
return true;
|
|
152
162
|
}
|
|
153
163
|
|
|
164
|
+
// eval promote only touches the mounted state_dir, unlike bare `eval`
|
|
165
|
+
// (vault ranking evaluation), which needs local embeddings and the vault.
|
|
166
|
+
if (command === "eval" && subCommand !== "promote") return true;
|
|
167
|
+
|
|
154
168
|
return command === "config" && ["init", "set"].includes(subCommand);
|
|
155
169
|
}
|
|
156
170
|
|
|
@@ -322,14 +336,29 @@ async function main() {
|
|
|
322
336
|
case "report":
|
|
323
337
|
await runReport(rawArgv.slice(1), { isJson });
|
|
324
338
|
break;
|
|
339
|
+
case "audit":
|
|
340
|
+
await runAudit(subCommand, commandArgs, { isJson, dryRun: isDryRun });
|
|
341
|
+
break;
|
|
325
342
|
case "scan":
|
|
326
343
|
await runScan(rawArgv.slice(1), { isJson });
|
|
327
344
|
break;
|
|
328
345
|
case "install":
|
|
329
346
|
await runInstall(rawArgv.slice(1), { isJson });
|
|
330
347
|
break;
|
|
348
|
+
case "outdated":
|
|
349
|
+
await runOutdated(rawArgv.slice(1), { isJson });
|
|
350
|
+
break;
|
|
351
|
+
case "update":
|
|
352
|
+
await runUpdate(rawArgv.slice(1), { isJson });
|
|
353
|
+
break;
|
|
331
354
|
case "eval":
|
|
332
|
-
|
|
355
|
+
if (subCommand === "promote") {
|
|
356
|
+
await runEvalPromote(commandArgs, { isJson, dryRun: isDryRun });
|
|
357
|
+
} else if (subCommand === "") {
|
|
358
|
+
await runEval({ isJson });
|
|
359
|
+
} else {
|
|
360
|
+
throw new Error(`usage: skillmux eval [promote --since <window> [--target <path>] [--dry-run] [--yes] [--json]]`);
|
|
361
|
+
}
|
|
333
362
|
break;
|
|
334
363
|
case "doctor":
|
|
335
364
|
await runDoctor({ isJson });
|
|
@@ -359,7 +388,7 @@ async function main() {
|
|
|
359
388
|
const suggestion = suggestCorrection(command, KNOWN_COMMANDS);
|
|
360
389
|
const msg = suggestion
|
|
361
390
|
? `Unknown command "${command}". Did you mean "${suggestion}"?`
|
|
362
|
-
: `usage: skillmux <serve|index|sync|init|project|target|core pin/unpin|report|scan|install|eval|doctor|skill which|local-vault init|config show|models download>`;
|
|
391
|
+
: `usage: skillmux <serve|index|sync|init|project|target|core pin/unpin|report|audit prune|scan|install|outdated|update|eval|doctor|skill which|local-vault init|config show|models download>`;
|
|
363
392
|
throw new Error(msg);
|
|
364
393
|
}
|
|
365
394
|
}
|
|
@@ -507,7 +536,7 @@ Default:
|
|
|
507
536
|
serve --transport http
|
|
508
537
|
|
|
509
538
|
Supported commands:
|
|
510
|
-
serve, index, doctor, report, scan, skill which
|
|
539
|
+
serve, index, doctor, report, audit prune, eval promote, scan, skill which
|
|
511
540
|
config show|get|validate|diff|status
|
|
512
541
|
|
|
513
542
|
Native skill management:
|
|
@@ -540,9 +569,16 @@ Init clients:
|
|
|
540
569
|
Init targets:
|
|
541
570
|
agent-skills, claude-code, codex, custom
|
|
542
571
|
|
|
572
|
+
Operations:
|
|
573
|
+
skillmux report [--server <url> | --db <path>] --since <window> [--json]
|
|
574
|
+
skillmux audit prune [--older-than <window>] [--dry-run] [--yes] [--json]
|
|
575
|
+
skillmux eval promote --since <window> [--target <path>] [--dry-run] [--yes] [--json]
|
|
576
|
+
skillmux outdated [--allow-local-source] [--json]
|
|
577
|
+
skillmux update [skill-id] [--yes] [--dry-run] [--force] [--allow-local-source] [--fail-on low|medium|high] [--json]
|
|
578
|
+
|
|
543
579
|
Commands:
|
|
544
|
-
serve, index, sync, init, project, target, core, report, scan, install,
|
|
545
|
-
local-vault, config, models, context, completions`);
|
|
580
|
+
serve, index, sync, init, project, target, core, report, audit, scan, install, outdated, update,
|
|
581
|
+
eval, doctor, skill, local-vault, config, models, context, completions`);
|
|
546
582
|
}
|
|
547
583
|
|
|
548
584
|
// ---------------------------------------------------------------------------
|
|
@@ -807,6 +843,11 @@ async function runSync(args: string[]): Promise<void> {
|
|
|
807
843
|
console.log(
|
|
808
844
|
`${targetName}: +${result.added.length} -${result.removed.length}${suffix}`,
|
|
809
845
|
);
|
|
846
|
+
if (result.skipped.length > 0) {
|
|
847
|
+
console.log(
|
|
848
|
+
` warning: refused to sync ${result.skipped.join(", ")} — skill directory contains a symlink`,
|
|
849
|
+
);
|
|
850
|
+
}
|
|
810
851
|
|
|
811
852
|
if (target.project_groups.length > 0) {
|
|
812
853
|
const allGroups = manifest.project ?? {};
|
|
@@ -821,6 +862,11 @@ async function runSync(args: string[]): Promise<void> {
|
|
|
821
862
|
console.log(
|
|
822
863
|
` ${projectResult.group} -> ${projectResult.pinDir}: +${projectResult.added.length} -${projectResult.removed.length}${suffix}`,
|
|
823
864
|
);
|
|
865
|
+
if (projectResult.skipped.length > 0) {
|
|
866
|
+
console.log(
|
|
867
|
+
` warning: refused to sync ${projectResult.skipped.join(", ")} — skill directory contains a symlink`,
|
|
868
|
+
);
|
|
869
|
+
}
|
|
824
870
|
}
|
|
825
871
|
}
|
|
826
872
|
}
|
|
@@ -1453,7 +1499,7 @@ async function runReport(
|
|
|
1453
1499
|
|
|
1454
1500
|
const db = dbPath
|
|
1455
1501
|
? new Database(dbPath, { readonly: true })
|
|
1456
|
-
:
|
|
1502
|
+
: openAudit(expandHome((await loadConfig()).state_dir));
|
|
1457
1503
|
const stats = getStats(db, since);
|
|
1458
1504
|
emitSuccess({ isJson: options.isJson }, stats, () =>
|
|
1459
1505
|
console.log(renderStatsText(stats)),
|
|
@@ -1592,12 +1638,20 @@ async function runInstall(
|
|
|
1592
1638
|
return;
|
|
1593
1639
|
}
|
|
1594
1640
|
|
|
1641
|
+
const commit = resolveCloneCommit(cloneDir);
|
|
1595
1642
|
const targetDir = installIntoVault(
|
|
1596
1643
|
vaultPath,
|
|
1597
1644
|
resolved.skillId,
|
|
1598
1645
|
resolved.dir,
|
|
1599
1646
|
force,
|
|
1600
1647
|
);
|
|
1648
|
+
writeSkillOrigin(targetDir, {
|
|
1649
|
+
source_url: source.url,
|
|
1650
|
+
skill_path: source.skillPath,
|
|
1651
|
+
commit,
|
|
1652
|
+
installed_at: new Date().toISOString(),
|
|
1653
|
+
content_hash: hashSkillContent(targetDir),
|
|
1654
|
+
});
|
|
1601
1655
|
emitSuccess(
|
|
1602
1656
|
{ isJson: options.isJson },
|
|
1603
1657
|
{ skill_id: resolved.skillId, installed_at: targetDir },
|