@mfdaves/okf-mcp 0.9.1 → 0.13.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +100 -0
- package/README.md +452 -26
- package/bin/okf-mcp.js +6 -1
- package/okf/okf-mcp/distribution/reference-bundle.md +10 -3
- package/okf/okf-mcp/index.md +2 -1
- package/okf/okf-mcp/interfaces/http-authoring-api.md +11 -2
- package/okf/okf-mcp/interfaces/mcp-http.md +64 -0
- package/okf/okf-mcp/interfaces/mcp-tools.md +25 -6
- package/okf/okf-mcp/overview/okf-mcp.md +16 -6
- package/okf/okf-mcp/policies/authoring-safety.md +28 -5
- package/okf/okf-mcp/runtime/indexer.md +14 -1
- package/okf/okf-mcp/runtime/mcp-server.md +16 -8
- package/okf/okf-mcp/specs/producer-api.md +16 -2
- package/package.json +17 -5
- package/server.json +38 -2
- package/src/assets.js +165 -73
- package/src/authoring/candidates.js +434 -0
- package/src/authoring/git-preflight.js +225 -0
- package/src/authoring/git-publish.js +783 -0
- package/src/authoring/live.js +757 -0
- package/src/authoring/paths.js +112 -0
- package/src/authoring/policy-hook.js +221 -0
- package/src/authoring/proposal-policy.js +159 -0
- package/src/authoring/receipts.js +373 -0
- package/src/authoring/render.js +38 -0
- package/src/authoring/shared.js +75 -0
- package/src/authoring/staging.js +216 -0
- package/src/authoring.js +386 -287
- package/src/cache/files.js +147 -0
- package/src/cache/lock.js +201 -0
- package/src/cache/paths.js +197 -0
- package/src/cache/store.js +703 -0
- package/src/catalog-text.js +19 -0
- package/src/cli/commands.js +448 -0
- package/src/cli/http-options.js +193 -0
- package/src/cli/options.js +554 -0
- package/src/cli/usage.js +96 -0
- package/src/cli.js +18 -932
- package/src/computation.js +99 -61
- package/src/context/assemble.js +567 -0
- package/src/context/cli-options.js +88 -0
- package/src/context/score.js +234 -0
- package/src/context/sections.js +274 -0
- package/src/embed/interface.js +160 -0
- package/src/embed/openai.js +715 -0
- package/src/embed/prepare.js +182 -0
- package/src/embed/units.js +96 -0
- package/src/error-details.js +53 -0
- package/src/git-source.js +112 -52
- package/src/graph.js +95 -37
- package/src/http-server.js +36 -17
- package/src/index.js +15 -0
- package/src/indexer/assemble.js +566 -0
- package/src/indexer/build.js +290 -0
- package/src/indexer/cache.js +81 -0
- package/src/indexer/edges.js +430 -0
- package/src/indexer/index.js +44 -0
- package/src/indexer/paths.js +178 -0
- package/src/indexer/resolve.js +282 -0
- package/src/indexer/update.js +720 -0
- package/src/indexer.js +3 -1068
- package/src/live-authoring.js +4 -2329
- package/src/markdown.js +21 -16
- package/src/mcp/auth.js +409 -0
- package/src/mcp/http.js +317 -0
- package/src/mcp/principal.js +59 -0
- package/src/mcp/registry.js +356 -0
- package/src/mcp/resources.js +45 -0
- package/src/mcp/server.js +567 -0
- package/src/mcp/tools/catalog.js +247 -0
- package/src/mcp/tools/computation.js +83 -0
- package/src/mcp/tools/context.js +133 -0
- package/src/mcp/tools/graph.js +167 -0
- package/src/mcp/tools/live-authoring.js +316 -0
- package/src/mcp/tools/migration.js +86 -0
- package/src/mcp/tools/producers.js +79 -0
- package/src/mcp/tools/proposals.js +225 -0
- package/src/mcp/tools/provenance.js +87 -0
- package/src/mcp/tools/remote.js +79 -0
- package/src/mcp/tools/search.js +242 -0
- package/src/mcp-server.js +3 -1611
- package/src/migration.js +431 -253
- package/src/parser.js +245 -130
- package/src/plugins.js +71 -48
- package/src/policy/write-policy.js +713 -0
- package/src/producer-publisher.js +202 -78
- package/src/producers.js +311 -103
- package/src/project.js +346 -93
- package/src/remote.js +96 -61
- package/src/search/cli-options.js +107 -0
- package/src/search/confidence.js +61 -0
- package/src/search/dense.js +304 -0
- package/src/search/embedding-cli.js +57 -0
- package/src/search/embedding-config.js +217 -0
- package/src/search/fusion.js +249 -0
- package/src/search/hybrid.js +526 -0
- package/src/search/languages.js +200 -0
- package/src/search/lexical.js +695 -0
- package/src/search/query.js +620 -0
- package/src/search/stem/among.js +43 -0
- package/src/search/stem/base-stemmer.js +311 -0
- package/src/search/stem/en-regions.js +456 -0
- package/src/search/stem/en-suffixes.js +664 -0
- package/src/search/stem/en-tables.js +176 -0
- package/src/search/stem/en.js +238 -0
- package/src/search/stem/it-regions.js +459 -0
- package/src/search/stem/it-suffixes.js +471 -0
- package/src/search/stem/it-tables.js +257 -0
- package/src/search/stem/it.js +155 -0
- package/src/search-index.js +2 -81
- package/src/search.js +2 -252
- package/src/store.js +134 -81
- package/src/tool-packs.js +346 -0
- package/src/v02.js +301 -105
- package/src/validation.js +24 -20
package/README.md
CHANGED
|
@@ -6,7 +6,7 @@ The part that is hard is writing to a catalog safely. A configured producer read
|
|
|
6
6
|
|
|
7
7
|
Reading is the ordinary half. It consumes an OKF bundle directory of Markdown files with YAML frontmatter. An optional workspace mode can federate several bundles. Concepts are exposed through CLI commands and MCP resources and tools for validation, structured search, graph navigation, provenance inspection, and proposal-based authoring.
|
|
8
8
|
|
|
9
|
-
The core intentionally has no database, embeddings, build step, or hosted-service dependency. It uses `js-yaml` for safe YAML, CommonMark for Markdown structure, MiniSearch for in-memory BM25+ text retrieval, and the official Model Context Protocol TypeScript SDK v2 for stdio
|
|
9
|
+
The core intentionally has no database, embeddings, build step, or hosted-service dependency. It uses `js-yaml` for safe YAML, CommonMark for Markdown structure, MiniSearch for in-memory BM25+ text retrieval, and the official Model Context Protocol TypeScript SDK v2 for MCP over stdio and Streamable HTTP. Local root mode makes no network calls. Optional remote loading fetches public Markdown concepts and only their explicitly referenced inert assets from GitHub. Nothing in the v0.2 computation support executes code or attests a receipt.
|
|
10
10
|
|
|
11
11
|
## OKF v0.2 Support And Extensions
|
|
12
12
|
|
|
@@ -37,7 +37,7 @@ Node 22 or newer is required.
|
|
|
37
37
|
Install from the GitHub release:
|
|
38
38
|
|
|
39
39
|
```bash
|
|
40
|
-
git clone --branch v0.
|
|
40
|
+
git clone --branch v0.12.0-rc.1 https://github.com/mfdaves/okf-mcp.git
|
|
41
41
|
cd okf-mcp
|
|
42
42
|
npm ci
|
|
43
43
|
node bin/okf-mcp.js --root ./path/to/okf validate
|
|
@@ -46,14 +46,14 @@ node bin/okf-mcp.js --root ./path/to/okf validate
|
|
|
46
46
|
Pin the published version for reproducible use:
|
|
47
47
|
|
|
48
48
|
```bash
|
|
49
|
-
npx -y @mfdaves/okf-mcp@0.
|
|
50
|
-
npx -y @mfdaves/okf-mcp@0.
|
|
49
|
+
npx -y @mfdaves/okf-mcp@0.12.0-rc.1 --version
|
|
50
|
+
npx -y @mfdaves/okf-mcp@0.12.0-rc.1 --root ./path/to/okf validate
|
|
51
51
|
```
|
|
52
52
|
|
|
53
53
|
For a persistent installation:
|
|
54
54
|
|
|
55
55
|
```bash
|
|
56
|
-
npm install --global @mfdaves/okf-mcp@0.
|
|
56
|
+
npm install --global @mfdaves/okf-mcp@0.12.0-rc.1
|
|
57
57
|
|
|
58
58
|
okf --version
|
|
59
59
|
okf --root ./path/to/okf validate
|
|
@@ -200,7 +200,7 @@ okf --project okf.project.yaml producer preview main-database
|
|
|
200
200
|
okf --project okf.project.yaml producer run main-database --write --actor process:nightly-refresh
|
|
201
201
|
```
|
|
202
202
|
|
|
203
|
-
`producer preview` exits `1` when the candidate is invalid; `producer run` exits `1` when nothing was published. Both print the same receipt the MCP tools return.
|
|
203
|
+
`producer preview` exits `1` when the candidate is invalid; `producer run` exits `1` when nothing was published. Both print the same receipt the MCP tools return. A producer that cannot generate may throw an error with its own `code` (lowercase, `^[a-z][a-z0-9_]{0,63}$`, for example `access_denied`) and a `details` object of scalar values; the receipt then carries one producer-layer diagnostic with that code and the bounded details (identifier keys, finite numbers, booleans, null, single-line strings up to 200 characters that hold no secret the producer read). The error's message is never reported. An error without such a code is reported as `producer_execution_failed`.
|
|
204
204
|
|
|
205
205
|
For a configured project, MCP exposes:
|
|
206
206
|
|
|
@@ -213,7 +213,7 @@ Preview and run accept only the configured producer name plus optional receipt d
|
|
|
213
213
|
Two rules keep a managed bundle stable across repeated runs:
|
|
214
214
|
|
|
215
215
|
- A producer restamps `generated.at` on every run. The host compares each candidate against the published file ignoring that stamp, so a source that did not change reports `unchanged` and leaves the bytes, the timestamps, and the manifest untouched. Publication-diff counts therefore describe the source, not the run.
|
|
216
|
-
- A manifest entry alone is not authority to destroy work. Any file the host would rewrite or remove must itself carry `generated` provenance, so a damaged, hand-merged, or forged manifest cannot make a producer delete hand-authored content.
|
|
216
|
+
- A manifest entry alone is not authority to destroy work. Any file the host would rewrite or remove must itself carry `generated` provenance, so a damaged, hand-merged, or forged manifest cannot make a producer delete hand-authored content. A reserved `index.md` or `log.md` cannot carry `generated` frontmatter, so for those the proof is that their bytes still match the manifest's digest; a hand-edited one stops the run with `producer_owned_file_modified`. A refusal names the file in its diagnostic's `details.path`.
|
|
217
217
|
|
|
218
218
|
Directories emptied by a stale deletion are removed with their last managed file.
|
|
219
219
|
|
|
@@ -226,6 +226,8 @@ Directories emptied by a stale deletion are removed with their last managed file
|
|
|
226
226
|
- `--write-token <token>`: bearer token for write endpoints; defaults to `OKF_WRITE_TOKEN`
|
|
227
227
|
- `--proposal-root <path>`: proposal JSON directory; defaults to `.okf-proposals` under the selected local root or project
|
|
228
228
|
|
|
229
|
+
Proposals are stored state, not a derived cache, so the proposal directory is never cleaned up automatically. A server started with `--git-commit` refuses to publish while the worktree has untracked files, so that directory must be ignored by Git: add `.okf-proposals/` to `.gitignore` (as this repository does), or pass `--proposal-root` with a directory Git already ignores. In project mode the proposal root must stay inside the project root.
|
|
230
|
+
|
|
229
231
|
## MCP Client Config
|
|
230
232
|
|
|
231
233
|
The npm-based examples below use the current published release. A source checkout can invoke its executable `bin/okf-mcp.js` with the same arguments.
|
|
@@ -239,7 +241,7 @@ Example client configuration:
|
|
|
239
241
|
"command": "npx",
|
|
240
242
|
"args": [
|
|
241
243
|
"-y",
|
|
242
|
-
"@mfdaves/okf-mcp@0.
|
|
244
|
+
"@mfdaves/okf-mcp@0.12.0-rc.1",
|
|
243
245
|
"--root",
|
|
244
246
|
"/absolute/path/to/okf",
|
|
245
247
|
"mcp"
|
|
@@ -258,7 +260,7 @@ Project config mode, with read-only project helpers but without proposal mutatio
|
|
|
258
260
|
"command": "npx",
|
|
259
261
|
"args": [
|
|
260
262
|
"-y",
|
|
261
|
-
"@mfdaves/okf-mcp@0.
|
|
263
|
+
"@mfdaves/okf-mcp@0.12.0-rc.1",
|
|
262
264
|
"--project",
|
|
263
265
|
"/absolute/path/to/repo/okf.project.yaml",
|
|
264
266
|
"mcp"
|
|
@@ -281,12 +283,280 @@ returned as MCP tool results with `isError: true`. Calls to tools that are not
|
|
|
281
283
|
enabled are rejected by SDK dispatch. Unexpected implementation errors are
|
|
282
284
|
masked instead of exposing internal details.
|
|
283
285
|
|
|
286
|
+
## MCP over HTTP
|
|
287
|
+
|
|
288
|
+
From 0.13.0-rc.1 the `mcp` command can serve the same tools, resources and
|
|
289
|
+
schemas over Streamable HTTP instead of stdio:
|
|
290
|
+
|
|
291
|
+
```bash
|
|
292
|
+
okf --root /path/to/okf mcp --http
|
|
293
|
+
# OKF MCP HTTP server listening on http://127.0.0.1:8766/mcp
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
| Flag | Meaning |
|
|
297
|
+
| --- | --- |
|
|
298
|
+
| `--http` | Serve MCP over Streamable HTTP. |
|
|
299
|
+
| `--host <host>` | Address to bind (default `127.0.0.1`). A non-loopback address requires `--auth`. |
|
|
300
|
+
| `--port <port>` | Port (default `8766`; `0` picks a free one). |
|
|
301
|
+
| `--public-url <url>` | The URL clients use for the MCP endpoint. Its path is the endpoint path (default `/mcp`; served with and without one trailing slash), its host is accepted in `Host` headers, and with introspection it is the token audience and the advertised `resource`, as written. |
|
|
302
|
+
| `--allowed-host <host>` | Another accepted `Host` header (hostname; the port is ignored); repeatable. `localhost`, `127.0.0.1` and `[::1]` are always accepted. |
|
|
303
|
+
| `--allowed-origin <origin>` | Another accepted browser `Origin` (hostname or origin URL); repeatable. Requests without an `Origin` header are not affected. |
|
|
304
|
+
| `--auth tokens:<file>`, `--auth introspect:<url>` | Bearer authentication, see below. |
|
|
305
|
+
| `--authorization-server <issuer>` | With introspection, the issuer named in the protected-resource metadata. |
|
|
306
|
+
| `--refresh-interval <ms>` | Check the files for changes made outside okf-mcp before a tool call or resource read at most this often (default `2000`; `0` checks before every one). |
|
|
307
|
+
|
|
308
|
+
The server answers:
|
|
309
|
+
|
|
310
|
+
- the MCP endpoint, through the SDK's `createMcpHandler`: protocol revision
|
|
311
|
+
`2026-07-28`, and 2025-era clients (including the `2025-11-25` `initialize`
|
|
312
|
+
handshake) through its stateless fallback. No session is kept: `GET` and
|
|
313
|
+
`DELETE` on the endpoint answer `405` to 2025-era clients.
|
|
314
|
+
- `GET /health` with `{"status":"ok"}`, also beside the endpoint when its
|
|
315
|
+
path is deeper (`/okf/mcp` → `/okf/health`).
|
|
316
|
+
- with introspection, the RFC 9728 protected-resource metadata at
|
|
317
|
+
`/.well-known/oauth-protected-resource` plus the path of `--public-url`
|
|
318
|
+
(for example `/.well-known/oauth-protected-resource/mcp`) and at
|
|
319
|
+
`/.well-known/oauth-protected-resource`.
|
|
320
|
+
|
|
321
|
+
Every request is checked against the accepted `Host` names (`403`
|
|
322
|
+
otherwise); requests to the MCP endpoint are also checked against the accepted
|
|
323
|
+
origins (`403`, without CORS headers) and refused above 1 MiB (`413`). For an
|
|
324
|
+
accepted browser origin the endpoint answers CORS: an `OPTIONS` preflight gets
|
|
325
|
+
`204` without authentication, with `Access-Control-Allow-Origin` (the exact
|
|
326
|
+
origin, `Vary: Origin`), `Access-Control-Allow-Methods: GET, POST, DELETE,
|
|
327
|
+
OPTIONS` and `Access-Control-Allow-Headers: Authorization, Content-Type,
|
|
328
|
+
Mcp-Session-Id, MCP-Protocol-Version, Last-Event-ID`; every other answer,
|
|
329
|
+
`401` and `403` included, carries the same `Access-Control-Allow-Origin` and
|
|
330
|
+
`Access-Control-Expose-Headers: WWW-Authenticate, Mcp-Session-Id`. The
|
|
331
|
+
protected-resource metadata is answered the same way (its preflight allows
|
|
332
|
+
`GET, HEAD, OPTIONS`), so a browser client can discover the authorization
|
|
333
|
+
server. Each request builds its MCP
|
|
334
|
+
server over the one shared catalog state, and tool calls from all clients run
|
|
335
|
+
one at a time through one queue.
|
|
336
|
+
|
|
337
|
+
### Principals and scopes
|
|
338
|
+
|
|
339
|
+
Every HTTP request runs for a principal: a subject, an actor, scopes, and
|
|
340
|
+
optionally a name and an email. `okf.read` is required for any call.
|
|
341
|
+
`okf.write` adds the live write tools (`okf_validate_changes`,
|
|
342
|
+
`okf_apply_changes`, when the server runs with `--write`), producer runs
|
|
343
|
+
(`okf_run_producer`) and, over HTTP, producer listing and previews
|
|
344
|
+
(`okf_list_producers`, `okf_preview_producer`, which run the producer with
|
|
345
|
+
the server's own source credentials), the stored proposals (`okf_list_proposals`,
|
|
346
|
+
`okf_get_proposal`, which can hold complete candidate Markdown) and, with
|
|
347
|
+
`--allow-remote-tool`, `load_remote_bundle`, which changes the index every
|
|
348
|
+
principal reads. Over stdio those flags alone enable them, as before. A principal without `okf.write` does not see those
|
|
349
|
+
tools in `tools/list`, and calling one fails as calling a disabled tool does.
|
|
350
|
+
|
|
351
|
+
Writes made for a principal stamp its actor in `generated.by` and are judged
|
|
352
|
+
by the write policy for that actor. With `--git-commit`, the principal's name
|
|
353
|
+
and email become the commit author; a missing one falls back to the
|
|
354
|
+
repository's configured Git identity, which also stays the committer. Author
|
|
355
|
+
data never fails or blocks a write: `<`, `>`, control characters and
|
|
356
|
+
surrounding whitespace are removed, a name made only of characters git
|
|
357
|
+
refuses (such as `'` or `"`) and an email that is not a plausible address are
|
|
358
|
+
dropped, and the dropped field falls back to the configured identity.
|
|
359
|
+
Producer runs keep
|
|
360
|
+
`process:<producer>`. `--write` still requires `--actor`, which names the
|
|
361
|
+
server process as over stdio; every authenticated HTTP principal writes as
|
|
362
|
+
its own actor.
|
|
363
|
+
|
|
364
|
+
Without `--auth`, every caller is an anonymous read-only principal. That mode
|
|
365
|
+
is allowed only on a loopback address, and `--write` is a usage error there.
|
|
366
|
+
`--authoring` is not available with `--http` in this release: which scope may
|
|
367
|
+
create and accept proposals over HTTP is not decided yet. Over stdio the
|
|
368
|
+
principal comes from the flags (`--actor`, and `okf.write` only with
|
|
369
|
+
`--write`), so stdio behaves as before.
|
|
370
|
+
|
|
371
|
+
### Token file
|
|
372
|
+
|
|
373
|
+
`--auth tokens:<file>` reads a YAML list of tokens by their SHA-256 hash;
|
|
374
|
+
plaintext tokens are never stored:
|
|
375
|
+
|
|
376
|
+
```yaml
|
|
377
|
+
tokens:
|
|
378
|
+
- sha256: 3f0c… # hex SHA-256 of the bearer token
|
|
379
|
+
subject: ci
|
|
380
|
+
actor: process:ci-nightly
|
|
381
|
+
scopes: [okf.read, okf.write]
|
|
382
|
+
```
|
|
383
|
+
|
|
384
|
+
`printf %s "$TOKEN" | sha256sum` prints the hash. The file must be a regular
|
|
385
|
+
file of at most 1 MiB; scopes are `okf.read`, `okf.write` or well-formed
|
|
386
|
+
names outside `okf.*` (for tool packs), and unknown fields, other scopes, invalid actors and
|
|
387
|
+
repeated hashes are startup errors that name the entry and field, never a
|
|
388
|
+
value. Listed tokens do not expire; remove an entry and restart the server to
|
|
389
|
+
revoke it.
|
|
390
|
+
|
|
391
|
+
### Token introspection
|
|
392
|
+
|
|
393
|
+
`--auth introspect:<url>` checks opaque bearer tokens against an external
|
|
394
|
+
OAuth 2.0 authorization server with RFC 7662 introspection. okf-mcp is only a
|
|
395
|
+
resource server: it never issues tokens, and clients obtain them from the
|
|
396
|
+
authorization server named in the protected-resource metadata.
|
|
397
|
+
|
|
398
|
+
```bash
|
|
399
|
+
OKF_INTROSPECTION_CLIENT_ID=okf-mcp OKF_INTROSPECTION_CLIENT_SECRET=… \
|
|
400
|
+
okf --root /srv/okf --write --actor process:okf-mcp --git-commit mcp --http \
|
|
401
|
+
--public-url https://mcp.example.com/mcp \
|
|
402
|
+
--auth introspect:https://auth.example.com/oauth/introspect \
|
|
403
|
+
--authorization-server https://auth.example.com
|
|
404
|
+
```
|
|
405
|
+
|
|
406
|
+
- okf-mcp posts `token=<access token>` as a form to the URL, with HTTP Basic
|
|
407
|
+
client credentials (form-urlencoded before base64, RFC 6749 section 2.3.1)
|
|
408
|
+
read from `OKF_INTROSPECTION_CLIENT_ID` and
|
|
409
|
+
`OKF_INTROSPECTION_CLIENT_SECRET` only. The URL must be `https`, or `http`
|
|
410
|
+
on a loopback host, without credentials, query or fragment; redirects are
|
|
411
|
+
not followed.
|
|
412
|
+
- A token is accepted when `active` is `true`, `aud` equals `--public-url`
|
|
413
|
+
(one trailing slash ignored on either side), `exp` is in the future and
|
|
414
|
+
`scope` holds `okf.read`. The principal's subject
|
|
415
|
+
is `sub`, its actor `human:<username>` (every character outside
|
|
416
|
+
`A-Z`, `a-z`, `0-9` and `._@+-` replaced by `-`, so a username can never
|
|
417
|
+
take the `provider/model` form), its scopes every well-formed entry of
|
|
418
|
+
`scope` (`okf.read` and `okf.write` gate the core tools, scopes outside
|
|
419
|
+
`okf.*` such as `odoo.read` gate [tool-pack](#tool-packs-experimental)
|
|
420
|
+
tools, other `okf.*` names are ignored), and its git author `name` and
|
|
421
|
+
`email`, each optional.
|
|
422
|
+
- A request without credentials gets `401` with
|
|
423
|
+
`WWW-Authenticate: Bearer realm="okf-mcp", scope="…", resource_metadata="…"`
|
|
424
|
+
and no `error` attribute (RFC 6750 section 3.1); a presented Bearer token
|
|
425
|
+
that is refused gets the same challenge with `error="invalid_token"`. The
|
|
426
|
+
`scope` lists every scope the server offers, the same list as
|
|
427
|
+
`scopes_supported`, so the authorization server narrows it to the user's
|
|
428
|
+
rights. A token without `okf.read` gets `403` with
|
|
429
|
+
`error="insufficient_scope", scope="okf.read"`. When the authorization server cannot be reached,
|
|
430
|
+
times out (5 s) or answers anything but `200` with JSON, the request gets
|
|
431
|
+
`503` with `{"error":"temporarily_unavailable"}` and no further detail.
|
|
432
|
+
- Answers are cached by the token's SHA-256 in two bounded caches: accepted
|
|
433
|
+
ones until the earlier of `exp` and 60 seconds (a revoked token keeps
|
|
434
|
+
working for up to a minute), refused ones for 30 seconds. At most 8
|
|
435
|
+
introspection requests run at once; a request that would need a ninth gets
|
|
436
|
+
`503` with `{"error":"temporarily_unavailable"}`.
|
|
437
|
+
|
|
438
|
+
Token values never appear in a response, an error or a log line, and the
|
|
439
|
+
caches keep only their hash; the verified token is handed on only to
|
|
440
|
+
[tool-pack](#tool-packs-experimental) handlers, as `ctx.subjectToken`.
|
|
441
|
+
|
|
442
|
+
### Index freshness
|
|
443
|
+
|
|
444
|
+
Over HTTP the server re-reads files changed outside okf-mcp (for example by
|
|
445
|
+
`git pull` or an editor) before a tool call or a resource read, at most once
|
|
446
|
+
per `--refresh-interval`; `initialize`, `ping` and `tools/list` never trigger
|
|
447
|
+
it. The check runs in the call queue, in the same order as the calls. Changes are detected by file size and modification time,
|
|
448
|
+
as for writes. An unchanged check costs about 35 ms at 5,000 concepts.
|
|
449
|
+
|
|
450
|
+
### Behind a reverse proxy
|
|
451
|
+
|
|
452
|
+
- Bind okf-mcp to a loopback address and let the proxy terminate TLS.
|
|
453
|
+
- Set `--public-url` to the external URL of the endpoint and forward that
|
|
454
|
+
path unchanged; forward the health path and
|
|
455
|
+
`/.well-known/oauth-protected-resource…` as well.
|
|
456
|
+
- Forward the `Host` header as the public host (accepted automatically), or
|
|
457
|
+
accept the host the proxy sends with `--allowed-host`.
|
|
458
|
+
- Forward the `Authorization` header unchanged.
|
|
459
|
+
- Responses can be Server-Sent Events streams: turn response buffering off
|
|
460
|
+
for the endpoint and allow long-lived responses.
|
|
461
|
+
- One process serves one root or one project; run one process per catalog.
|
|
462
|
+
|
|
463
|
+
## Tool packs (experimental)
|
|
464
|
+
|
|
465
|
+
Experimental from 0.13.0-rc.1 and outside the 1.0 freeze: the interface may
|
|
466
|
+
change in any release.
|
|
467
|
+
|
|
468
|
+
A tool pack is an npm package that adds read tools to the MCP server, next to
|
|
469
|
+
the core tools. Like a producer, it is listed by exact package name in the
|
|
470
|
+
trusted project config and loaded only from the project's `node_modules`:
|
|
471
|
+
|
|
472
|
+
```yaml
|
|
473
|
+
toolPacks:
|
|
474
|
+
- package: "@example/okf-data-pack"
|
|
475
|
+
timeoutMs: 30000 # optional, per call; default 30000, at most 300000
|
|
476
|
+
config:
|
|
477
|
+
url: https://data.example.com
|
|
478
|
+
clientIdEnv: DATA_CLIENT_ID
|
|
479
|
+
clientSecretEnv: DATA_CLIENT_SECRET
|
|
480
|
+
```
|
|
481
|
+
|
|
482
|
+
The package exports:
|
|
483
|
+
|
|
484
|
+
```js
|
|
485
|
+
exports.okfToolPack = {
|
|
486
|
+
apiVersion: "1",
|
|
487
|
+
id: "data", // ^[a-z][a-z0-9_]{2,63}$
|
|
488
|
+
validateConfig(config) { return { valid: true, config }; },
|
|
489
|
+
serialize: false, // optional, see below
|
|
490
|
+
tools: {
|
|
491
|
+
data_search: { // ^[a-z][a-z0-9_]{2,63}$
|
|
492
|
+
description: "…",
|
|
493
|
+
annotations: { readOnlyHint: true },
|
|
494
|
+
inputSchema: { type: "object", properties: { … }, additionalProperties: false },
|
|
495
|
+
outputSchema: { type: "object", … }, // optional
|
|
496
|
+
scope: "data.read",
|
|
497
|
+
async handler(args, ctx) { return { records: [] }; },
|
|
498
|
+
},
|
|
499
|
+
},
|
|
500
|
+
};
|
|
501
|
+
```
|
|
502
|
+
|
|
503
|
+
- **Loading.** Packs load when the server starts with `--project`, in config
|
|
504
|
+
order. A package that is not installed under the project's `node_modules`,
|
|
505
|
+
a descriptor the host does not accept, a configuration `validateConfig`
|
|
506
|
+
refuses, an inline secret in the config (`password`, `secret`, `token`,
|
|
507
|
+
`apiKey`, `credential(s)`), or a tool name that repeats a core tool or
|
|
508
|
+
another pack's tool stops the server with an error naming the package and
|
|
509
|
+
tool. Descriptions and schemas come from the package, never from config.
|
|
510
|
+
- **Listing and access.** Pack tools are listed after the core tools, packs
|
|
511
|
+
in config order and each pack's tools in the order it declares them. A
|
|
512
|
+
principal sees a pack tool when it holds `okf.read` and the tool's `scope`;
|
|
513
|
+
calling a tool it does not see fails as calling a disabled tool does. Over
|
|
514
|
+
stdio the principal is the process's operator and holds every scope outside
|
|
515
|
+
`okf.*` that the configured packs declare (an `okf.write` scope still needs
|
|
516
|
+
`--write`). A tool that sets `requiresSubjectToken: true` (because it acts
|
|
517
|
+
for the caller elsewhere) is listed and callable only when the request
|
|
518
|
+
carries a subject token, that is over HTTP with `--auth`; over stdio and
|
|
519
|
+
anonymous HTTP it is hidden and refused like a disabled tool.
|
|
520
|
+
- **Calls.** Arguments are validated against `inputSchema` exactly like core
|
|
521
|
+
tool arguments. `ctx` holds `principal` (a frozen copy of subject, actor,
|
|
522
|
+
scopes, name and email), `subjectToken` (over HTTP the caller's verified
|
|
523
|
+
bearer token, absent over stdio), `config` (the normalised config
|
|
524
|
+
`validateConfig` returned, else the configured one; a fresh copy per call),
|
|
525
|
+
`getSecret(name)` (only for environment variables named by config keys
|
|
526
|
+
ending in `Env`) and `signal`, which aborts when `timeoutMs` passes or the
|
|
527
|
+
request is cancelled. A handler returns a JSON value (at most 1 MiB;
|
|
528
|
+
returned as structured content when the tool declares an `outputSchema`).
|
|
529
|
+
- **Errors.** A handler may throw an error with a snake_case `code` and a
|
|
530
|
+
`details` object; the caller receives
|
|
531
|
+
`{ "error": "Tool execution failed", "tool", "message", "code", "details" }`
|
|
532
|
+
with a fixed message, the code, and only bounded scalar details: at most
|
|
533
|
+
16 entries under identifier keys, values null, booleans, finite numbers or
|
|
534
|
+
single-line strings, a string longer than 500 characters cut to 500 ending
|
|
535
|
+
in `…`, and no string holding the caller's token or a secret the pack read.
|
|
536
|
+
The text content repeats this for clients that read only text: the fixed
|
|
537
|
+
message, then one JSON line with `code` and `details`. The error's own
|
|
538
|
+
message is never forwarded. Any other failure is
|
|
539
|
+
`tool_pack_execution_failed`; the host's own codes are `tool_pack_timeout`,
|
|
540
|
+
`tool_pack_cancelled`, `tool_pack_result_too_large`,
|
|
541
|
+
`tool_pack_secret_not_allowed` and `tool_pack_secret_unavailable`.
|
|
542
|
+
- **Concurrency.** Pack tools are read tools and run outside the call queue
|
|
543
|
+
that serialises knowledge reads and writes, so a slow pack never blocks a
|
|
544
|
+
write. A pack that needs the queue sets `serialize: true`.
|
|
545
|
+
- **Scopes over HTTP.** The server advertises `okf.read`, `okf.write` and
|
|
546
|
+
every scope its packs declare, each once, in config and declaration order:
|
|
547
|
+
in the protected-resource metadata's `scopes_supported` and in the `scope`
|
|
548
|
+
of the `401` challenge, so a client signing in asks for all of them. An
|
|
549
|
+
authorization server puts into a token only the scopes asked for at sign-in,
|
|
550
|
+
so users who signed in before a pack was added must sign in again to obtain
|
|
551
|
+
its scope.
|
|
552
|
+
|
|
284
553
|
## MCP Registry Metadata
|
|
285
554
|
|
|
286
|
-
`server.json` describes the npm package
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
555
|
+
`server.json` describes the npm package `io.github.mfdaves/okf-mcp` twice:
|
|
556
|
+
as a stdio server, and as a Streamable HTTP server at
|
|
557
|
+
`http://127.0.0.1:8766/mcp` started with the extra `--http` argument.
|
|
558
|
+
Registry-aware clients should prompt for an absolute OKF root path, pass it
|
|
559
|
+
through `--root`, and append the fixed `mcp` command.
|
|
290
560
|
|
|
291
561
|
## Concept Identity And Extensions
|
|
292
562
|
|
|
@@ -450,6 +720,37 @@ The server writes nothing unless every candidate is valid, revision checks still
|
|
|
450
720
|
|
|
451
721
|
Add `--git-commit` as server policy to create one commit per successful batch. A detected Git worktree must be completely clean and have a configured identity before publication. Active Git filter attributes and `assume-unchanged`/`skip-worktree` index flags block the operation so validation cannot execute configured filters or overlook hidden user changes; replace-object resolution is disabled so hidden replacement history cannot alter the parent tree. The server builds an isolated index from the validated Markdown bytes, creates that exact tree with `commit-tree`, publishes it with a compare-and-swap ref update, synchronizes only the affected ordinary-index paths afterward, and never pushes. Concurrent unrelated staged entries cannot enter the commit. A determinate commit failure leaves matching valid files as working-tree-only; an ambiguous ref-update timeout is reported as unknown unless the resulting commit tree can be proven. Non-Git catalogs are written normally. Every response makes the repository root, commit state, index state, target-byte state, and persistence boundary explicit.
|
|
452
722
|
|
|
723
|
+
### Write policy
|
|
724
|
+
|
|
725
|
+
A write policy narrows what each actor may write through live batches and proposals. It is trusted configuration: declare it as `writePolicy` in `okf.project.yaml`, or pass `--policy <file>` (the same mapping as a YAML file) with `--root` and `--write` or `--authoring`. Tool and HTTP arguments cannot name, replace, or relax it.
|
|
726
|
+
|
|
727
|
+
```yaml
|
|
728
|
+
writePolicy:
|
|
729
|
+
defaultEffect: deny # allow (default) or deny when no rule decides
|
|
730
|
+
rules:
|
|
731
|
+
- name: humans
|
|
732
|
+
actors: ["human:*"]
|
|
733
|
+
- name: nightly-seo
|
|
734
|
+
actors: ["process:nightly-*", "openai/*"]
|
|
735
|
+
bundles: ["customers-*"]
|
|
736
|
+
paths: ["findings/**"]
|
|
737
|
+
types: ["SEO Finding"]
|
|
738
|
+
requiredFields: ["sources", "stale_after"]
|
|
739
|
+
defaultStatus: draft # stamped on creates that set no status
|
|
740
|
+
maxConceptsPerBatch: 20
|
|
741
|
+
- name: no-drafts
|
|
742
|
+
actors: ["**"]
|
|
743
|
+
paths: ["drafts/**"]
|
|
744
|
+
effect: deny
|
|
745
|
+
applyToProducers: false # true also judges producer runs
|
|
746
|
+
```
|
|
747
|
+
|
|
748
|
+
For each concept in a batch, the first rule whose `actors` match the configured `--actor` and whose other selectors (when present) match decides. `effect` is `allow` by default; `requiredFields`, `defaultStatus`, and `maxConceptsPerBatch` apply to allow rules. Patterns use a fixed glob subset: `*` within one segment, `**` as a whole segment for any depth, and literal segments. They are never compiled to regular expressions, and absolute, empty, `.` and `..` segments are rejected when the policy loads. A malformed policy is a project error, so no writable server starts on it. The policy judges only the concepts a batch creates or updates; existing content is never retro-validated.
|
|
749
|
+
|
|
750
|
+
Violations are structured: `{ code: "policy_violation", rule, reason, field?, path?, bundleId?, uri?, limit?, count? }`, where `rule` is the rule's `name`, else its index, and `null` when `defaultEffect` decided. `reason` is one of `actor_denied`, `bundle_denied`, `path_denied`, `type_denied`, `missing_required_field`, or `batch_too_large`. `okf_validate_changes` reports them read-only in `policyViolations` and as `layer: "policy"` diagnostics; `okf_apply_changes` refuses such a batch with status `rejected` before staging or touching any file. Receipts carry `policyViolations` only when a policy is configured; messages name rules, paths, and URIs, never catalog text.
|
|
751
|
+
|
|
752
|
+
Scope. Live batches (`okf_validate_changes`, `okf_apply_changes`) are judged against `--actor`. Proposals are judged twice. When `okf_propose_concept`, `okf_propose_update`, `okf_propose_attested_computation`, `okf_propose_v02_migration` (every child update and the `index.md` version declaration, before any is stored) or their HTTP equivalents would store one, it is judged against the proposing actor (`--actor`, which `--authoring` also accepts); a refused proposal returns `created: false` with `policyViolations` and is not stored. When `okf_accept_proposal` or `POST /v1/proposals/<id>/accept` would write it, it is judged against the actor recorded on the proposal (`proposedBy`), else the accepting actor; a refusal returns `accepted: false` before any file is written, and an accepted create the matching rule gives a `defaultStatus` is published with that status. A write with no actor (an HTTP call, or a proposal stored before an actor or a policy was configured) is judged as the literal actor `unknown`: `*` and `**` match it and a rule can name it, so legacy proposals are judged as `unknown` at acceptance unless the accepting server has an actor. `okf serve` takes the policy from `okf.project.yaml`, or from `--policy <file>` with `--root` and `--authoring`. Producers publish under `process:<producer name>` and are exempt unless `applyToProducers: true`, in which case `okf_preview_producer` reports the violations of the created and updated concept files read-only and `okf_run_producer` / `producer run` refuse before publication. Producer output is never rewritten, so `defaultStatus` does not apply to it: a required `status` must be in the produced file, or the run is refused with `missing_required_field`. The reserved `index.md` and `log.md` files of a producer bundle are not judged.
|
|
753
|
+
|
|
453
754
|
## Authoring Concepts
|
|
454
755
|
|
|
455
756
|
The reviewable proposal workflow remains available through MCP tools started with `--authoring` and through the HTTP API. Clients never need direct local file access.
|
|
@@ -513,6 +814,8 @@ Safety rules:
|
|
|
513
814
|
|
|
514
815
|
## HTTP API
|
|
515
816
|
|
|
817
|
+
The REST authoring API below is separate from [MCP over HTTP](#mcp-over-http) (`mcp --http`).
|
|
818
|
+
|
|
516
819
|
Start the HTTP server:
|
|
517
820
|
|
|
518
821
|
```bash
|
|
@@ -607,17 +910,48 @@ Use `list_remote_bundles` to inspect what was loaded.
|
|
|
607
910
|
- `limit`
|
|
608
911
|
- `offset`
|
|
609
912
|
|
|
610
|
-
`list_concepts` also accepts a text `query` and applies it
|
|
611
|
-
listing filters. Text search tokenizes case-insensitively
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
913
|
+
`list_concepts` also accepts a text `query` (and `mode`) and applies it
|
|
914
|
+
together with its listing filters. Text search tokenizes case-insensitively,
|
|
915
|
+
folds diacritics, drops stop words and single letters, and stems every term
|
|
916
|
+
with the catalog's languages (English and Italian at 1.0). `mode` is `auto`
|
|
917
|
+
by default since 0.10: the concepts matching every term come first, in their
|
|
918
|
+
own order, and when fewer than `autoFill` of them exist (default 3) the
|
|
919
|
+
concepts matching only some of the terms follow them (the response's `mode`
|
|
920
|
+
is then `auto`; it is `any` when nothing matches every term and `all` when
|
|
921
|
+
nothing was appended). This keeps one long document that happens to contain
|
|
922
|
+
every term from hiding the shorter documents that match most of them.
|
|
923
|
+
`autoFill` is a library search option and an `okf.project.yaml` key
|
|
924
|
+
(`autoFill: 0` restores the 0.10 rule: fall back only when nothing matches
|
|
925
|
+
every term), never a tool argument. `all` restores the pre-0.10
|
|
926
|
+
behaviour and `any` ranks anything matching one term. BM25+ ranks title,
|
|
927
|
+
type, tags, aliases, description, path, body, and an exact field (ids,
|
|
928
|
+
paths, aliases, raw title tokens) that is never stemmed, so identifiers and
|
|
929
|
+
SKUs still match literally. Double-quoted text is matched literally (no
|
|
930
|
+
stemming or accent folding, stop words and single letters kept) against ids,
|
|
931
|
+
paths, aliases, titles and raw tokens, and unquoted tokens containing digits
|
|
932
|
+
or hyphens stay literal as well; stems only ever reach the text fields.
|
|
933
|
+
Frontmatter remains available through exact structured filters but is not
|
|
934
|
+
copied into the text index. Scores are
|
|
935
|
+
relative within a result set and are not a stable cross-version scale; use
|
|
936
|
+
ordering and `relevance` instead.
|
|
937
|
+
|
|
938
|
+
Query responses report `mode` (the one that ran), `queryTerms`,
|
|
939
|
+
`droppedTerms`, `allTermsTotal` (concepts matching every term, also under
|
|
940
|
+
`any`) and `lowConfidence`, true when `mode` is `any` (the any-term ranking produced the page)
|
|
941
|
+
and its top result matched fewer than half of the terms. Each result carries
|
|
942
|
+
`coverage` (share of query terms matched), `relevance` (score over the top
|
|
943
|
+
score) and, in full detail, `matchedTerms`. Opt-in floors `minCoverage` and
|
|
944
|
+
`minRelevance` drop weaker results; `freshnessPrior` down-weights deprecated
|
|
945
|
+
(0.5), stale (0.7), and draft (0.85) concepts. Compact results contain
|
|
946
|
+
`uri`, `title`, `type`, `description`, `coverage`, and `relevance`; title,
|
|
947
|
+
type, and description are bounded, while full results remain lossless.
|
|
948
|
+
|
|
949
|
+
Queries are bounded to 512 characters and 16 terms after stop words. Prefix
|
|
950
|
+
expansion and fuzzy matching are intentionally disabled. The stemming
|
|
951
|
+
languages come from the project config key `languages` (`auto`, `none`, or
|
|
952
|
+
codes such as `[it, en]`) or `--languages` in `--root` mode; `auto` detects
|
|
953
|
+
each bundle's languages from its documents. Punctuation-only queries return
|
|
954
|
+
no matches. Tags and types are matched case-insensitively. Arbitrary
|
|
621
955
|
frontmatter filters support exact scalar matching and array-contains
|
|
622
956
|
matching. `relationType` selects concepts with an outgoing relation of that
|
|
623
957
|
type.
|
|
@@ -650,6 +984,97 @@ latency, Recall@10, MRR@10, and representative rankings. Search indexes are
|
|
|
650
984
|
process-local and keyed to the parsed OKF index, so remote loads and accepted
|
|
651
985
|
proposals receive a fresh index automatically.
|
|
652
986
|
|
|
987
|
+
### Budgeted context and section anchors (0.11)
|
|
988
|
+
|
|
989
|
+
`get_context` (MCP, read-only) and `okf context <query> [--budget <tokens>]
|
|
990
|
+
[--limit <n>] [--provenance compact|full|none]` (CLI, JSON) answer a question
|
|
991
|
+
with the sections that matter instead of whole concepts. Candidates come from
|
|
992
|
+
the same search as `search_concepts` (same filters, `mode`, `minCoverage`,
|
|
993
|
+
`minRelevance`, `lowConfidence`, `queryTerms`). Their sections are scored on
|
|
994
|
+
their own text (BM25; the concept's type, tags and path add at most 20%) and
|
|
995
|
+
ordered by the candidate concept rank; within a concept, sections whose text
|
|
996
|
+
matches the query come before sections found only through that metadata, and
|
|
997
|
+
a strongly matching section may move ahead of up to four weaker concepts.
|
|
998
|
+
Sections are deduplicated and packed in that order while they fit
|
|
999
|
+
`budgetTokens` (default 1,200, never exceeded). The budget covers the whole
|
|
1000
|
+
response as delivered: `usedTokens` is `ceil(characters / 4)` of the entire
|
|
1001
|
+
JSON result, every field included (the candidate `concepts`, `queryTerms`,
|
|
1002
|
+
the counters and every section field, scoring fields too), and `get_context`
|
|
1003
|
+
returns that JSON compact, so the text an agent receives is what was
|
|
1004
|
+
measured. Sections are packed into what the rest of the response leaves;
|
|
1005
|
+
when even the fields around them do not fit, the candidate concept list is
|
|
1006
|
+
shortened from its tail (`omittedConcepts`), and only a budget smaller than
|
|
1007
|
+
the query echo and the field names can be exceeded (then no section and no
|
|
1008
|
+
concept is returned). Each section's `tokens` is the same estimate for that
|
|
1009
|
+
section, and the estimator is named in the response. Before 0.11.0-rc.2 only
|
|
1010
|
+
the sections' content fields were counted, so responses ran at 1.6-1.7 times
|
|
1011
|
+
the budget. Provenance is
|
|
1012
|
+
`compact` by default (`status`, `freshness`, `verified` as a boolean,
|
|
1013
|
+
`sourceCount`); `provenance: "full"` returns `status`, `staleAfter`,
|
|
1014
|
+
`freshness`, `trustTier`, `verified`, `generated` and `sources`, and
|
|
1015
|
+
`provenance: "none"` (or `includeProvenance: false`) spends the whole budget
|
|
1016
|
+
on the sections. The library exposes
|
|
1017
|
+
`assembleContext(index, options)`, `conceptSections(doc)` and
|
|
1018
|
+
`sectionUnits(index)`.
|
|
1019
|
+
|
|
1020
|
+
Section anchors are an okf-mcp extension; OKF v0.2 defines none and nothing
|
|
1021
|
+
in a bundle needs them. A concept is cut at its document-level headings: the
|
|
1022
|
+
text before the first heading (with the title and description) is addressed
|
|
1023
|
+
by the concept URI alone, every other section by `uri#slug`. The heading
|
|
1024
|
+
text is first normalised (the line breaks of a setext or wrapped heading and
|
|
1025
|
+
every whitespace run become one space, then trimmed); the slug then follows
|
|
1026
|
+
GitHub's rule: lower-cased, every character other than a letter, a combining
|
|
1027
|
+
mark, a digit, an underscore, a hyphen or a space removed, and each space
|
|
1028
|
+
turned into a hyphen (so `## Consentito / Allowed` gives
|
|
1029
|
+
`consentito--allowed`, as on GitHub). A repeated heading takes `-1`, `-2`,
|
|
1030
|
+
... in document order. A reader that drops the fragment still lands on the
|
|
1031
|
+
concept. An anchor depends only on the heading texts up to its own: edits to
|
|
1032
|
+
any section body, whitespace-only edits and edits to later sections leave it
|
|
1033
|
+
unchanged, while renaming the heading, or adding a heading with the same text
|
|
1034
|
+
before it, changes it.
|
|
1035
|
+
|
|
1036
|
+
## Embeddings (0.11, opt-in)
|
|
1037
|
+
|
|
1038
|
+
Dense retrieval is off unless configured, and the default install makes no network calls. 0.11 ships the embedder interface, a zero-dependency client for the OpenAI embeddings API and compatible servers (Ollama, vLLM, LM Studio), a vector cache kept outside the bundle under `$XDG_CACHE_HOME/okf-mcp/<sha256 of the root>/` (`~/.cache/okf-mcp/` or `%LOCALAPPDATA%\okf-mcp\`), so a populated cache never dirties a Git worktree and can be deleted at any time, and hybrid search over it.
|
|
1039
|
+
|
|
1040
|
+
Enable it in `okf.project.yaml` (or, for `--root` mode, in a JSON or YAML file passed with `--embedder-config <file>`, which may also be a bare embedder block; `--cache-dir <dir>` overrides the cache location):
|
|
1041
|
+
|
|
1042
|
+
```yaml
|
|
1043
|
+
embedder:
|
|
1044
|
+
type: openai
|
|
1045
|
+
baseUrl: https://api.openai.com # or a local OpenAI-compatible server
|
|
1046
|
+
model: text-embedding-3-small
|
|
1047
|
+
apiKeyEnv: OPENAI_API_KEY # the key is read only from this variable
|
|
1048
|
+
dimensions: 512
|
|
1049
|
+
# optional: batchSize, timeoutMs, maxRetries, concurrency, maxInputChars
|
|
1050
|
+
cacheDir: ../okf-vectors # optional; relative to the project root, never inside it
|
|
1051
|
+
fusion:
|
|
1052
|
+
alpha: 0.4 # weight of the dense side (default 0.4, chosen on the development queries)
|
|
1053
|
+
normalization: tmm # tmm (default), lexical-tmm or minmax
|
|
1054
|
+
embedUnits: concepts # concepts (default) or sections (one vector per section; better on synonym queries)
|
|
1055
|
+
```
|
|
1056
|
+
|
|
1057
|
+
Embedder settings are never accepted as tool arguments. At startup the server validates the embedder, discovers the dimension when it is not configured, opens the cache and embeds the missing units in the background; it never fails to start because of the embedder. `search_concepts` then fuses the lexical ranking with cosine similarity over the cache by convex combination of normalised scores: each side is scaled by its best score above a lower bound: by default the theoretical minimum of its scoring function (0 for the lexical score, -1 for cosine; `tmm`); `lexical-tmm` uses the theoretical minimum on the lexical side only and `minmax` the observed minimum on both, as in Bruch et al., arXiv:2210.11934 (`hybrid: false` per call keeps it lexical and makes no request). Remote bundles stay lexical-only: their text is never embedded. `mode`, `coverage`, `allTermsTotal` and the `minCoverage`/`minRelevance` floors keep their lexical meaning (an explicit `mode: "all"` only re-ranks the all-term matches): the lexical hits are fused with the 200 nearest concepts by cosine, a fixed candidate set so every page of a query paginates the same ranking, and each candidate carries its own lexical evidence (matched terms, coverage, score), which the floors and `lowConfidence` (computed on the displayed top result) read. When the all-term pass ran and the query holds quoted or identifier-like terms, the all-term matches stay ahead of every semantic neighbour. `relevance` and `score` become fused and each result adds `lexicalScore`, `denseScore` (cosine) and `fusedScore`. The response adds `hybrid`, `alpha`, `normalization`, `embedder: { id, remote, dimension, degraded, code?, serverCode? }`, `cache: { rows, dir? }` (`dir` only on a `--write` server), `usage: { promptTokens, requests }` for the query encoding, and `timings: { queryEncodingMs, searchMs, totalMs }`. Whenever the embedder is missing, failing or the cache is empty, search runs lexical-only with `embedder.degraded: true`, a fixed `code` and, for HTTP failures, the enumerated `serverCode`; no server-supplied text is ever returned. After a failed query encoding the server stays lexical for 30 seconds before trying again. After each successful `okf_apply_changes` batch only the changed units are embedded (one request when the batch fits) and the receipt carries `embedding: { units, embedded, skipped, embedder, cache, usage, durationMs }`. The read-only `embedding_status` tool, listed only when an embedder is configured, reports the embedder, the cache, the startup and last preparation receipts and the token usage. Receipts report `embedder.remote: true` whenever text leaves the machine. The cache is keyed by the embedder identity `openai:<endpoint>:<model>:<dimension>:<maxInputChars>`, so a change of endpoint, model, dimension or truncation length rebuilds it. The library exposes `createOpenAiEmbedder`, `openVectorCache`, `resolveCacheDir`, `prepareEmbeddings`, `openEmbeddingRuntime`, `denseSearch` and `fuseRankings`; `node scripts/eval-search.js --embedder <file>` scores hybrid runs on the retrieval fixtures.
|
|
1058
|
+
|
|
1059
|
+
Long or identifier-dense documents: the client truncates every input at `maxInputChars` (default 24,000 characters) and plans requests with a conservative estimate of one token per 3 characters, because generated contracts and column lists run at about 2.6 characters per token and overflow the provider's 8,192-token input limit at the old 32,000-character default (the default change rebuilds caches keyed with 32000). If the provider still refuses a request as `invalid_request`, the client splits the batch in halves until the refused inputs stand alone (a bounded number of extra requests), retries each refused input once at half its length (never below 2,000 characters) and, if it is refused again, skips that unit: the preparation receipt lists it under `skipped: [{ id, reason: "input_rejected" }]`, every other unit is embedded and search stays hybrid. `embedding_status` reports `skipped: { count, ids }` (the first five ids) for the last preparation, and the startup receipt carries the same field; a skipped unit is retried by the next preparation and is found by its lexical score meanwhile. A request whose every input is refused is a configuration problem and still fails as before.
|
|
1060
|
+
|
|
1061
|
+
Adaptive fusion weight (0.12): `fusion.alpha: adaptive` decides the weight of the semantic side once per query, from what the lexical pass already knows, so every page of a query uses the same weight: `fusion.adaptive.low` when the query holds quoted or identifier-like terms and some concept matches every term (the all-term matches still stay ahead of every semantic neighbour), `mid` when at least `autoFill` concepts match every term, `high` when no concept matches every term and none matches even half of them, and in between linear in the best lexical coverage, from `high` at 0.5 to `mid` at 1. The weights default to the development-split choice `{ low: 0.1, mid: 0.5, high: 0.9 }`; `high` may not be below `mid`. Hybrid responses report `alphaUsed` and `alphaReason` (`fixed`, `literal_terms`, `all_terms`, `weak_lexical` or `interpolated`) and `embedding_status` reports the setting and its weights. The default stays the fixed 0.4, which ranks exactly as 0.11 did: on the fixtures the adaptive weight raised alpinaria Recall@5 from 0.830 to 0.838 (synonyms 0.542 to 0.604) but lowered atlas from 0.882 to 0.870, so it missed its pre-registered conditions (`test/fixtures/retrieval/reports/hybrid-0.12-2026-10-25.md`).
|
|
1062
|
+
|
|
1063
|
+
```yaml
|
|
1064
|
+
fusion:
|
|
1065
|
+
alpha: adaptive # or a number between 0 and 1 (default 0.4)
|
|
1066
|
+
adaptive: { low: 0.1, mid: 0.5, high: 0.9 }
|
|
1067
|
+
```
|
|
1068
|
+
|
|
1069
|
+
Contextual units (0.12): `embedUnits: contextual-sections` embeds each section after a one-line prefix naming its concept, `title · type · path` (at most 160 characters; a long title is shortened first), so a bare "Limits" or "Steps" section carries the system it belongs to; `contextual-concepts` puts `type · path` before the concept text. Anchors are those of `sections`. The prefixed texts have their own hashes, so switching unit sets re-embeds every unit once (the preparation receipt, the startup receipt and `embedding_status` name the unit set; `embedded` and `pruned` count the switch), and afterwards a body edit re-embeds only its section (a title, type or path edit re-embeds every section of that concept). `concepts` stays the default: on the development queries no other unit set beat it by the pre-registered rule (mean dev Recall@5 0.844 for concepts, 0.843 contextual-concepts, 0.841 contextual-sections, 0.834 sections), and on the fixtures' short sections the prefix costs 56% more prompt tokens (6.6% for contextual-concepts).
|
|
1070
|
+
|
|
1071
|
+
Confidence (0.12): every `search_concepts` and `get_context` response to a query carries `confidence: { coverage, topSimilarity, lowConfidence, reasons }`. `coverage` is the share of query terms the top result matched; `topSimilarity` is the best cosine similarity between the query and any candidate's units when the semantic side ran (`null` otherwise); `lowConfidence` (also the top-level field) is true when the any-term fallback produced the results and the top result matched fewer than half of the terms (reason `weak_lexical_match`), or when the top result matched fewer than half of the terms and `topSimilarity` is below `confidence.minSimilarity` (reason `low_similarity`). Lexical-only responses have no similarity and keep the 0.10 rule. `get_context` takes `abstainOnLowConfidence: true` to return no sections for a low-confidence query (the response keeps the confidence block and the candidate concepts and says `abstained: true`). The similarity threshold depends on the embedding model; the default, 0, was chosen on the development queries for `text-embedding-3-small`: no threshold up to 0.8 flagged a single extra unanswerable query there, because every unanswerable query whose top result matched fewer than half of the terms was already an any-term fallback. A deployment can set its own:
|
|
1072
|
+
|
|
1073
|
+
```yaml
|
|
1074
|
+
confidence:
|
|
1075
|
+
minSimilarity: 0.3 # cosine below which a weakly matched top result is flagged
|
|
1076
|
+
```
|
|
1077
|
+
|
|
653
1078
|
## Graph Behavior
|
|
654
1079
|
|
|
655
1080
|
As an okf-mcp graph projection, Markdown links become `markdown_link` edges, extension `relations` become typed `relation` edges, and standard v0.2 path-valued fields become `resource`, `source`, `computation`, `executor`, and `attester` edges. Internal concept references resolve to canonical nodes; explicitly referenced non-Markdown files resolve to okf-mcp `okf-asset://` nodes; URLs and scope descriptors remain unfetched external or opaque leaves.
|
|
@@ -780,8 +1205,9 @@ Generated output is regular Markdown/YAML OKF and is validated by the same index
|
|
|
780
1205
|
|
|
781
1206
|
## Limitations
|
|
782
1207
|
|
|
783
|
-
- MCP
|
|
1208
|
+
- MCP is served over stdio and, with `mcp --http`, over Streamable HTTP. The separate HTTP authoring API (`serve`) is not MCP over HTTP.
|
|
784
1209
|
- MCP protocol compatibility follows the pinned official SDK v2 dependency.
|
|
785
|
-
- There is no file watcher.
|
|
1210
|
+
- There is no file watcher. Over stdio, restart the server after external file changes; over HTTP, changes are picked up before a request at most once per `--refresh-interval`. Concepts accepted through MCP authoring refresh the MCP server index immediately.
|
|
1211
|
+
- With `mcp --http`, proposal authoring (`--authoring`) is not available, a revoked introspected token stays valid for up to 60 seconds, and one process still serves one root or project for every principal.
|
|
786
1212
|
- The HTTP API is a lightweight built-in server, not a full hosted multi-tenant service.
|
|
787
1213
|
- OKF v0.2 computation support is static inspection and preflight only; no computation or attester is executed.
|
package/bin/okf-mcp.js
CHANGED
|
@@ -7,7 +7,12 @@ const debug = process.argv.includes("--debug");
|
|
|
7
7
|
const args = process.argv.slice(2).filter((arg) => arg !== "--debug");
|
|
8
8
|
|
|
9
9
|
main(args).catch((error) => {
|
|
10
|
-
const message =
|
|
10
|
+
const message =
|
|
11
|
+
debug && error && error.stack
|
|
12
|
+
? error.stack
|
|
13
|
+
: error && error.message
|
|
14
|
+
? error.message
|
|
15
|
+
: String(error);
|
|
11
16
|
process.stderr.write(message + "\n");
|
|
12
17
|
process.exitCode = exitCodeForError(error);
|
|
13
18
|
});
|