@panaversity/ksor 0.0.22 → 0.0.24

package/dist/gateway.d.mts

@@ -0,0 +1,144 @@
+import { CallToolResult, McpServer as McpServer$1, StandardSchemaWithJSON } from "@modelcontextprotocol/server";
+import { z as z$1 } from "zod";
+import pg from "pg";
+//#region ../content-gateway/dist/gateway-api-D9COH1tR.d.mts
+//#region src/instructions.d.ts
+/**
+ * Has the owner said what this record is FOR yet?
+ *
+ * The MCP door already answers honestly when they have not — it replaces the
+ * template with a plain statement that the scope is unstated. But the operator
+ * starting the server was told nothing, so a record serving with no declared
+ * identity looked exactly like one that had been described. The boot report is
+ * where that belongs, beside the abstention posture: both are answers to "how
+ * much should I trust what this thing says".
+ */
+declare function recordIsUndescribed(authored: string): boolean;
+declare function composeInstructions(authored: string): string;
+//#endregion
+//#region ../content/dist/index.d.mts
+//#endregion
+//#region src/lib/abstain.d.ts
+/**
+ * The abstention gates (oracle SC/lib/abstain.py, exact). "Not in this
+ * corpus" is a correct answer, never an error, never a licence to fall back
+ * on model knowledge.
+ *
+ * Floors come from the instance frontmatter, never module globals, and are
+ * calibrated per corpus per embedding space — never copy a calibrated
+ * constant between corpora.
+ */
+interface AbstainConfig {
+  /**
+   * A calibrated number gates (abstain below it); `null` = no gate declared,
+   * honest absence surfaced on /health; `"uncalibrated"` = a floor was
+   * DECLARED but not measured, which REFUSES every serve until it is pasted
+   * (the "fail closed once a floor is declared" invariant, representable).
+   */
+  readonly vectorFloor: number | null | "uncalibrated";
+  /**
+   * Degraded-path ts_rank_cd floor; null = abstain only on zero matches.
+   * Recorded negative result (oracle, measured on 416 in-corpus gold + 38
+   * OOC probes): ts_rank_cd does NOT separate in/out-of-corpus — every
+   * leak-stopping floor false-abstained 67–98% (0.1333 → FA 255/379;
+   * 0.7 → FA 355/379). The shipped instance keeps keyword_floor null;
+   * never set one by intuition, recalibrate per corpus.
+   */
+  readonly keywordFloor: number | null;
+}
+interface ContentInstance {
+  /** The corpus identity — instance.md's `name:` (the path/name IS the identity). */
+  readonly name: string;
+  readonly corpusId: string;
+  readonly tenantId: string;
+  /** The NAME of the env var holding the DSN; the composition root resolves it. */
+  readonly dsnEnv: string;
+  readonly abstain: AbstainConfig;
+  readonly maximumResponseCharacters: number;
+  /** The authored agent-surface instructions (the body, edge-trimmed). */
+  readonly instructions: string;
+  /** Transport name (registry key, never persisted). */
+  readonly embeddingProvider: string;
+  /** The record's reader audiences, least- to most-restricted; empty = no model. */
+  /** The Postgres text-search configuration the keyword arm stems with. */
+  readonly textSearchConfig: string;
+  readonly audiences: readonly string[];
+  /** The tier a document takes when it declares none; null = none declared. */
+  readonly defaultVisibility: string | null;
+  /** model + dim are the persisted IDENTITY of the embedding space. */
+  readonly embeddingModel: string;
+  readonly embeddingDim: number;
+}
+interface KeyRing {
+  readonly keys: ReadonlyMap<string, Buffer>;
+  /** Mint with the active key; keep validating old ids until their tokens age out (≤ TTL). */
+  readonly active: string;
+}
+declare const MAX_SEARCH_K = 50;
+interface ServiceContext {
+  readonly pool: pg.Pool;
+  readonly instance: ContentInstance;
+  readonly ring: KeyRing;
+  /** sha256 of instance.md — the deployment binding snapshots carry. */
+  readonly instanceDigest: string;
+  /** The query-embed pipeline (cache + breaker + timeout live behind it); returns a pgvector literal or a raw vector. */
+  readonly embedQuery: (query: string) => Promise<readonly number[] | string>;
+  /** The verified caller, or null → audited as "anonymous". */
+  readonly actor?: () => string | null;
+  /**
+   * The audience tier this door serves. null = the record's least-privileged
+   * tier, which is the safe default: a door that cannot establish who is asking
+   * must not hand out the restricted half of the record. Ignored entirely when
+   * the instance declares no `audiences:` model.
+   */
+  readonly audience?: string | null;
+}
+/**
+ * The largest outline a caller may ASK for. The tool schema and the service
+ * both derive from it, so the ceiling is one number rather than three
+ * hand-copied ones.
+ */
+declare const MAX_OUTLINE_LIMIT = 5e3;
+//#endregion
+//#region src/tools.d.ts
+/**
+ * The framework text every tool description must carry.
+ *
+ * A registration file puts its own prose ABOVE one of these. It is a template
+ * literal in adopter code, so nothing stops someone omitting it — which is why
+ * the door verifies its own served surface at boot rather than trusting that
+ * nobody did.
+ */
+declare const FLOOR: Readonly<Record<"search" | "outline" | "read", string>>;
+/** Every ksor tool is a read: no writes, safe to retry, closed world. */
+declare const READ_ONLY: {
+  readonly readOnlyHint: true;
+  readonly destructiveHint: false;
+  readonly idempotentHint: true;
+  readonly openWorldHint: false;
+};
+declare const SEARCH_OUTPUT: StandardSchemaWithJSON;
+declare const OUTLINE_OUTPUT: StandardSchemaWithJSON;
+declare const READ_OUTPUT: StandardSchemaWithJSON;
+interface SearchArgs {
+  readonly query: string;
+  readonly k: number;
+}
+declare function searchHandler(ctx: ServiceContext): (args: SearchArgs) => Promise<CallToolResult>;
+interface OutlineArgs {
+  readonly node?: string | undefined;
+  readonly depth?: number | undefined;
+  readonly limit: number;
+  readonly offset?: number | undefined;
+}
+declare function outlineHandler(ctx: ServiceContext): (args: OutlineArgs) => Promise<CallToolResult>;
+interface ReadArgs {
+  readonly slug: string;
+  readonly heading?: string | undefined;
+  readonly from_heading?: string | undefined;
+  readonly snapshot_token?: string | undefined;
+  readonly token_budget?: number | undefined;
+}
+declare function readHandler(ctx: ServiceContext): (args: ReadArgs) => Promise<CallToolResult>;
+//#endregion
+export { FLOOR, MAX_OUTLINE_LIMIT, MAX_SEARCH_K, McpServer$1 as McpServer, OUTLINE_OUTPUT, type OutlineArgs, READ_ONLY, READ_OUTPUT, type ReadArgs, SEARCH_OUTPUT, type SearchArgs, type ServiceContext, composeInstructions, outlineHandler, readHandler, recordIsUndescribed, searchHandler, z$1 as z };

package/dist/gateway.mjs

@@ -0,0 +1,2 @@
+import { L as z$1, O as readHandler, a as MAX_OUTLINE_LIMIT, d as READ_OUTPUT, f as SEARCH_OUTPUT, j as searchHandler, k as recordIsUndescribed, l as OUTLINE_OUTPUT, o as MAX_SEARCH_K, r as FLOOR, s as McpServer$1, u as READ_ONLY, w as outlineHandler, y as composeInstructions } from "./gateway-api-BF06IsJ--D-eI--yB.mjs";
+export { FLOOR, MAX_OUTLINE_LIMIT, MAX_SEARCH_K, McpServer$1 as McpServer, OUTLINE_OUTPUT, READ_ONLY, READ_OUTPUT, SEARCH_OUTPUT, composeInstructions, outlineHandler, readHandler, recordIsUndescribed, searchHandler, z$1 as z };

package/docs/deploying.md

@@ -0,0 +1,233 @@
+---
+title: Deploying
+status: draft
+---
+
+# Deploying a Knowledge System of Record
+
+A KSoR publishes two surfaces from one record, and they deploy differently
+because they are different things:
+
+| surface      | what it is                                       | how it deploys                         |
+| ------------ | ------------------------------------------------ | -------------------------------------- |
+| **the site** | a fully static export — HTML, `llms.txt`, search | upload a folder to any static host     |
+| **the door** | the MCP server, a live process reading Postgres  | run a container that listens on a port |
+
+`ksor init` emits everything both need: `Dockerfile` and `.dockerignore` for the
+door, and a `vercel.json` that puts the two behind one domain. This page is the
+walkthrough, executed rather than described.
+
+**Publishing is a third thing, and it is not on this page's critical path.**
+Neither surface ingests. The door serves whatever generation is already active
+in the database, and the site renders `knowledge/` from the repository. Getting
+content INTO the database is [ingesting.md](./ingesting.md), and it is a deploy
+step you run, never something a booting container does.
+
+## The shape
+
+```
+                     ┌───────────────────────────────┐
+   a reader  ────────▶  /            the site        │  static files
+                     │  /docs/…                      │
+                     ├───────────────────────────────┤
+   an agent  ────────▶  /mcp         the door        │  container ──▶ Postgres
+                     │  /health /ready               │
+                     │  /.well-known/oauth-…         │
+                     └───────────────────────────────┘
+```
+
+One domain, two services. An agent that finds `https://your-host/mcp` and a
+person who opens `https://your-host/` are reading the same record.
+
+## The container is the portable artifact
+
+The emitted `Dockerfile` names no host. It installs the pinned
+`@panaversity/ksor` from your `package.json`, honours `$PORT`, and runs
+`ksor serve`:
+
+```sh
+docker build -t my-record .
+docker run --rm -p 8080:80 --env-file .env my-record
+```
+
+That runs on Cloud Run, Fly, Render, ECS, Kubernetes, or a VPS with no changes.
+`vercel.json` **points at this same file** rather than replacing it, which is
+what keeps the host a choice — the artifact is yours, and moving it is a
+redeploy, not a rewrite.
+
+What the image deliberately does NOT contain (see `.dockerignore`):
+
+- **`.env`** — configuration arrives from the environment at run time. A DSN
+  baked into a layer is published to anyone who can pull the image.
+- **`knowledge/`** — the door reads Postgres. Copying the corpus in would
+  suggest the container reads it, and it never does.
+- **`system/`** — that is the other surface, built and hosted separately.
+
+## Deploying both surfaces to Vercel
+
+The emitted `vercel.json` declares both services and routes between them:
+
+```json
+{
+  "services": {
+    "site": { "root": ".", "buildCommand": "pnpm build", "outputDirectory": "system/site/out" },
+    "door": { "root": ".", "runtime": "container", "entrypoint": "Dockerfile" }
+  },
+  "rewrites": [
+    { "source": "/mcp(.*)", "destination": { "service": "door" } },
+    { "source": "/.well-known/oauth-protected-resource(.*)", "destination": { "service": "door" } },
+    { "source": "/health", "destination": { "service": "door" } },
+    { "source": "/ready", "destination": { "service": "door" } },
+    { "source": "/(.*)", "destination": { "service": "site" } }
+  ]
+}
+```
+
+Deploy from the repository **root**, never from `system/site/`:
+
+```sh
+vercel deploy --prod
+```
+
+Two things about this file are worth knowing before you edit it.
+
+**The catch-all must stay last.** Rewrites match in order, so a `/(.*)` rule
+moved above the door's routes silently sends `POST /mcp` to the static site,
+where it becomes a 404 and reads like the door is down.
+
+**Do not add a project-level `trailingSlash`.** The site's own Next config
+already sets it and exports `index.html` directories, so it buys nothing — and
+at the project level it applies to the door too, where it redirects `POST /mcp`
+to `/mcp/` with a 308. Found exactly that way: every door route 308ed until the
+setting came out.
+
+## Configuration
+
+The door takes everything from the environment. Set these on the deployment, not
+in a file:
+
+| variable                | why                                                                      |
+| ----------------------- | ------------------------------------------------------------------------ |
+| `KSOR_DB_URL`           | the record's Postgres store                                              |
+| `GEMINI_API_KEY`        | embeds the incoming query, so retrieval works at all                     |
+| `KSOR_SNAPSHOT_KEYS`    | `kid=secret` — **required in practice**, see below                       |
+| `KSOR_ALLOWED_HOSTS`    | the host you serve on (DNS-rebind defence)                               |
+| `KSOR_MCP_RESOURCE_URL` | this record's canonical URL, e.g. `https://your-host/mcp`                |
+| auth, one of two        | `KSOR_SSO_URL` + audiences, **or** `KSOR_ALLOW_PUBLIC_UNAUTHENTICATED=1` |
+
+`KSOR_SNAPSHOT_KEYS` is listed as a production knob but behaves as a
+requirement on any host that scales to zero. Unset, the signing key is generated
+**per process** — so a citation minted before a scale-down stops validating
+after it, with a single instance and no replicas involved. Generate one:
+
+```sh
+KSOR_SNAPSHOT_KEYS="k1=$(openssl rand -hex 32)"
+```
+
+The value is `kid=secret`, not `kid:secret`, and the first entry is the active
+one. Multiple entries let you rotate without invalidating outstanding
+citations.
+
+### The site build needs the DSN too
+
+Once `instance.md` declares a `database:` block, `pnpm build` runs
+`pnpm export-denylist` first — it asks the database what has been withdrawn and
+writes `.ksor-denylist.json`. Without `KSOR_DB_URL` the build **refuses**:
+
+```
+KSOR_DB_URL is unset, and instance.md declares a database
+  why: a takedown lives in that database. Without it this build cannot tell
+       'nothing is denied' from 'nobody asked'
+```
+
+That refusal is the design working. A takedown reaches the door instantly (it is
+a row) and reaches the site at its next build (it reads a file), so a site built
+without the DSN would keep publishing what the door already refuses. Set
+`KSOR_DB_URL` on the site build as well as on the door.
+
+## Authorization, or the deliberate absence of it
+
+`ksor serve` **refuses to boot unauthenticated on a public bind.** There is no
+auth-off default, and that refusal is the last real step of a deployment. Two
+ways past it:
+
+- **Configure the SSO door** — `KSOR_SSO_URL`, `KSOR_MCP_RESOURCE_URL`,
+  `KSOR_JWT_ALLOWED_AUDIENCES`. Worked recipes for two different authorization
+  servers: [authorization.md](./authorization.md).
+- **Set `KSOR_ALLOW_PUBLIC_UNAUTHENTICATED=1`** — a deliberate decision that
+  serves your whole record to anyone who can reach the port. Correct for a
+  genuinely public record, or behind your own gateway. Never as a way to get a
+  deploy green.
+
+Check which one you got. `/health` says so plainly:
+
+```json
+{
+  "corpus_id": "book",
+  "abstain_gate": "OFF (no floor declared — will not refuse out-of-corpus questions)",
+  "embedding_space": "gemini-embedding-001/d1536 ok",
+  "auth": "disabled"
+}
+```
+
+`"auth":"disabled"` on a public host means the second option is in effect.
+
+## What a cold start costs
+
+The door is built for a runtime that suspends it. It holds **no idle database
+connections** — the pool minimum is 0 and an unused connection closes after 10s
+— so a quiet instance keeps nothing open against a serverless Postgres, and the
+first request after a suspend both wakes the database and retries the connect
+rather than failing.
+
+Measured against a live deployment on Vercel, Neon behind it, an 81-document
+record of 6,963 chunks:
+
+|                                        |           |
+| -------------------------------------- | --------- |
+| warm request                           | **~1.7s** |
+| cold start (container + database wake) | **~9.1s** |
+
+Most of the cold number is the two wakes, not ksor. If that matters for your
+readers, the levers are your host's minimum instance count and your database's
+suspend delay — both outside this tool.
+
+`SIGTERM` drains and exits within 8s (`KSOR_DRAIN_TIMEOUT_MS`), inside the ~10s
+a scale-to-zero runtime usually allows before `SIGKILL`.
+
+## Verifying a deployment
+
+In order, because each answers a different question:
+
+```sh
+B=https://your-host
+
+curl -s $B/health          # the door booted, and on which posture
+curl -s $B/ready           # the database answers
+curl -s -o /dev/null -w '%{http_code}\n' $B/   # the site is served
+
+curl -s -X POST $B/mcp \
+  -H 'content-type: application/json' \
+  -H 'accept: application/json, text/event-stream' \
+  -d '{"jsonrpc":"2.0","id":1,"method":"initialize","params":{
+       "protocolVersion":"2025-11-25","capabilities":{},
+       "clientInfo":{"name":"probe","version":"1"}}}'
+```
+
+If `/health` answers but a search returns nothing, the door is fine and the
+**record was never published** — go to [ingesting.md](./ingesting.md). A first
+deploy with no ingest serves an empty record, which is the single most common
+"it deployed but does not work".
+
+If `/mcp` returns HTML, the catch-all rewrite is matching before the door's
+route.
+
+## Deploying anywhere else
+
+Nothing above is Vercel-specific except `vercel.json`. On any other host:
+
+1. Build the site — `pnpm build` — and upload `system/site/out/` as static files.
+2. Build and run the container, giving it the environment above.
+3. Put the door at `/mcp` on the same domain, or a different one; if it is a
+   different domain, set `KSOR_MCP_RESOURCE_URL` to the door's real URL, since
+   that value is the record's identity in a token's audience, not a guess.

package/docs/index.md

@@ -32,6 +32,18 @@ instead of their training memory. The corpus grows with each implemented verb.
   export the manifest the site build reads), `ksor calibrate` (measure the
   abstention floor) and `ksor gc` (reap retired generations). Only `ksor dev` and `ksor build` remain designed, not
   implemented: each prints an honest notice and exits `2`.
+  - **[tool-surface.md](./tool-surface.md)** — shaping what agents see.
+    `system/gateways/content.ts` is emitted, adopter-owned and deletable; it
+    decides tool names, what the record says it covers, and how much of the
+    caller's context an answer costs. Includes the measurements.
+  - **[deploying.md](./deploying.md)** — getting both surfaces onto a host. The
+    scaffold emits a `Dockerfile` that names no vendor, and a `vercel.json` that
+    points at it to put the site and the MCP door behind one domain. Includes
+    what a cold start costs, measured.
+  - **[ingesting.md](./ingesting.md)** — publishing the record and keeping it
+    current. Serving does not publish, so a first deploy with no ingest serves
+    an empty record; this is the page that explains why, where ingest belongs
+    (never inside the container), and how the abstention gate gets turned on.
   - **[authorization.md](./authorization.md)** — putting the record behind an
     authorization server, with worked recipes for two of them, executed rather
     than written. `ksor serve` refuses to boot unauthenticated on a public bind,

package/docs/ingesting.md

@@ -0,0 +1,159 @@
+---
+title: Ingesting
+status: draft
+---
+
+# Publishing the record — `ksor ingest`
+
+Serving does not publish. That split is deliberate and it is the thing most
+worth understanding before a first deployment: `ksor serve` opens a port against
+whatever generation is already active, and `ksor ingest` is what makes a
+generation exist. A container that ingested on boot would pay the whole record's
+embedding cost on every cold start and would need write credentials at runtime.
+
+So **a first deploy with no ingest serves an empty record.** It is not broken;
+nothing was ever published to it.
+
+## The order, once
+
+```sh
+pnpm provision   # ksor schema --apply, then ksor grant
+pnpm refresh     # ksor ingest --flip, then ksor gc
+```
+
+`provision` is separate because applying DDL and granting ingest are acts an
+operator performs, not side effects of starting a server. Both are re-runnable
+and report what they found — an applied schema says "already applied", an
+existing grant says "already granted".
+
+Then, after every change to `knowledge/`:
+
+```sh
+pnpm refresh
+```
+
+## What a generation is
+
+Each ingest builds a **fresh generation** — invisible until activated — and
+carries every unchanged embedding forward from the last complete one, matched by
+content hash. Only changed or previously-failed chunks are re-embedded.
+
+`--flip` swaps the active pointer. The previous generation stays as a rollback
+target; `ksor gc` reaps the ones nothing points at any more.
+
+Three consequences worth knowing:
+
+- **Re-ingest is cheap.** An ordinary edit makes a handful of provider calls,
+  not a corpus-worth.
+- **An unchanged record costs nothing at all.** Ingest compares what it just
+  read against the generation already serving and, when they are identical at
+  the same commit, consumes no generation and writes no rows:
+  `unchanged — generation N already serves this corpus`.
+- **Reordering is free.** Changing `order:` frontmatter re-ingests with no
+  embedding at all; only the ordering moved.
+
+A flip that would drop more than `KSOR_MAX_SHRINK` of the record (default
+`0.15`, i.e. 15%) **refuses**. When a large deletion is intended, say so:
+`KSOR_ALLOW_SHRINK=1`.
+
+## Where ingest runs — not on the host
+
+Ingest is a long job. It embeds every new chunk through the provider, and that
+is bounded by the provider's throughput rather than by anything ksor does.
+
+**Measured:** an 81-document book — 6,963 chunks — took **about 50 minutes**
+against a remote Postgres on a first, cold ingest with nothing to carry forward.
+
+Compare that with the request timeouts of the platforms people reach for first:
+a serverless function caps out in the region of 300–800 seconds depending on
+plan. Ingest is **an order of magnitude past that**, so it cannot be a route, a
+build step, or anything else the platform is allowed to kill.
+
+Run it where nothing is watching a clock:
+
+- **From your machine**, with `KSOR_DB_URL` pointing at the deployment's
+  database. This is the honest default for a record one person maintains.
+- **From CI**, as a job triggered on changes to `knowledge/` — the right shape
+  once more than one person edits, because it makes publishing an auditable
+  event rather than something someone did locally.
+
+It does not matter whether the process that ingests is the process that serves.
+They meet in the database and nowhere else.
+
+### If an ingest is interrupted
+
+Nothing is corrupted — an unactivated generation is invisible by construction.
+Run it again. The next attempt finds the incomplete generation's work and
+carries forward everything that was already embedded, including from a
+generation that was still `building` when it died, so a resumed run pays only
+for what the first one had not reached.
+
+## Endpoints, poolers, and what actually matters
+
+If your provider offers both a **pooled** and a **direct** endpoint (Neon's
+`-pooler` host, or anything on port 6432), ksor detects which one you gave it
+and says so in the boot report. That line is **informational**: it classifies,
+it never transforms. The hazard it descends from — a transaction pooler and
+server-side prepared statements — cannot arise here, because node-postgres does
+not auto-prepare.
+
+For the record: the 6,963-chunk ingest above ran through a **pooled** endpoint
+without incident, and the same DSN serves. Use whichever your provider gives
+you, and reach for the direct endpoint only if you actually hit pooler
+connection limits under a parallel ingest — not pre-emptively.
+
+`KSOR_DB_POOLED_ENDPOINT=1` forces the classification when your host name does
+not announce itself.
+
+## Turning the abstention gate on
+
+This is a separate act, done once, **after** the record is serving — because it
+is a measurement of this corpus in this embedding space, and there is nothing to
+measure until the corpus is in there.
+
+```sh
+pnpm exec ksor calibrate --instance instance.md
+```
+
+It prints a recommended `vector_floor`. Paste it in with the date you measured
+it, and restart:
+
+```yaml
+retrieval:
+  vector_floor: 0.55 # measured by ksor calibrate on 2026-08-23
+```
+
+Until you do, `/health` reports the gate as `OFF (no floor declared — will not
+refuse out-of-corpus questions)` and every search envelope carries
+`gate: "off"`. That is honest absence, and an agent reading the envelope knows
+an answer is not evidence of coverage.
+
+**Never copy a floor from another corpus.** The number means nothing away from
+the corpus and embedding space it was measured in.
+
+If calibrate reports `NOT separable`, read what it names underneath: it lists
+the out-of-corpus probes that scored at or above your weakest in-corpus
+question. Usually one of them is a question your record actually answers, and it
+belongs on the in-corpus side — moving it separates the measurement. Sometimes
+it is a genuine near-miss the corpus cannot separate, and then the floor
+correctly stays uncalibrated.
+
+## Withdrawing a document
+
+A takedown is a row, not a file, so it reaches the door immediately:
+
+```sh
+pnpm exec ksor takedown --instance instance.md <stable-id> \
+  --reason "legal request 2026-08" --actor "j.smith"
+```
+
+`--actor` is required, and there is no default. A name taken from the
+environment reads like a person and is whatever the shell happened to be
+(`runner` under CI, `root` in a container) — worse than no name at all in the
+one row that exists to record who did this.
+
+**The site stops at its next build.** It reads `.ksor-denylist.json`, which
+`pnpm build` refreshes via `pnpm export-denylist`. So after a takedown, rebuild
+and redeploy the site, or the human surface keeps publishing what the agent
+surface already refuses. See [deploying.md](./deploying.md) for why that build
+needs `KSOR_DB_URL`.