@panaversity/ksor 0.0.19 → 0.0.21

package/dist/cli.mjs

@@ -15,7 +15,7 @@ import { bodyLimit } from "hono/body-limit";
 import { execFileSync, spawnSync } from "node:child_process";
 import { parseArgs } from "node:util";
 import { readFile, readdir, stat } from "node:fs/promises";
-//#region ../content-gateway/dist/main-DDeyGVjK.mjs
+//#region ../content-gateway/dist/main-DKA8sPwv.mjs
 /**
 * A connection could not be ESTABLISHED in time — retryable.
 *
@@ -3508,6 +3508,27 @@ const POS_TTL_S = 60;
 function isBadToken(err) {
 	return err instanceof errors.JWTExpired || err instanceof errors.JWTClaimValidationFailed || err instanceof errors.JWTInvalid || err instanceof errors.JWSInvalid || err instanceof errors.JWSSignatureVerificationFailed || err instanceof errors.JOSEAlgNotAllowed || err instanceof errors.JOSENotSupported;
 }
+/**
+* The `iss` a token CLAIMS, read without verifying anything.
+*
+* Sound for exactly one purpose: REFUSING. A token that passes this check still
+* has its signature verified in full, so an attacker gains nothing by lying here
+* — the worst they achieve is being refused for a different reason. It must
+* never be used to admit anything.
+*
+* Returns null when the payload is not readable JSON, which sends the token down
+* the ordinary path rather than inventing a verdict about it.
+*/
+function claimedIssuer(token) {
+	try {
+		const payload = token.split(".")[1];
+		if (payload === void 0 || payload === "") return null;
+		const iss = JSON.parse(Buffer.from(payload, "base64url").toString("utf8")).iss;
+		return typeof iss === "string" && iss !== "" ? iss : null;
+	} catch {
+		return null;
+	}
+}
 function describeError(err) {
 	return err instanceof Error ? `${err.name}: ${err.message}` : String(err);
 }
@@ -3571,6 +3592,13 @@ function createVerify(config, deps, jwksOf) {
 		if (hit !== void 0 && hit.until > now()) return hit.identity;
 		let claims = null;
 		if (token.split(".").length === 3) {
+			if (config.issuer !== null) {
+				const claimed = claimedIssuer(token);
+				if (claimed !== null && claimed !== config.issuer) {
+					reject(key);
+					throw new TokenVerifyError(`token iss ${JSON.stringify(claimed)} is not this record's authorization server ${JSON.stringify(config.issuer)}`, { transient: false });
+				}
+			}
 			try {
 				claims = await verifyJwt(token);
 			} catch (err) {
@@ -4546,7 +4574,7 @@ async function withPgRetry(op, options = {}) {
 	throw lastError;
 }
 //#endregion
-//#region ../content/dist/commands-DcPJJlNb.mjs
+//#region ../content/dist/commands-FLQD4HUi.mjs
 /**
 * EVAL-LOCKED constants, quarried verbatim from the oracle
 * (sor-agentfactory @ b554f91, config.py) — changing any of these is a
@@ -6562,19 +6590,42 @@ async function allocateRun(client, opts) {
 *
 * Returns 0 when there is no complete embedded generation — the first ingest.
 */
-async function bestCarrySource(client, opts) {
-	const gen = (await client.query(`
-    SELECT max(c.generation) AS gen FROM chunks c
-    JOIN ingestion_runs r ON r.tenant_id = c.tenant_id AND r.generation = c.generation
-    WHERE c.tenant_id = $1 AND r.corpus_id = $2
-      AND r.state IN ('ready', 'active', 'retired')
-      AND c.generation <> $3 AND c.embedding_status = 'embedded'
+/**
+* Every generation whose vectors this build may copy, best source FIRST.
+*
+* Ordered by how much the source has been vetted, then by recency:
+*
+*   1. complete runs (`ready` / `active` / `retired`), newest first
+*   2. ABANDONED runs (`building`), newest first
+*
+* The second group used to be excluded outright, and that threw away work an
+* operator had already paid for: a killed `ksor ingest` leaves its generation in
+* `building`, so the rerun carried NOTHING and re-embedded the whole corpus.
+* Reproduced live — an 81-document book killed at 4,736 of 6,963 chunks, whose
+* rerun reported `carried 0, pending 6963` (issue #97).
+*
+* Nothing about an abandoned run makes its vectors wrong. An embedding is a pure
+* function of (embed input, model); the match key in `carryForward` establishes
+* identity on its own; and carry only ever fills `pending` rows, so a later pass
+* can never overwrite an earlier, better-vetted one. The run's STATE therefore
+* decides priority, not eligibility — which is what ordering expresses and
+* exclusion could not.
+*/
+async function carrySources(client, opts) {
+	return (await client.query(`
+    SELECT DISTINCT c.generation AS gen,
+           CASE WHEN r.state IN ('ready','active','retired') THEN 0 ELSE 1 END AS rank
+      FROM chunks c
+      JOIN ingestion_runs r ON r.tenant_id = c.tenant_id AND r.generation = c.generation
+     WHERE c.tenant_id = $1 AND r.corpus_id = $2
+       AND r.state IN ('ready', 'active', 'retired', 'building')
+       AND c.generation <> $3 AND c.embedding_status = 'embedded'
+     ORDER BY rank, gen DESC
     `, [
 		opts.tenantId,
 		opts.corpusId,
 		opts.excludeGeneration
-	])).rows[0]?.gen ?? null;
-	return gen === null ? 0 : Number(gen);
+	])).rows.map((r) => Number(r.gen));
 }
 /**
 * Copy embeddings for chunks whose ENTIRE embed input is unchanged (hash +
@@ -8530,17 +8581,19 @@ async function buildStructure(client, opts) {
 		fromGeneration: active,
 		modelId
 	});
-	const newest = await bestCarrySource(client, {
+	for (const source of await carrySources(client, {
 		tenantId,
 		corpusId: opts.corpusId,
 		excludeGeneration: generation
-	});
-	if (newest !== 0 && newest !== active) carried += await carryForward(client, {
-		tenantId,
-		generation,
-		fromGeneration: newest,
-		modelId
-	});
+	})) {
+		if (source === active) continue;
+		carried += await carryForward(client, {
+			tenantId,
+			generation,
+			fromGeneration: source,
+			modelId
+		});
+	}
 	const health = await generationHealth(client, {
 		tenantId,
 		generation
@@ -8852,6 +8905,31 @@ async function buildGeneration(pool, instance, options) {
 function pct(fraction) {
 	return `${(fraction * 100).toFixed(0)}%`;
 }
+/**
+* Out-of-corpus probes scoring at or above the weakest in-corpus question,
+* worst first — the ones that decided the verdict. Empty when the measurement
+* separated, because then nothing held it open.
+*/
+function overlappingProbes(report) {
+	if (report.separable) return [];
+	const weakest = report.low_tail[0]?.score;
+	if (weakest === void 0) return [];
+	return report.detail.filter((d) => !d.in_corpus && d.score >= weakest).toSorted((a, b) => b.score - a.score);
+}
+/**
+* The guidance itself, or null when there is nothing to say.
+*
+* Deliberately names BOTH readings. The overlapping probe is sometimes a
+* question the record covers — mislabelled, and the measurement is fine once it
+* moves — and sometimes a genuine near-miss the corpus simply cannot separate,
+* in which case the floor stays uncalibrated and that is the correct outcome.
+* Asserting either one alone would send half the readers the wrong way.
+*/
+function overlapAdvice(report) {
+	const overlapping = overlappingProbes(report);
+	if (overlapping.length === 0) return null;
+	return "these out-of-corpus probes scored at or above your weakest in-corpus question:\n" + overlapping.map((d) => `  ${d.score.toFixed(3)}  ${d.query}\n`).join("") + "  ^ look at these first. Either the record COVERS one — move it to the\n    in-corpus side, because a probe the record answers is not out of corpus\n    — or it genuinely does not separate, and the floor stays uncalibrated.\n";
+}
 /** One transaction (tenant GUC + ingest role): list collectables, then reap each. */
 async function runGc(pool, instance, options = {}) {
 	const dryRun = options.dryRun === true;
@@ -9329,6 +9407,8 @@ async function calibrateCommand(args) {
 		minChars: values["min-chars"] === void 0 ? void 0 : intFlag("--min-chars", values["min-chars"])
 	}));
 	process.stdout.write(renderReport(report) + "\n");
+	const advice = overlapAdvice(report);
+	if (advice !== null) process.stdout.write(advice);
 	return 0;
 }
 async function grantCommand(args) {

package/docs/authorization.md

@@ -0,0 +1,197 @@
+---
+title: Authorization
+status: draft
+---
+
+# Putting the record behind an authorization server
+
+`ksor serve` refuses to boot unauthenticated on a public bind. That is the whole
+posture, and it means the last step of a deployment is standing up an
+authorization server and pointing the door at it.
+
+This page is two worked recipes, both executed against real servers rather than
+written from their documentation, plus what an agent does to obtain a token. The
+mechanism is standard OAuth 2.0 — nothing here is specific to either product, and
+that is the point: two different implementations are shown because a single one
+proves nothing about neutrality.
+
+## What the door needs
+
+Three variables, and one more you should set even though it is optional:
+
+| variable                     | what it is                                                                    | where the value comes from                                                                          |
+| ---------------------------- | ----------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------- |
+| `KSOR_SSO_URL`               | your authorization server's base URL                                          | the AS itself; for OIDC it is the issuer, the URL whose `/.well-known/openid-configuration` answers |
+| `KSOR_MCP_RESOURCE_URL`      | **this record's** canonical URL — the identifier a token must be audienced at | you choose it; it is the public URL agents reach the door on                                        |
+| `KSOR_JWT_ALLOWED_AUDIENCES` | which audiences are accepted, comma-separated                                 | normally exactly `KSOR_MCP_RESOURCE_URL`                                                            |
+| `KSOR_SSO_ISSUER`            | the issuer to enforce                                                         | the `issuer` field of the AS's discovery document                                                   |
+
+`KSOR_MCP_RESOURCE_URL` is **not** a place the door fetches anything from. It is
+the name of this resource, in the RFC 8707 sense: a token minted for a different
+resource is refused even when the signature is perfect and the issuer is right.
+That is what stops a token issued for some other service being replayed at your
+record.
+
+The door finds the signing keys by discovery, in this order, and says at boot
+which one it used:
+
+```
+1. KSOR_JWKS_URL                        you stated the URI outright
+2. /.well-known/oauth-authorization-server   RFC 8414 metadata
+3. /.well-known/openid-configuration         OIDC discovery
+4. <KSOR_SSO_URL>/api/auth/jwks              a vendor default, reported as a GUESS
+```
+
+**Set `KSOR_SSO_ISSUER`.** Without it, a token from a _different_ authorization
+server produces an unknown key id, which is indistinguishable from key-rotation
+lag — so the door answers `503`, the client retries a credential that can never
+work, and a misconfiguration reads as an outage. With the issuer declared, that
+same token is refused `401` before any key is fetched.
+
+## Recipe: Keycloak
+
+Run it:
+
+```sh
+docker run -d --name kc -p 8180:8080 \
+  -e KC_BOOTSTRAP_ADMIN_USERNAME=admin -e KC_BOOTSTRAP_ADMIN_PASSWORD=admin \
+  quay.io/keycloak/keycloak:26.0 start-dev
+```
+
+Get an admin token, then create a client for the agent. `client_credentials` is
+the machine-to-machine shape — an agent is not a person and has no browser:
+
+```sh
+KC=http://127.0.0.1:8180
+ADM=$(curl -s -X POST "$KC/realms/master/protocol/openid-connect/token" \
+  -d client_id=admin-cli -d username=admin -d password=admin -d grant_type=password \
+  | jq -r .access_token)
+
+curl -s -X POST "$KC/admin/realms/master/clients" -H "Authorization: Bearer $ADM" \
+  -H 'Content-Type: application/json' -d '{
+    "clientId":"ksor-agent","protocol":"openid-connect","publicClient":false,
+    "serviceAccountsEnabled":true,"standardFlowEnabled":false,"secret":"agent-secret"}'
+```
+
+Keycloak does not put your resource in the token's `aud` on its own. Add an
+audience mapper — this is the step people miss, and its absence looks exactly
+like a rejected token:
+
+```sh
+CID=$(curl -s "$KC/admin/realms/master/clients?clientId=ksor-agent" \
+  -H "Authorization: Bearer $ADM" | jq -r '.[0].id')
+
+curl -s -X POST "$KC/admin/realms/master/clients/$CID/protocol-mappers/models" \
+  -H "Authorization: Bearer $ADM" -H 'Content-Type: application/json' -d '{
+    "name":"ksor-resource-audience","protocol":"openid-connect",
+    "protocolMapper":"oidc-audience-mapper",
+    "config":{"included.custom.audience":"https://records.example.com/mcp",
+              "access.token.claim":"true"}}'
+```
+
+Point the door at it:
+
+```sh
+export KSOR_SSO_URL=http://127.0.0.1:8180/realms/master
+export KSOR_SSO_ISSUER=http://127.0.0.1:8180/realms/master
+export KSOR_MCP_RESOURCE_URL=https://records.example.com/mcp
+export KSOR_JWT_ALLOWED_AUDIENCES=https://records.example.com/mcp
+ksor serve --instance instance.md
+```
+
+The boot block tells you whether discovery worked:
+
+```
+auth     bearer tokens, verified against the record's authorization server
+keys     openid-configuration — http://127.0.0.1:8180/realms/master/protocol/openid-connect/certs
+```
+
+If that line says `guess` instead of naming a discovery document, the AS did not
+publish metadata where the door looked, and you should set `KSOR_JWKS_URL`
+yourself rather than rely on the vendor default.
+
+## Recipe: Ory Hydra
+
+A different implementation, and a different way of asking for the audience —
+which is the neutrality proof. Hydra takes the RFC 8707 `audience` parameter on
+the token request, so no mapper is involved:
+
+```sh
+docker run -d --name hydra -p 4444:4444 -p 4445:4445 \
+  -e DSN=memory -e URLS_SELF_ISSUER=http://127.0.0.1:4444 \
+  -e SECRETS_SYSTEM=change-me-0000000000000000000000 \
+  -e STRATEGIES_ACCESS_TOKEN=jwt \
+  oryd/hydra:v2.2.0 serve all --dev
+
+curl -s -X POST http://127.0.0.1:4445/admin/clients -H 'Content-Type: application/json' -d '{
+  "client_id":"ksor-agent","client_secret":"agent-secret",
+  "grant_types":["client_credentials"],"token_endpoint_auth_method":"client_secret_post",
+  "audience":["https://records.example.com/mcp"],"access_token_strategy":"jwt"}'
+```
+
+Only the two SSO variables change:
+
+```sh
+export KSOR_SSO_URL=http://127.0.0.1:4444
+export KSOR_SSO_ISSUER=http://127.0.0.1:4444
+```
+
+Hydra publishes its keys at `/.well-known/jwks.json` rather than Keycloak's
+`/protocol/openid-connect/certs`. Nothing in ksor knows that; discovery reads it
+from the metadata document, which is why the door works against both unmodified.
+
+## What an agent does
+
+Ask the token endpoint for a token audienced at the record, then send it as an
+ordinary bearer:
+
+```sh
+# Hydra — the audience is a request parameter
+curl -s -X POST http://127.0.0.1:4444/oauth2/token \
+  -d grant_type=client_credentials -d client_id=ksor-agent -d client_secret=agent-secret \
+  -d audience=https://records.example.com/mcp
+
+# Keycloak — the audience comes from the mapper, so the request is plain
+curl -s -X POST "$KC/realms/master/protocol/openid-connect/token" \
+  -d client_id=ksor-agent -d client_secret=agent-secret -d grant_type=client_credentials
+```
+
+With the MCP TypeScript SDK:
+
+```ts
+const transport = new StreamableHTTPClientTransport(new URL("https://records.example.com/mcp"), {
+  requestInit: { headers: { authorization: `Bearer ${accessToken}` } },
+});
+```
+
+A client that does not know where to authenticate can find out: an unauthorized
+request answers `401` with a pointer to this record's metadata, per RFC 9728.
+
+```
+www-authenticate: Bearer resource_metadata="https://records.example.com/.well-known/oauth-protected-resource/mcp"
+```
+
+## What the door refuses, and how it says so
+
+| what you send                                                    | answer                                                                      |
+| ---------------------------------------------------------------- | --------------------------------------------------------------------------- |
+| no token                                                         | `401`, with the `resource_metadata` pointer                                 |
+| a malformed or unsigned token                                    | `401`, `error="invalid_token"`                                              |
+| a token for a **different resource**                             | `401` — the audience binding, and the reason it exists                      |
+| a token from a **different issuer** (with `KSOR_SSO_ISSUER` set) | `401`, before any key is fetched                                            |
+| a token from a different issuer (issuer NOT set)                 | `503` — indistinguishable from rotation lag, which is why you should set it |
+| an expired token                                                 | `401`                                                                       |
+| a valid token                                                    | the record answers, and the answer carries its citations                    |
+
+A genuine key-rotation lag stays a `503` on purpose: it is transient, retrying is
+the right response, and the refusal is never cached — a valid bearer is
+re-admitted the instant the key set catches up.
+
+## Before a public bind
+
+- Auth configured as above, **or** `KSOR_ALLOW_PUBLIC_UNAUTHENTICATED=1` set
+  deliberately — the door will not come up on a public address without one of
+  them, and the second is a decision, not a default.
+- `KSOR_ALLOWED_HOSTS` set to the host you serve on.
+- `KSOR_SNAPSHOT_KEYS` shared across every replica. Unset means a key per
+  process, so a citation minted by one replica fails on another.

package/docs/index.md

@@ -32,6 +32,10 @@ instead of their training memory. The corpus grows with each implemented verb.
   export the manifest the site build reads), `ksor calibrate` (measure the
   abstention floor) and `ksor gc` (reap retired generations). Only `ksor dev` and `ksor build` remain designed, not
   implemented: each prints an honest notice and exits `2`.
+  - **[authorization.md](./authorization.md)** — putting the record behind an
+    authorization server, with worked recipes for two of them, executed rather
+    than written. `ksor serve` refuses to boot unauthenticated on a public bind,
+    so this is the last step of a deployment, not an optional hardening pass.
 - Exit codes are a contract: `1` refused (first stderr line is a stable
   slug such as `error: bad-name`, followed by a remedy), `2` designed but
   not implemented, `3` the environment cannot run ksor

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@panaversity/ksor",
-  "version": "0.0.19",
+  "version": "0.0.21",
   "description": "Knowledge System of Record — compile governed markdown into a static site for people and an MCP server for AI agents, with citations and measured abstention.",
   "keywords": [
     "abstention",