@panaversity/ksor 0.0.11 → 0.0.13

package/CHANGELOG.md

@@ -1,5 +1,205 @@
 # @panaversity/ksor
 
+## 0.0.13
+
+### Patch Changes
+
+- 8c5013b: A provider outage is never reported as "the record does not cover this"
+
+  When the embedding provider is down, the vector arm does not run — so an empty
+  result says nothing about coverage. It says we could not look. That distinction
+  was fixed once for records with a calibrated floor, and the condition was the
+  bug: it left the case out that matters most.
+
+  An **uncalibrated** record is the default state of every fresh scaffold. There
+  the emptiness came from the keyword arm, which abstains when it returns no rows
+  — and it returns nothing for almost every natural-language question, because
+  `websearch_to_tsquery` ANDs its terms (measured 12 of 12 on real questions). So
+  during any outage an uncalibrated record answered every question with
+  `abstained: true`, while the tool description instructs the agent to state that
+  as fact and never fall back on its own knowledge.
+
+  It reached this release because the existing test asked a question the keyword
+  arm could answer, so the degraded path served real hits and looked correct. Ask
+  the way a person asks and it did not. That case is now covered.
+
+  Found live against the published 0.0.12 with an invalid key — the same state a
+  rejected CI key had produced that morning, which is how likely this is.
+
+  **`ksor calibrate` also stops blessing a floor on far-domain evidence alone.**
+  The built-in out-of-corpus probes are all far-domain — dinner, taxes, boiling an
+  egg — and a shipped set cannot be scope-adjacent, because adjacency depends on a
+  corpus it has never seen. Far-domain probes score low against anything, so the
+  margin comes out inflated. Measured on one record, changing only the probe set:
+  built-ins reported "separable, margin 0.072" and recommended a floor; eight
+  scope-adjacent near-misses reported "NOT separable, margin -0.030" — and that
+  floor then answered six of the eight live, with citations. The tool already knew
+  to say "widen the probe set", but said it only on the not-separable branch, which
+  is when it is least needed. It now says it whenever the built-ins are used.
+
+- 692d296: `ksor ingest` says how much of the record no search will return
+
+  A chunk shorter than the navigation threshold is stored, embedded and readable —
+  and excluded from every retrieval arm by the serving predicate. That rule exists
+  for a good reason: a "See also: [a] [b] [c]" block should never be a search hit.
+  But it decides by LENGTH ALONE, so a short _substantive_ paragraph is caught by
+  it too — and a policy handbook is made of short substantive statements.
+
+  Measured on a realistic five-document operations handbook with real embeddings:
+  **10 of 16 chunks unsearchable, and one entire document that `outline` lists and
+  `read` returns in full but `search` can never find.** A complete policy
+  statement — "Probation: six months, with a written review at three and six" —
+  is 191 characters, so the record treats it as navigation. The ingest line
+  reported a cheerful `16 chunks; embedded 16` and said nothing.
+
+  It says it now:
+
+  ```
+  ingest: generation 1 — 2 nodes, 4 chunks; embedded 4, carried 0, failed 0
+    not searchable: 3 of 4 chunk(s) (75%) are shorter than the navigation
+      threshold — stored and readable, but no search returns them
+    FOUND ONLY BY NAME: knowledge/onboarding:prose — no searchable chunk at all
+  ```
+
+  This does **not** change the threshold, and nothing that was searchable stops
+  being so. Where that line belongs needs a gold-set measurement, which is issue
+  #55. What is fixed here is the silence — because the silence is what let a
+  record ship most of itself unfindable, and told its owner everything was fine.
+
+  The count is computed with the serving predicate's own admission test, and a db
+  test compares it against what the SQL actually admits: a report the database
+  disagrees with would be worse than none.
+
+- c4976dd: Two defects introduced in 0.0.12, found by verifying the published package live
+
+  **The discovery document became invalid exactly when a record became real.** The
+  MCP registry schema caps `ServerDetail.description` at **100 characters**
+  (2025-12-11). 0.0.12 started generating that description from the record's own
+  prose — a real improvement over the hard-coded sentence it replaced — and capped
+  it at 300. The unfilled placeholder is 88 characters and validates; a described
+  record's title plus scope sentence is routinely 150-350 and does not. So
+  `/.well-known/mcp/server.json` passed validation until the owner did the thing
+  the scaffold asks for, then silently stopped, with nothing in the build to say
+  so. It is now assembled inside the schema's budget and trimmed at a word
+  boundary rather than mid-word.
+
+  **The boot report reassured the operator in the one configuration that needs a
+  warning.** `KSOR_ALLOW_PUBLIC_UNAUTHENTICATED=1` permits an unauthenticated
+  public bind, and the auth line kept printing `DISABLED — 0.0.0.0 only, and a
+public bind will refuse to boot` — false on both counts, at the moment the whole
+  record is being served to anyone who can reach the port. It now says that,
+  naming the variable responsible.
+
+  Both shipped in 0.0.12 and both were mine; the aligned boot report and the
+  self-describing discovery document are otherwise unchanged.
+
+## 0.0.12
+
+### Patch Changes
+
+- 36e4a4c: The scaffold documents what a CLIENT has to do to reach a public MCP door
+
+  `ksor serve` implements the OAuth Resource Server handshake — an
+  unauthenticated request gets a 401 carrying
+  `WWW-Authenticate: Bearer resource_metadata="…"`, and that document names the
+  record's resource identifier and its authorization server, so a client discovers
+  where to authenticate instead of being told. None of it was written down
+  anywhere an adopter or their agent reads. The operator half was documented (the
+  three environment variables); the half their agents actually execute was not.
+
+  The scaffold's `AGENTS.md` now walks the three steps, and names the failure that
+  goes wrong quietly: a token minted for a different audience is a perfectly valid
+  token, and this door rejects it, so `aud` against `KSOR_JWT_ALLOWED_AUDIENCES` is
+  the first thing to compare when a client authenticates fine and still gets 401.
+  It also records the two behaviours a client author has to know and could not have
+  guessed — RS256 only, no opaque-token introspection, and an unknown key id
+  answering 503 rather than 401, because during a key rotation the token is
+  probably good and retrying beats sending the user back through a login.
+
+  This closes one of the three items named in issue #26; the worked provider
+  recipes and the introspection/rotation policy remain open there.
+
+- 125970c: Every 401 from the MCP door carries its `WWW-Authenticate` challenge, not just the first
+
+  Only the missing-token branch emitted `WWW-Authenticate: Bearer
+resource_metadata="…"`. A token that failed verification — expired, wrong
+  audience, no subject, bad signature — came back as a bare 401. That is the most
+  common 401 a real client will ever see, because tokens expire mid-conversation,
+  and it left the client with no pointer back to the resource-metadata document:
+  it could not re-discover the authorization server it had just been talking to.
+  Only a caller that had never sent a token was told where to go.
+
+  The MCP authorization spec requires `WWW-Authenticate` on a 401 without
+  qualification. Every 401 now carries it, with RFC 6750's `error="invalid_token"`
+  so a client refreshes rather than retrying the dead token.
+
+  A **503** stays deliberately unchallenged: an unreachable key set is our outage,
+  not the token's fault, and challenging there would send a user whose token is
+  perfectly good back through a login over a key-fetch failure.
+
+  Found by adversarially checking the release that documented this door. The
+  adversarial auth suite missed it by asserting the STATUS of each rejection and
+  never the header — it now sweeps every 401-producing token and asserts the
+  challenge on each, with the 503 as the negative control.
+
+- 0a94e31: A 503 refusal no longer puts the database host and user on the wire
+
+  When the deferred boot checks fail, `/mcp` refuses with the thrown error's
+  message in full under `data.detail`. For the three authored failures that is the
+  point — a too-old schema, a governance violation and a text-search mismatch each
+  carry a multi-line remedy the operator has to act on. But the catch treated every
+  error alike, and `pg` writes the host, its resolved address, the port and the
+  database user into its connection and authentication failures. Those went out
+  verbatim to any caller who could reach the door.
+
+  What may leave is now decided in one place and by TYPE, not by inspecting
+  message text: a class we wrote is a class whose words we control. A driver error
+  is refused with its class named and its text withheld, and the caller is told
+  which kind of failure it is — infrastructure, not their request.
+
+  The full text still reaches the operator, deliberately: the refusal says the
+  reason is in the server's logs, and the deferred-boot line recorded only the
+  error's NAME, so before this the real message existed nowhere. That is also why
+  the boot checks are not sanitised at their source — reducing a driver error to a
+  class name early would destroy the one copy anyone can act on.
+
+  **The test that covered this was holding it in place.** It asserted that
+  `http.ts` contains the literal string `data: { detail: message }` — so the leak
+  was pinned by an assertion with reasoning attached. Grepping source is the right
+  instrument for "does this check run before dispatch", because position is a
+  property of source, and the wrong one for "what does the response contain".
+  Response contents are now asserted against real bodies, including a `pg`-shaped
+  connection failure whose host, address, port and user must all be absent.
+
+  Verified live: a gateway pointed at an unreachable database answers
+  `the content store is unavailable (Error)` with no host, port, user or database
+  name anywhere in the body, while the server log carries
+  `connect ECONNREFUSED 127.0.0.1:59999` in full.
+
+- 1dd6211: The dimension ceiling says which shape it applies to, instead of blaming pgvector
+
+  `ksor schema` refuses an embedding dimension above 2000 with
+  "(pgvector vector + HNSW ceiling)". The refusal is right and the reason was
+  wrong: pgvector indexes a `vector` to 2000, but a **`halfvec` to 4000**, via an
+  expression index on the cast — verified live against a real database, where
+  `hnsw ((embedding::halfvec(3072)) halfvec_cosine_ops)` plans an Index Scan.
+
+  The old wording read as pgvector's own limit, so an adopter whose model emits
+  more than 2000 dimensions could conclude it was unusable here, over a wall that
+  is not one. The message now names the shape the ceiling belongs to — this schema
+  declares `VECTOR(dim)` columns and indexes one directly — and the constant
+  carries why raising it is a decision rather than an edit: every query site would
+  have to use the same cast as the index or fall silently back to a sequential
+  scan, and the halfvec arm's float16 rounding lands on the score the abstention
+  gate reads.
+
+  The same claim is corrected in the scaffold's `AGENTS.md`, which gains the reason
+  `dim: 1536` is the shipped default: `gemini-embedding-001` emits 3072 and ksor
+  asks it for 1536, which per Google's published MTEB table costs nothing
+  measurable — 1536 scores 68.17 against 2048's 68.16.
+
+  The 2000 refusal is unchanged. Issue #49 records the decision it now points at.
+
 ## 0.0.11
 
 ### Patch Changes

package/dist/cli.mjs

@@ -16,7 +16,7 @@ import { bodyLimit } from "hono/body-limit";
 import { execFileSync, spawnSync } from "node:child_process";
 import { parseArgs } from "node:util";
 import { readFile, readdir, stat } from "node:fs/promises";
-//#region ../content-gateway/dist/main-3DN4fOiE.mjs
+//#region ../content-gateway/dist/main-B1G1VdMp.mjs
 /**
 * A connection could not be ESTABLISHED in time — retryable.
 *
@@ -448,6 +448,8 @@ const CHUNK_POLICY$1 = "heading-aware-1500-content-only-v5";
 * The Markdown BODY below the frontmatter is the authored agent-surface
 * instructions — byte-preserved, stripped only at the edges.
 */
+/** Mirrors `schema.ts`'s ceiling, so a bad `dim:` is refused when instance.md is
+*  PARSED rather than when the DDL is rendered. The why lives there. */
 const EMBED_DIM_MAX$1$1 = 2e3;
 const SUPPORTED_FORMATS$1$1 = [1];
 var InstanceParseError$1 = class extends Error {
@@ -541,7 +543,7 @@ const groupSchemas$1 = {
 	embedding: z.object({
 		provider: z.string().min(1).default("gemini"),
 		model: z.string().min(1).default(EMBED_MODEL$1),
-		dim: z.coerce.number().int().min(1).max(EMBED_DIM_MAX$1$1).default(EMBED_DIM$1)
+		dim: z.coerce.number().int().min(1).max(EMBED_DIM_MAX$1$1, { error: `at most ${EMBED_DIM_MAX$1$1}: this schema declares VECTOR columns and indexes one directly, and pgvector's HNSW takes a vector to ${EMBED_DIM_MAX$1$1}` }).default(EMBED_DIM$1)
 	}),
 	retrieval: z.object({
 		/**
@@ -1112,11 +1114,24 @@ g AS (
           WHERE tenant_id = $1 AND corpus_id = $2)
     ) AS gen
 )`;
-const ARM_WHERE$1 = `
+/**
+* The arm predicate, built for the parameter numbering of the query that uses
+* it — a FUNCTION, not a string the caller renumbers afterwards.
+*
+* It used to be derived with `ARM_WHERE.replaceAll("$5", "$4")`, which works
+* only while the predicate happens to contain exactly one placeholder and no
+* other text matching it. Adding any second parameter to the predicate breaks
+* every derived query silently — and when it breaks, the failure arrives as a
+* driver error that the serving layer correctly reduces to "content store
+* temporarily unavailable", which tells you nothing about the cause. Found by
+* tripping over it while trying a fix for issue #59; taking the number as an
+* argument makes the coupling visible instead of textual.
+*/
+const armWhere$1 = (kindsParam) => `
         c.tenant_id = $1 AND c.generation = g.gen
           AND c.embedding_status = 'embedded' AND ${SERVABLE$1}
           AND n.status = 'published'
-          AND ($5::text[] IS NULL OR n.kind = ANY($5::text[]))
+          AND (${kindsParam}::text[] IS NULL OR n.kind = ANY(${kindsParam}::text[]))
           AND ${DENY$1}
           AND ${AUDIENCE_ALLOWED$1}`;
 const JOINS$1 = `
@@ -1125,6 +1140,15 @@ const JOINS$1 = `
         JOIN sources s ON s.source_id = c.source_id AND s.tenant_id = c.tenant_id
                       AND s.generation = c.generation
         JOIN content_nodes n ON n.node_id = s.node_id AND n.tenant_id = s.tenant_id`;
+/**
+* Exported ONLY so a test can EXPLAIN the real thing.
+*
+* The index regression this fixes was announced as fixed once before, and came
+* back through a different clause, because nothing ever asserted the RESULT —
+* that the plan opens `idx_chunks_hnsw`. A timing threshold would be flaky and
+* would not have caught it either; the plan shape is the property that matters
+* (issue #59).
+*/
 const HYBRID_SQL$1 = `
 WITH RECURSIVE ${GEN_CTE$1}, ${DENIED_CTE$1},
     -- The top-k is taken by a PLAIN \`ORDER BY <distance> LIMIT\`, and the rank
@@ -1145,7 +1169,7 @@ WITH RECURSIVE ${GEN_CTE$1}, ${DENIED_CTE$1},
         FROM (
             SELECT c.chunk_id, g.gen, (c.embedding <=> $3::vector) AS dist
             ${JOINS$1}
-            WHERE ${ARM_WHERE$1}
+            WHERE ${armWhere$1("$5")}
             ORDER BY c.embedding <=> $3::vector, c.chunk_id
             LIMIT $6
         ) ranked),
@@ -1154,7 +1178,7 @@ WITH RECURSIVE ${GEN_CTE$1}, ${DENIED_CTE$1},
                row_number() OVER (ORDER BY ts_rank_cd(c.search_tsv,
                    websearch_to_tsquery($9::regconfig, $4)) DESC, c.chunk_id) AS r
         ${JOINS$1}
-        WHERE ${ARM_WHERE$1}
+        WHERE ${armWhere$1("$5")}
           AND c.search_tsv @@ websearch_to_tsquery($9::regconfig, $4)
         ORDER BY r LIMIT $6),
     fused AS (
@@ -1177,10 +1201,10 @@ WITH RECURSIVE ${GEN_CTE$1.replace("$8", "$6")}, ${DENIED_CTE$1}
            ts_rank_cd(c.search_tsv, websearch_to_tsquery($7::regconfig, $3)) AS score,
            g.gen, n.permalink
     ${JOINS$1}
-    WHERE ${ARM_WHERE$1.replaceAll("$5", "$4")}
+    WHERE ${armWhere$1("$4")}
       AND c.search_tsv @@ websearch_to_tsquery($7::regconfig, $3)
     ORDER BY score DESC, c.chunk_id LIMIT $5`;
-`${GEN_CTE$1.replace("$8", "$5")}${DENIED_CTE$1}${JOINS$1}${ARM_WHERE$1.replaceAll("$5", "$4")}`;
+`${GEN_CTE$1.replace("$8", "$5")}${DENIED_CTE$1}${JOINS$1}${armWhere$1("$4")}`;
 const HIT_COLUMNS = 9;
 /** Serialize a query vector as a pgvector literal. */
 function vectorLiteral$1(vector) {
@@ -2602,7 +2626,7 @@ async function search(ctx, query, k = 10) {
 			const r = await client.query("SELECT active_generation FROM corpora WHERE tenant_id = $1 AND corpus_id = $2", [inst.tenantId, inst.corpusId]);
 			return Number(r.rows[0]?.active_generation ?? 0) === 0;
 		}, audienceScope(ctx));
-		const reason = embedFailed && inst.abstain.vectorFloor !== null ? "unavailable" : unpublished ? "unpublished" : "abstained";
+		const reason = embedFailed ? "unavailable" : unpublished ? "unpublished" : "abstained";
 		return {
 			ok: false,
 			abstained: reason === "abstained",
@@ -3593,9 +3617,10 @@ function withoutSdkResponseModeWarning(body) {
 * it survivable (`buildAuth` refuses a non-loopback bind without auth, so the
 * only way to read this line is on a host that cannot be reached from outside).
 */
-function authPosture(mode, host) {
-	if (mode === "disabled") return `DISABLED — ${host} only, and a public bind will refuse to boot`;
-	return "bearer tokens, verified against the record's authorization server";
+function authPosture(mode, host, publicUnauthenticated) {
+	if (mode !== "disabled") return "bearer tokens, verified against the record's authorization server";
+	if (publicUnauthenticated) return `UNAUTHENTICATED and bound to ${host} — KSOR_ALLOW_PUBLIC_UNAUTHENTICATED=1 is set, so the whole record is served to anyone who can reach this port`;
+	return `DISABLED — ${host} only, and a public bind will refuse to boot`;
 }
 /**
 * What the record will refuse. `null` means no floor is declared and the gate
@@ -3723,6 +3748,54 @@ async function compose(instancePath, version) {
 	};
 }
 /**
+* The body of a deferred-boot refusal, and the one decision inside it: whose
+* message may go on the wire.
+*
+* Three AUTHORED failures carry a remedy written for the operator —
+* `SchemaVersionError`, `GovernanceGateError`, `TextSearchConfigMismatch`. Their
+* whole multi-line message is the point; a caller that receives only the first
+* line has been told a problem exists and not how to end it.
+*
+* Everything else reaching this catch is infrastructure — most of it straight
+* from `pg`, whose connection and authentication failures name the host, the
+* resolved address, the port and the database user. Those went out verbatim
+* under `data.detail`, because the catch treated every error alike.
+*
+* So the split is by TYPE, not by message inspection: a class we wrote is a
+* class whose text we control. A driver error is refused with its class named
+* and its text withheld — the operator finds the real message in the server's
+* own logs, which is where an infrastructure fault belongs.
+*/
+/**
+* Did WE write this error's text?
+*
+* Named classes only. Matching on message prose would put the decision back
+* inside the strings it is meant to police, and a reworded driver error would
+* quietly re-open the leak.
+*/
+function isAuthored(error) {
+	return error instanceof SchemaVersionError || error instanceof GovernanceGateError$1 || error instanceof TextSearchConfigMismatch;
+}
+function refusalBody(error) {
+	if (isAuthored(error)) return {
+		jsonrpc: "2.0",
+		error: {
+			code: -32001,
+			message: `this record cannot be served: ${error.message.split("\n")[0] ?? ""}`,
+			data: { detail: error.message }
+		},
+		id: null
+	};
+	return {
+		jsonrpc: "2.0",
+		error: {
+			code: -32001,
+			message: `this record cannot be served: the content store is unavailable (${error instanceof Error ? error.name : "Error"}). The reason is in this server's logs; it is withheld here because a driver error names the database host and user.`
+		},
+		id: null
+	};
+}
+/**
 * The MCP door: the SDK v2 HTTP entry (Request → Response, stateless)
 * behind Hono, serving the 2026-07-28 revision with 2025-era clients still
 * answered through the stateless fallback. Modern exchanges are buffered JSON;
@@ -3886,16 +3959,8 @@ async function runHttp(composition) {
 		if (verifyBoot !== null) try {
 			await verifyBoot();
 		} catch (error) {
-			const message = error instanceof Error ? error.message : String(error);
-			return new Response(JSON.stringify({
-				jsonrpc: "2.0",
-				error: {
-					code: -32001,
-					message: `this record cannot be served: ${message.split("\n")[0]}`,
-					data: { detail: message }
-				},
-				id: null
-			}), {
+			console.error(`refusing requests — boot checks failing: ${error instanceof Error ? error.stack ?? error.message : String(error)}`);
+			return new Response(JSON.stringify(refusalBody(error)), {
 				status: 503,
 				headers: { "content-type": "application/json" }
 			});
@@ -3939,7 +4004,7 @@ async function runHttp(composition) {
 				identity = await auth.verify(token);
 			} catch (error) {
 				const transient = error instanceof TokenVerifyError && error.transient;
-				return c.json({ error: transient ? "token verification temporarily unavailable" : "invalid token" }, transient ? 503 : 401);
+				return c.json({ error: transient ? "token verification temporarily unavailable" : "invalid token" }, transient ? 503 : 401, transient ? {} : { "www-authenticate": `Bearer error="invalid_token", resource_metadata="${resourceMetadataUrl}"` });
 			}
 			bearer = token;
 		}
@@ -3979,7 +4044,7 @@ async function runHttp(composition) {
 		s.once("error", reject);
 	});
 	if (recordIsUndescribed(instance.instructions)) console.error(bootLine("identity", UNDESCRIBED_RECORD));
-	console.error(bootLine("auth", authPosture(auth.mode, bind.host)));
+	console.error(bootLine("auth", authPosture(auth.mode, bind.host, process.env["KSOR_ALLOW_PUBLIC_UNAUTHENTICATED"] === "1" && !loopback)));
 	for (const line of keyLines) console.error(line);
 	console.error(bootLine("abstain", abstainPosture(instance.abstain.vectorFloor)));
 	console.error(bootLine("serving", `http://${bind.host}:${bind.port}/mcp`));
@@ -4389,7 +4454,7 @@ async function withPgRetry(op, options = {}) {
 	throw lastError;
 }
 //#endregion
-//#region ../content/dist/commands-ulU-h9ei.mjs
+//#region ../content/dist/commands-LOfF_iLT.mjs
 /**
 * EVAL-LOCKED constants, quarried verbatim from the oracle
 * (sor-agentfactory @ b554f91, config.py) — changing any of these is a
@@ -4425,6 +4490,8 @@ const HARD_MAX_CHARS = 4e3;
 * The Markdown BODY below the frontmatter is the authored agent-surface
 * instructions — byte-preserved, stripped only at the edges.
 */
+/** Mirrors `schema.ts`'s ceiling, so a bad `dim:` is refused when instance.md is
+*  PARSED rather than when the DDL is rendered. The why lives there. */
 const EMBED_DIM_MAX$1 = 2e3;
 const SUPPORTED_FORMATS$1 = [1];
 var InstanceParseError = class extends Error {
@@ -4518,7 +4585,7 @@ const groupSchemas = {
 	embedding: z.object({
 		provider: z.string().min(1).default("gemini"),
 		model: z.string().min(1).default(EMBED_MODEL),
-		dim: z.coerce.number().int().min(1).max(EMBED_DIM_MAX$1).default(EMBED_DIM)
+		dim: z.coerce.number().int().min(1).max(EMBED_DIM_MAX$1, { error: `at most ${EMBED_DIM_MAX$1}: this schema declares VECTOR columns and indexes one directly, and pgvector's HNSW takes a vector to ${EMBED_DIM_MAX$1}` }).default(EMBED_DIM)
 	}),
 	retrieval: z.object({
 		/**
@@ -4942,7 +5009,29 @@ async function runIngest(pool, tenantId, op) {
 * fresh DDL is rendered from the instance that will fill it, never
 * hand-edited.
 */
-/** pgvector vector + HNSW ceiling. */
+/**
+* The largest embedding dimension this schema will render.
+*
+* It is the ceiling for the shape we USE, not pgvector's ceiling: `schema.sql`
+* declares two `VECTOR(dim)` columns and indexes one of them directly, and
+* pgvector's HNSW and IVFFlat take a `vector` to 2000. They take a `halfvec` to
+* **4000**, reachable by indexing an expression — `hnsw ((embedding::halfvec(N))
+* halfvec_cosine_ops)` — which we do not do, so 2000 binds here.
+*
+* Said precisely because the old wording ("pgvector vector + HNSW ceiling")
+* read as pgvector's own limit and sent a reader off to change providers over a
+* wall that is not one (verified live against a real database, 2026-08-21:
+* a halfvec(3072) expression index plans an Index Scan).
+*
+* Raising it is a decision, not a constant: every query site would have to use
+* the same cast as the index or fall silently back to a sequential scan, and
+* the halfvec arm's float16 rounding lands on the score the abstention gate
+* reads. Recorded in issue #49, along with the evidence for staying at 1536 —
+* Google's published MTEB table runs 128..2048 and is FLAT at the top of that
+* range (1536 scores 68.17, 2048 scores 68.16), so there is no gradient to
+* climb toward the ceiling. It carries no 3072 row, so the cost of the
+* truncation itself is unpublished; do not infer one.
+*/
 const EMBED_DIM_MAX = 2e3;
 /** The schema version schema.sql declares — parsed from the DDL so code and
 * the applied database share ONE source (a drift test pins the coupling). */
@@ -4989,7 +5078,7 @@ function verifyTemplate(text, dim) {
 }
 /** The pure core: render the given template text at the given dimension. */
 function renderSchemaText(text, dim, textSearchConfig = SHIPPED_TS_CONFIG) {
-	if (!Number.isInteger(dim) || dim < 1 || dim > 2e3) throw new Error(`dim must be an integer in 1..${EMBED_DIM_MAX} (pgvector vector + HNSW ceiling), got ${JSON.stringify(dim)}`);
+	if (!Number.isInteger(dim) || dim < 1 || dim > 2e3) throw new Error(`dim must be an integer in 1..${EMBED_DIM_MAX} — this schema indexes a vector column directly, and pgvector's HNSW takes a vector to 2000 — got ${JSON.stringify(dim)}`);
 	verifyTemplate(text, EMBED_DIM);
 	const withTs = renderTsConfig(text, textSearchConfig);
 	if (dim === 1536) return withTs;
@@ -5127,11 +5216,24 @@ g AS (
           WHERE tenant_id = $1 AND corpus_id = $2)
     ) AS gen
 )`;
-const ARM_WHERE = `
+/**
+* The arm predicate, built for the parameter numbering of the query that uses
+* it — a FUNCTION, not a string the caller renumbers afterwards.
+*
+* It used to be derived with `ARM_WHERE.replaceAll("$5", "$4")`, which works
+* only while the predicate happens to contain exactly one placeholder and no
+* other text matching it. Adding any second parameter to the predicate breaks
+* every derived query silently — and when it breaks, the failure arrives as a
+* driver error that the serving layer correctly reduces to "content store
+* temporarily unavailable", which tells you nothing about the cause. Found by
+* tripping over it while trying a fix for issue #59; taking the number as an
+* argument makes the coupling visible instead of textual.
+*/
+const armWhere = (kindsParam) => `
         c.tenant_id = $1 AND c.generation = g.gen
           AND c.embedding_status = 'embedded' AND ${SERVABLE}
           AND n.status = 'published'
-          AND ($5::text[] IS NULL OR n.kind = ANY($5::text[]))
+          AND (${kindsParam}::text[] IS NULL OR n.kind = ANY(${kindsParam}::text[]))
           AND ${DENY}
           AND ${AUDIENCE_ALLOWED}`;
 const JOINS = `
@@ -5140,14 +5242,14 @@ const JOINS = `
         JOIN sources s ON s.source_id = c.source_id AND s.tenant_id = c.tenant_id
                       AND s.generation = c.generation
         JOIN content_nodes n ON n.node_id = s.node_id AND n.tenant_id = s.tenant_id`;
-`${GEN_CTE}${DENIED_CTE}${JOINS}${ARM_WHERE}${JOINS}${ARM_WHERE}`;
-`${GEN_CTE.replace("$8", "$6")}${DENIED_CTE}${JOINS}${ARM_WHERE.replaceAll("$5", "$4")}`;
+`${GEN_CTE}${DENIED_CTE}${JOINS}${armWhere("$5")}${JOINS}${armWhere("$5")}`;
+`${GEN_CTE.replace("$8", "$6")}${DENIED_CTE}${JOINS}${armWhere("$4")}`;
 /** The calibrator's standalone top-1 signal (the read path gets it free from HYBRID_SQL). */
 const TOP_ONE_SQL = `
 WITH RECURSIVE ${GEN_CTE.replace("$8", "$5")}, ${DENIED_CTE}
     SELECT 1 - (c.embedding <=> $3::vector) AS score
     ${JOINS}
-    WHERE ${ARM_WHERE.replaceAll("$5", "$4")}
+    WHERE ${armWhere("$4")}
     ORDER BY c.embedding <=> $3::vector, c.chunk_id
     LIMIT 1`;
 /** Serialize a query vector as a pgvector literal. */
@@ -5729,6 +5831,29 @@ const BUILT_IN_OOC = [
 * measurement had an easier question set than production would.
 */
 const SYNTHESIZED_CAVEAT = "CAVEAT: synthesized queries are written FROM the passages they are then scored against, so they share vocabulary a reader's question will not. This door measures an UPPER BOUND on separation — treat the floor below as provisional until it has been checked against questions the corpus did not write (--queries-file), and re-run if real questions score under it.";
+/**
+* What a separability verdict is worth when the probes came from the binary.
+*
+* Every entry in BUILT_IN_OOC is far-domain — dinner, taxes, football, boiling
+* an egg. Those score low against ANY corpus, so max-OOC comes out artificially
+* low and the margin is inflated from that end, exactly as synthesized in-corpus
+* queries inflate it from the other. Measured on one record, changing ONLY the
+* probe set: built-ins reported "separable, margin 0.072" and recommended a
+* floor; eight scope-adjacent near-misses on the same corpus and the same
+* in-corpus questions reported "NOT separable, margin -0.030". The recommended
+* floor then answered six of those eight near-misses live, with citations
+* (2026-08-21).
+*
+* The tool already knows this — the not-separable branch tells the operator to
+* "widen the probe set (scope-adjacent near-misses, not only far-domain
+* questions)". It said so only AFTER weak probes had failed to bless a floor,
+* which is the one case where the advice is least needed.
+*
+* A shipped set cannot be scope-adjacent, because adjacency depends on a corpus
+* the binary has never seen. So this is stated whenever built-ins are used, on
+* BOTH branches, rather than pretending a better default exists.
+*/
+const BUILT_IN_OOC_CAVEAT = "CAVEAT: the out-of-corpus probes are the BUILT-IN set, which is entirely far-domain — a shipped set cannot be scope-adjacent, because adjacency depends on a corpus it has never seen. Far-domain probes score low against anything, so this margin is an OVER-estimate and a floor it blesses may still answer near-misses just outside your scope. Re-run with --ooc-file naming questions a reader might plausibly ask that this record does NOT cover, and trust that verdict over this one.";
 const QUERIES_FILE_CAVEAT = "CAVEAT: --queries-file floors are measured on human/gold-derived queries — section-weighted eval targets, NOT per-node passage samples — so this floor's low tail is a different distribution than the synthesized door's; record 'door: queries-file' beside the number and never compare the two doors' floors as interchangeable.";
 /**
 * The report dict, assembled from every scored query. `in_corpus_queries` /
@@ -5770,6 +5895,7 @@ function buildReport(detail, meta, targetPrecision = .95, now = /* @__PURE__ */
 		paste_why,
 		margin: pythonRound(marginOf(points), 4),
 		separable,
+		ooc_source: meta.oocSource,
 		target: rec.target,
 		measured_at: now.toISOString().slice(0, 10),
 		low_tail,
@@ -5790,6 +5916,7 @@ function renderReport(report) {
 	const gen = report.generation === null ? "unknown (no generation pinned)" : String(report.generation);
 	lines.push(`\nmeasured on generation ${gen} (${how}), model ${report.model}, door: ${report.door}`);
 	lines.push(report.door === "queries-file" ? QUERIES_FILE_CAVEAT : SYNTHESIZED_CAVEAT);
+	if (report.ooc_source === "built-in") lines.push(BUILT_IN_OOC_CAVEAT);
 	lines.push(`AURC = ${pythonFloatRepr(report.aurc)}  (lower = better separation)`);
 	lines.push(`separation margin: ${pythonFormatFixed(report.margin, 3)} (over ${report.in_corpus_queries} in-corpus / ${report.ooc_probes} out-of-corpus probes)`);
 	if (z) lines.push(`zero-FA floor (never refuse a real question): ${pythonFormatFixed(z.floor, 3)} -> coverage ${pythonFormatFixed(z.coverage, 3)}, ooc leak ${pythonFormatFixed(z.risk, 3)}`);
@@ -6044,13 +6171,15 @@ async function runCalibration(pool, options) {
 		}
 		inQueries = normalizeQueries(synthesized);
 	}
+	const oocSource = options.oocProbes === void 0 ? "built-in" : "provided";
 	const ooc = normalizeQueries(options.oocProbes ?? BUILT_IN_OOC);
 	return buildReport([...await scoreQueries(pool, scope, options.provider, inQueries, true), ...await scoreQueries(pool, scope, options.provider, ooc, false)], {
 		generation: counted.measured,
 		pinned: generation !== null,
 		model: options.provider.modelId,
 		dim: options.provider.dim,
-		door
+		door,
+		oocSource
 	}, options.targetPrecision ?? .95);
 }
 /**
@@ -7996,6 +8125,15 @@ async function drain(pending, io) {
 */
 const CHUNK_INSERT_PREFIX = "INSERT INTO chunks (tenant_id, generation, source_id, ordinal, content, chunk_hash, heading_path, heading_path_text, anchor, labels, embedding_status) VALUES ";
 const CHUNK_PARAMS = 10;
+/**
+* Dense character count — whitespace removed, exactly as the serving
+* predicate's `length(regexp_replace(c.content, '\s', '', 'g'))` computes it.
+* Written here rather than approximated so the ingest report and the SQL admit
+* the same chunks.
+*/
+function denseLength(content) {
+	return content.replace(/\s/g, "").length;
+}
 /** 500 rows × 10 params stays far under Postgres's 65535 bind-parameter cap. */
 const CHUNK_ROWS_PER_STATEMENT = 500;
 /**
@@ -8040,6 +8178,8 @@ async function buildStructure(client, opts) {
 	const treeRoot = resolve(opts.treeRoot);
 	let nSources = 0;
 	let nChunks = 0;
+	let nUnsearchable = 0;
+	const unsearchableSources = [];
 	const chunkRows = [];
 	for (const f of manifest.files) {
 		const src = opts.files.get(f.path);
@@ -8063,6 +8203,7 @@ async function buildStructure(client, opts) {
 			manifest.source_commit
 		]);
 		nSources += 1;
+		let sourceServable = 0;
 		for (const chunk of chunkText(body)) {
 			chunkRows.push([
 				tenantId,
@@ -8077,7 +8218,10 @@ async function buildStructure(client, opts) {
 				JSON.stringify({ source_type: chunk.sourceType })
 			]);
 			nChunks += 1;
+			if (chunk.sourceType === "prose" && denseLength(chunk.content) >= 24) sourceServable += 1;
+			else nUnsearchable += 1;
 		}
+		if (sourceServable === 0 && nChunks > 0) unsearchableSources.push(sid);
 	}
 	for (let i = 0; i < chunkRows.length; i += CHUNK_ROWS_PER_STATEMENT) {
 		const slice = chunkRows.slice(i, i + CHUNK_ROWS_PER_STATEMENT);
@@ -8116,7 +8260,9 @@ async function buildStructure(client, opts) {
 		sources: nSources,
 		chunks: nChunks,
 		carried,
-		pending: health.pending
+		pending: health.pending,
+		unsearchable: nUnsearchable,
+		unsearchableSources
 	};
 }
 /**
@@ -8299,6 +8445,8 @@ async function buildGeneration(pool, instance, options) {
 				ready: true,
 				centroids: 0,
 				flipped: false,
+				unsearchable: 0,
+				unsearchableSources: [],
 				refusal: null,
 				health: {
 					ok: true,
@@ -8403,6 +8551,8 @@ async function buildGeneration(pool, instance, options) {
 		ready: fin.ready,
 		centroids: fin.centroids,
 		flipped: fin.flipped,
+		unsearchable: stats.unsearchable,
+		unsearchableSources: stats.unsearchableSources,
 		refusal: fin.refusal,
 		health: fin.health,
 		unchanged: false
@@ -8742,6 +8892,16 @@ async function ingestCommand(args) {
 	}
 	process.stdout.write(sourceCommit === "unspecified" ? "source: unspecified — knowledge/ is not in a git repository, so this generation cannot be traced back to a reviewed commit\n" : `source: ${sourceCommit}\n`);
 	process.stdout.write(`ingest: generation ${report.generation} — ${report.nodes} nodes, ${report.chunks} chunks; embedded ${report.embedded}, carried ${report.carried}, failed ${report.failed}\n`);
+	if (report.unsearchable > 0) {
+		const pct = Math.round(report.unsearchable / Math.max(report.chunks, 1) * 100);
+		process.stdout.write(`  not searchable: ${report.unsearchable} of ${report.chunks} chunk(s) (${pct}%) are shorter than the navigation threshold — stored and readable, but no search returns them\n`);
+		if (report.unsearchableSources.length > 0) {
+			const named = report.unsearchableSources.slice(0, 10).join(", ");
+			const more = report.unsearchableSources.length - Math.min(10, report.unsearchableSources.length);
+			process.stdout.write(`  FOUND ONLY BY NAME: ${named}${more > 0 ? `, and ${more} more` : ""} — no searchable chunk at all; lengthen these sections or read them by slug
+`);
+		}
+	}
 	if (report.refusal !== null) return fail$1(REFUSED, report.refusal);
 	const governance = await withPool(dsn, (pool) => assertGovernanceServable(pool, instance, report.generation).then(() => null, (error) => error instanceof Error ? error.message : String(error)));
 	if (governance !== null) return fail$1(REFUSED, `generation ${report.generation} was built and NOT activated — no surface could serve it\n  ${governance.split("\n").join("\n  ")}\n  note: generation ${report.generation} is left behind, un-activated; \`ksor gc\` reaps it once the grace window passes. The previously active generation still serves.`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@panaversity/ksor",
-  "version": "0.0.11",
+  "version": "0.0.13",
   "description": "Knowledge System of Record — compile governed markdown into a static site for people and an MCP server for AI agents, with citations and measured abstention.",
   "keywords": [
     "abstention",

package/templates/scaffold/AGENTS.md

@@ -65,8 +65,13 @@ Stand it up in this order (each step's errors explain how to fix themselves):
    `provider: gemini`, `model: gemini-embedding-001`, `dim: 1536`; write it out
    only to pin the space explicitly or to change it — and note that model and
    dim are the PERSISTED identity of the embedding space, so changing either
-   later means re-embedding the whole corpus. Keep `dim` at or below 2000: the
-   pgvector HNSW index refuses more, and `gemini-embedding-001` can emit 3072.
+   later means re-embedding the whole corpus. Keep `dim` at or below 2000 — the
+   schema indexes a `vector` column directly and pgvector's HNSW takes a
+   `vector` to 2000. `gemini-embedding-001` emits 3072 by default, so ksor asks
+   it for 1536. Google's published MTEB table runs 128–2048 and is flat at the
+   top of it — 1536 scores 68.17 against 2048's 68.16 — so there is no gradient
+   to climb toward the ceiling; going the other way, 768 costs 0.18 if you want
+   the storage back.
 
    Leave `retrieval:` out for now — the gate is off and the server says so.
    Turning it on is step 4, AFTER the record is serving.
@@ -227,7 +232,7 @@ bind, set `KSOR_ALLOWED_HOSTS` / `KSOR_ALLOWED_ORIGINS`; on more than one
 replica, set a shared `KSOR_SNAPSHOT_KEYS` (unset ⇒ a per-process key, so a
 search token minted by one replica fails on another).
 
-Two things worth being deliberate about:
+Three things worth being deliberate about:
 
 - **`KSOR_ALLOW_PUBLIC_UNAUTHENTICATED=1` serves your whole record to anyone
   who can reach the port.** It exists for deployments fronted by your own
@@ -244,6 +249,62 @@ Two things worth being deliberate about:
   `KSOR_JWKS_URL` only to override that, or when your SSO publishes no metadata
   at all.
 
+### What a CLIENT has to do
+
+Once the SSO door is configured (the three variables above), the server is an
+OAuth **Resource Server**, which means a client is not told the authorization
+server — it discovers it. Nothing here needs configuring beyond those variables;
+this is what your agents will experience, and what to check when one cannot
+connect. With `KSOR_AUTH_DISABLED=1` — the local default `.env.example` ships —
+none of it applies: there is no challenge and the metadata document answers 404,
+because there is no authorization server to point at.
+
+1. The client calls `POST /mcp` with no token and gets **401** carrying
+
+   ```
+   WWW-Authenticate: Bearer resource_metadata="https://<your-host>/.well-known/oauth-protected-resource/mcp"
+   ```
+
+   That header is the whole handshake: it names a DOCUMENT, not the resource.
+
+2. The client fetches that document and finds the record's resource identifier
+   and its authorization server:
+
+   ```json
+   {
+     "resource": "https://<your-host>/mcp",
+     "authorization_servers": ["https://your-sso.example.com"]
+   }
+   ```
+
+   Those two values are `KSOR_MCP_RESOURCE_URL` and `KSOR_SSO_URL`.
+
+3. The client gets a token from that authorization server, asking for THIS
+   record as the resource (RFC 8707: `resource=https://<your-host>/mcp`), and
+   sends it as `Authorization: Bearer <token>`.
+
+Every 401 carries that same header, not just the first one — including the one
+your clients will hit most often, a token that expired mid-conversation. It
+arrives as `Bearer error="invalid_token", resource_metadata="…"`, so a client
+knows to refresh rather than to retry the dead token. A **503** is deliberately
+_not_ challenged: an unreachable key set is our outage, not your token's fault,
+and telling a client to re-authenticate over it would send a perfectly good user
+back through a login.
+
+The one thing that goes wrong here goes wrong quietly: a token minted for a
+different audience is a perfectly valid token, and this door rejects it. The
+`aud` claim must match one of `KSOR_JWT_ALLOWED_AUDIENCES` — normally the same
+value as `KSOR_MCP_RESOURCE_URL` — because a bearer accepted for any audience is
+a bearer stolen from one service and replayed against this one. If a client
+authenticates fine and still gets 401, compare its token's `aud` against that
+list before looking anywhere else.
+
+Tokens must be signed **RS256**; nothing else is accepted, and opaque tokens are
+not supported (there is no introspection call). When your SSO rotates its
+signing keys, an unknown key id answers **503**, not 401 — the token may well be
+good and the door's key set merely stale, so a client should retry rather than
+send the user back through a login.
+
 ## Withdrawing a document — `ksor takedown`
 
 A takedown is the one governance act that must reach EVERY surface at once.

package/templates/scaffold/system/site/lib/shared.ts

@@ -89,12 +89,38 @@ function readInstanceScope(): string | null {
   for (const para of afterHeading.split(/\n[ \t]*\n/)) {
     const one = para.trim().replace(/\s+/g, " ");
     if (one === "" || one.startsWith("#") || one.startsWith("-") || one.startsWith(">")) continue;
-    const sentence = /^(.+?[.!?])(\s|$)/.exec(one)?.[1] ?? one;
-    return sentence.length > 300 ? `${sentence.slice(0, 297)}...` : sentence;
+    return sentence(one);
   }
   return null;
 }
 
+/**
+ * The MCP registry schema caps `ServerDetail.description` at **100 characters**
+ * (`ServerDetail.description.maxLength`, 2025-12-11). Over it, the document a
+ * validating client reads is invalid — and the failure was shaped exactly
+ * wrong: the unfilled placeholder is 88 characters and validates, so the
+ * document became invalid the moment an owner did the thing the scaffold asks
+ * for and wrote a real scope sentence. Silent, and only for real records.
+ *
+ * A hard truncation would publish a sentence cut mid-word, so this trims at a
+ * word boundary and marks it, and the whole description is assembled inside the
+ * budget rather than clipped after the fact.
+ */
+const DESCRIPTION_MAX = 100;
+
+function sentence(one: string): string {
+  const first = /^(.+?[.!?])(\s|$)/.exec(one)?.[1] ?? one;
+  return first;
+}
+
+/** Fit `text` inside `max`, breaking on a word rather than mid-word. */
+function fit(text: string, max: number): string {
+  if (text.length <= max) return text;
+  const cut = text.slice(0, max - 1);
+  const at = cut.lastIndexOf(" ");
+  return `${(at > max / 2 ? cut.slice(0, at) : cut).replace(/[,;:.\s]+$/, "")}\u2026`;
+}
+
 /** null until the owner has written one — never a guess. */
 export const appScope: string | null = readInstanceScope();
 
@@ -103,9 +129,11 @@ export const appScope: string | null = readInstanceScope();
  * registry document and anything else that needs one cannot drift apart.
  */
 export function recordDescription(): string {
-  return appScope === null
-    ? `${appTitle} — its owner has not yet described what this record covers.`
-    : `${appTitle} — ${appScope}`;
+  const whole =
+    appScope === null
+      ? `${appTitle} — its owner has not yet described what this record covers.`
+      : `${appTitle} — ${appScope}`;
+  return fit(whole, DESCRIPTION_MAX);
 }
 
 /**