@panaversity/ksor 0.0.16 → 0.0.18

package/CHANGELOG.md

@@ -1,5 +1,77 @@
 # @panaversity/ksor
 
+## 0.0.18
+
+### Patch Changes
+
+- ea049fd: A takedown can no longer stop applying without saying so
+
+  Two ways a recorded withdrawal quietly stopped covering what it was recorded to
+  cover. Both were found by attacking the door before exposing it publicly, and
+  both were reproduced end to end against a real database.
+
+  **A denial matched nothing after the document moved.** `takedown_denylist`
+  records a `stable_id`, and the serving predicate matches those rows against the
+  documents in the generation being served — so an id that no longer exists denies
+  nothing. The default stable_id is derived from the file's path, which means an
+  ordinary rename or move of a withdrawn document was enough: search, read,
+  outline and the site all served it again, with no error anywhere. Adding an
+  `index.md` beside a withdrawn section did the same, by changing the section's id.
+
+  Serving now refuses in that state, and so does the ingest that would create it —
+  the same check at both ends, so a generation where a withdrawal has stopped
+  applying cannot be published _or_ served:
+
+  ```
+  2 takedown(s) match no document in generation 7: knowledge/legal/notice.md, …
+    why: … an id that no longer exists denies NOTHING — so a withdrawn document
+    that was renamed, moved, or had an index.md added beside it is served again
+    fix: point the denial at where the document lives now, or retire it
+    deliberately — never guess which one, because the tool cannot tell a rename
+    from a deletion
+  ```
+
+  Refusing rather than re-pointing automatically is the whole point: a tool that
+  guessed would eventually guess that a withdrawn document had been deleted when
+  it had been renamed.
+
+  **A withdrawn section did not cover its own directory.** When a section has no
+  `index.md` and its documents all live one level further down, it had no file to
+  name its own directory, so only the subdirectory was exported to the site. A
+  document written directly under the withdrawn section published to `/docs` and
+  `llms.txt` in the window before the next ingest. The section's directory is now
+  derived from its own identity, which for an index-less section is its path.
+
+## 0.0.17
+
+### Patch Changes
+
+- 44feada: Installing ksor no longer pulls 32 MB of vendor SDK
+
+  `npx @panaversity/ksor init` installed 54 MB across 52 packages. 32 MB of that
+  was `@google/genai` and its dependencies — carried by every adopter, including
+  the ones who only ever run `init` and `dev` and never reach a served rung.
+
+  It existed to make two HTTP calls, both already wrapped behind one
+  structurally-typed client boundary. Those calls are now spoken directly:
+
+  ```
+  before   54 MB   52 packages
+  after    22 MB   22 packages
+  ```
+
+  Nothing about the embedding changed, and that was checked first rather than
+  assumed: the SDK and the REST endpoint return **byte-identical vectors** for the
+  same text, model, dimensionality and task type — a maximum per-component
+  difference of 0.000e+0 at 1536 dimensions. So stored embeddings stay valid and a
+  calibrated `vector_floor` keeps its meaning. Had they differed by a rounding
+  step, this would have quietly invalidated abstention on every existing record.
+
+  The provider seam is unchanged: a deployment that prefers an SDK can still
+  supply one through `clientFactory`. The single live call to the real vendor
+  stays where it was, as the tripwire for API drift, and now meets Gemini with
+  nothing in between.
+
 ## 0.0.16
 
 ### Patch Changes

package/dist/cli.mjs

@@ -7,7 +7,6 @@ import { z } from "zod";
 import pg from "pg";
 import path, { basename, dirname, join, resolve, sep } from "node:path";
 import { createHash, createHmac, randomBytes, timingSafeEqual } from "node:crypto";
-import { GoogleGenAI } from "@google/genai";
 import { AsyncLocalStorage } from "node:async_hooks";
 import { createRemoteJWKSet, errors, jwtVerify } from "jose";
 import { serve } from "@hono/node-server";
@@ -16,7 +15,7 @@ import { bodyLimit } from "hono/body-limit";
 import { execFileSync, spawnSync } from "node:child_process";
 import { parseArgs } from "node:util";
 import { readFile, readdir, stat } from "node:fs/promises";
-//#region ../content-gateway/dist/main-BFWvQ6vY.mjs
+//#region ../content-gateway/dist/main-Deg5kd9y.mjs
 /**
 * A connection could not be ESTABLISHED in time — retryable.
 *
@@ -1526,8 +1525,58 @@ var FakeEmbeddingProvider$1 = class {
 	}
 	reset() {}
 };
+const DEFAULT_BASE$1 = "https://generativelanguage.googleapis.com/v1beta";
 /**
-* The Gemini transport — the ONE place `@google/genai` is imported (converted
+* An HTTP-shaped failure carrying the status the retry classifier reads.
+*
+* `isRetryable` in `gemini.ts` asks for a numeric `status` and nothing else, by
+* design — it was written to survive SDK refactors. This keeps that contract
+* when the SDK is gone.
+*/
+var GeminiHttpError$1 = class extends Error {
+	status;
+	constructor(status, detail) {
+		super(`Gemini API error ${status}: ${detail}`);
+		this.name = "GeminiHttpError";
+		this.status = status;
+	}
+};
+/** One POST, with the key in a HEADER — never the query string, which is logged. */
+async function post$1(opts, apiKey, path, body, timeoutMs) {
+	const res = await (opts.fetchImpl ?? fetch)(`${opts.baseUrl ?? DEFAULT_BASE$1}${path}`, {
+		method: "POST",
+		headers: {
+			"x-goog-api-key": apiKey,
+			"content-type": "application/json"
+		},
+		body: JSON.stringify(body),
+		signal: AbortSignal.timeout(timeoutMs)
+	});
+	const text = await res.text();
+	if (!res.ok) {
+		let detail = text.slice(0, 300);
+		try {
+			const message = JSON.parse(text).error?.message;
+			if (typeof message === "string") detail = message;
+		} catch {}
+		throw new GeminiHttpError$1(res.status, detail);
+	}
+	return JSON.parse(text);
+}
+/** The embedding half of the slice, spoken over `batchEmbedContents`. */
+function geminiRestEmbedClient$1(apiKey, opts = {}) {
+	return { models: { async embedContent(params) {
+		const payload = { requests: params.contents.map((text) => ({
+			model: `models/${params.model}`,
+			content: { parts: [{ text }] },
+			taskType: params.config.taskType,
+			outputDimensionality: params.config.outputDimensionality
+		})) };
+		return { embeddings: (await post$1(opts, apiKey, `/models/${params.model}:batchEmbedContents`, payload, params.config.httpOptions.timeout)).embeddings ?? [] };
+	} } };
+}
+/**
+* The Gemini transport — the ONE place the vendor is spoken to (converted
 * from the oracle's sor_content/lib/providers/gemini.py; decision 6).
 * Identity (model, dim, task labels) is CONSTRUCTOR-INJECTED — this module
 * never imports config, so the same adapter serves any Gemini embedding
@@ -1547,7 +1596,7 @@ var FakeEmbeddingProvider$1 = class {
 *   clock. (The oracle's one divergence — a sync query-intent embed keeping
 *   the batch clock, an eval-harness case — has no TS call site.)
 * - The oracle's "has been closed" stale-client RuntimeError predicate is a
-*   Python-SDK failure mode with no @google/genai JS equivalent; `reset()`
+*   Python-SDK failure mode with no JS equivalent; `reset()`
 *   keeps its drop-never-close contract regardless.
 */
 function httpStatusOf$1(exc) {
@@ -1601,7 +1650,7 @@ var GeminiEmbeddingProvider$1 = class {
 		this.queryTaskLabel = opts.queryTaskLabel;
 		this.documentTimeoutMs = Math.trunc(opts.documentTimeoutS * 1e3);
 		this.queryTimeoutMs = Math.trunc(opts.queryTimeoutS * 1e3);
-		this.clientFactory = opts.clientFactory ?? (() => new GoogleGenAI({ apiKey: opts.apiKey }));
+		this.clientFactory = opts.clientFactory ?? (() => geminiRestEmbedClient$1(opts.apiKey));
 	}
 	get recipe() {
 		return `${this.modelId}/d${this.dim}/${this.documentTaskLabel}`;
@@ -1781,7 +1830,8 @@ async function assertGovernanceServable$1(pool, instance, targetGeneration) {
 		if (generation === 0) return {
 			generation,
 			builtAt: null,
-			restricted: 0
+			restricted: 0,
+			orphaned: []
 		};
 		return {
 			generation,
@@ -1790,13 +1840,27 @@ async function assertGovernanceServable$1(pool, instance, targetGeneration) {
 				instance.corpusId,
 				generation
 			])).rows[0]?.schema_version ?? null,
-			restricted: declaresModel ? 0 : Number((await client.query("SELECT count(*)::int AS n FROM content_nodes WHERE tenant_id = $1 AND generation = $2 AND visibility IS NOT NULL", [instance.tenantId, generation])).rows[0].n)
+			restricted: declaresModel ? 0 : Number((await client.query("SELECT count(*)::int AS n FROM content_nodes WHERE tenant_id = $1 AND generation = $2 AND visibility IS NOT NULL", [instance.tenantId, generation])).rows[0].n),
+			orphaned: (await client.query("SELECT d.stable_id FROM takedown_denylist d WHERE d.tenant_id = $1 AND d.corpus_id = $2   AND NOT EXISTS (SELECT 1 FROM content_nodes n                    WHERE n.tenant_id = d.tenant_id AND n.corpus_id = d.corpus_id                      AND n.generation = $3 AND n.stable_id = d.stable_id) ORDER BY d.stable_id", [
+				instance.tenantId,
+				instance.corpusId,
+				generation
+			])).rows.map((r) => r.stable_id)
 		};
 	});
 	if (state.generation === 0) return;
 	if (declaresModel && (state.builtAt === null || compareSchemaVersion$1(state.builtAt, "2.2") < 0)) throw new GovernanceGateError$1(`generation ${state.generation} was built against schema ${state.builtAt ?? "(before 2.4, which is when a generation started recording this)"}, older than 2.2 — the version that put visibility on the node row\n  why: instance.md declares an audience model, but the documents in this generation carry no visibility at all. Every one of them would be served at default_visibility — the WIDEST tier — including any document whose frontmatter restricts it
   fix: rebuild the record so its governance reaches the database:
     ksor ingest --instance instance.md --knowledge knowledge --flip`);
+	if (state.orphaned.length > 0) {
+		const named = state.orphaned.slice(0, 5).join(", ");
+		const more = state.orphaned.length - Math.min(5, state.orphaned.length);
+		throw new GovernanceGateError$1(`${state.orphaned.length} takedown(s) match no document in generation ${state.generation}: ${named}${more > 0 ? `, and ${more} more` : ""}\n  why: a denial is recorded against a stable_id, and the serving predicate matches it against the documents in this generation. An id that no longer exists denies NOTHING — so a withdrawn document that was renamed, moved, or had an index.md added beside it is served again by search, read, outline and the site, with no error anywhere. The denial is meant to be immune to reorganization; this is the state where it is not
+  fix: point the denial at where the document lives now, or retire it deliberately — never guess which one, because the tool cannot tell a rename from a deletion:
+    ksor takedown --instance instance.md --stable-id <the new id> --reason <why> --actor <who>
+    ksor takedown --instance instance.md --revoke <the old id> --actor <who>
+  (ksor takedown --list shows what is recorded)`);
+	}
 	if (!declaresModel && state.restricted > 0) throw new GovernanceGateError$1(`${state.restricted} document(s) in generation ${state.generation} declare visibility:, but instance.md declares no audiences:
   why: an author restricted those documents and nothing would enforce it — this door would serve them in full to every caller, and the frontmatter key saying otherwise would be the only trace. The site refuses to BUILD in this exact state (ksor-visibility-without-audiences); the door must not serve in it
   fix: declare the model in instance.md (audiences: least-restricted first, plus default_visibility:), or remove the visibility: keys and re-ingest`);
@@ -4452,7 +4516,7 @@ async function withPgRetry(op, options = {}) {
 	throw lastError;
 }
 //#endregion
-//#region ../content/dist/commands-B3-NrLiw.mjs
+//#region ../content/dist/commands-DcPJJlNb.mjs
 /**
 * EVAL-LOCKED constants, quarried verbatim from the oracle
 * (sor-agentfactory @ b554f91, config.py) — changing any of these is a
@@ -5423,8 +5487,72 @@ var FakeEmbeddingProvider = class {
 	}
 	reset() {}
 };
+const DEFAULT_BASE = "https://generativelanguage.googleapis.com/v1beta";
 /**
-* The Gemini transport — the ONE place `@google/genai` is imported (converted
+* An HTTP-shaped failure carrying the status the retry classifier reads.
+*
+* `isRetryable` in `gemini.ts` asks for a numeric `status` and nothing else, by
+* design — it was written to survive SDK refactors. This keeps that contract
+* when the SDK is gone.
+*/
+var GeminiHttpError = class extends Error {
+	status;
+	constructor(status, detail) {
+		super(`Gemini API error ${status}: ${detail}`);
+		this.name = "GeminiHttpError";
+		this.status = status;
+	}
+};
+/** One POST, with the key in a HEADER — never the query string, which is logged. */
+async function post(opts, apiKey, path, body, timeoutMs) {
+	const res = await (opts.fetchImpl ?? fetch)(`${opts.baseUrl ?? DEFAULT_BASE}${path}`, {
+		method: "POST",
+		headers: {
+			"x-goog-api-key": apiKey,
+			"content-type": "application/json"
+		},
+		body: JSON.stringify(body),
+		signal: AbortSignal.timeout(timeoutMs)
+	});
+	const text = await res.text();
+	if (!res.ok) {
+		let detail = text.slice(0, 300);
+		try {
+			const message = JSON.parse(text).error?.message;
+			if (typeof message === "string") detail = message;
+		} catch {}
+		throw new GeminiHttpError(res.status, detail);
+	}
+	return JSON.parse(text);
+}
+/** The embedding half of the slice, spoken over `batchEmbedContents`. */
+function geminiRestEmbedClient(apiKey, opts = {}) {
+	return { models: { async embedContent(params) {
+		const payload = { requests: params.contents.map((text) => ({
+			model: `models/${params.model}`,
+			content: { parts: [{ text }] },
+			taskType: params.config.taskType,
+			outputDimensionality: params.config.outputDimensionality
+		})) };
+		return { embeddings: (await post(opts, apiKey, `/models/${params.model}:batchEmbedContents`, payload, params.config.httpOptions.timeout)).embeddings ?? [] };
+	} } };
+}
+/** The text half of the slice, spoken over `generateContent`. */
+function geminiRestTextClient(apiKey, opts = {}) {
+	return { models: { async generateContent(params) {
+		const payload = {
+			contents: [{ parts: [{ text: params.contents }] }],
+			generationConfig: {
+				temperature: params.config.temperature,
+				maxOutputTokens: params.config.maxOutputTokens,
+				thinkingConfig: params.config.thinkingConfig
+			}
+		};
+		return { text: ((await post(opts, apiKey, `/models/${params.model}:generateContent`, payload, 12e4)).candidates?.[0]?.content?.parts ?? []).map((p) => p.text ?? "").join("") };
+	} } };
+}
+/**
+* The Gemini transport — the ONE place the vendor is spoken to (converted
 * from the oracle's sor_content/lib/providers/gemini.py; decision 6).
 * Identity (model, dim, task labels) is CONSTRUCTOR-INJECTED — this module
 * never imports config, so the same adapter serves any Gemini embedding
@@ -5444,7 +5572,7 @@ var FakeEmbeddingProvider = class {
 *   clock. (The oracle's one divergence — a sync query-intent embed keeping
 *   the batch clock, an eval-harness case — has no TS call site.)
 * - The oracle's "has been closed" stale-client RuntimeError predicate is a
-*   Python-SDK failure mode with no @google/genai JS equivalent; `reset()`
+*   Python-SDK failure mode with no JS equivalent; `reset()`
 *   keeps its drop-never-close contract regardless.
 */
 function httpStatusOf(exc) {
@@ -5498,7 +5626,7 @@ var GeminiEmbeddingProvider = class {
 		this.queryTaskLabel = opts.queryTaskLabel;
 		this.documentTimeoutMs = Math.trunc(opts.documentTimeoutS * 1e3);
 		this.queryTimeoutMs = Math.trunc(opts.queryTimeoutS * 1e3);
-		this.clientFactory = opts.clientFactory ?? (() => new GoogleGenAI({ apiKey: opts.apiKey }));
+		this.clientFactory = opts.clientFactory ?? (() => geminiRestEmbedClient(opts.apiKey));
 	}
 	get recipe() {
 		return `${this.modelId}/d${this.dim}/${this.documentTaskLabel}`;
@@ -5541,7 +5669,7 @@ var GeminiTextGenerator = class {
 	client = null;
 	constructor(opts) {
 		this.model = opts.model ?? "gemini-2.5-flash";
-		this.clientFactory = opts.clientFactory ?? (() => new GoogleGenAI({ apiKey: opts.apiKey }));
+		this.clientFactory = opts.clientFactory ?? (() => geminiRestTextClient(opts.apiKey));
 	}
 	getClient() {
 		this.client ??= this.clientFactory();
@@ -6312,7 +6440,8 @@ async function assertGovernanceServable(pool, instance, targetGeneration) {
 		if (generation === 0) return {
 			generation,
 			builtAt: null,
-			restricted: 0
+			restricted: 0,
+			orphaned: []
 		};
 		return {
 			generation,
@@ -6321,13 +6450,27 @@ async function assertGovernanceServable(pool, instance, targetGeneration) {
 				instance.corpusId,
 				generation
 			])).rows[0]?.schema_version ?? null,
-			restricted: declaresModel ? 0 : Number((await client.query("SELECT count(*)::int AS n FROM content_nodes WHERE tenant_id = $1 AND generation = $2 AND visibility IS NOT NULL", [instance.tenantId, generation])).rows[0].n)
+			restricted: declaresModel ? 0 : Number((await client.query("SELECT count(*)::int AS n FROM content_nodes WHERE tenant_id = $1 AND generation = $2 AND visibility IS NOT NULL", [instance.tenantId, generation])).rows[0].n),
+			orphaned: (await client.query("SELECT d.stable_id FROM takedown_denylist d WHERE d.tenant_id = $1 AND d.corpus_id = $2   AND NOT EXISTS (SELECT 1 FROM content_nodes n                    WHERE n.tenant_id = d.tenant_id AND n.corpus_id = d.corpus_id                      AND n.generation = $3 AND n.stable_id = d.stable_id) ORDER BY d.stable_id", [
+				instance.tenantId,
+				instance.corpusId,
+				generation
+			])).rows.map((r) => r.stable_id)
 		};
 	});
 	if (state.generation === 0) return;
 	if (declaresModel && (state.builtAt === null || compareSchemaVersion(state.builtAt, "2.2") < 0)) throw new GovernanceGateError(`generation ${state.generation} was built against schema ${state.builtAt ?? "(before 2.4, which is when a generation started recording this)"}, older than 2.2 — the version that put visibility on the node row\n  why: instance.md declares an audience model, but the documents in this generation carry no visibility at all. Every one of them would be served at default_visibility — the WIDEST tier — including any document whose frontmatter restricts it
   fix: rebuild the record so its governance reaches the database:
     ksor ingest --instance instance.md --knowledge knowledge --flip`);
+	if (state.orphaned.length > 0) {
+		const named = state.orphaned.slice(0, 5).join(", ");
+		const more = state.orphaned.length - Math.min(5, state.orphaned.length);
+		throw new GovernanceGateError(`${state.orphaned.length} takedown(s) match no document in generation ${state.generation}: ${named}${more > 0 ? `, and ${more} more` : ""}\n  why: a denial is recorded against a stable_id, and the serving predicate matches it against the documents in this generation. An id that no longer exists denies NOTHING — so a withdrawn document that was renamed, moved, or had an index.md added beside it is served again by search, read, outline and the site, with no error anywhere. The denial is meant to be immune to reorganization; this is the state where it is not
+  fix: point the denial at where the document lives now, or retire it deliberately — never guess which one, because the tool cannot tell a rename from a deletion:
+    ksor takedown --instance instance.md --stable-id <the new id> --reason <why> --actor <who>
+    ksor takedown --instance instance.md --revoke <the old id> --actor <who>
+  (ksor takedown --list shows what is recorded)`);
+	}
 	if (!declaresModel && state.restricted > 0) throw new GovernanceGateError(`${state.restricted} document(s) in generation ${state.generation} declare visibility:, but instance.md declares no audiences:
   why: an author restricted those documents and nothing would enforce it — this door would serve them in full to every caller, and the frontmatter key saying otherwise would be the only trace. The site refuses to BUILD in this exact state (ksor-visibility-without-audiences); the door must not serve in it
   fix: declare the model in instance.md (audiences: least-restricted first, plus default_visibility:), or remove the visibility: keys and re-ingest`);
@@ -6745,7 +6888,10 @@ async function deniedStableIds(pool, instance) {
 *
 * The seed's OWN file counts when the seed has children, and only then — see
 * the SQL comment: a container's index.md names its directory, a leaf's file
-* names its parent's.
+* names its parent's. An index-less container has no such file, so the one it
+* would have had is synthesized from its (path-derived) "#section" id — without
+* it, a section whose descendants all live one level down contributed only the
+* subdirectory and left its own level publishable (issue #86).
 */
 async function deniedSubtreeDirs(pool, instance) {
 	const paths = await runRead(pool, instance.tenantId, async (client) => {
@@ -6753,7 +6899,7 @@ async function deniedSubtreeDirs(pool, instance) {
          SELECT active_generation AS g FROM corpora WHERE tenant_id = $1 AND corpus_id = $2
        ),
        seed AS (
-         SELECT n.node_id
+         SELECT n.node_id, n.stable_id
            FROM takedown_denylist d
            JOIN content_nodes n ON n.tenant_id = d.tenant_id AND n.stable_id = d.stable_id
            JOIN gen ON n.generation = gen.g
@@ -6791,7 +6937,28 @@ async function deniedSubtreeDirs(pool, instance) {
                  WHERE NOT EXISTS (SELECT 1 FROM content_nodes kid
                                     JOIN gen ON kid.generation = gen.g
                                    WHERE kid.tenant_id = $1 AND kid.parent_id = s2.node_id)
-              )`, [instance.tenantId, instance.corpusId])).rows.map((r) => String(r.origin_path));
+              )
+        UNION
+        -- An INDEX-LESS container has no file at all, so the join above drops
+        -- it however many descendants it has, and a section whose files all
+        -- live one level down contributed only the SUBdirectory — leaving a
+        -- document written directly under the withdrawn section publishable
+        -- (issue #86). The round-10 "seed counts when it has children" rule was
+        -- right and could not fire here, because there was nothing to count.
+        --
+        -- So the index.md it WOULD have had is synthesized, and the container
+        -- then names its directory exactly as an index-bearing one does. This is
+        -- not the stable_id prefix matching decision 14 rejects: that fails
+        -- because a sor_id: override decouples an id from its path, and a
+        -- "#section" id is generated from the path with no frontmatter in
+        -- reach: there is no index file to carry an override
+        -- (adapters/plain-tree.ts:214-219).
+        SELECT substring(sd.stable_id from '^(.*)#section$') || '/index.md'
+          FROM seed sd
+         WHERE sd.stable_id LIKE '%#section'
+           AND EXISTS (SELECT 1 FROM content_nodes kid
+                        JOIN gen ON kid.generation = gen.g
+                       WHERE kid.tenant_id = $1 AND kid.parent_id = sd.node_id)`, [instance.tenantId, instance.corpusId])).rows.map((r) => String(r.origin_path));
 	});
 	const dirs = /* @__PURE__ */ new Set();
 	for (const raw of paths) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@panaversity/ksor",
-  "version": "0.0.16",
+  "version": "0.0.18",
   "description": "Knowledge System of Record — compile governed markdown into a static site for people and an MCP server for AI agents, with citations and measured abstention.",
   "keywords": [
     "abstention",
@@ -51,7 +51,6 @@
     "provenance": true
   },
   "dependencies": {
-    "@google/genai": "^2.17.1",
     "@hono/node-server": "2.1.1",
     "@modelcontextprotocol/server": "^2.0.0",
     "@types/pg": "^8.21.0",