@panaversity/ksor 0.0.14 → 0.0.15

package/CHANGELOG.md

@@ -1,5 +1,59 @@
 # @panaversity/ksor
 
+## 0.0.15
+
+### Patch Changes
+
+- 5b076e6: Short documents reach search again — navigation is a shape, not a length
+
+  A record could be fully ingested, report "embedded 16, failed 0", and still be
+  unable to answer questions it plainly contained. Sections were classified as
+  navigation by LENGTH — anything under 250 characters — and navigation is
+  excluded from every retrieval arm. On a handbook that inverts the intent,
+  because a handbook's most valuable statements are its shortest.
+
+  Walked on 0.0.14 with three ordinary policy statements — a refund window, an
+  escalation path, a badge rule, 200-300 characters each. Three of four chunks
+  were unsearchable, and:
+
+  > **Q.** "how long does a buyer have to send something back"
+  > **A.** the scaffold's placeholder page — against a record stating _thirty days_
+
+  The answer was in the corpus, correctly ingested, readable by slug, and
+  unreachable by search.
+
+  Navigation is now decided by shape: a section is navigation when link lines are
+  most of it, or when what remains after them is too short to answer anything —
+  the same floor the serving predicate already applies. Length is no longer
+  consulted, so a 180-character link list is navigation and a 51-character fact is
+  not, which is the ordering length had backwards.
+
+  Measured on an authored handbook gold set with real embeddings, paired: short
+  substantive facts went **0/9 to 9/9 at rank 1**, the long-prose control held at
+  **4/4**, and the link-list page was returned **0** times. That last number is the
+  one that matters — admitting everything would have improved the first two and
+  made the product worse.
+
+  **To pick this up, re-run `ksor ingest`.** Chunks are re-classified on every
+  build and unchanged content is not re-embedded, so the upgrade costs a build,
+  not an embedding bill. `CHUNK_POLICY` moves to v6 because it is persisted
+  provenance and the behaviour it labels has changed.
+
+- 5763e8b: Internal: a pool test that raced Postgres, and a comment that had it backwards
+
+  No adopter-visible behaviour changes.
+
+  `idle.db.test.ts` sampled `pg_stat_activity` immediately after a previous test's
+  `pool.end()`. Those are two different clocks — `end()` resolves when the client
+  socket closes, while the row disappears only once the server-side backend
+  actually exits — so the suite was order-coupled through the database and went
+  red in CI on a branch that changed nothing but a document. Each test now waits
+  for a quiet database before it starts, and states that it does.
+
+  The comment added in the previous release explaining the `env.example` guard fix
+  described the rename backwards: the TEMPLATE holds `env.example` and
+  `materialize.ts` maps it to `.env.example` on emit, not the other way round.
+
 ## 0.0.14
 
 ### Patch Changes

package/dist/cli.mjs

@@ -16,7 +16,7 @@ import { bodyLimit } from "hono/body-limit";
 import { execFileSync, spawnSync } from "node:child_process";
 import { parseArgs } from "node:util";
 import { readFile, readdir, stat } from "node:fs/promises";
-//#region ../content-gateway/dist/main-VL9lXhmT.mjs
+//#region ../content-gateway/dist/main-wfyAlzsD.mjs
 /**
 * A connection could not be ESTABLISHED in time — retryable.
 *
@@ -430,7 +430,7 @@ const EMBED_DIM$1 = 1536;
 const EMBED_TASK_DOCUMENT$1 = "RETRIEVAL_DOCUMENT";
 const EMBED_TASK_QUERY$1 = "RETRIEVAL_QUERY";
 /** bump ⇒ provenance (v5: CommonMark fences). All char limits count CODE POINTS (Python len parity). */
-const CHUNK_POLICY$1 = "heading-aware-1500-content-only-v5";
+const CHUNK_POLICY$1 = "heading-aware-1500-content-only-v6";
 /**
 * The kernel's view of `instance.md` (adapted from oracle SC/instance.py
 * under decision 11 — an adaptation, not a port: ksor has ONE instance file
@@ -4452,7 +4452,7 @@ async function withPgRetry(op, options = {}) {
 	throw lastError;
 }
 //#endregion
-//#region ../content/dist/commands-BRK58wVc.mjs
+//#region ../content/dist/commands-_8HvMWfD.mjs
 /**
 * EVAL-LOCKED constants, quarried verbatim from the oracle
 * (sor-agentfactory @ b554f91, config.py) — changing any of these is a
@@ -4467,7 +4467,7 @@ const EMBED_DIM = 1536;
 const EMBED_TASK_DOCUMENT = "RETRIEVAL_DOCUMENT";
 const EMBED_TASK_QUERY = "RETRIEVAL_QUERY";
 /** bump ⇒ provenance (v5: CommonMark fences). All char limits count CODE POINTS (Python len parity). */
-const CHUNK_POLICY = "heading-aware-1500-content-only-v5";
+const CHUNK_POLICY = "heading-aware-1500-content-only-v6";
 const MAX_CHARS = 1500;
 /** < Gemini's 2048-token embed input. */
 const HARD_MAX_CHARS = 4e3;
@@ -7882,11 +7882,48 @@ function cleanBody(rawBody) {
 function teachingBody(content) {
 	return pyStrip(pySplitLines(content, false).filter((ln) => !HEADING.test(ln)).join("\n"));
 }
+/**
+* A line that is navigation rather than prose: strip its list marker and what
+* remains is nothing but links.
+*
+* Deliberately narrow. A line with prose AROUND a link ("Claim within thirty
+* days; see the [expenses page](x) to file.") is prose, because the sentence is
+* the content and the link is incidental.
+*/
+const NAV_LINE = /^(?:[-*+]\s+|\d+[.)]\s+)?(?:\[[^\]]*\]\([^)]*\)|<https?:\/\/[^>]*>|https?:\/\/\S+)(?:[\s,;·|>—–-]*(?:\[[^\]]*\]\([^)]*\)|<https?:\/\/[^>]*>|https?:\/\/\S+))*[\s.,;:]*$/;
+/**
+* Is this segment NAVIGATION — a thing that points at content rather than
+* being content?
+*
+* The oracle answered this with length: under NAV_MAX_CHARS (250) meant nav.
+* On the curriculum corpus it was tuned against, that proxy holds — a short
+* segment there really is a link list. On a handbook it inverts, because a
+* handbook's most valuable statements are its shortest ("Six months, with a
+* written review at three and six"), and `nav` is excluded from search. Issue
+* #55, walked live on 0.0.14: three of four chunks in an ordinary policy
+* record were unsearchable, and a question the record plainly answered was
+* served the scaffold's placeholder instead.
+*
+* So the question is asked about SHAPE, which is what "navigation" always
+* meant. A segment is nav when link lines are most of it, or when what is left
+* after them is too little to answer anything (MIN_CONTENT_CHARS — the same
+* floor the serving predicate applies, so this never labels `prose` something
+* search would refuse to return anyway).
+*
+* Length is no longer consulted. A 180-character link list is nav and a
+* 51-character fact is prose, which is the ordering length got backwards.
+*/
+function isNavShaped(content) {
+	const lines = pySplitLines(teachingBody(content), false).map((ln) => pyStrip(ln)).filter((ln) => ln !== "");
+	if (lines.length === 0) return true;
+	if (lines.filter((ln) => NAV_LINE.test(ln)).length * 2 > lines.length) return true;
+	return cpLen(lines.filter((ln) => !NAV_LINE.test(ln)).join(" ")) < 24;
+}
 function classify(content, headingPath) {
 	if (JSX_ASSESS.test(content)) return "assessment";
 	const leaf = headingPath.length > 0 ? headingPath[headingPath.length - 1] : "";
 	if (JSX_EMBED.test(content) || content.includes("docs.google.com/presentation") || leaf.includes("Teaching Aid")) return "embed";
-	if (cpLen(teachingBody(content)) < 250) return "nav";
+	if (isNavShaped(content)) return "nav";
 	return "prose";
 }
 /** A segment DOMINATED by a line-leading widget (with < NAV_MAX_CHARS of
@@ -8006,7 +8043,7 @@ function chunkText(text, maxChars = MAX_CHARS) {
 			} else prefix += seg.text;
 			continue;
 		}
-		const segIsNav = cpLen(teachingBody(seg.text)) < 250;
+		const segIsNav = isNavShaped(seg.text);
 		const segMarker = segmentMarkerType(seg.text);
 		for (const piece of subsplit(seg.text, maxChars)) {
 			let sourceType;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@panaversity/ksor",
-  "version": "0.0.14",
+  "version": "0.0.15",
   "description": "Knowledge System of Record — compile governed markdown into a static site for people and an MCP server for AI agents, with citations and measured abstention.",
   "keywords": [
     "abstention",