@panaversity/ksor 0.0.8 → 0.0.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (3) hide show
  1. package/CHANGELOG.md +19 -0
  2. package/dist/cli.mjs +1498 -1490
  3. package/package.json +1 -1
package/dist/cli.mjs CHANGED
@@ -16,7 +16,7 @@ import { bodyLimit } from "hono/body-limit";
16
16
  import { execFileSync, spawnSync } from "node:child_process";
17
17
  import { parseArgs } from "node:util";
18
18
  import { readFile, readdir, stat } from "node:fs/promises";
19
- //#region ../content-gateway/dist/main-fIdtI02D.mjs
19
+ //#region ../content-gateway/dist/main-tBbDvU1w.mjs
20
20
  /**
21
21
  * A connection could not be ESTABLISHED in time — retryable.
22
22
  *
@@ -4082,7 +4082,7 @@ async function withPgRetry(op, options = {}) {
4082
4082
  throw lastError;
4083
4083
  }
4084
4084
  //#endregion
4085
- //#region ../content/dist/commands-BxgkBvyW.mjs
4085
+ //#region ../content/dist/commands-1ZBNjWKb.mjs
4086
4086
  /**
4087
4087
  * EVAL-LOCKED constants, quarried verbatim from the oracle
4088
4088
  * (sor-agentfactory @ b554f91, config.py) — changing any of these is a
@@ -5849,1609 +5849,1609 @@ async function assertGovernanceServable(pool, instance, targetGeneration) {
5849
5849
  why: an author restricted those documents and nothing would enforce it — this door would serve them in full to every caller, and the frontmatter key saying otherwise would be the only trace. The site refuses to BUILD in this exact state (ksor-visibility-without-audiences); the door must not serve in it
5850
5850
  fix: declare the model in instance.md (audiences: least-restricted first, plus default_visibility:), or remove the visibility: keys and re-ingest`);
5851
5851
  }
5852
+ /** §5 rule 2: snapshot-token TTL (30 min) + 10 min = 40 min from retirement. */
5853
+ const GC_GRACE_MS = 24e5;
5852
5854
  /**
5853
- * The §7 row for a governance act, written INSIDE the same transaction as the
5854
- * act. `logRead` deliberately covers only the four serving actions; a takedown
5855
- * is a write-plane act, and separating the two writes would allow a denial with
5856
- * no row proving it happened the one outcome the ledger exists to prevent.
5855
+ * Poison-chunk tolerance (oracle review: poison-chunk-wedge): one
5856
+ * deterministically-failing chunk must not wedge every future flip forever. A
5857
+ * generation is servable if a SMALL fraction failed the read path already
5858
+ * filters to `embedded`, so a quarantined chunk is simply absent, not
5859
+ * corrupt. Above the fraction, a real ingest break is signalled by
5860
+ * withholding readiness.
5857
5861
  */
5858
- async function recordAct(client, instance, detail, actor, action = "takedown_applied") {
5859
- await client.query("INSERT INTO retrieval_log (tenant_id, corpus_id, actor, action, detail) VALUES ($1, $2, $3, $5, $4::jsonb)", [
5860
- instance.tenantId,
5861
- instance.corpusId,
5862
- actor,
5863
- JSON.stringify(detail),
5864
- action
5862
+ const MAX_FAILED_FRACTION = .02;
5863
+ const LOCK_SQL = "SELECT pg_advisory_xact_lock(hashtextextended('sor-ingest:' || $1, 0))";
5864
+ /**
5865
+ * Take the tenant lock, allocate generation max+1 (monotonic per corpus,
5866
+ * never reused), open the building run. The corpora row seeds at
5867
+ * active_generation=0 (nothing active) on first ingest.
5868
+ *
5869
+ * `manifestSha256` fills `instance_bundle_sha256` — ksor has no bundle
5870
+ * transport (the CLI reads the local repo), so the recorded digest is of the
5871
+ * manifest this build actually consumed: the closest honest provenance.
5872
+ */
5873
+ async function allocateRun(client, opts) {
5874
+ await client.query(LOCK_SQL, [opts.tenantId]);
5875
+ await client.query("INSERT INTO corpora (tenant_id, corpus_id, active_generation) VALUES ($1, $2, 0) ON CONFLICT (tenant_id, corpus_id) DO NOTHING", [opts.tenantId, opts.corpusId]);
5876
+ const next = await client.query("SELECT COALESCE(max(generation), 0) + 1 AS next FROM ingestion_runs WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId]);
5877
+ const generation = Number(next.rows[0].next);
5878
+ const run = await client.query("INSERT INTO ingestion_runs (tenant_id, corpus_id, generation, state, source_commit, instance_bundle_sha256, schema_version) VALUES ($1, $2, $3, 'building', $4, $5, $6) RETURNING run_id", [
5879
+ opts.tenantId,
5880
+ opts.corpusId,
5881
+ generation,
5882
+ opts.sourceCommit,
5883
+ opts.manifestSha256,
5884
+ schemaVersion()
5865
5885
  ]);
5886
+ return {
5887
+ runId: Number(run.rows[0].run_id),
5888
+ generation
5889
+ };
5866
5890
  }
5867
5891
  /**
5868
- * Deny a node (or its subtree) and record the act.
5892
+ * The generation to carry embeddings FROM: the newest COMPLETE one holding
5893
+ * embedded chunks.
5869
5894
  *
5870
- * The audit row is written in the SAME transaction as the denial: a takedown
5871
- * that happened without a row proving it happened is exactly the shape the
5872
- * §7 ledger exists to prevent.
5895
+ * WHY NOT ONLY THE ACTIVE ONE: the eval-before-flip design means a candidate
5896
+ * is often built, measured, and deliberately NOT served; ACTIVE then points
5897
+ * at an OLD generation and the next candidate re-embeds the whole corpus —
5898
+ * measured 2026-08-02: generation 4 re-embedded 5,915 chunks while
5899
+ * generation 3 held near-identical content, because generation 1 was still
5900
+ * active.
5901
+ *
5902
+ * Two constraints a rewrite once dropped (oracle review of PR #420):
5903
+ * CORPUS-SCOPED via the run-table join (chunks carry no corpus_id), and
5904
+ * COMPLETE RUNS ONLY (ready/active/retired) — a crashed `building` queue's
5905
+ * half-drained vectors never qualify.
5906
+ *
5907
+ * Returns 0 when there is no complete embedded generation — the first ingest.
5873
5908
  */
5874
- async function applyTakedown(pool, instance, opts) {
5875
- return runIngest(pool, instance.tenantId, async (client) => {
5876
- const resolves = ((await client.query(`SELECT 1 FROM content_nodes n
5877
- JOIN corpora c ON c.tenant_id = n.tenant_id AND c.corpus_id = $2
5878
- WHERE n.tenant_id = $1 AND n.stable_id = $3 AND n.generation = c.active_generation
5879
- LIMIT 1`, [
5880
- instance.tenantId,
5881
- instance.corpusId,
5882
- opts.stableId
5883
- ])).rowCount ?? 0) > 0;
5884
- const changed = (await client.query("INSERT INTO takedown_denylist (tenant_id, corpus_id, stable_id, scope, reason) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (tenant_id, corpus_id, stable_id) DO UPDATE SET scope = EXCLUDED.scope, reason = EXCLUDED.reason WHERE takedown_denylist.scope IS DISTINCT FROM EXCLUDED.scope OR takedown_denylist.reason IS DISTINCT FROM EXCLUDED.reason RETURNING stable_id", [
5885
- instance.tenantId,
5886
- instance.corpusId,
5887
- opts.stableId,
5888
- opts.scope,
5889
- opts.reason
5890
- ])).rowCount === 1;
5891
- await recordAct(client, instance, {
5892
- stable_id: opts.stableId,
5893
- scope: opts.scope,
5894
- reason: opts.reason,
5895
- change: changed ? "applied" : "unchanged"
5896
- }, opts.actor);
5897
- return {
5898
- stableId: opts.stableId,
5899
- scope: opts.scope,
5900
- changed,
5901
- resolves
5902
- };
5903
- });
5904
- }
5905
- /** Lift a denial. The ledger keeps the row that recorded imposing it. */
5906
- async function revokeTakedown(pool, instance, opts) {
5907
- return runIngest(pool, instance.tenantId, async (client) => {
5908
- const changed = ((await client.query("DELETE FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2 AND stable_id = $3", [
5909
- instance.tenantId,
5910
- instance.corpusId,
5911
- opts.stableId
5912
- ])).rowCount ?? 0) > 0;
5913
- await recordAct(client, instance, {
5914
- stable_id: opts.stableId,
5915
- change: changed ? "revoked" : "not-denied"
5916
- }, opts.actor, "takedown_revoked");
5917
- return {
5918
- stableId: opts.stableId,
5919
- scope: "node",
5920
- changed
5921
- };
5922
- });
5923
- }
5924
- async function readLedger(pool, instance, limit) {
5925
- return runAuditRead(pool, instance.tenantId, async (client) => {
5926
- return (await client.query("SELECT action, actor, generation, detail, created_at FROM retrieval_log WHERE tenant_id = $1 AND corpus_id = $2 ORDER BY created_at DESC, id DESC LIMIT $3", [
5927
- instance.tenantId,
5928
- instance.corpusId,
5929
- limit
5930
- ])).rows.map((row) => ({
5931
- action: String(row.action),
5932
- actor: String(row.actor),
5933
- generation: row.generation === null ? null : Number(row.generation),
5934
- detail: row.detail ?? {},
5935
- createdAt: row.created_at
5936
- }));
5937
- });
5909
+ async function bestCarrySource(client, opts) {
5910
+ const gen = (await client.query(`
5911
+ SELECT max(c.generation) AS gen FROM chunks c
5912
+ JOIN ingestion_runs r ON r.tenant_id = c.tenant_id AND r.generation = c.generation
5913
+ WHERE c.tenant_id = $1 AND r.corpus_id = $2
5914
+ AND r.state IN ('ready', 'active', 'retired')
5915
+ AND c.generation <> $3 AND c.embedding_status = 'embedded'
5916
+ `, [
5917
+ opts.tenantId,
5918
+ opts.corpusId,
5919
+ opts.excludeGeneration
5920
+ ])).rows[0]?.gen ?? null;
5921
+ return gen === null ? 0 : Number(gen);
5938
5922
  }
5939
5923
  /**
5940
- * Every stable_id a build must not publish, with `subtree` denials EXPANDED to
5941
- * their actual descendants by the same `parent_id` walk the serving side uses.
5924
+ * Copy embeddings for chunks whose ENTIRE embed input is unchanged (hash +
5925
+ * heading path + node title). Cost change survives the generational
5926
+ * rebuild. Returns rows carried.
5942
5927
  *
5943
- * The site cannot do this itself: it has no tree, so it matched a prefix — and
5944
- * a section's stable_id ends in `/index` (or `#section`), so the prefix never
5945
- * matched its children and every descendant of a subtree takedown kept
5946
- * publishing. Decision 14 records exactly why a prefix is wrong here; the fix
5947
- * is to resolve the walk where the tree lives and hand over a flat list
5948
- * (round-2 review of #43).
5928
+ * `modelId` is REQUIRED, never defaulted here: the vendor transport is
5929
+ * irrelevant to the space (the same model through two providers is the same
5930
+ * space), and a silent module default is exactly how a model bump would
5931
+ * carry stale vectors unnoticed.
5949
5932
  */
5950
- async function deniedStableIds(pool, instance) {
5951
- return runRead(pool, instance.tenantId, async (client) => {
5952
- return (await client.query(`WITH RECURSIVE gen AS (
5953
- SELECT active_generation AS g FROM corpora WHERE tenant_id = $1 AND corpus_id = $2
5954
- ),
5955
- seed AS (
5956
- SELECT n.node_id, n.stable_id, d.scope
5957
- FROM takedown_denylist d
5958
- JOIN content_nodes n ON n.tenant_id = d.tenant_id AND n.stable_id = d.stable_id
5959
- JOIN gen ON n.generation = gen.g
5960
- WHERE d.tenant_id = $1 AND d.corpus_id = $2
5961
- ),
5962
- walk AS (
5963
- SELECT node_id, stable_id, scope FROM seed
5964
- UNION ALL
5965
- SELECT c.node_id, c.stable_id, w.scope
5966
- FROM content_nodes c
5967
- JOIN walk w ON c.parent_id = w.node_id
5968
- JOIN gen ON c.generation = gen.g
5969
- WHERE c.tenant_id = $1 AND w.scope = 'subtree'
5970
- )
5971
- SELECT DISTINCT stable_id FROM walk
5972
- UNION
5973
- -- Denials naming a stable_id no CURRENT generation carries are still
5974
- -- denied: identity outlives any one generation (decision 14).
5975
- SELECT stable_id FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2`, [instance.tenantId, instance.corpusId])).rows.map((r) => String(r.stable_id)).sort();
5976
- });
5933
+ async function carryForward(client, opts) {
5934
+ if (opts.fromGeneration < 1) return 0;
5935
+ return (await client.query(`
5936
+ UPDATE chunks new SET embedding = old.embedding, embedding_status = 'embedded',
5937
+ embedded_at = old.embedded_at, embedding_model = old.embedding_model
5938
+ FROM chunks old, sources os, content_nodes onode, sources ns, content_nodes nnode
5939
+ WHERE new.tenant_id = $1 AND new.generation = $2
5940
+ AND new.embedding_status = 'pending'
5941
+ AND old.tenant_id = new.tenant_id AND old.generation = $3
5942
+ AND old.embedding_status = 'embedded'
5943
+ -- R-1 gate (oracle review: carry-model-gate-r1): carry ONLY vectors from the CURRENT
5944
+ -- embedding model. Without this, a model bump silently carries every old-model vector
5945
+ -- forward (pending→0, flip → corpus-wide nonsense cosine vs the new query model, zero
5946
+ -- errors). A model change now correctly leaves the old vectors pending → they re-embed.
5947
+ AND old.embedding_model = $4
5948
+ AND old.source_id = new.source_id
5949
+ AND old.chunk_hash = new.chunk_hash
5950
+ AND old.heading_path_text IS NOT DISTINCT FROM new.heading_path_text
5951
+ AND os.source_id = old.source_id AND os.tenant_id = old.tenant_id
5952
+ AND os.generation = old.generation
5953
+ AND onode.node_id = os.node_id AND onode.tenant_id = os.tenant_id
5954
+ AND ns.source_id = new.source_id AND ns.tenant_id = new.tenant_id
5955
+ AND ns.generation = new.generation
5956
+ AND nnode.node_id = ns.node_id AND nnode.tenant_id = ns.tenant_id
5957
+ AND onode.title = nnode.title
5958
+ `, [
5959
+ opts.tenantId,
5960
+ opts.generation,
5961
+ opts.fromGeneration,
5962
+ opts.modelId
5963
+ ])).rowCount ?? 0;
5977
5964
  }
5978
5965
  /**
5979
- * The knowledge-relative DIRECTORIES that `--subtree` denials govern.
5980
- *
5981
- * Derived from the DESCENDANTS' `sources.origin_path`, never from the denied
5982
- * node's own id or path, because neither works:
5983
- *
5984
- * a section has no source `knowledge/policies#section` is synthetic — the
5985
- * tree node for a directory. Joining `sources` on
5986
- * the denied node itself yields nothing, and a
5987
- * section is the ordinary target of `--subtree`.
5988
- * a leaf's directory is not `--subtree` on one document would emit that
5989
- * its subtree document's directory and deny every sibling.
5990
- *
5991
- * So: walk the descendants, take the directory of each one's file, and keep the
5992
- * SHALLOWEST — a directory that contains another in the set is the subtree
5993
- * root, and `startsWith` then covers subdirectories added later too. A denial
5994
- * with no descendants contributes nothing, which is correct: its subtree is
5995
- * itself, and the flat id list already holds it.
5996
- *
5997
- * The seed's OWN file counts when the seed has children, and only then — see
5998
- * the SQL comment: a container's index.md names its directory, a leaf's file
5999
- * names its parent's.
5966
+ * avg(embedding) per node over servable prose — rows the routing arm reads
5967
+ * (never aggregate at query time again). nav/embed/assessment chunks never
5968
+ * pollute routing centroids.
6000
5969
  */
6001
- async function deniedSubtreeDirs(pool, instance) {
6002
- const paths = await runRead(pool, instance.tenantId, async (client) => {
6003
- return (await client.query(`WITH RECURSIVE gen AS (
6004
- SELECT active_generation AS g FROM corpora WHERE tenant_id = $1 AND corpus_id = $2
6005
- ),
6006
- seed AS (
6007
- SELECT n.node_id
6008
- FROM takedown_denylist d
6009
- JOIN content_nodes n ON n.tenant_id = d.tenant_id AND n.stable_id = d.stable_id
6010
- JOIN gen ON n.generation = gen.g
6011
- WHERE d.tenant_id = $1 AND d.corpus_id = $2 AND d.scope = 'subtree'
6012
- ),
6013
- walk AS (
6014
- SELECT node_id FROM seed
6015
- UNION ALL
6016
- SELECT c.node_id
6017
- FROM content_nodes c
6018
- JOIN walk w ON c.parent_id = w.node_id
6019
- JOIN gen ON c.generation = gen.g
6020
- WHERE c.tenant_id = $1
6021
- )
6022
- SELECT DISTINCT s.origin_path
6023
- FROM walk w
6024
- JOIN content_nodes n ON n.node_id = w.node_id
6025
- JOIN sources s ON s.tenant_id = n.tenant_id AND s.generation = n.generation
6026
- AND s.node_id = n.node_id
6027
- -- The seed's own file counts only when the seed HAS CHILDREN.
6028
- --
6029
- -- Excluding every seed stopped a LEAF denial emitting its parent
6030
- -- directory and denying every sibling — right for a leaf, wrong for a
6031
- -- container. A section's own index.md is the file that names the
6032
- -- section's DIRECTORY, so a section whose other descendants all live
6033
- -- one level down contributed only the subdirectory, and a document
6034
- -- written directly under the withdrawn section published to /docs and
6035
- -- llms.txt (round-10 review of PR 43).
6036
- --
6037
- -- "Has children" is the right test, not "kind = section": it is the
6038
- -- property that decides whether the node's directory is its subtree
6039
- -- or its parent's.
6040
- WHERE w.node_id NOT IN (
6041
- SELECT s2.node_id FROM seed s2
6042
- WHERE NOT EXISTS (SELECT 1 FROM content_nodes kid
6043
- JOIN gen ON kid.generation = gen.g
6044
- WHERE kid.tenant_id = $1 AND kid.parent_id = s2.node_id)
6045
- )`, [instance.tenantId, instance.corpusId])).rows.map((r) => String(r.origin_path));
6046
- });
6047
- const dirs = /* @__PURE__ */ new Set();
6048
- for (const raw of paths) {
6049
- const normalized = raw.replace(/\\/g, "/");
6050
- const slash = normalized.lastIndexOf("/");
6051
- dirs.add(slash === -1 ? "/" : `${normalized.slice(0, slash)}/`);
6052
- }
6053
- const all = [...dirs];
6054
- return all.filter((dir) => !all.some((other) => other !== dir && dir.startsWith(other))).sort();
5970
+ async function materializeCentroids(client, opts) {
5971
+ await client.query("DELETE FROM node_centroids WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation]);
5972
+ return (await client.query(`
5973
+ INSERT INTO node_centroids (tenant_id, generation, node_id, stable_id, chunk_count, embedding)
5974
+ SELECT c.tenant_id, c.generation, n.node_id, n.stable_id, count(*), avg(c.embedding)
5975
+ FROM chunks c
5976
+ JOIN sources s ON s.source_id = c.source_id AND s.tenant_id = c.tenant_id
5977
+ AND s.generation = c.generation
5978
+ JOIN content_nodes n ON n.node_id = s.node_id AND n.tenant_id = s.tenant_id
5979
+ WHERE c.tenant_id = $1 AND c.generation = $2 AND c.embedding_status = 'embedded'
5980
+ AND c.labels->>'source_type' = 'prose'
5981
+ GROUP BY c.tenant_id, c.generation, n.node_id, n.stable_id
5982
+ `, [opts.tenantId, opts.generation])).rowCount ?? 0;
6055
5983
  }
6056
- async function listTakedowns(pool, instance) {
6057
- return runRead(pool, instance.tenantId, async (client) => {
6058
- return (await client.query("SELECT stable_id, scope, reason, created_at FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2 ORDER BY created_at, stable_id", [instance.tenantId, instance.corpusId])).rows.map((r) => ({
6059
- stableId: String(r.stable_id),
6060
- scope: String(r.scope),
6061
- reason: String(r.reason),
6062
- createdAt: r.created_at
6063
- }));
6064
- });
5984
+ /**
5985
+ * The ready gate, factored pure: zero PENDING (the queue drained) + some
5986
+ * embedded content + failures within tolerance. The read path serves only
5987
+ * `embedded`, so a failed chunk is quarantined, not corrupt.
5988
+ */
5989
+ function generationReady(health) {
5990
+ if (health.pending !== 0 || health.embedded === 0) return false;
5991
+ const total = health.embedded + health.failed;
5992
+ return health.failed / total <= MAX_FAILED_FRACTION;
6065
5993
  }
6066
- function denylistManifest(corpusId, stableIds, now, source = "database", deniedSubtrees = []) {
5994
+ async function generationHealth(client, opts) {
5995
+ const row = (await client.query("SELECT count(*) FILTER (WHERE embedding_status = 'embedded') AS embedded, count(*) FILTER (WHERE embedding_status = 'pending') AS pending, count(*) FILTER (WHERE embedding_status = 'failed') AS failed FROM chunks WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation])).rows[0];
6067
5996
  return {
6068
- format: 1,
6069
- corpus_id: corpusId,
6070
- source,
6071
- denied_subtrees: [...deniedSubtrees].sort(),
6072
- exported_at: now.toISOString(),
6073
- denied: stableIds.map((stable_id) => ({
6074
- stable_id,
6075
- scope: "node"
6076
- }))
5997
+ generation: opts.generation,
5998
+ embedded: Number(row.embedded),
5999
+ pending: Number(row.pending),
6000
+ failed: Number(row.failed)
6077
6001
  };
6078
6002
  }
6003
+ function addedSlugs(delta) {
6004
+ return [...delta.newSlugs].filter((s) => !delta.priorSlugs.has(s)).sort();
6005
+ }
6006
+ function removedSlugs(delta) {
6007
+ return [...delta.priorSlugs].filter((s) => !delta.newSlugs.has(s)).sort();
6008
+ }
6079
6009
  /**
6080
- * The plain-tree corpus adapter ANY folder of Markdown becomes a corpus.
6081
- * Converted from the oracle (sor-agentfactory @ b554f91,
6082
- * ingest/adapters/plain_tree.py); the kernel cannot tell this manifest from
6083
- * any other adapter's.
6084
- *
6085
- * Conventions (deliberately minimal — an operator can satisfy them with a bare
6086
- * folder):
6087
- * - directories become `section` nodes; `.md`/`.mdx` files become `document`
6088
- * nodes;
6089
- * - `index.md` (or `README.md`) inside a directory is that SECTION's own
6090
- * content, not a child;
6091
- * - ordering: frontmatter `position` (or `sidebar_position`) wins, else name
6092
- * sort;
6093
- * - titles: frontmatter `title`, else the filename humanized;
6094
- * - stable ids: frontmatter `sor_id`, else the tree-relative path;
6095
- * - hidden entries (leading `.` or `_`) and ALL symlinks are skipped LOUDLY
6096
- * (reported through `onSkip`, console by default — never silent); symlinks
6097
- * are never followed, so a link cannot walk out of the tree or cycle it;
6098
- * - a directory carrying MORE than one index-named file (index.md +
6099
- * README.md …) fails loud: which one is the section's own content is
6100
- * ambiguous, and silently dropping the loser is exactly the corpus
6101
- * corruption this adapter must never commit.
6102
- *
6103
- * The oracle's `publish_bundle` (deterministic tgz staging) is a separate
6104
- * slice and is not converted here.
6010
+ * Net fractional drop in node count vs the prior generation. Zero when the
6011
+ * prior generation is empty (a FIRST ingest has nothing to shrink from) so
6012
+ * the guard never trips on it.
6105
6013
  */
6106
- const INDEX_NAMES = [
6107
- "index.md",
6108
- "index.mdx",
6109
- "README.md"
6110
- ];
6111
- /** Frontmatter-position fallback for entries that declare none (oracle plain_tree.py:107,114). */
6112
- const POSITION_FALLBACK = 1e4;
6113
- /** Walk a directory on disk → manifest + sources. Fail-loud on emptiness and ambiguity. */
6114
- async function buildManifest(treeRoot, options) {
6115
- const rootPath = treeRoot.length > 1 ? treeRoot.replace(/\/+$/, "") : treeRoot;
6116
- let isDir = false;
6117
- try {
6118
- isDir = (await stat(rootPath)).isDirectory();
6119
- } catch {
6120
- isDir = false;
6121
- }
6122
- if (!isDir) throw new ManifestError(`plain-tree root ${rootPath} is not a directory`);
6123
- return buildManifestFromTree(await readTree(rootPath), {
6124
- ...options,
6125
- rootPath
6126
- });
6014
+ function shrinkFraction(priorCount, newCount) {
6015
+ if (priorCount === 0) return 0;
6016
+ return Math.max(0, priorCount - newCount) / priorCount;
6127
6017
  }
6128
6018
  /**
6129
- * Load a directory into an in-memory tree. lstat semantics throughout: a
6130
- * symlink is represented as a symlink even one named `index.md` never
6131
- * followed, never read (the oracle's docstring contract; its `_index_of`
6132
- * incidentally followed a symlinked index via `is_file()`, which this port
6133
- * deliberately does not reproduce). Non-markdown files are invisible to the
6134
- * walk, exactly as the oracle's suffix filter makes them.
6019
+ * True when the corpus shrank by MORE than the tolerated fraction — the flip
6020
+ * should be refused unless the drop is explicitly acknowledged. A first
6021
+ * ingest is always safe.
6135
6022
  */
6136
- async function readTree(dirPath) {
6137
- const dirents = await readdir(dirPath, { withFileTypes: true });
6138
- const entries = [];
6139
- for (const d of dirents) if (d.isSymbolicLink()) entries.push({
6140
- kind: "symlink",
6141
- name: d.name
6142
- });
6143
- else if (d.isDirectory()) entries.push(await readTree(join(dirPath, d.name)));
6144
- else if (d.isFile() && isDoc(d.name)) entries.push({
6145
- kind: "file",
6146
- name: d.name,
6147
- text: await readFile(join(dirPath, d.name), "utf8")
6148
- });
6023
+ function shrinkUnsafe(priorCount, newCount, maxShrink) {
6024
+ return priorCount > 0 && shrinkFraction(priorCount, newCount) > maxShrink;
6025
+ }
6026
+ /** Read the node-slug sets of the active generation and the candidate; the caller decides. */
6027
+ async function flipDelta(client, opts) {
6028
+ const raw = (await client.query("SELECT active_generation FROM corpora WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId])).rows[0]?.active_generation ?? null;
6029
+ const prior = raw === null ? 0 : Number(raw);
6030
+ const nodesOf = async (generation) => {
6031
+ if (generation < 1) return /* @__PURE__ */ new Set();
6032
+ const res = await client.query("SELECT stable_id FROM content_nodes WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, generation]);
6033
+ return new Set(res.rows.map((r) => String(r.stable_id)));
6034
+ };
6149
6035
  return {
6150
- kind: "dir",
6151
- name: basename(dirPath),
6152
- entries
6036
+ priorGeneration: prior,
6037
+ priorSlugs: await nodesOf(prior),
6038
+ newSlugs: await nodesOf(opts.newGeneration)
6153
6039
  };
6154
6040
  }
6155
- /** The pure walk: tree → manifest + {manifest path → source path}. */
6156
- function buildManifestFromTree(root, options) {
6157
- const rootName = root.name;
6158
- const rootPath = options.rootPath ?? rootName;
6159
- const onSkip = options.onSkip ?? ((line) => console.log(line));
6160
- const nodes = [];
6161
- const files = [];
6162
- const sources = /* @__PURE__ */ new Map();
6163
- const skipped = [];
6164
- const fullPath = (relSegs, name) => `${rootPath}/${[...relSegs, name].join("/")}`;
6165
- const addFile = (nodeSid, fileSegs) => {
6166
- const rel = fileSegs.join("/");
6167
- const manifestPath = `${rootName}/${rel}`;
6168
- files.push(manifestFile({
6169
- path: manifestPath,
6170
- node: nodeSid
6171
- }));
6172
- sources.set(manifestPath, `${rootPath}/${rel}`);
6173
- };
6174
- const walk = (dir, relSegs, parentSid) => {
6175
- const entries = [...dir.entries].sort((a, b) => codePointCompare(a.name.toLowerCase(), b.name.toLowerCase()));
6176
- const docs = [];
6177
- const dirs = [];
6178
- for (const e of entries) if (e.kind === "symlink") skipped.push(`${fullPath(relSegs, e.name)} (symlink)`);
6179
- else if (e.kind === "file" && isDoc(e.name)) docs.push(e);
6180
- else if (e.kind === "dir") dirs.push(e);
6181
- const ordered = [];
6182
- for (const f of docs) {
6183
- if (f.name.startsWith(".") || f.name.startsWith("_")) {
6184
- skipped.push(fullPath(relSegs, f.name));
6185
- continue;
6186
- }
6187
- if (INDEX_NAMES.includes(f.name)) continue;
6188
- ordered.push({
6189
- position: positionOf(frontmatterMeta(f.text), POSITION_FALLBACK),
6190
- nameLower: f.name.toLowerCase(),
6191
- entry: f
6192
- });
6193
- }
6194
- for (const d of dirs) {
6195
- if (d.name.startsWith(".") || d.name.startsWith("_")) {
6196
- skipped.push(fullPath(relSegs, d.name));
6197
- continue;
6198
- }
6199
- const index = indexOf(d, fullPath(relSegs, d.name));
6200
- const dirMeta = index === null ? {} : frontmatterMeta(index.text);
6201
- ordered.push({
6202
- position: positionOf(dirMeta, POSITION_FALLBACK),
6203
- nameLower: d.name.toLowerCase(),
6204
- entry: d
6205
- });
6206
- }
6207
- ordered.sort((x, y) => x.position - y.position || codePointCompare(x.nameLower, y.nameLower));
6208
- let position = 0;
6209
- for (const { entry } of ordered) {
6210
- position += 1;
6211
- if (entry.kind === "dir") {
6212
- const dirSegs = [...relSegs, entry.name];
6213
- const index = indexOf(entry, fullPath(relSegs, entry.name));
6214
- const meta = index === null ? {} : frontmatterMeta(index.text);
6215
- const sid = index === null ? `${rootName}/${dirSegs.join("/")}#section` : stableIdOf(rootName, [...dirSegs, index.name], meta);
6216
- nodes.push(manifestNode({
6217
- stable_id: sid,
6218
- slug: slugify(entry.name),
6219
- title: titleOf(meta, entry.name),
6220
- kind: "section",
6221
- parent: parentSid,
6222
- position,
6223
- governance: index === null ? NO_GOVERNANCE : governanceFromFrontmatter(meta, index.text)
6224
- }));
6225
- if (index !== null) addFile(sid, [...dirSegs, index.name]);
6226
- walk(entry, dirSegs, sid);
6227
- } else {
6228
- const meta = frontmatterMeta(entry.text);
6229
- const stem = stemOf(entry.name);
6230
- const sid = stableIdOf(rootName, [...relSegs, entry.name], meta);
6231
- nodes.push(manifestNode({
6232
- stable_id: sid,
6233
- slug: slugify(stem),
6234
- title: titleOf(meta, stem),
6235
- kind: "document",
6236
- parent: parentSid,
6237
- position,
6238
- governance: governanceFromFrontmatter(meta, entry.text)
6239
- }));
6240
- addFile(sid, [...relSegs, entry.name]);
6241
- }
6242
- }
6243
- };
6244
- const rootIndex = indexOf(root, rootPath);
6245
- if (rootIndex !== null) {
6246
- const meta = frontmatterMeta(rootIndex.text);
6247
- const sid = stableIdOf(rootName, [rootIndex.name], meta);
6248
- nodes.push(manifestNode({
6249
- stable_id: sid,
6250
- slug: slugify(rootName),
6251
- title: titleOf(meta, rootName),
6252
- kind: "document",
6253
- position: 0,
6254
- governance: governanceFromFrontmatter(meta, rootIndex.text)
6255
- }));
6256
- addFile(sid, [rootIndex.name]);
6257
- }
6258
- walk(root, [], null);
6259
- for (const s of skipped) onSkip(`plain-tree: skipped ${s}`);
6260
- if (files.length === 0) throw new ManifestError(`plain-tree root ${rootPath} contains no Markdown`);
6261
- const manifest = {
6262
- format: 1,
6263
- corpus_id: options.corpusId,
6264
- source_commit: options.sourceCommit,
6265
- nodes,
6266
- files
6267
- };
6268
- parseManifest(JSON.stringify(manifestToJson(manifest)));
6269
- return {
6270
- manifest,
6271
- sources
6272
- };
6273
- }
6274
- /** Python `p.suffix in (".md", ".mdx")` parity: a dotfile named exactly ".md" has NO suffix. */
6275
- function isDoc(name) {
6276
- const dot = name.lastIndexOf(".");
6277
- if (dot <= 0) return false;
6278
- const suffix = name.slice(dot);
6279
- return suffix === ".md" || suffix === ".mdx";
6280
- }
6281
- function indexOf(dir, dirPath) {
6282
- const present = [];
6283
- for (const name of INDEX_NAMES) {
6284
- const hit = dir.entries.find((e) => e.kind === "file" && e.name === name);
6285
- if (hit !== void 0) present.push(hit);
6286
- }
6287
- if (present.length > 1) throw new ManifestError(`ambiguous section index in ${dirPath}: [${present.map((p) => `'${p.name}'`).join(", ")}] — keep exactly one`);
6288
- return present[0] ?? null;
6289
- }
6290
- function stableIdOf(rootName, fileSegs, meta) {
6291
- const sid = meta["sor_id"];
6292
- if (typeof sid === "string" && sid.trim() !== "") return sid.trim();
6293
- return `${rootName}/${withoutSuffix(fileSegs.join("/"))}`;
6294
- }
6295
- /** Python Path.with_suffix("") parity: strip the LAST suffix only; a dotfile has none. */
6296
- function withoutSuffix(rel) {
6297
- const slash = rel.lastIndexOf("/");
6298
- const name = rel.slice(slash + 1);
6299
- const dot = name.lastIndexOf(".");
6300
- if (dot <= 0) return rel;
6301
- return rel.slice(0, slash + 1) + name.slice(0, dot);
6302
- }
6303
- function stemOf(name) {
6304
- return withoutSuffix(name);
6305
- }
6306
- function slugify(text) {
6307
- const slug = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
6308
- if (slug !== "") return slug;
6309
- return "x-" + createHash("sha256").update(text, "utf8").digest("hex").slice(0, 8);
6041
+ /**
6042
+ * Activation + run-state bookkeeping + the ledger row. The health gate is the
6043
+ * CALLER's duty — never partially activate. Serialized under the tenant
6044
+ * advisory lock (re-taken here: the allocate lock died with its own txn) and
6045
+ * MONOTONIC: only ever advances.
6046
+ */
6047
+ async function flip(client, opts) {
6048
+ await client.query(LOCK_SQL, [opts.tenantId]);
6049
+ if (!(await client.query("UPDATE corpora SET rollback_generation = active_generation, active_generation = $1, updated_at = now() WHERE tenant_id = $2 AND corpus_id = $3 AND active_generation < $1", [
6050
+ opts.toGeneration,
6051
+ opts.tenantId,
6052
+ opts.corpusId
6053
+ ])).rowCount) throw new Error(`flip to generation ${opts.toGeneration} refused: active_generation is already >= it (an out-of-order or duplicate flip — refusing to regress the served corpus)`);
6054
+ await client.query("UPDATE ingestion_runs SET state = 'retired', finished_at = now() WHERE tenant_id = $1 AND corpus_id = $2 AND state = 'active'", [opts.tenantId, opts.corpusId]);
6055
+ await client.query("UPDATE ingestion_runs SET state = 'active', finished_at = COALESCE(finished_at, now()) WHERE tenant_id = $1 AND corpus_id = $2 AND generation = $3", [
6056
+ opts.tenantId,
6057
+ opts.corpusId,
6058
+ opts.toGeneration
6059
+ ]);
6060
+ await client.query("INSERT INTO retrieval_log (tenant_id, corpus_id, generation, actor, action, detail) VALUES ($1, $2, $3, 'sor-ingest', 'generation_activated', '{}')", [
6061
+ opts.tenantId,
6062
+ opts.corpusId,
6063
+ opts.toGeneration
6064
+ ]);
6310
6065
  }
6311
- const CASED = /\p{Cased}/u;
6312
6066
  /**
6313
- * Python str.title() parity (the oracle's `_humanize`): a cased character
6314
- * following an uncased one uppercases, following a cased one lowercases —
6315
- * apostrophe quirk included ("rock'n'roll" "Rock'N'Roll"). Node titles are
6316
- * carry-forward join keys, so the quirk is load-bearing, not cosmetic.
6067
+ * The §5 algebra: not active, not rollback, past token grace since
6068
+ * retirement, ≥2 complete generations REMAIN after collection; abandoned
6069
+ * builds reap on heartbeat staleness alone (they were never served, no token
6070
+ * can reference them).
6317
6071
  */
6318
- function humanize(stem) {
6319
- const spaced = stem.replace(/[-_]+/g, " ").trim();
6320
- let out = "";
6321
- let prevCased = false;
6322
- for (const ch of spaced) {
6323
- const cased = CASED.test(ch);
6324
- out += cased ? prevCased ? ch.toLowerCase() : ch.toUpperCase() : ch;
6325
- prevCased = cased;
6072
+ async function collectableGenerations(client, opts) {
6073
+ const ts = opts.now ?? /* @__PURE__ */ new Date();
6074
+ const pointer = await client.query("SELECT active_generation, rollback_generation FROM corpora WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId]);
6075
+ if (pointer.rows.length === 0) return [];
6076
+ const active = Number(pointer.rows[0].active_generation);
6077
+ const rollbackRaw = pointer.rows[0].rollback_generation;
6078
+ const rollbackGen = rollbackRaw === null ? null : Number(rollbackRaw);
6079
+ const runs = await client.query("SELECT generation, state, finished_at, heartbeat_at FROM ingestion_runs WHERE tenant_id = $1 AND corpus_id = $2 AND state <> 'reaped' ORDER BY generation", [opts.tenantId, opts.corpusId]);
6080
+ const complete = runs.rows.filter((r) => [
6081
+ "ready",
6082
+ "active",
6083
+ "retired"
6084
+ ].includes(String(r.state)));
6085
+ const out = [];
6086
+ let remaining = complete.length;
6087
+ for (const row of runs.rows) {
6088
+ const gen = Number(row.generation);
6089
+ const state = String(row.state);
6090
+ const finishedAt = row.finished_at;
6091
+ const heartbeatAt = row.heartbeat_at;
6092
+ if (state === "building") {
6093
+ if (heartbeatAt !== null && ts.getTime() - heartbeatAt.getTime() > 864e5) out.push(gen);
6094
+ continue;
6095
+ }
6096
+ if (gen === active || rollbackGen !== null && gen === rollbackGen) continue;
6097
+ if (finishedAt === null || ts.getTime() - finishedAt.getTime() < GC_GRACE_MS) continue;
6098
+ if (remaining - 1 < 2) continue;
6099
+ remaining -= 1;
6100
+ out.push(gen);
6326
6101
  }
6327
6102
  return out;
6328
6103
  }
6329
- /** Python `str(meta.get("title") or _humanize(...))` — falsy titles fall back. */
6330
- function titleOf(meta, fallbackStem) {
6331
- const t = meta["title"];
6332
- if (t === void 0 || t === null || t === "" || t === 0 || t === false) return humanize(fallbackStem);
6333
- return String(t);
6334
- }
6335
- function positionOf(meta, fallback) {
6336
- for (const key of ["position", "sidebar_position"]) {
6337
- const val = meta[key];
6338
- if (typeof val === "number" && Number.isFinite(val)) return Math.trunc(val);
6339
- }
6340
- return fallback;
6341
- }
6342
- /** Python compares strings by code point; JS `<` compares UTF-16 units — they differ on astral names. */
6343
- function codePointCompare(a, b) {
6344
- const as = [...a];
6345
- const bs = [...b];
6346
- const n = Math.min(as.length, bs.length);
6347
- for (let i = 0; i < n; i++) {
6348
- const d = (as[i]?.codePointAt(0) ?? 0) - (bs[i]?.codePointAt(0) ?? 0);
6349
- if (d !== 0) return d;
6350
- }
6351
- return as.length - bs.length;
6104
+ /**
6105
+ * Delete one generation's rows (chunks cascade from sources) and mark the run
6106
+ * reaped. NEVER touches takedown_denylist or retrieval_log — the ledger and
6107
+ * denylist outlive the content they governed (§5).
6108
+ */
6109
+ async function reap(client, opts) {
6110
+ for (const sql of [
6111
+ "DELETE FROM node_centroids WHERE tenant_id = $1 AND generation = $2",
6112
+ "DELETE FROM slug_aliases WHERE tenant_id = $1 AND generation = $2",
6113
+ "DELETE FROM sources WHERE tenant_id = $1 AND generation = $2"
6114
+ ]) await client.query(sql, [opts.tenantId, opts.generation]);
6115
+ for (;;) if (!(await client.query("DELETE FROM content_nodes n WHERE n.tenant_id = $1 AND n.generation = $2 AND NOT EXISTS (SELECT 1 FROM content_nodes ch WHERE ch.parent_id = n.node_id AND ch.tenant_id = n.tenant_id AND ch.generation = n.generation)", [opts.tenantId, opts.generation])).rowCount) break;
6116
+ await client.query("UPDATE ingestion_runs SET state = 'reaped' WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation]);
6352
6117
  }
6353
- /** Re-exported so every reader of a document agrees where its frontmatter ENDS. */
6354
- const FRONTMATTER$1 = /^\uFEFF?---\r?\n([\s\S]*?)\r?\n---[ \t]*\r?\n?/;
6355
- const YAML_BOOLS = {
6356
- yes: true,
6357
- Yes: true,
6358
- YES: true,
6359
- no: false,
6360
- No: false,
6361
- NO: false,
6362
- true: true,
6363
- True: true,
6364
- TRUE: true,
6365
- false: false,
6366
- False: false,
6367
- FALSE: false,
6368
- on: true,
6369
- On: true,
6370
- ON: true,
6371
- off: false,
6372
- Off: false,
6373
- OFF: false
6374
- };
6375
6118
  /**
6376
- * Minimal PyYAML-compatible frontmatter reader for the FOUR scalar keys this
6377
- * adapter consumes (`title`, `position`, `sidebar_position`, `sor_id`) the
6378
- * kernel discards every other frontmatter key at build time (taxonomy comes
6379
- * from the manifest), so a YAML dependency would buy nothing (guard rule 5).
6380
- * Scope, deliberately narrow pending a shared markdown module: top-level
6381
- * `key: scalar` pairs only; nested/indented structure is ignored. Mirroring
6382
- * the oracle's error path (`parse_frontmatter` catches YAMLError → `{}`), a
6383
- * document PyYAML would refuse — an UNQUOTED value containing ": ", a block
6384
- * scalar, an anchor/alias/tag, a non-mapping line — yields an EMPTY meta, so
6385
- * titles fall back to the humanized filename instead of a half-read mapping.
6119
+ * The §7 row for a governance act, written INSIDE the same transaction as the
6120
+ * act. `logRead` deliberately covers only the four serving actions; a takedown
6121
+ * is a write-plane act, and separating the two writes would allow a denial with
6122
+ * no row proving it happened the one outcome the ledger exists to prevent.
6386
6123
  */
6387
- function frontmatterMeta(text) {
6388
- const block = FRONTMATTER$1.exec(text)?.[1];
6389
- if (block === void 0) return {};
6390
- const meta = {};
6391
- for (const line of block.split(/\r?\n/)) {
6392
- if (line.trim() === "" || line.trimStart().startsWith("#")) continue;
6393
- if (/^[ \t]/.test(line)) continue;
6394
- const kv = /^([^\s:]+):(?:[ \t]+(.*))?$/.exec(line);
6395
- const key = kv?.[1];
6396
- if (key === void 0) return {};
6397
- const parsed = scalarValue((kv?.[2] ?? "").trim());
6398
- if (!parsed.ok) return {};
6399
- meta[key] = parsed.value;
6400
- }
6401
- return meta;
6402
- }
6403
- function scalarValue(raw) {
6404
- if (raw === "") return {
6405
- ok: true,
6406
- value: null
6407
- };
6408
- const dq = /^"(.*)"$/.exec(raw);
6409
- if (dq !== null) return {
6410
- ok: true,
6411
- value: (dq[1] ?? "").replace(/\\"/g, "\"").replace(/\\\\/g, "\\")
6412
- };
6413
- const sq = /^'(.*)'$/.exec(raw);
6414
- if (sq !== null) return {
6415
- ok: true,
6416
- value: (sq[1] ?? "").replace(/''/g, "'")
6417
- };
6418
- const plain = raw.replace(/[ \t]+#.*$/, "").trim();
6419
- if (Object.hasOwn(YAML_BOOLS, plain)) return {
6420
- ok: true,
6421
- value: YAML_BOOLS[plain]
6422
- };
6423
- if (plain === "~" || /^(?:null|Null|NULL)$/.test(plain)) return {
6424
- ok: true,
6425
- value: null
6426
- };
6427
- if (/^[-+]?[0-9][0-9_]*$/.test(plain)) return {
6428
- ok: true,
6429
- value: Number.parseInt(plain.replaceAll("_", ""), 10)
6430
- };
6431
- if (/^[-+]?(?:\.[0-9]+|[0-9][0-9_]*\.[0-9_]*)(?:[eE][-+]?[0-9]+)?$/.test(plain)) return {
6432
- ok: true,
6433
- value: Number.parseFloat(plain.replaceAll("_", ""))
6434
- };
6435
- if (/:[ \t]/.test(plain) || plain.endsWith(":")) return {
6436
- ok: false,
6437
- value: null
6438
- };
6439
- if (/^[|>&*!{[]/.test(plain)) return {
6440
- ok: false,
6441
- value: null
6442
- };
6443
- return {
6444
- ok: true,
6445
- value: plain
6446
- };
6124
+ async function recordAct(client, instance, detail, actor, action = "takedown_applied") {
6125
+ await client.query("INSERT INTO retrieval_log (tenant_id, corpus_id, actor, action, detail) VALUES ($1, $2, $3, $5, $4::jsonb)", [
6126
+ instance.tenantId,
6127
+ instance.corpusId,
6128
+ actor,
6129
+ JSON.stringify(detail),
6130
+ action
6131
+ ]);
6447
6132
  }
6448
6133
  /**
6449
- * The governance a document declares about itself, read once and carried onto
6450
- * the record.
6451
- *
6452
- * Before this module the ingest adapter kept four frontmatter keys and dropped
6453
- * the rest, so `visibility`, `status`, `owner` and `provenance` existed only in
6454
- * markdown — and every surface re-derived them independently. The site enforced
6455
- * `visibility:`; the MCP door could not, because the record did not carry it.
6456
- * One reader, one shape, persisted on `content_nodes` (schema 2.2).
6134
+ * Deny a node (or its subtree) and record the act.
6457
6135
  *
6458
- * The vocabulary is deliberately NOT closed here. A record that declares an
6459
- * audience the instance does not know is a corpus error the checker names; the
6460
- * ingest path's job is to carry what was written, faithfully, so the serving
6461
- * door can make the decision with the instance in hand. Refusing unknown values
6462
- * here would put the audience model in two places again.
6136
+ * The audit row is written in the SAME transaction as the denial: a takedown
6137
+ * that happened without a row proving it happened is exactly the shape the
6138
+ * §7 ledger exists to prevent.
6463
6139
  */
6464
- const NO_GOVERNANCE = {
6465
- visibility: null,
6466
- docStatus: null,
6467
- owner: null,
6468
- provenance: null,
6469
- supersededBy: null
6470
- };
6471
- function scalar(meta, key) {
6472
- const raw = meta[key];
6473
- if (typeof raw === "string") {
6474
- const trimmed = raw.trim();
6475
- return trimmed === "" ? null : trimmed;
6476
- }
6477
- if (typeof raw === "boolean") return raw ? "true" : "false";
6478
- if (typeof raw === "number") return String(raw);
6479
- return null;
6140
+ async function applyTakedown(pool, instance, opts) {
6141
+ return runIngest(pool, instance.tenantId, async (client) => {
6142
+ const resolves = ((await client.query(`SELECT 1 FROM content_nodes n
6143
+ JOIN corpora c ON c.tenant_id = n.tenant_id AND c.corpus_id = $2
6144
+ WHERE n.tenant_id = $1 AND n.stable_id = $3 AND n.generation = c.active_generation
6145
+ LIMIT 1`, [
6146
+ instance.tenantId,
6147
+ instance.corpusId,
6148
+ opts.stableId
6149
+ ])).rowCount ?? 0) > 0;
6150
+ const changed = (await client.query("INSERT INTO takedown_denylist (tenant_id, corpus_id, stable_id, scope, reason) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (tenant_id, corpus_id, stable_id) DO UPDATE SET scope = EXCLUDED.scope, reason = EXCLUDED.reason WHERE takedown_denylist.scope IS DISTINCT FROM EXCLUDED.scope OR takedown_denylist.reason IS DISTINCT FROM EXCLUDED.reason RETURNING stable_id", [
6151
+ instance.tenantId,
6152
+ instance.corpusId,
6153
+ opts.stableId,
6154
+ opts.scope,
6155
+ opts.reason
6156
+ ])).rowCount === 1;
6157
+ await recordAct(client, instance, {
6158
+ stable_id: opts.stableId,
6159
+ scope: opts.scope,
6160
+ reason: opts.reason,
6161
+ change: changed ? "applied" : "unchanged"
6162
+ }, opts.actor);
6163
+ return {
6164
+ stableId: opts.stableId,
6165
+ scope: opts.scope,
6166
+ changed,
6167
+ resolves
6168
+ };
6169
+ });
6170
+ }
6171
+ /** Lift a denial. The ledger keeps the row that recorded imposing it. */
6172
+ async function revokeTakedown(pool, instance, opts) {
6173
+ return runIngest(pool, instance.tenantId, async (client) => {
6174
+ const changed = ((await client.query("DELETE FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2 AND stable_id = $3", [
6175
+ instance.tenantId,
6176
+ instance.corpusId,
6177
+ opts.stableId
6178
+ ])).rowCount ?? 0) > 0;
6179
+ await recordAct(client, instance, {
6180
+ stable_id: opts.stableId,
6181
+ change: changed ? "revoked" : "not-denied"
6182
+ }, opts.actor, "takedown_revoked");
6183
+ return {
6184
+ stableId: opts.stableId,
6185
+ scope: "node",
6186
+ changed
6187
+ };
6188
+ });
6189
+ }
6190
+ async function readLedger(pool, instance, limit) {
6191
+ return runAuditRead(pool, instance.tenantId, async (client) => {
6192
+ return (await client.query("SELECT action, actor, generation, detail, created_at FROM retrieval_log WHERE tenant_id = $1 AND corpus_id = $2 ORDER BY created_at DESC, id DESC LIMIT $3", [
6193
+ instance.tenantId,
6194
+ instance.corpusId,
6195
+ limit
6196
+ ])).rows.map((row) => ({
6197
+ action: String(row.action),
6198
+ actor: String(row.actor),
6199
+ generation: row.generation === null ? null : Number(row.generation),
6200
+ detail: row.detail ?? {},
6201
+ createdAt: row.created_at
6202
+ }));
6203
+ });
6480
6204
  }
6481
- const BLOCK_LIST = (key) => new RegExp(`^${key}:[ \\t]*\\r?\\n((?:[ \\t]*-[ \\t]+.*\\r?\\n?)+)`, "m");
6482
6205
  /**
6483
- * Values of a simple `key:` block list, the one nested shape the record's
6484
- * grammar uses (`provenance:` here, `audiences:` in instance.md). The scalar
6485
- * reader deliberately ignores indented lines, so without this a provenance list
6486
- * would vanish silently the failure mode this whole module exists to end.
6206
+ * Every stable_id a build must not publish, with `subtree` denials EXPANDED to
6207
+ * their actual descendants by the same `parent_id` walk the serving side uses.
6208
+ *
6209
+ * The site cannot do this itself: it has no tree, so it matched a prefix — and
6210
+ * a section's stable_id ends in `/index` (or `#section`), so the prefix never
6211
+ * matched its children and every descendant of a subtree takedown kept
6212
+ * publishing. Decision 14 records exactly why a prefix is wrong here; the fix
6213
+ * is to resolve the walk where the tree lives and hand over a flat list
6214
+ * (round-2 review of #43).
6487
6215
  */
6488
- function frontmatterListValues(text, key) {
6489
- const block = FRONTMATTER$1.exec(text)?.[1];
6490
- if (block === void 0) return null;
6491
- const m = BLOCK_LIST(key).exec(block + "\n");
6492
- if (m === null) return null;
6493
- const items = (m[1] ?? "").split(/\r?\n/).map((line) => /^[ \t]*-[ \t]+(.*)$/.exec(line)?.[1] ?? "").map((v) => v.trim().replace(/^["']|["']$/g, "").trim()).filter((v) => v !== "");
6494
- return items.length > 0 ? items : null;
6216
+ async function deniedStableIds(pool, instance) {
6217
+ return runRead(pool, instance.tenantId, async (client) => {
6218
+ return (await client.query(`WITH RECURSIVE gen AS (
6219
+ SELECT active_generation AS g FROM corpora WHERE tenant_id = $1 AND corpus_id = $2
6220
+ ),
6221
+ seed AS (
6222
+ SELECT n.node_id, n.stable_id, d.scope
6223
+ FROM takedown_denylist d
6224
+ JOIN content_nodes n ON n.tenant_id = d.tenant_id AND n.stable_id = d.stable_id
6225
+ JOIN gen ON n.generation = gen.g
6226
+ WHERE d.tenant_id = $1 AND d.corpus_id = $2
6227
+ ),
6228
+ walk AS (
6229
+ SELECT node_id, stable_id, scope FROM seed
6230
+ UNION ALL
6231
+ SELECT c.node_id, c.stable_id, w.scope
6232
+ FROM content_nodes c
6233
+ JOIN walk w ON c.parent_id = w.node_id
6234
+ JOIN gen ON c.generation = gen.g
6235
+ WHERE c.tenant_id = $1 AND w.scope = 'subtree'
6236
+ )
6237
+ SELECT DISTINCT stable_id FROM walk
6238
+ UNION
6239
+ -- Denials naming a stable_id no CURRENT generation carries are still
6240
+ -- denied: identity outlives any one generation (decision 14).
6241
+ SELECT stable_id FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2`, [instance.tenantId, instance.corpusId])).rows.map((r) => String(r.stable_id)).sort();
6242
+ });
6495
6243
  }
6496
6244
  /**
6497
- * Read the governance keys from an already-parsed scalar map plus the raw
6498
- * document text (which the list reader needs). Unknown keys are ignored, as
6499
- * they always were this module widens what the record carries, it does not
6500
- * narrow what a document may say.
6245
+ * The knowledge-relative DIRECTORIES that `--subtree` denials govern.
6246
+ *
6247
+ * Derived from the DESCENDANTS' `sources.origin_path`, never from the denied
6248
+ * node's own id or path, because neither works:
6249
+ *
6250
+ * a section has no source `knowledge/policies#section` is synthetic — the
6251
+ * tree node for a directory. Joining `sources` on
6252
+ * the denied node itself yields nothing, and a
6253
+ * section is the ordinary target of `--subtree`.
6254
+ * a leaf's directory is not `--subtree` on one document would emit that
6255
+ * its subtree document's directory and deny every sibling.
6256
+ *
6257
+ * So: walk the descendants, take the directory of each one's file, and keep the
6258
+ * SHALLOWEST — a directory that contains another in the set is the subtree
6259
+ * root, and `startsWith` then covers subdirectories added later too. A denial
6260
+ * with no descendants contributes nothing, which is correct: its subtree is
6261
+ * itself, and the flat id list already holds it.
6262
+ *
6263
+ * The seed's OWN file counts when the seed has children, and only then — see
6264
+ * the SQL comment: a container's index.md names its directory, a leaf's file
6265
+ * names its parent's.
6501
6266
  */
6502
- var GovernanceParseError = class extends Error {
6503
- name = "GovernanceParseError";
6504
- };
6505
- function governanceFromFrontmatter(meta, text) {
6506
- if (frontmatterListValues(text, "visibility") !== null) throw new GovernanceParseError("a document declares `visibility:` as a LIST — a document belongs to exactly one tier. Write a single value, e.g. `visibility: internal`.");
6507
- const declaredInText = /^visibility:[ \t]*(.*)$/m.exec(FRONTMATTER$1.exec(text)?.[1] ?? "");
6508
- if (declaredInText !== null && scalar(meta, "visibility") === null) {
6509
- const written = declaredInText[1]?.trim() ?? "";
6510
- throw new GovernanceParseError(written === "" ? "a document declares `visibility:` with no readable value — an unreadable tier reads as no tier, and no tier is the default tier, which is how a restricted document gets served. Write a single value, e.g. `visibility: internal`." : `a document declares \`visibility: ${written}\` but this reader could not resolve it — usually because ANOTHER key in the same frontmatter is a shape it cannot read (a flow list like \`tags: [a, b]\`, or an unquoted value containing ": "). An unresolved tier would be served at the default. Quote the other value, or write it as a block list.`);
6511
- }
6512
- const provenanceScalar = scalar(meta, "provenance");
6513
- const provenanceList = frontmatterListValues(text, "provenance");
6514
- return {
6515
- visibility: scalar(meta, "visibility"),
6516
- docStatus: scalar(meta, "status"),
6517
- owner: scalar(meta, "owner"),
6518
- provenance: provenanceList ?? (provenanceScalar === null ? null : [provenanceScalar]),
6519
- supersededBy: scalar(meta, "superseded_by")
6520
- };
6521
- }
6522
- const SUPPORTED_FORMATS = [1];
6523
- /** The manifest is malformed — named precisely; a bad manifest never half-ingests. */
6524
- var ManifestError = class extends Error {
6525
- constructor(message) {
6526
- super(message);
6527
- this.name = "ManifestError";
6528
- }
6529
- };
6530
- /** Mirrors the oracle dataclass defaults (parent/summary/permalink None, position 0, keywords ()). */
6531
- function manifestNode(init) {
6532
- return {
6533
- stable_id: init.stable_id,
6534
- slug: init.slug,
6535
- title: init.title,
6536
- kind: init.kind,
6537
- parent: init.parent ?? null,
6538
- position: init.position ?? 0,
6539
- summary: init.summary ?? null,
6540
- keywords: init.keywords ?? [],
6541
- permalink: init.permalink ?? null,
6542
- governance: init.governance ?? NO_GOVERNANCE
6543
- };
6544
- }
6545
- function manifestFile(init) {
6546
- return {
6547
- path: init.path,
6548
- node: init.node,
6549
- title: init.title ?? null
6550
- };
6551
- }
6552
- function parseManifest(text) {
6553
- let raw;
6554
- try {
6555
- raw = JSON.parse(text);
6556
- } catch (exc) {
6557
- throw new ManifestError(`manifest.json is not valid JSON: ${exc instanceof Error ? exc.message : String(exc)}`);
6267
+ async function deniedSubtreeDirs(pool, instance) {
6268
+ const paths = await runRead(pool, instance.tenantId, async (client) => {
6269
+ return (await client.query(`WITH RECURSIVE gen AS (
6270
+ SELECT active_generation AS g FROM corpora WHERE tenant_id = $1 AND corpus_id = $2
6271
+ ),
6272
+ seed AS (
6273
+ SELECT n.node_id
6274
+ FROM takedown_denylist d
6275
+ JOIN content_nodes n ON n.tenant_id = d.tenant_id AND n.stable_id = d.stable_id
6276
+ JOIN gen ON n.generation = gen.g
6277
+ WHERE d.tenant_id = $1 AND d.corpus_id = $2 AND d.scope = 'subtree'
6278
+ ),
6279
+ walk AS (
6280
+ SELECT node_id FROM seed
6281
+ UNION ALL
6282
+ SELECT c.node_id
6283
+ FROM content_nodes c
6284
+ JOIN walk w ON c.parent_id = w.node_id
6285
+ JOIN gen ON c.generation = gen.g
6286
+ WHERE c.tenant_id = $1
6287
+ )
6288
+ SELECT DISTINCT s.origin_path
6289
+ FROM walk w
6290
+ JOIN content_nodes n ON n.node_id = w.node_id
6291
+ JOIN sources s ON s.tenant_id = n.tenant_id AND s.generation = n.generation
6292
+ AND s.node_id = n.node_id
6293
+ -- The seed's own file counts only when the seed HAS CHILDREN.
6294
+ --
6295
+ -- Excluding every seed stopped a LEAF denial emitting its parent
6296
+ -- directory and denying every sibling — right for a leaf, wrong for a
6297
+ -- container. A section's own index.md is the file that names the
6298
+ -- section's DIRECTORY, so a section whose other descendants all live
6299
+ -- one level down contributed only the subdirectory, and a document
6300
+ -- written directly under the withdrawn section published to /docs and
6301
+ -- llms.txt (round-10 review of PR 43).
6302
+ --
6303
+ -- "Has children" is the right test, not "kind = section": it is the
6304
+ -- property that decides whether the node's directory is its subtree
6305
+ -- or its parent's.
6306
+ WHERE w.node_id NOT IN (
6307
+ SELECT s2.node_id FROM seed s2
6308
+ WHERE NOT EXISTS (SELECT 1 FROM content_nodes kid
6309
+ JOIN gen ON kid.generation = gen.g
6310
+ WHERE kid.tenant_id = $1 AND kid.parent_id = s2.node_id)
6311
+ )`, [instance.tenantId, instance.corpusId])).rows.map((r) => String(r.origin_path));
6312
+ });
6313
+ const dirs = /* @__PURE__ */ new Set();
6314
+ for (const raw of paths) {
6315
+ const normalized = raw.replace(/\\/g, "/");
6316
+ const slash = normalized.lastIndexOf("/");
6317
+ dirs.add(slash === -1 ? "/" : `${normalized.slice(0, slash)}/`);
6558
6318
  }
6559
- if (typeof raw !== "object" || raw === null || Array.isArray(raw)) throw new ManifestError("manifest.json must be an object");
6560
- const obj = raw;
6561
- const fmt = obj["format"];
6562
- if (typeof fmt !== "number" || !SUPPORTED_FORMATS.includes(fmt)) throw new ManifestError(`manifest format ${JSON.stringify(fmt)} unsupported (supported: ${SUPPORTED_FORMATS.join(", ")})`);
6563
- const corpusId = topString(obj, "corpus_id");
6564
- const sourceCommit = topString(obj, "source_commit");
6565
- const nodes = entriesOf(obj, "nodes").map((n, i) => manifestNode({
6566
- stable_id: req(n, "stable_id", i),
6567
- slug: req(n, "slug", i),
6568
- title: req(n, "title", i),
6569
- kind: req(n, "kind", i),
6570
- parent: optString(n["parent"]),
6571
- position: toPosition(n["position"], i),
6572
- summary: optString(n["summary"]),
6573
- keywords: toKeywords(n["keywords"], i),
6574
- permalink: optString(n["permalink"]),
6575
- governance: toGovernance(n["governance"], i)
6576
- }));
6577
- const files = entriesOf(obj, "files").map((f, i) => manifestFile({
6578
- path: req(f, "path", i),
6579
- node: req(f, "node", i),
6580
- title: optString(f["title"])
6581
- }));
6582
- validate(nodes, files);
6319
+ const all = [...dirs];
6320
+ return all.filter((dir) => !all.some((other) => other !== dir && dir.startsWith(other))).sort();
6321
+ }
6322
+ async function listTakedowns(pool, instance) {
6323
+ return runRead(pool, instance.tenantId, async (client) => {
6324
+ return (await client.query("SELECT stable_id, scope, reason, created_at FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2 ORDER BY created_at, stable_id", [instance.tenantId, instance.corpusId])).rows.map((r) => ({
6325
+ stableId: String(r.stable_id),
6326
+ scope: String(r.scope),
6327
+ reason: String(r.reason),
6328
+ createdAt: r.created_at
6329
+ }));
6330
+ });
6331
+ }
6332
+ function denylistManifest(corpusId, stableIds, now, source = "database", deniedSubtrees = []) {
6583
6333
  return {
6584
- format: fmt,
6334
+ format: 1,
6585
6335
  corpus_id: corpusId,
6586
- source_commit: sourceCommit,
6587
- nodes,
6588
- files
6336
+ source,
6337
+ denied_subtrees: [...deniedSubtrees].sort(),
6338
+ exported_at: now.toISOString(),
6339
+ denied: stableIds.map((stable_id) => ({
6340
+ stable_id,
6341
+ scope: "node"
6342
+ }))
6589
6343
  };
6590
6344
  }
6591
6345
  /**
6592
- * The inverse of `governanceToJson`. Absent NO_GOVERNANCE, which is what a
6593
- * corpus that declares nothing has always meant. A present-but-wrong shape is
6594
- * REFUSED rather than silently dropped: dropping it would serve a document at
6595
- * the instance default, and for a `visibility:` that means serving a restricted
6596
- * document to everyone.
6346
+ * The plain-tree corpus adapter ANY folder of Markdown becomes a corpus.
6347
+ * Converted from the oracle (sor-agentfactory @ b554f91,
6348
+ * ingest/adapters/plain_tree.py); the kernel cannot tell this manifest from
6349
+ * any other adapter's.
6350
+ *
6351
+ * Conventions (deliberately minimal — an operator can satisfy them with a bare
6352
+ * folder):
6353
+ * - directories become `section` nodes; `.md`/`.mdx` files become `document`
6354
+ * nodes;
6355
+ * - `index.md` (or `README.md`) inside a directory is that SECTION's own
6356
+ * content, not a child;
6357
+ * - ordering: frontmatter `position` (or `sidebar_position`) wins, else name
6358
+ * sort;
6359
+ * - titles: frontmatter `title`, else the filename humanized;
6360
+ * - stable ids: frontmatter `sor_id`, else the tree-relative path;
6361
+ * - hidden entries (leading `.` or `_`) and ALL symlinks are skipped LOUDLY
6362
+ * (reported through `onSkip`, console by default — never silent); symlinks
6363
+ * are never followed, so a link cannot walk out of the tree or cycle it;
6364
+ * - a directory carrying MORE than one index-named file (index.md +
6365
+ * README.md …) fails loud: which one is the section's own content is
6366
+ * ambiguous, and silently dropping the loser is exactly the corpus
6367
+ * corruption this adapter must never commit.
6368
+ *
6369
+ * The oracle's `publish_bundle` (deterministic tgz staging) is a separate
6370
+ * slice and is not converted here.
6597
6371
  */
6598
- function toGovernance(raw, index) {
6599
- if (raw === void 0 || raw === null) return NO_GOVERNANCE;
6600
- if (typeof raw !== "object" || Array.isArray(raw)) throw new ManifestError(`entry ${index}: 'governance' must be an object`);
6601
- const g = raw;
6602
- const str = (key) => {
6603
- const val = g[key];
6604
- if (val === void 0 || val === null) return null;
6605
- if (typeof val !== "string" || val === "") throw new ManifestError(`entry ${index}: 'governance.${key}' must be a non-empty string`);
6606
- return val;
6607
- };
6608
- const provenanceRaw = g["provenance"];
6609
- let provenance = null;
6610
- if (provenanceRaw !== void 0 && provenanceRaw !== null) {
6611
- if (!Array.isArray(provenanceRaw) || provenanceRaw.some((v) => typeof v !== "string")) throw new ManifestError(`entry ${index}: 'governance.provenance' must be a list of strings`);
6612
- provenance = provenanceRaw;
6372
+ const INDEX_NAMES = [
6373
+ "index.md",
6374
+ "index.mdx",
6375
+ "README.md"
6376
+ ];
6377
+ /** Frontmatter-position fallback for entries that declare none (oracle plain_tree.py:107,114). */
6378
+ const POSITION_FALLBACK = 1e4;
6379
+ /** Walk a directory on disk manifest + sources. Fail-loud on emptiness and ambiguity. */
6380
+ async function buildManifest(treeRoot, options) {
6381
+ const rootPath = treeRoot.length > 1 ? treeRoot.replace(/\/+$/, "") : treeRoot;
6382
+ let isDir = false;
6383
+ try {
6384
+ isDir = (await stat(rootPath)).isDirectory();
6385
+ } catch {
6386
+ isDir = false;
6613
6387
  }
6614
- return {
6615
- visibility: str("visibility"),
6616
- docStatus: str("doc_status"),
6617
- owner: str("owner"),
6618
- provenance,
6619
- supersededBy: str("superseded_by")
6620
- };
6621
- }
6622
- function topString(obj, key) {
6623
- const val = obj[key];
6624
- if (typeof val !== "string" || !val) throw new ManifestError(`manifest.${key} must be a non-empty string`);
6625
- return val;
6626
- }
6627
- function entriesOf(obj, key) {
6628
- const raw = obj[key] ?? [];
6629
- if (!Array.isArray(raw)) throw new ManifestError(`manifest.${key} must be an array`);
6630
- return raw.map((entry, i) => {
6631
- if (typeof entry !== "object" || entry === null || Array.isArray(entry)) throw new ManifestError(`entry ${i}: must be an object`);
6632
- return entry;
6388
+ if (!isDir) throw new ManifestError(`plain-tree root ${rootPath} is not a directory`);
6389
+ return buildManifestFromTree(await readTree(rootPath), {
6390
+ ...options,
6391
+ rootPath
6633
6392
  });
6634
6393
  }
6635
- function req(obj, key, index) {
6636
- const val = obj[key];
6637
- if (typeof val !== "string" || !val) throw new ManifestError(`entry ${index}: '${key}' must be a non-empty string`);
6638
- return val;
6639
- }
6640
- function optString(val) {
6641
- return typeof val === "string" ? val : null;
6642
- }
6643
- /** Python `int(...)` parity: truncate finite numbers, parse integer strings, refuse the rest loudly. */
6644
- function toPosition(val, index) {
6645
- if (val === void 0) return 0;
6646
- if (typeof val === "number" && Number.isFinite(val)) return Math.trunc(val);
6647
- if (typeof val === "boolean") return val ? 1 : 0;
6648
- if (typeof val === "string" && /^[+-]?\d+$/.test(val.trim())) return Number.parseInt(val.trim(), 10);
6649
- throw new ManifestError(`entry ${index}: position must be an integer, got ${JSON.stringify(val)}`);
6650
- }
6651
- function toKeywords(val, index) {
6652
- if (val === void 0 || val === null) return [];
6653
- if (!Array.isArray(val)) throw new ManifestError(`entry ${index}: keywords must be an array of strings`);
6654
- return val.map((k, j) => {
6655
- if (typeof k !== "string") throw new ManifestError(`entry ${index}: keywords[${j}] must be a string`);
6656
- return k;
6394
+ /**
6395
+ * Load a directory into an in-memory tree. lstat semantics throughout: a
6396
+ * symlink is represented as a symlink even one named `index.md` never
6397
+ * followed, never read (the oracle's docstring contract; its `_index_of`
6398
+ * incidentally followed a symlinked index via `is_file()`, which this port
6399
+ * deliberately does not reproduce). Non-markdown files are invisible to the
6400
+ * walk, exactly as the oracle's suffix filter makes them.
6401
+ */
6402
+ async function readTree(dirPath) {
6403
+ const dirents = await readdir(dirPath, { withFileTypes: true });
6404
+ const entries = [];
6405
+ for (const d of dirents) if (d.isSymbolicLink()) entries.push({
6406
+ kind: "symlink",
6407
+ name: d.name
6408
+ });
6409
+ else if (d.isDirectory()) entries.push(await readTree(join(dirPath, d.name)));
6410
+ else if (d.isFile() && isDoc(d.name)) entries.push({
6411
+ kind: "file",
6412
+ name: d.name,
6413
+ text: await readFile(join(dirPath, d.name), "utf8")
6657
6414
  });
6415
+ return {
6416
+ kind: "dir",
6417
+ name: basename(dirPath),
6418
+ entries
6419
+ };
6658
6420
  }
6659
- function validate(nodes, files) {
6660
- const seen = /* @__PURE__ */ new Set();
6661
- const dupes = /* @__PURE__ */ new Set();
6662
- for (const n of nodes) {
6663
- if (seen.has(n.stable_id)) dupes.add(n.stable_id);
6664
- seen.add(n.stable_id);
6665
- }
6666
- if (dupes.size > 0) throw new ManifestError(`duplicate node stable_id(s): [${[...dupes].sort().map((d) => `'${d}'`).join(", ")}]`);
6667
- for (const n of nodes) if (n.parent !== null && !seen.has(n.parent)) throw new ManifestError(`node '${n.stable_id}': unknown parent '${n.parent}'`);
6668
- for (const f of files) if (!seen.has(f.node)) throw new ManifestError(`file '${f.path}': unknown node '${f.node}'`);
6669
- const paths = /* @__PURE__ */ new Set();
6670
- for (const f of files) {
6671
- if (paths.has(f.path)) throw new ManifestError("duplicate file paths in manifest");
6672
- paths.add(f.path);
6673
- }
6674
- const siblingSlugs = /* @__PURE__ */ new Map();
6675
- for (const n of nodes) {
6676
- const parent = n.parent ?? "";
6677
- const bySlug = siblingSlugs.get(parent) ?? /* @__PURE__ */ new Map();
6678
- const owners = bySlug.get(n.slug) ?? [];
6679
- owners.push(n.stable_id);
6680
- bySlug.set(n.slug, owners);
6681
- siblingSlugs.set(parent, bySlug);
6682
- }
6683
- for (const bySlug of siblingSlugs.values()) for (const [slug, owners] of bySlug) if (owners.length > 1) throw new ManifestError(`sibling slug collision: ${owners.map((o) => `'${o}'`).join(", ")} all slug to '${slug}' under the same parent — rename one (a slug is a node's URL segment and must be unique among siblings)`);
6684
- }
6685
- /** Parents before children (insert order for the FK); a cycle fails loudly. */
6686
- function topological(nodes) {
6687
- const byId = new Map(nodes.map((n) => [n.stable_id, n]));
6688
- const out = [];
6689
- const state = /* @__PURE__ */ new Map();
6690
- const visit = (n) => {
6691
- const mark = state.get(n.stable_id) ?? 0;
6692
- if (mark === 2) return;
6693
- if (mark === 1) throw new ManifestError(`parent cycle at '${n.stable_id}'`);
6694
- state.set(n.stable_id, 1);
6695
- if (n.parent !== null) {
6696
- const parent = byId.get(n.parent);
6697
- if (parent === void 0) throw new ManifestError(`node '${n.stable_id}': unknown parent '${n.parent}'`);
6698
- visit(parent);
6421
+ /** The pure walk: tree → manifest + {manifest path → source path}. */
6422
+ function buildManifestFromTree(root, options) {
6423
+ const rootName = root.name;
6424
+ const rootPath = options.rootPath ?? rootName;
6425
+ const onSkip = options.onSkip ?? ((line) => console.log(line));
6426
+ const nodes = [];
6427
+ const files = [];
6428
+ const sources = /* @__PURE__ */ new Map();
6429
+ const skipped = [];
6430
+ const fullPath = (relSegs, name) => `${rootPath}/${[...relSegs, name].join("/")}`;
6431
+ const addFile = (nodeSid, fileSegs) => {
6432
+ const rel = fileSegs.join("/");
6433
+ const manifestPath = `${rootName}/${rel}`;
6434
+ files.push(manifestFile({
6435
+ path: manifestPath,
6436
+ node: nodeSid
6437
+ }));
6438
+ sources.set(manifestPath, `${rootPath}/${rel}`);
6439
+ };
6440
+ const walk = (dir, relSegs, parentSid) => {
6441
+ const entries = [...dir.entries].sort((a, b) => codePointCompare(a.name.toLowerCase(), b.name.toLowerCase()));
6442
+ const docs = [];
6443
+ const dirs = [];
6444
+ for (const e of entries) if (e.kind === "symlink") skipped.push(`${fullPath(relSegs, e.name)} (symlink)`);
6445
+ else if (e.kind === "file" && isDoc(e.name)) docs.push(e);
6446
+ else if (e.kind === "dir") dirs.push(e);
6447
+ const ordered = [];
6448
+ for (const f of docs) {
6449
+ if (f.name.startsWith(".") || f.name.startsWith("_")) {
6450
+ skipped.push(fullPath(relSegs, f.name));
6451
+ continue;
6452
+ }
6453
+ if (INDEX_NAMES.includes(f.name)) continue;
6454
+ ordered.push({
6455
+ position: positionOf(frontmatterMeta(f.text), POSITION_FALLBACK),
6456
+ nameLower: f.name.toLowerCase(),
6457
+ entry: f
6458
+ });
6459
+ }
6460
+ for (const d of dirs) {
6461
+ if (d.name.startsWith(".") || d.name.startsWith("_")) {
6462
+ skipped.push(fullPath(relSegs, d.name));
6463
+ continue;
6464
+ }
6465
+ const index = indexOf(d, fullPath(relSegs, d.name));
6466
+ const dirMeta = index === null ? {} : frontmatterMeta(index.text);
6467
+ ordered.push({
6468
+ position: positionOf(dirMeta, POSITION_FALLBACK),
6469
+ nameLower: d.name.toLowerCase(),
6470
+ entry: d
6471
+ });
6472
+ }
6473
+ ordered.sort((x, y) => x.position - y.position || codePointCompare(x.nameLower, y.nameLower));
6474
+ let position = 0;
6475
+ for (const { entry } of ordered) {
6476
+ position += 1;
6477
+ if (entry.kind === "dir") {
6478
+ const dirSegs = [...relSegs, entry.name];
6479
+ const index = indexOf(entry, fullPath(relSegs, entry.name));
6480
+ const meta = index === null ? {} : frontmatterMeta(index.text);
6481
+ const sid = index === null ? `${rootName}/${dirSegs.join("/")}#section` : stableIdOf(rootName, [...dirSegs, index.name], meta);
6482
+ nodes.push(manifestNode({
6483
+ stable_id: sid,
6484
+ slug: slugify(entry.name),
6485
+ title: titleOf(meta, entry.name),
6486
+ kind: "section",
6487
+ parent: parentSid,
6488
+ position,
6489
+ governance: index === null ? NO_GOVERNANCE : governanceFromFrontmatter(meta, index.text)
6490
+ }));
6491
+ if (index !== null) addFile(sid, [...dirSegs, index.name]);
6492
+ walk(entry, dirSegs, sid);
6493
+ } else {
6494
+ const meta = frontmatterMeta(entry.text);
6495
+ const stem = stemOf(entry.name);
6496
+ const sid = stableIdOf(rootName, [...relSegs, entry.name], meta);
6497
+ nodes.push(manifestNode({
6498
+ stable_id: sid,
6499
+ slug: slugify(stem),
6500
+ title: titleOf(meta, stem),
6501
+ kind: "document",
6502
+ parent: parentSid,
6503
+ position,
6504
+ governance: governanceFromFrontmatter(meta, entry.text)
6505
+ }));
6506
+ addFile(sid, [...relSegs, entry.name]);
6507
+ }
6699
6508
  }
6700
- state.set(n.stable_id, 2);
6701
- out.push(n);
6702
6509
  };
6703
- for (const n of nodes) visit(n);
6704
- return out;
6705
- }
6706
- /**
6707
- * The one canonical JSON emitter for every adapter (re-homed from the oracle's
6708
- * `_to_json`, adapters/docusaurus_sidebar.py:410): node keys whose value is
6709
- * null/empty/zero are omitted EXCEPT `position`, which is always emitted; file
6710
- * dicts carry `title` only when set. Adapters round-trip the result through
6711
- * `parseManifest` before writing, so an adapter can never emit what ingest
6712
- * would refuse.
6713
- */
6714
- function manifestToJson(m) {
6510
+ const rootIndex = indexOf(root, rootPath);
6511
+ if (rootIndex !== null) {
6512
+ const meta = frontmatterMeta(rootIndex.text);
6513
+ const sid = stableIdOf(rootName, [rootIndex.name], meta);
6514
+ nodes.push(manifestNode({
6515
+ stable_id: sid,
6516
+ slug: slugify(rootName),
6517
+ title: titleOf(meta, rootName),
6518
+ kind: "document",
6519
+ position: 0,
6520
+ governance: governanceFromFrontmatter(meta, rootIndex.text)
6521
+ }));
6522
+ addFile(sid, [rootIndex.name]);
6523
+ }
6524
+ walk(root, [], null);
6525
+ for (const s of skipped) onSkip(`plain-tree: skipped ${s}`);
6526
+ if (files.length === 0) throw new ManifestError(`plain-tree root ${rootPath} contains no Markdown`);
6527
+ const manifest = {
6528
+ format: 1,
6529
+ corpus_id: options.corpusId,
6530
+ source_commit: options.sourceCommit,
6531
+ nodes,
6532
+ files
6533
+ };
6534
+ parseManifest(JSON.stringify(manifestToJson(manifest)));
6715
6535
  return {
6716
- format: m.format,
6717
- corpus_id: m.corpus_id,
6718
- source_commit: m.source_commit,
6719
- nodes: m.nodes.map(nodeToJson),
6720
- files: m.files.map((f) => ({
6721
- path: f.path,
6722
- node: f.node,
6723
- ...f.title ? { title: f.title } : {}
6724
- }))
6536
+ manifest,
6537
+ sources
6725
6538
  };
6726
6539
  }
6727
- function nodeToJson(n) {
6728
- const fields = [
6729
- ["stable_id", n.stable_id],
6730
- ["slug", n.slug],
6731
- ["title", n.title],
6732
- ["kind", n.kind],
6733
- ["parent", n.parent],
6734
- ["position", n.position],
6735
- ["summary", n.summary],
6736
- ["keywords", n.keywords],
6737
- ["permalink", n.permalink]
6738
- ];
6739
- const out = {};
6740
- for (const [key, val] of fields) {
6741
- const omit = val === null || val === 0 || Array.isArray(val) && val.length === 0;
6742
- if (key === "position" || !omit) out[key] = val;
6540
+ /** Python `p.suffix in (".md", ".mdx")` parity: a dotfile named exactly ".md" has NO suffix. */
6541
+ function isDoc(name) {
6542
+ const dot = name.lastIndexOf(".");
6543
+ if (dot <= 0) return false;
6544
+ const suffix = name.slice(dot);
6545
+ return suffix === ".md" || suffix === ".mdx";
6546
+ }
6547
+ function indexOf(dir, dirPath) {
6548
+ const present = [];
6549
+ for (const name of INDEX_NAMES) {
6550
+ const hit = dir.entries.find((e) => e.kind === "file" && e.name === name);
6551
+ if (hit !== void 0) present.push(hit);
6743
6552
  }
6744
- const gov = governanceToJson(n.governance);
6745
- if (Object.keys(gov).length > 0) out["governance"] = gov;
6746
- return out;
6553
+ if (present.length > 1) throw new ManifestError(`ambiguous section index in ${dirPath}: [${present.map((p) => `'${p.name}'`).join(", ")}] — keep exactly one`);
6554
+ return present[0] ?? null;
6747
6555
  }
6748
- function governanceToJson(g) {
6749
- const out = {};
6750
- if (g.visibility !== null) out["visibility"] = g.visibility;
6751
- if (g.docStatus !== null) out["doc_status"] = g.docStatus;
6752
- if (g.owner !== null) out["owner"] = g.owner;
6753
- if (g.provenance !== null && g.provenance.length > 0) out["provenance"] = g.provenance;
6754
- if (g.supersededBy !== null) out["superseded_by"] = g.supersededBy;
6755
- return out;
6556
+ function stableIdOf(rootName, fileSegs, meta) {
6557
+ const sid = meta["sor_id"];
6558
+ if (typeof sid === "string" && sid.trim() !== "") return sid.trim();
6559
+ return `${rootName}/${withoutSuffix(fileSegs.join("/"))}`;
6560
+ }
6561
+ /** Python Path.with_suffix("") parity: strip the LAST suffix only; a dotfile has none. */
6562
+ function withoutSuffix(rel) {
6563
+ const slash = rel.lastIndexOf("/");
6564
+ const name = rel.slice(slash + 1);
6565
+ const dot = name.lastIndexOf(".");
6566
+ if (dot <= 0) return rel;
6567
+ return rel.slice(0, slash + 1) + name.slice(0, dot);
6568
+ }
6569
+ function stemOf(name) {
6570
+ return withoutSuffix(name);
6571
+ }
6572
+ function slugify(text) {
6573
+ const slug = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
6574
+ if (slug !== "") return slug;
6575
+ return "x-" + createHash("sha256").update(text, "utf8").digest("hex").slice(0, 8);
6756
6576
  }
6577
+ const CASED = /\p{Cased}/u;
6757
6578
  /**
6758
- * Carried verbatim from the oracle's build step (ingest/build.py:109):
6759
- * `source_id = f.path.removesuffix(".md") + ":prose"`. The ".md"-only strip is
6760
- * a deliberate quirk `a/b.md``a/b:prose` while `a/b.mdx` KEEPS its suffix
6761
- * `a/b.mdx:prose`. Persisted source rows and carry-forward joins already pin
6762
- * this shape; changing it is a policy decision, never a tidy-up.
6579
+ * Python str.title() parity (the oracle's `_humanize`): a cased character
6580
+ * following an uncased one uppercases, following a cased one lowercases —
6581
+ * apostrophe quirk included ("rock'n'roll""Rock'N'Roll"). Node titles are
6582
+ * carry-forward join keys, so the quirk is load-bearing, not cosmetic.
6763
6583
  */
6764
- function sourceId(path) {
6765
- return (path.endsWith(".md") ? path.slice(0, -3) : path) + ":prose";
6584
+ function humanize(stem) {
6585
+ const spaced = stem.replace(/[-_]+/g, " ").trim();
6586
+ let out = "";
6587
+ let prevCased = false;
6588
+ for (const ch of spaced) {
6589
+ const cased = CASED.test(ch);
6590
+ out += cased ? prevCased ? ch.toLowerCase() : ch.toUpperCase() : ch;
6591
+ prevCased = cased;
6592
+ }
6593
+ return out;
6766
6594
  }
6767
- /** Python len(): Unicode code points, not UTF-16 units. Every limit comparison
6768
- * and the HARD_MAX_CHARS slice go through code points or the policy silently
6769
- * changes on astral-plane text (emoji, musical symbols, CJK extensions). */
6770
- function cpLen(s) {
6771
- let n = 0;
6772
- for (const _ch of s) n += 1;
6773
- return n;
6595
+ /** Python `str(meta.get("title") or _humanize(...))` falsy titles fall back. */
6596
+ function titleOf(meta, fallbackStem) {
6597
+ const t = meta["title"];
6598
+ if (t === void 0 || t === null || t === "" || t === 0 || t === false) return humanize(fallbackStem);
6599
+ return String(t);
6774
6600
  }
6775
- /** Python str.splitlines() boundary set (full code-point scan, 2026-08-19):
6776
- * \n \v \f \r \x1c \x1d \x1e \x85 \u2028 \u2029, with \r\n as one boundary.
6777
- * A naive split(/\r?\n/) changes segmentation on \x85, \u2028 etc. All
6778
- * boundaries are BMP, so a UTF-16 walk cannot land inside a surrogate pair. */
6779
- const LINE_BOUNDARY = /* @__PURE__ */ new Set([
6780
- "\n",
6781
- "\v",
6782
- "\f",
6783
- "\r",
6784
- "",
6785
- "",
6786
- "",
6787
- "…",
6788
- "\u2028",
6789
- "\u2029"
6790
- ]);
6791
- function pySplitLines(text, keepends) {
6792
- const out = [];
6793
- let start = 0;
6794
- let i = 0;
6795
- while (i < text.length) {
6796
- const ch = text[i];
6797
- if (LINE_BOUNDARY.has(ch)) {
6798
- let end = i + 1;
6799
- if (ch === "\r" && text[end] === "\n") end += 1;
6800
- out.push(keepends ? text.slice(start, end) : text.slice(start, i));
6801
- start = end;
6802
- i = end;
6803
- } else i += 1;
6601
+ function positionOf(meta, fallback) {
6602
+ for (const key of ["position", "sidebar_position"]) {
6603
+ const val = meta[key];
6604
+ if (typeof val === "number" && Number.isFinite(val)) return Math.trunc(val);
6804
6605
  }
6805
- if (start < text.length) out.push(text.slice(start));
6806
- return out;
6606
+ return fallback;
6807
6607
  }
6808
- /** Python's whitespace set str.isspace() == str.strip() == re \s for str
6809
- * patterns (verified identical by full code-point scan, 2026-08-19). Note the
6810
- * two-way mismatch with JS: \x1c-\x1f and \x85 are whitespace only here;
6811
- * \ufeff is whitespace to JS trim()/\s but NOT to Python. */
6812
- const PY_SPACE = /* @__PURE__ */ new Set(" \n\v\f\r …\xA0            \u2028\u2029   ");
6813
- function pyStrip(s) {
6814
- let a = 0;
6815
- let b = s.length;
6816
- while (a < b && PY_SPACE.has(s[a])) a += 1;
6817
- while (b > a && PY_SPACE.has(s[b - 1])) b -= 1;
6818
- return s.slice(a, b);
6608
+ /** Python compares strings by code point; JS `<` compares UTF-16 units — they differ on astral names. */
6609
+ function codePointCompare(a, b) {
6610
+ const as = [...a];
6611
+ const bs = [...b];
6612
+ const n = Math.min(as.length, bs.length);
6613
+ for (let i = 0; i < n; i++) {
6614
+ const d = (as[i]?.codePointAt(0) ?? 0) - (bs[i]?.codePointAt(0) ?? 0);
6615
+ if (d !== 0) return d;
6616
+ }
6617
+ return as.length - bs.length;
6819
6618
  }
6820
- /** Character-class text for Python \s (same set as PY_SPACE, for regexes). */
6821
- const WS = "\\t\\n\\v\\f\\r\\x1c-\\x1f \\x85\\xa0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000";
6822
- /** Character-class text for Python \w: L* ∪ Nd ∪ Nl ∪ No ∪ {_} — i.e.
6823
- * str.isalnum() plus underscore (spot-verified: é 中 Ⅰ ½ yes; 😀 and combining
6824
- * marks no). Used where the oracle wrote \w or \b (JS \w/\b are ASCII-only). */
6825
- const WORD = "\\p{L}\\p{N}_";
6826
- const HEADING = new RegExp(`^(#{1,4})[${WS}]+([^\\n]*?)[${WS}]*$`);
6827
- const FENCE = new RegExp(`^ {0,3}(\`{3,}|~{3,})([^\\n]*?)[${WS}]*$`);
6828
- const EXPLICIT_ID = new RegExp(`[${WS}]*\\{#([${WORD}-]+)\\}[${WS}]*$`, "u");
6829
- const SLUG_RUN = /[^a-z0-9]+/g;
6830
- const JSX_ASSESS = new RegExp(`(?:^|(?<=\\n))[${WS}]*<(?:Quiz|Flashcards)(?![${WORD}])`, "u");
6831
- const JSX_EMBED = new RegExp(`(?:^|(?<=\\n))[${WS}]*<(?:iframe|AICheck|AICheckField|ProjectCard|CapstoneWorkbook)(?![${WORD}])`, "u");
6832
- const STYLE_OPEN = new RegExp(`^[${WS}]*<style(?![${WORD}])`, "iu");
6833
- const CLASS_ATTR_G = new RegExp(`[${WS}]*className=(?:"[^"]*"|'[^']*'|\\{[^{}]*\\})`, "gu");
6834
- const LAYOUT_NAMES = "div|span|section|figure|article|header|footer|main|aside";
6835
- const LAYOUT_TAG_G = new RegExp(`<[${WS}]*(\\/?)[${WS}]*(${LAYOUT_NAMES})(?![${WORD}])([^<>]*?)(\\/?)[${WS}]*>`, "gu");
6836
- const LAYOUT_TAG_PROBE = new RegExp(`<[${WS}]*(\\/?)[${WS}]*(${LAYOUT_NAMES})(?![${WORD}])([^<>]*?)(\\/?)[${WS}]*>`, "u");
6837
- const INLINE_CODE_G = /* @__PURE__ */ new RegExp("`[^`\\n]+`", "g");
6838
- const BLANK_RUN_G = /* @__PURE__ */ new RegExp("\\n{3,}", "g");
6839
- const NUL = "\0";
6840
- const BLANK_SEP = new RegExp(`(\\n[${WS}]*\\n)`);
6841
- const sha256 = (s) => createHash("sha256").update(s, "utf8").digest("hex");
6842
- function slug(title, cap = 60) {
6843
- const s = title.toLowerCase().replace(SLUG_RUN, "-").replace(/^-+|-+$/g, "");
6844
- if (s.length <= cap) return s;
6845
- const cut = s.lastIndexOf("-", cap - 1);
6846
- return s.slice(0, cut > 0 ? cut : cap).replace(/^-+|-+$/g, "");
6619
+ /** Re-exported so every reader of a document agrees where its frontmatter ENDS. */
6620
+ const FRONTMATTER$1 = /^\uFEFF?---\r?\n([\s\S]*?)\r?\n---[ \t]*\r?\n?/;
6621
+ const YAML_BOOLS = {
6622
+ yes: true,
6623
+ Yes: true,
6624
+ YES: true,
6625
+ no: false,
6626
+ No: false,
6627
+ NO: false,
6628
+ true: true,
6629
+ True: true,
6630
+ TRUE: true,
6631
+ false: false,
6632
+ False: false,
6633
+ FALSE: false,
6634
+ on: true,
6635
+ On: true,
6636
+ ON: true,
6637
+ off: false,
6638
+ Off: false,
6639
+ OFF: false
6640
+ };
6641
+ /**
6642
+ * Minimal PyYAML-compatible frontmatter reader for the FOUR scalar keys this
6643
+ * adapter consumes (`title`, `position`, `sidebar_position`, `sor_id`) the
6644
+ * kernel discards every other frontmatter key at build time (taxonomy comes
6645
+ * from the manifest), so a YAML dependency would buy nothing (guard rule 5).
6646
+ * Scope, deliberately narrow pending a shared markdown module: top-level
6647
+ * `key: scalar` pairs only; nested/indented structure is ignored. Mirroring
6648
+ * the oracle's error path (`parse_frontmatter` catches YAMLError → `{}`), a
6649
+ * document PyYAML would refuse — an UNQUOTED value containing ": ", a block
6650
+ * scalar, an anchor/alias/tag, a non-mapping line — yields an EMPTY meta, so
6651
+ * titles fall back to the humanized filename instead of a half-read mapping.
6652
+ */
6653
+ function frontmatterMeta(text) {
6654
+ const block = FRONTMATTER$1.exec(text)?.[1];
6655
+ if (block === void 0) return {};
6656
+ const meta = {};
6657
+ for (const line of block.split(/\r?\n/)) {
6658
+ if (line.trim() === "" || line.trimStart().startsWith("#")) continue;
6659
+ if (/^[ \t]/.test(line)) continue;
6660
+ const kv = /^([^\s:]+):(?:[ \t]+(.*))?$/.exec(line);
6661
+ const key = kv?.[1];
6662
+ if (key === void 0) return {};
6663
+ const parsed = scalarValue((kv?.[2] ?? "").trim());
6664
+ if (!parsed.ok) return {};
6665
+ meta[key] = parsed.value;
6666
+ }
6667
+ return meta;
6847
6668
  }
6848
- function headingPathText(path) {
6849
- return path.map((p) => slug(p)).filter((s) => s !== "").join("/");
6669
+ function scalarValue(raw) {
6670
+ if (raw === "") return {
6671
+ ok: true,
6672
+ value: null
6673
+ };
6674
+ const dq = /^"(.*)"$/.exec(raw);
6675
+ if (dq !== null) return {
6676
+ ok: true,
6677
+ value: (dq[1] ?? "").replace(/\\"/g, "\"").replace(/\\\\/g, "\\")
6678
+ };
6679
+ const sq = /^'(.*)'$/.exec(raw);
6680
+ if (sq !== null) return {
6681
+ ok: true,
6682
+ value: (sq[1] ?? "").replace(/''/g, "'")
6683
+ };
6684
+ const plain = raw.replace(/[ \t]+#.*$/, "").trim();
6685
+ if (Object.hasOwn(YAML_BOOLS, plain)) return {
6686
+ ok: true,
6687
+ value: YAML_BOOLS[plain]
6688
+ };
6689
+ if (plain === "~" || /^(?:null|Null|NULL)$/.test(plain)) return {
6690
+ ok: true,
6691
+ value: null
6692
+ };
6693
+ if (/^[-+]?[0-9][0-9_]*$/.test(plain)) return {
6694
+ ok: true,
6695
+ value: Number.parseInt(plain.replaceAll("_", ""), 10)
6696
+ };
6697
+ if (/^[-+]?(?:\.[0-9]+|[0-9][0-9_]*\.[0-9_]*)(?:[eE][-+]?[0-9]+)?$/.test(plain)) return {
6698
+ ok: true,
6699
+ value: Number.parseFloat(plain.replaceAll("_", ""))
6700
+ };
6701
+ if (/:[ \t]/.test(plain) || plain.endsWith(":")) return {
6702
+ ok: false,
6703
+ value: null
6704
+ };
6705
+ if (/^[|>&*!{[]/.test(plain)) return {
6706
+ ok: false,
6707
+ value: null
6708
+ };
6709
+ return {
6710
+ ok: true,
6711
+ value: plain
6712
+ };
6850
6713
  }
6851
- /** CommonMark fence tracking (v5 — the correctness core). A backtick fence
6852
- * whose info string contains a backtick is NOT a fence; closing needs the same
6853
- * char, a run at least as long, and nothing but whitespace after. */
6854
- function fenceStep(line, fence) {
6855
- const m = FENCE.exec(line);
6856
- if (fence === null) {
6857
- if (m !== null) {
6858
- const marker = m[1];
6859
- const info = m[2];
6860
- if (marker[0] === "`" && info.includes("`")) return null;
6861
- return [marker[0], marker.length];
6862
- }
6863
- return null;
6714
+ /**
6715
+ * The governance a document declares about itself, read once and carried onto
6716
+ * the record.
6717
+ *
6718
+ * Before this module the ingest adapter kept four frontmatter keys and dropped
6719
+ * the rest, so `visibility`, `status`, `owner` and `provenance` existed only in
6720
+ * markdown and every surface re-derived them independently. The site enforced
6721
+ * `visibility:`; the MCP door could not, because the record did not carry it.
6722
+ * One reader, one shape, persisted on `content_nodes` (schema 2.2).
6723
+ *
6724
+ * The vocabulary is deliberately NOT closed here. A record that declares an
6725
+ * audience the instance does not know is a corpus error the checker names; the
6726
+ * ingest path's job is to carry what was written, faithfully, so the serving
6727
+ * door can make the decision with the instance in hand. Refusing unknown values
6728
+ * here would put the audience model in two places again.
6729
+ */
6730
+ const NO_GOVERNANCE = {
6731
+ visibility: null,
6732
+ docStatus: null,
6733
+ owner: null,
6734
+ provenance: null,
6735
+ supersededBy: null
6736
+ };
6737
+ function scalar(meta, key) {
6738
+ const raw = meta[key];
6739
+ if (typeof raw === "string") {
6740
+ const trimmed = raw.trim();
6741
+ return trimmed === "" ? null : trimmed;
6742
+ }
6743
+ if (typeof raw === "boolean") return raw ? "true" : "false";
6744
+ if (typeof raw === "number") return String(raw);
6745
+ return null;
6746
+ }
6747
+ const BLOCK_LIST = (key) => new RegExp(`^${key}:[ \\t]*\\r?\\n((?:[ \\t]*-[ \\t]+.*\\r?\\n?)+)`, "m");
6748
+ /**
6749
+ * Values of a simple `key:` block list, the one nested shape the record's
6750
+ * grammar uses (`provenance:` here, `audiences:` in instance.md). The scalar
6751
+ * reader deliberately ignores indented lines, so without this a provenance list
6752
+ * would vanish silently — the failure mode this whole module exists to end.
6753
+ */
6754
+ function frontmatterListValues(text, key) {
6755
+ const block = FRONTMATTER$1.exec(text)?.[1];
6756
+ if (block === void 0) return null;
6757
+ const m = BLOCK_LIST(key).exec(block + "\n");
6758
+ if (m === null) return null;
6759
+ const items = (m[1] ?? "").split(/\r?\n/).map((line) => /^[ \t]*-[ \t]+(.*)$/.exec(line)?.[1] ?? "").map((v) => v.trim().replace(/^["']|["']$/g, "").trim()).filter((v) => v !== "");
6760
+ return items.length > 0 ? items : null;
6761
+ }
6762
+ /**
6763
+ * Read the governance keys from an already-parsed scalar map plus the raw
6764
+ * document text (which the list reader needs). Unknown keys are ignored, as
6765
+ * they always were — this module widens what the record carries, it does not
6766
+ * narrow what a document may say.
6767
+ */
6768
+ var GovernanceParseError = class extends Error {
6769
+ name = "GovernanceParseError";
6770
+ };
6771
+ function governanceFromFrontmatter(meta, text) {
6772
+ if (frontmatterListValues(text, "visibility") !== null) throw new GovernanceParseError("a document declares `visibility:` as a LIST — a document belongs to exactly one tier. Write a single value, e.g. `visibility: internal`.");
6773
+ const declaredInText = /^visibility:[ \t]*(.*)$/m.exec(FRONTMATTER$1.exec(text)?.[1] ?? "");
6774
+ if (declaredInText !== null && scalar(meta, "visibility") === null) {
6775
+ const written = declaredInText[1]?.trim() ?? "";
6776
+ throw new GovernanceParseError(written === "" ? "a document declares `visibility:` with no readable value — an unreadable tier reads as no tier, and no tier is the default tier, which is how a restricted document gets served. Write a single value, e.g. `visibility: internal`." : `a document declares \`visibility: ${written}\` but this reader could not resolve it — usually because ANOTHER key in the same frontmatter is a shape it cannot read (a flow list like \`tags: [a, b]\`, or an unquoted value containing ": "). An unresolved tier would be served at the default. Quote the other value, or write it as a block list.`);
6777
+ }
6778
+ const provenanceScalar = scalar(meta, "provenance");
6779
+ const provenanceList = frontmatterListValues(text, "provenance");
6780
+ return {
6781
+ visibility: scalar(meta, "visibility"),
6782
+ docStatus: scalar(meta, "status"),
6783
+ owner: scalar(meta, "owner"),
6784
+ provenance: provenanceList ?? (provenanceScalar === null ? null : [provenanceScalar]),
6785
+ supersededBy: scalar(meta, "superseded_by")
6786
+ };
6787
+ }
6788
+ const SUPPORTED_FORMATS = [1];
6789
+ /** The manifest is malformed — named precisely; a bad manifest never half-ingests. */
6790
+ var ManifestError = class extends Error {
6791
+ constructor(message) {
6792
+ super(message);
6793
+ this.name = "ManifestError";
6794
+ }
6795
+ };
6796
+ /** Mirrors the oracle dataclass defaults (parent/summary/permalink None, position 0, keywords ()). */
6797
+ function manifestNode(init) {
6798
+ return {
6799
+ stable_id: init.stable_id,
6800
+ slug: init.slug,
6801
+ title: init.title,
6802
+ kind: init.kind,
6803
+ parent: init.parent ?? null,
6804
+ position: init.position ?? 0,
6805
+ summary: init.summary ?? null,
6806
+ keywords: init.keywords ?? [],
6807
+ permalink: init.permalink ?? null,
6808
+ governance: init.governance ?? NO_GOVERNANCE
6809
+ };
6810
+ }
6811
+ function manifestFile(init) {
6812
+ return {
6813
+ path: init.path,
6814
+ node: init.node,
6815
+ title: init.title ?? null
6816
+ };
6817
+ }
6818
+ function parseManifest(text) {
6819
+ let raw;
6820
+ try {
6821
+ raw = JSON.parse(text);
6822
+ } catch (exc) {
6823
+ throw new ManifestError(`manifest.json is not valid JSON: ${exc instanceof Error ? exc.message : String(exc)}`);
6824
+ }
6825
+ if (typeof raw !== "object" || raw === null || Array.isArray(raw)) throw new ManifestError("manifest.json must be an object");
6826
+ const obj = raw;
6827
+ const fmt = obj["format"];
6828
+ if (typeof fmt !== "number" || !SUPPORTED_FORMATS.includes(fmt)) throw new ManifestError(`manifest format ${JSON.stringify(fmt)} unsupported (supported: ${SUPPORTED_FORMATS.join(", ")})`);
6829
+ const corpusId = topString(obj, "corpus_id");
6830
+ const sourceCommit = topString(obj, "source_commit");
6831
+ const nodes = entriesOf(obj, "nodes").map((n, i) => manifestNode({
6832
+ stable_id: req(n, "stable_id", i),
6833
+ slug: req(n, "slug", i),
6834
+ title: req(n, "title", i),
6835
+ kind: req(n, "kind", i),
6836
+ parent: optString(n["parent"]),
6837
+ position: toPosition(n["position"], i),
6838
+ summary: optString(n["summary"]),
6839
+ keywords: toKeywords(n["keywords"], i),
6840
+ permalink: optString(n["permalink"]),
6841
+ governance: toGovernance(n["governance"], i)
6842
+ }));
6843
+ const files = entriesOf(obj, "files").map((f, i) => manifestFile({
6844
+ path: req(f, "path", i),
6845
+ node: req(f, "node", i),
6846
+ title: optString(f["title"])
6847
+ }));
6848
+ validate(nodes, files);
6849
+ return {
6850
+ format: fmt,
6851
+ corpus_id: corpusId,
6852
+ source_commit: sourceCommit,
6853
+ nodes,
6854
+ files
6855
+ };
6856
+ }
6857
+ /**
6858
+ * The inverse of `governanceToJson`. Absent → NO_GOVERNANCE, which is what a
6859
+ * corpus that declares nothing has always meant. A present-but-wrong shape is
6860
+ * REFUSED rather than silently dropped: dropping it would serve a document at
6861
+ * the instance default, and for a `visibility:` that means serving a restricted
6862
+ * document to everyone.
6863
+ */
6864
+ function toGovernance(raw, index) {
6865
+ if (raw === void 0 || raw === null) return NO_GOVERNANCE;
6866
+ if (typeof raw !== "object" || Array.isArray(raw)) throw new ManifestError(`entry ${index}: 'governance' must be an object`);
6867
+ const g = raw;
6868
+ const str = (key) => {
6869
+ const val = g[key];
6870
+ if (val === void 0 || val === null) return null;
6871
+ if (typeof val !== "string" || val === "") throw new ManifestError(`entry ${index}: 'governance.${key}' must be a non-empty string`);
6872
+ return val;
6873
+ };
6874
+ const provenanceRaw = g["provenance"];
6875
+ let provenance = null;
6876
+ if (provenanceRaw !== void 0 && provenanceRaw !== null) {
6877
+ if (!Array.isArray(provenanceRaw) || provenanceRaw.some((v) => typeof v !== "string")) throw new ManifestError(`entry ${index}: 'governance.provenance' must be a list of strings`);
6878
+ provenance = provenanceRaw;
6879
+ }
6880
+ return {
6881
+ visibility: str("visibility"),
6882
+ docStatus: str("doc_status"),
6883
+ owner: str("owner"),
6884
+ provenance,
6885
+ supersededBy: str("superseded_by")
6886
+ };
6887
+ }
6888
+ function topString(obj, key) {
6889
+ const val = obj[key];
6890
+ if (typeof val !== "string" || !val) throw new ManifestError(`manifest.${key} must be a non-empty string`);
6891
+ return val;
6892
+ }
6893
+ function entriesOf(obj, key) {
6894
+ const raw = obj[key] ?? [];
6895
+ if (!Array.isArray(raw)) throw new ManifestError(`manifest.${key} must be an array`);
6896
+ return raw.map((entry, i) => {
6897
+ if (typeof entry !== "object" || entry === null || Array.isArray(entry)) throw new ManifestError(`entry ${i}: must be an object`);
6898
+ return entry;
6899
+ });
6900
+ }
6901
+ function req(obj, key, index) {
6902
+ const val = obj[key];
6903
+ if (typeof val !== "string" || !val) throw new ManifestError(`entry ${index}: '${key}' must be a non-empty string`);
6904
+ return val;
6905
+ }
6906
+ function optString(val) {
6907
+ return typeof val === "string" ? val : null;
6908
+ }
6909
+ /** Python `int(...)` parity: truncate finite numbers, parse integer strings, refuse the rest loudly. */
6910
+ function toPosition(val, index) {
6911
+ if (val === void 0) return 0;
6912
+ if (typeof val === "number" && Number.isFinite(val)) return Math.trunc(val);
6913
+ if (typeof val === "boolean") return val ? 1 : 0;
6914
+ if (typeof val === "string" && /^[+-]?\d+$/.test(val.trim())) return Number.parseInt(val.trim(), 10);
6915
+ throw new ManifestError(`entry ${index}: position must be an integer, got ${JSON.stringify(val)}`);
6916
+ }
6917
+ function toKeywords(val, index) {
6918
+ if (val === void 0 || val === null) return [];
6919
+ if (!Array.isArray(val)) throw new ManifestError(`entry ${index}: keywords must be an array of strings`);
6920
+ return val.map((k, j) => {
6921
+ if (typeof k !== "string") throw new ManifestError(`entry ${index}: keywords[${j}] must be a string`);
6922
+ return k;
6923
+ });
6924
+ }
6925
+ function validate(nodes, files) {
6926
+ const seen = /* @__PURE__ */ new Set();
6927
+ const dupes = /* @__PURE__ */ new Set();
6928
+ for (const n of nodes) {
6929
+ if (seen.has(n.stable_id)) dupes.add(n.stable_id);
6930
+ seen.add(n.stable_id);
6931
+ }
6932
+ if (dupes.size > 0) throw new ManifestError(`duplicate node stable_id(s): [${[...dupes].sort().map((d) => `'${d}'`).join(", ")}]`);
6933
+ for (const n of nodes) if (n.parent !== null && !seen.has(n.parent)) throw new ManifestError(`node '${n.stable_id}': unknown parent '${n.parent}'`);
6934
+ for (const f of files) if (!seen.has(f.node)) throw new ManifestError(`file '${f.path}': unknown node '${f.node}'`);
6935
+ const paths = /* @__PURE__ */ new Set();
6936
+ for (const f of files) {
6937
+ if (paths.has(f.path)) throw new ManifestError("duplicate file paths in manifest");
6938
+ paths.add(f.path);
6864
6939
  }
6865
- if (m !== null && m[1][0] === fence[0] && m[1].length >= fence[1] && m[2] === "") return null;
6866
- return fence;
6867
- }
6868
- /** Drop prose-level `<style>…</style>` CSS pure presentation; classify would
6869
- * size a multi-KB CSS wall as prose and serve it (the oracle's field-test #3
6870
- * "about" doc opened with ~2KB of it). Runs BEFORE contentHash + chunkText, so
6871
- * chunks reassemble the CLEANED body byte-exact and only files that actually
6872
- * held a block re-embed. FENCE-SAFE: a `<style>` shown as example code inside
6873
- * a fence is content and survives. See STYLE_OPEN for the locked fast-path
6874
- * quirk (^-anchored, no multiline). */
6875
- function stripStyleBlocks(text) {
6876
- if (!STYLE_OPEN.test(text)) return text;
6877
- const out = [];
6878
- let fence = null;
6879
- let dropping = false;
6880
- for (const line of pySplitLines(text, true)) {
6881
- if (dropping) {
6882
- if (line.toLowerCase().includes("</style>")) dropping = false;
6883
- continue;
6884
- }
6885
- if (fence === null && STYLE_OPEN.test(line)) {
6886
- if (!line.toLowerCase().includes("</style>")) dropping = true;
6887
- continue;
6888
- }
6889
- out.push(line);
6890
- fence = fenceStep(line, fence);
6940
+ const siblingSlugs = /* @__PURE__ */ new Map();
6941
+ for (const n of nodes) {
6942
+ const parent = n.parent ?? "";
6943
+ const bySlug = siblingSlugs.get(parent) ?? /* @__PURE__ */ new Map();
6944
+ const owners = bySlug.get(n.slug) ?? [];
6945
+ owners.push(n.stable_id);
6946
+ bySlug.set(n.slug, owners);
6947
+ siblingSlugs.set(parent, bySlug);
6891
6948
  }
6892
- return out.join("");
6949
+ for (const bySlug of siblingSlugs.values()) for (const [slug, owners] of bySlug) if (owners.length > 1) throw new ManifestError(`sibling slug collision: ${owners.map((o) => `'${o}'`).join(", ")} all slug to '${slug}' under the same parent — rename one (a slug is a node's URL segment and must be unique among siblings)`);
6893
6950
  }
6894
- const rstripSpacesTabs = (s) => {
6895
- let b = s.length;
6896
- while (b > 0 && (s[b - 1] === " " || s[b - 1] === " ")) b -= 1;
6897
- return s.slice(0, b);
6898
- };
6899
- /** Remove `style={{ ... }}` attributes by BRACE MATCHING, not regex: the value
6900
- * is a JS object literal and the oracle measured 168 of them spanning lines.
6901
- * An unbalanced opener leaves the rest of the text verbatim — a stripper must
6902
- * never eat the rest of a document to satisfy itself. (The pre-attr rstrip of
6903
- * spaces/tabs also glues `<div ` + a KEPT unbalanced `style={{` opener into
6904
- * `<divstyle={{` — an oracle quirk locked by fixture strip-malformed-kept.) */
6905
- function stripStyleAttr(text) {
6951
+ /** Parents before children (insert order for the FK); a cycle fails loudly. */
6952
+ function topological(nodes) {
6953
+ const byId = new Map(nodes.map((n) => [n.stable_id, n]));
6906
6954
  const out = [];
6907
- let i = 0;
6908
- for (;;) {
6909
- const j = text.indexOf("style={{", i);
6910
- if (j < 0) break;
6911
- out.push(rstripSpacesTabs(text.slice(i, j)));
6912
- let k = j + 6;
6913
- let depth = 0;
6914
- let closed = false;
6915
- while (k < text.length) {
6916
- const ch = text[k];
6917
- if (ch === "{") depth += 1;
6918
- else if (ch === "}") {
6919
- depth -= 1;
6920
- if (depth === 0) {
6921
- k += 1;
6922
- closed = true;
6923
- break;
6924
- }
6925
- }
6926
- k += 1;
6927
- }
6928
- if (!closed) {
6929
- out.push(text.slice(j));
6930
- return out.join("");
6955
+ const state = /* @__PURE__ */ new Map();
6956
+ const visit = (n) => {
6957
+ const mark = state.get(n.stable_id) ?? 0;
6958
+ if (mark === 2) return;
6959
+ if (mark === 1) throw new ManifestError(`parent cycle at '${n.stable_id}'`);
6960
+ state.set(n.stable_id, 1);
6961
+ if (n.parent !== null) {
6962
+ const parent = byId.get(n.parent);
6963
+ if (parent === void 0) throw new ManifestError(`node '${n.stable_id}': unknown parent '${n.parent}'`);
6964
+ visit(parent);
6931
6965
  }
6932
- i = k;
6933
- }
6934
- out.push(text.slice(i));
6935
- return out.join("");
6966
+ state.set(n.stable_id, 2);
6967
+ out.push(n);
6968
+ };
6969
+ for (const n of nodes) visit(n);
6970
+ return out;
6936
6971
  }
6937
- /** Drop layout tags left BARE by the attribute strip, KEEPING each closer
6938
- * paired with its opener: a closer carries no attributes, so `</div>` cannot
6939
- * say whether it belongs to a removed wrapper or to a kept `<div id="x">`.
6940
- * Track depth; a closer is removed iff its opener was. On any mismatch the tag
6941
- * is KEPT a stray tag beats eaten content. `stack` is owned by the CALLER
6942
- * and persists across every prose segment of one document: a styled wrapper
6943
- * around a code fence splits at the fence, and its opener/closer land in
6944
- * different segments (228 of them in the oracle's corpus). */
6945
- function dropBareLayoutTags(segment, stack) {
6946
- return segment.replace(LAYOUT_TAG_G, (m0, closing, name, rawAttrs, selfClosing) => {
6947
- const attrs = pyStrip(rawAttrs);
6948
- if (closing !== "") {
6949
- const top = stack[stack.length - 1];
6950
- if (top !== void 0 && top[0] === name) return stack.pop()[1] ? "" : m0;
6951
- return m0;
6952
- }
6953
- if (selfClosing !== "") return attrs === "" ? "" : m0;
6954
- stack.push([name, attrs === ""]);
6955
- return attrs === "" ? "" : m0;
6956
- });
6972
+ /**
6973
+ * The one canonical JSON emitter for every adapter (re-homed from the oracle's
6974
+ * `_to_json`, adapters/docusaurus_sidebar.py:410): node keys whose value is
6975
+ * null/empty/zero are omitted EXCEPT `position`, which is always emitted; file
6976
+ * dicts carry `title` only when set. Adapters round-trip the result through
6977
+ * `parseManifest` before writing, so an adapter can never emit what ingest
6978
+ * would refuse.
6979
+ */
6980
+ function manifestToJson(m) {
6981
+ return {
6982
+ format: m.format,
6983
+ corpus_id: m.corpus_id,
6984
+ source_commit: m.source_commit,
6985
+ nodes: m.nodes.map(nodeToJson),
6986
+ files: m.files.map((f) => ({
6987
+ path: f.path,
6988
+ node: f.node,
6989
+ ...f.title ? { title: f.title } : {}
6990
+ }))
6991
+ };
6957
6992
  }
6958
- /** Restore NUL<n>NUL placeholders — equivalent to the oracle's
6959
- * re.sub(r"\x00(\d+)\x00", ...): a leftmost scan where a match is a NUL, a
6960
- * maximal non-empty ASCII digit run, and a closing NUL; anything else stays
6961
- * verbatim (backtracking cannot produce any other match for this pattern). */
6962
- function restoreHeld(seg, held) {
6963
- let out = "";
6964
- let i = 0;
6965
- for (;;) {
6966
- const a = seg.indexOf(NUL, i);
6967
- if (a < 0) break;
6968
- let b = a + 1;
6969
- while (b < seg.length && seg[b] >= "0" && seg[b] <= "9") b += 1;
6970
- if (b > a + 1 && seg[b] === NUL) {
6971
- const d = seg.slice(a + 1, b);
6972
- const h = held[Number(d)];
6973
- if (h === void 0) throw new Error(`inline-code placeholder NUL${d}NUL has no held span (held ${held.length}): the source document contains a literal NUL-digit-NUL sequence, which collides with the stripper's placeholder scheme. Remove NUL control characters from the file.`);
6974
- out += seg.slice(i, a) + h;
6975
- i = b + 1;
6976
- } else {
6977
- out += seg.slice(i, a + 1);
6978
- i = a + 1;
6979
- }
6993
+ function nodeToJson(n) {
6994
+ const fields = [
6995
+ ["stable_id", n.stable_id],
6996
+ ["slug", n.slug],
6997
+ ["title", n.title],
6998
+ ["kind", n.kind],
6999
+ ["parent", n.parent],
7000
+ ["position", n.position],
7001
+ ["summary", n.summary],
7002
+ ["keywords", n.keywords],
7003
+ ["permalink", n.permalink]
7004
+ ];
7005
+ const out = {};
7006
+ for (const [key, val] of fields) {
7007
+ const omit = val === null || val === 0 || Array.isArray(val) && val.length === 0;
7008
+ if (key === "position" || !omit) out[key] = val;
6980
7009
  }
6981
- return out + seg.slice(i);
7010
+ const gov = governanceToJson(n.governance);
7011
+ if (Object.keys(gov).length > 0) out["governance"] = gov;
7012
+ return out;
6982
7013
  }
6983
- /** Strip presentation markup from ONE non-fenced segment. Inline code is
6984
- * protected first (`<div>` written in prose backticks is a lesson). `stack` is
6985
- * the document-wide layout-tag stack (see dropBareLayoutTags). */
6986
- function stripProsePresentation(segment, stack) {
6987
- const held = [];
6988
- let seg = segment.replace(INLINE_CODE_G, (m0) => {
6989
- held.push(m0);
6990
- return `${NUL}${held.length - 1}${NUL}`;
6991
- });
6992
- seg = stripStyleAttr(seg).replace(CLASS_ATTR_G, "");
6993
- seg = dropBareLayoutTags(seg, stack);
6994
- seg = seg.replace(BLANK_RUN_G, "\n\n");
6995
- return restoreHeld(seg, held);
7014
+ function governanceToJson(g) {
7015
+ const out = {};
7016
+ if (g.visibility !== null) out["visibility"] = g.visibility;
7017
+ if (g.docStatus !== null) out["doc_status"] = g.docStatus;
7018
+ if (g.owner !== null) out["owner"] = g.owner;
7019
+ if (g.provenance !== null && g.provenance.length > 0) out["provenance"] = g.provenance;
7020
+ if (g.supersededBy !== null) out["superseded_by"] = g.supersededBy;
7021
+ return out;
6996
7022
  }
6997
- /** Drop layout markup — `style={{…}}`, `className="…"`, and the `<div>`/
6998
- * `<span>` wrappers they leave bare while KEEPING every character of the
6999
- * text inside them ("The Third Era of AI Tools" is content, `af-hero-eyebrow`
7000
- * is a CSS hook; the oracle measured a student being served the wrapper).
7001
- * A layout tag is only stripped once it is BARE: attributes go first, and a
7002
- * wrapper with nothing left was pure layout by construction, while
7003
- * `<div id="x">` keeps its tag — no tag allowlist to drift. Deliberately never
7004
- * touches capitalised components (<Quiz> is curriculum), inline SVG, or
7005
- * `<details>`/`<summary>` (semantic HTML). FENCE- and inline-code-safe. ONE
7006
- * layout-tag stack is threaded through every prose segment of the document so
7007
- * pairing survives a fence split. */
7008
- function stripPresentationJsx(text) {
7009
- if (!text.includes("className=") && !text.includes("style={{") && !LAYOUT_TAG_PROBE.test(text)) return text;
7010
- const out = [];
7011
- let buf = [];
7012
- let fence = null;
7013
- const stack = [];
7014
- for (const line of pySplitLines(text, true)) {
7015
- const nxt = fenceStep(line, fence);
7016
- if (fence === null && nxt === null) buf.push(line);
7017
- else {
7018
- if (buf.length > 0) {
7019
- out.push(stripProsePresentation(buf.join(""), stack));
7020
- buf = [];
7021
- }
7022
- out.push(line);
7023
- }
7024
- fence = nxt;
7023
+ /**
7024
+ * Carried verbatim from the oracle's build step (ingest/build.py:109):
7025
+ * `source_id = f.path.removesuffix(".md") + ":prose"`. The ".md"-only strip is
7026
+ * a deliberate quirk `a/b.md` `a/b:prose` while `a/b.mdx` KEEPS its suffix
7027
+ * `a/b.mdx:prose`. Persisted source rows and carry-forward joins already pin
7028
+ * this shape; changing it is a policy decision, never a tidy-up.
7029
+ */
7030
+ function sourceId(path) {
7031
+ return (path.endsWith(".md") ? path.slice(0, -3) : path) + ":prose";
7032
+ }
7033
+ /** Python len(): Unicode code points, not UTF-16 units. Every limit comparison
7034
+ * and the HARD_MAX_CHARS slice go through code points or the policy silently
7035
+ * changes on astral-plane text (emoji, musical symbols, CJK extensions). */
7036
+ function cpLen(s) {
7037
+ let n = 0;
7038
+ for (const _ch of s) n += 1;
7039
+ return n;
7040
+ }
7041
+ /** Python str.splitlines() boundary set (full code-point scan, 2026-08-19):
7042
+ * \n \v \f \r \x1c \x1d \x1e \x85 \u2028 \u2029, with \r\n as one boundary.
7043
+ * A naive split(/\r?\n/) changes segmentation on \x85, \u2028 etc. All
7044
+ * boundaries are BMP, so a UTF-16 walk cannot land inside a surrogate pair. */
7045
+ const LINE_BOUNDARY = /* @__PURE__ */ new Set([
7046
+ "\n",
7047
+ "\v",
7048
+ "\f",
7049
+ "\r",
7050
+ "",
7051
+ "",
7052
+ "",
7053
+ "…",
7054
+ "\u2028",
7055
+ "\u2029"
7056
+ ]);
7057
+ function pySplitLines(text, keepends) {
7058
+ const out = [];
7059
+ let start = 0;
7060
+ let i = 0;
7061
+ while (i < text.length) {
7062
+ const ch = text[i];
7063
+ if (LINE_BOUNDARY.has(ch)) {
7064
+ let end = i + 1;
7065
+ if (ch === "\r" && text[end] === "\n") end += 1;
7066
+ out.push(keepends ? text.slice(start, end) : text.slice(start, i));
7067
+ start = end;
7068
+ i = end;
7069
+ } else i += 1;
7025
7070
  }
7026
- if (buf.length > 0) out.push(stripProsePresentation(buf.join(""), stack));
7027
- return out.join("");
7028
- }
7029
- /**
7030
- * The body-cleaning pipeline every ingest runs BEFORE the skip-gate hash and
7031
- * chunking, as ONE ordered unit so the order cannot regress. CRLF→LF is
7032
- * normalized FIRST — the strippers are \n-anchored (BLANK_RUN_G = /\n{3,}/),
7033
- * so normalizing AFTER them left a CRLF checkout's blank runs un-collapsed and
7034
- * every chunk_hash + content_hash diverged from an LF checkout, re-embedding
7035
- * the whole file while content_hash claimed nothing changed (review,
7036
- * 2026-08-19). Then style blocks and presentation JSX are stripped so served
7037
- * chunks reassemble the CLEANED body byte-exact. A bare \r (no following \n)
7038
- * stays content.
7039
- */
7040
- function cleanBody(rawBody) {
7041
- return stripPresentationJsx(stripStyleBlocks(rawBody.replaceAll("\r\n", "\n")));
7071
+ if (start < text.length) out.push(text.slice(start));
7072
+ return out;
7042
7073
  }
7043
- /** Heading text never counts toward the nav/prose size test the
7044
- * "content-only" in the policy name. Note: joins on \n, so exotic line
7045
- * boundaries are normalized before the length is taken (as in the oracle). */
7046
- function teachingBody(content) {
7047
- return pyStrip(pySplitLines(content, false).filter((ln) => !HEADING.test(ln)).join("\n"));
7074
+ /** Python's whitespace set str.isspace() == str.strip() == re \s for str
7075
+ * patterns (verified identical by full code-point scan, 2026-08-19). Note the
7076
+ * two-way mismatch with JS: \x1c-\x1f and \x85 are whitespace only here;
7077
+ * \ufeff is whitespace to JS trim()/\s but NOT to Python. */
7078
+ const PY_SPACE = /* @__PURE__ */ new Set(" \n\v\f\r …\xA0            \u2028\u2029   ");
7079
+ function pyStrip(s) {
7080
+ let a = 0;
7081
+ let b = s.length;
7082
+ while (a < b && PY_SPACE.has(s[a])) a += 1;
7083
+ while (b > a && PY_SPACE.has(s[b - 1])) b -= 1;
7084
+ return s.slice(a, b);
7048
7085
  }
7049
- function classify(content, headingPath) {
7050
- if (JSX_ASSESS.test(content)) return "assessment";
7051
- const leaf = headingPath.length > 0 ? headingPath[headingPath.length - 1] : "";
7052
- if (JSX_EMBED.test(content) || content.includes("docs.google.com/presentation") || leaf.includes("Teaching Aid")) return "embed";
7053
- if (cpLen(teachingBody(content)) < 250) return "nav";
7054
- return "prose";
7086
+ /** Character-class text for Python \s (same set as PY_SPACE, for regexes). */
7087
+ const WS = "\\t\\n\\v\\f\\r\\x1c-\\x1f \\x85\\xa0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000";
7088
+ /** Character-class text for Python \w: L* Nd Nl ∪ No ∪ {_} — i.e.
7089
+ * str.isalnum() plus underscore (spot-verified: é ½ yes; 😀 and combining
7090
+ * marks no). Used where the oracle wrote \w or \b (JS \w/\b are ASCII-only). */
7091
+ const WORD = "\\p{L}\\p{N}_";
7092
+ const HEADING = new RegExp(`^(#{1,4})[${WS}]+([^\\n]*?)[${WS}]*$`);
7093
+ const FENCE = new RegExp(`^ {0,3}(\`{3,}|~{3,})([^\\n]*?)[${WS}]*$`);
7094
+ const EXPLICIT_ID = new RegExp(`[${WS}]*\\{#([${WORD}-]+)\\}[${WS}]*$`, "u");
7095
+ const SLUG_RUN = /[^a-z0-9]+/g;
7096
+ const JSX_ASSESS = new RegExp(`(?:^|(?<=\\n))[${WS}]*<(?:Quiz|Flashcards)(?![${WORD}])`, "u");
7097
+ const JSX_EMBED = new RegExp(`(?:^|(?<=\\n))[${WS}]*<(?:iframe|AICheck|AICheckField|ProjectCard|CapstoneWorkbook)(?![${WORD}])`, "u");
7098
+ const STYLE_OPEN = new RegExp(`^[${WS}]*<style(?![${WORD}])`, "iu");
7099
+ const CLASS_ATTR_G = new RegExp(`[${WS}]*className=(?:"[^"]*"|'[^']*'|\\{[^{}]*\\})`, "gu");
7100
+ const LAYOUT_NAMES = "div|span|section|figure|article|header|footer|main|aside";
7101
+ const LAYOUT_TAG_G = new RegExp(`<[${WS}]*(\\/?)[${WS}]*(${LAYOUT_NAMES})(?![${WORD}])([^<>]*?)(\\/?)[${WS}]*>`, "gu");
7102
+ const LAYOUT_TAG_PROBE = new RegExp(`<[${WS}]*(\\/?)[${WS}]*(${LAYOUT_NAMES})(?![${WORD}])([^<>]*?)(\\/?)[${WS}]*>`, "u");
7103
+ const INLINE_CODE_G = /* @__PURE__ */ new RegExp("`[^`\\n]+`", "g");
7104
+ const BLANK_RUN_G = /* @__PURE__ */ new RegExp("\\n{3,}", "g");
7105
+ const NUL = "\0";
7106
+ const BLANK_SEP = new RegExp(`(\\n[${WS}]*\\n)`);
7107
+ const sha256 = (s) => createHash("sha256").update(s, "utf8").digest("hex");
7108
+ function slug(title, cap = 60) {
7109
+ const s = title.toLowerCase().replace(SLUG_RUN, "-").replace(/^-+|-+$/g, "");
7110
+ if (s.length <= cap) return s;
7111
+ const cut = s.lastIndexOf("-", cap - 1);
7112
+ return s.slice(0, cut > 0 ? cut : cap).replace(/^-+|-+$/g, "");
7055
7113
  }
7056
- /** A segment DOMINATED by a line-leading widget (with < NAV_MAX_CHARS of
7057
- * teaching body before it) labels EVERY fragment — a char-sliced widget must
7058
- * not leak as prose. */
7059
- function segmentMarkerType(span) {
7060
- for (const [re, label] of [[JSX_ASSESS, "assessment"], [JSX_EMBED, "embed"]]) {
7061
- const m = re.exec(span);
7062
- if (m !== null && cpLen(teachingBody(span.slice(0, m.index))) < 250) return label;
7063
- }
7064
- return null;
7114
+ function headingPathText(path) {
7115
+ return path.map((p) => slug(p)).filter((s) => s !== "").join("/");
7065
7116
  }
7066
- /** Walk lines; headings count only OUTSIDE fences; every line lands in exactly
7067
- * one segment (byte-exact). H1 records a title but never enters the path. */
7068
- function segmentText(text) {
7069
- const segments = [];
7070
- const titles = /* @__PURE__ */ new Map();
7071
- const anchors = /* @__PURE__ */ new Map();
7072
- let buf = [];
7073
- let curPath = [];
7074
- let curAnchor = null;
7075
- let fence = null;
7076
- const flush = () => {
7077
- if (buf.length > 0) {
7078
- segments.push({
7079
- path: [...curPath],
7080
- anchor: curAnchor,
7081
- text: buf.join("")
7082
- });
7083
- buf = [];
7084
- }
7085
- };
7086
- for (const line of pySplitLines(text, true)) {
7087
- const m = fence === null ? HEADING.exec(line) : null;
7117
+ /** CommonMark fence tracking (v5 the correctness core). A backtick fence
7118
+ * whose info string contains a backtick is NOT a fence; closing needs the same
7119
+ * char, a run at least as long, and nothing but whitespace after. */
7120
+ function fenceStep(line, fence) {
7121
+ const m = FENCE.exec(line);
7122
+ if (fence === null) {
7088
7123
  if (m !== null) {
7089
- flush();
7090
- const level = m[1].length;
7091
- const rawTitle = m[2];
7092
- const idM = EXPLICIT_ID.exec(rawTitle);
7093
- const title = idM !== null ? pyStrip(rawTitle.replace(EXPLICIT_ID, "")) : rawTitle;
7094
- titles.set(level, title);
7095
- anchors.set(level, idM !== null ? idM[1] : null);
7096
- const deeper = [...titles.keys()].filter((lv) => lv > level);
7097
- for (const lv of deeper) {
7098
- titles.delete(lv);
7099
- anchors.delete(lv);
7100
- }
7101
- curPath = [...titles.keys()].sort((a, b) => a - b).filter((lv) => lv >= 2 && lv <= level).map((lv) => titles.get(lv));
7102
- curAnchor = curPath.length > 0 ? anchors.get(level) || slug(title) : null;
7124
+ const marker = m[1];
7125
+ const info = m[2];
7126
+ if (marker[0] === "`" && info.includes("`")) return null;
7127
+ return [marker[0], marker.length];
7103
7128
  }
7104
- buf.push(line);
7105
- if (m === null) fence = fenceStep(line, fence);
7129
+ return null;
7106
7130
  }
7107
- flush();
7108
- return segments;
7131
+ if (m !== null && m[1][0] === fence[0] && m[1].length >= fence[1] && m[2] === "") return null;
7132
+ return fence;
7109
7133
  }
7110
- /** Split on blank-line runs, greedy-pack, but flush ONLY outside a fence — a
7111
- * flush never lands between an open fence and its close. The separator is
7112
- * CAPTURED, so blank-line runs ride along as parts and concatenation is
7113
- * lossless; nothing is trimmed. */
7114
- function subsplit(span, maxChars) {
7115
- if (cpLen(span) <= maxChars) return [span];
7116
- const parts = span.split(BLANK_SEP);
7117
- const pieces = [];
7118
- let buf = "";
7119
- let bufLen = 0;
7134
+ /** Drop prose-level `<style>…</style>` CSS pure presentation; classify would
7135
+ * size a multi-KB CSS wall as prose and serve it (the oracle's field-test #3
7136
+ * "about" doc opened with ~2KB of it). Runs BEFORE contentHash + chunkText, so
7137
+ * chunks reassemble the CLEANED body byte-exact and only files that actually
7138
+ * held a block re-embed. FENCE-SAFE: a `<style>` shown as example code inside
7139
+ * a fence is content and survives. See STYLE_OPEN for the locked fast-path
7140
+ * quirk (^-anchored, no multiline). */
7141
+ function stripStyleBlocks(text) {
7142
+ if (!STYLE_OPEN.test(text)) return text;
7143
+ const out = [];
7120
7144
  let fence = null;
7121
- for (const part of parts) {
7122
- const partLen = cpLen(part);
7123
- if (buf !== "" && fence === null && bufLen + partLen > maxChars) {
7124
- pieces.push(buf);
7125
- buf = "";
7126
- bufLen = 0;
7145
+ let dropping = false;
7146
+ for (const line of pySplitLines(text, true)) {
7147
+ if (dropping) {
7148
+ if (line.toLowerCase().includes("</style>")) dropping = false;
7149
+ continue;
7127
7150
  }
7128
- buf += part;
7129
- bufLen += partLen;
7130
- for (const line of pySplitLines(part, true)) fence = fenceStep(line, fence);
7151
+ if (fence === null && STYLE_OPEN.test(line)) {
7152
+ if (!line.toLowerCase().includes("</style>")) dropping = true;
7153
+ continue;
7154
+ }
7155
+ out.push(line);
7156
+ fence = fenceStep(line, fence);
7131
7157
  }
7132
- if (buf !== "") pieces.push(buf);
7133
- const out = [];
7134
- for (const piece of pieces) out.push(...enforceCeiling(piece));
7135
- return out;
7158
+ return out.join("");
7136
7159
  }
7137
- /** The ONLY place mid-line/mid-fence slicing can happen (a pathological single
7138
- * paragraph or giant fence). Slices by CODE POINTS — 4000 UTF-16 units would
7139
- * be a different policy and could split a surrogate pair. */
7140
- function enforceCeiling(piece) {
7141
- if (cpLen(piece) <= 4e3) return [piece];
7142
- const cps = [...piece];
7160
+ const rstripSpacesTabs = (s) => {
7161
+ let b = s.length;
7162
+ while (b > 0 && (s[b - 1] === " " || s[b - 1] === " ")) b -= 1;
7163
+ return s.slice(0, b);
7164
+ };
7165
+ /** Remove `style={{ ... }}` attributes by BRACE MATCHING, not regex: the value
7166
+ * is a JS object literal and the oracle measured 168 of them spanning lines.
7167
+ * An unbalanced opener leaves the rest of the text verbatim — a stripper must
7168
+ * never eat the rest of a document to satisfy itself. (The pre-attr rstrip of
7169
+ * spaces/tabs also glues `<div ` + a KEPT unbalanced `style={{` opener into
7170
+ * `<divstyle={{` — an oracle quirk locked by fixture strip-malformed-kept.) */
7171
+ function stripStyleAttr(text) {
7143
7172
  const out = [];
7144
- for (let i = 0; i < cps.length; i += HARD_MAX_CHARS) out.push(cps.slice(i, i + HARD_MAX_CHARS).join(""));
7145
- return out;
7146
- }
7147
- function chunkText(text, maxChars = MAX_CHARS) {
7148
- const chunks = [];
7149
- let prefix = "";
7150
- const emit = (content, path, anchor, sourceType) => {
7151
- chunks.push({
7152
- ordinal: chunks.length,
7153
- content,
7154
- chunkHash: sha256(content),
7155
- headingPath: [...path],
7156
- anchor,
7157
- sourceType
7158
- });
7159
- };
7160
- for (const seg of segmentText(text)) {
7161
- if (pyStrip(seg.text) === "") {
7162
- if (chunks.length > 0) {
7163
- const last = chunks[chunks.length - 1];
7164
- const content = last.content + seg.text;
7165
- chunks[chunks.length - 1] = {
7166
- ...last,
7167
- content,
7168
- chunkHash: sha256(content)
7169
- };
7170
- } else prefix += seg.text;
7171
- continue;
7172
- }
7173
- const segIsNav = cpLen(teachingBody(seg.text)) < 250;
7174
- const segMarker = segmentMarkerType(seg.text);
7175
- for (const piece of subsplit(seg.text, maxChars)) {
7176
- let sourceType;
7177
- if (segMarker !== null) sourceType = segMarker;
7178
- else {
7179
- sourceType = classify(piece, seg.path);
7180
- if (sourceType === "nav" && !segIsNav) sourceType = "prose";
7173
+ let i = 0;
7174
+ for (;;) {
7175
+ const j = text.indexOf("style={{", i);
7176
+ if (j < 0) break;
7177
+ out.push(rstripSpacesTabs(text.slice(i, j)));
7178
+ let k = j + 6;
7179
+ let depth = 0;
7180
+ let closed = false;
7181
+ while (k < text.length) {
7182
+ const ch = text[k];
7183
+ if (ch === "{") depth += 1;
7184
+ else if (ch === "}") {
7185
+ depth -= 1;
7186
+ if (depth === 0) {
7187
+ k += 1;
7188
+ closed = true;
7189
+ break;
7190
+ }
7181
7191
  }
7182
- const content = prefix !== "" ? prefix + piece : piece;
7183
- prefix = "";
7184
- emit(content, seg.path, seg.anchor, sourceType);
7192
+ k += 1;
7193
+ }
7194
+ if (!closed) {
7195
+ out.push(text.slice(j));
7196
+ return out.join("");
7185
7197
  }
7198
+ i = k;
7186
7199
  }
7187
- if (prefix !== "") emit(prefix, [], null, "nav");
7188
- return chunks;
7189
- }
7190
- /** §5 rule 2: snapshot-token TTL (30 min) + 10 min = 40 min from retirement. */
7191
- const GC_GRACE_MS = 24e5;
7192
- /**
7193
- * Poison-chunk tolerance (oracle review: poison-chunk-wedge): one
7194
- * deterministically-failing chunk must not wedge every future flip forever. A
7195
- * generation is servable if a SMALL fraction failed — the read path already
7196
- * filters to `embedded`, so a quarantined chunk is simply absent, not
7197
- * corrupt. Above the fraction, a real ingest break is signalled by
7198
- * withholding readiness.
7199
- */
7200
- const MAX_FAILED_FRACTION = .02;
7201
- const LOCK_SQL = "SELECT pg_advisory_xact_lock(hashtextextended('sor-ingest:' || $1, 0))";
7202
- /**
7203
- * Take the tenant lock, allocate generation max+1 (monotonic per corpus,
7204
- * never reused), open the building run. The corpora row seeds at
7205
- * active_generation=0 (nothing active) on first ingest.
7206
- *
7207
- * `manifestSha256` fills `instance_bundle_sha256` — ksor has no bundle
7208
- * transport (the CLI reads the local repo), so the recorded digest is of the
7209
- * manifest this build actually consumed: the closest honest provenance.
7210
- */
7211
- async function allocateRun(client, opts) {
7212
- await client.query(LOCK_SQL, [opts.tenantId]);
7213
- await client.query("INSERT INTO corpora (tenant_id, corpus_id, active_generation) VALUES ($1, $2, 0) ON CONFLICT (tenant_id, corpus_id) DO NOTHING", [opts.tenantId, opts.corpusId]);
7214
- const next = await client.query("SELECT COALESCE(max(generation), 0) + 1 AS next FROM ingestion_runs WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId]);
7215
- const generation = Number(next.rows[0].next);
7216
- const run = await client.query("INSERT INTO ingestion_runs (tenant_id, corpus_id, generation, state, source_commit, instance_bundle_sha256, schema_version) VALUES ($1, $2, $3, 'building', $4, $5, $6) RETURNING run_id", [
7217
- opts.tenantId,
7218
- opts.corpusId,
7219
- generation,
7220
- opts.sourceCommit,
7221
- opts.manifestSha256,
7222
- schemaVersion()
7223
- ]);
7224
- return {
7225
- runId: Number(run.rows[0].run_id),
7226
- generation
7227
- };
7228
- }
7229
- /**
7230
- * The generation to carry embeddings FROM: the newest COMPLETE one holding
7231
- * embedded chunks.
7232
- *
7233
- * WHY NOT ONLY THE ACTIVE ONE: the eval-before-flip design means a candidate
7234
- * is often built, measured, and deliberately NOT served; ACTIVE then points
7235
- * at an OLD generation and the next candidate re-embeds the whole corpus —
7236
- * measured 2026-08-02: generation 4 re-embedded 5,915 chunks while
7237
- * generation 3 held near-identical content, because generation 1 was still
7238
- * active.
7239
- *
7240
- * Two constraints a rewrite once dropped (oracle review of PR #420):
7241
- * CORPUS-SCOPED via the run-table join (chunks carry no corpus_id), and
7242
- * COMPLETE RUNS ONLY (ready/active/retired) — a crashed `building` queue's
7243
- * half-drained vectors never qualify.
7244
- *
7245
- * Returns 0 when there is no complete embedded generation — the first ingest.
7246
- */
7247
- async function bestCarrySource(client, opts) {
7248
- const gen = (await client.query(`
7249
- SELECT max(c.generation) AS gen FROM chunks c
7250
- JOIN ingestion_runs r ON r.tenant_id = c.tenant_id AND r.generation = c.generation
7251
- WHERE c.tenant_id = $1 AND r.corpus_id = $2
7252
- AND r.state IN ('ready', 'active', 'retired')
7253
- AND c.generation <> $3 AND c.embedding_status = 'embedded'
7254
- `, [
7255
- opts.tenantId,
7256
- opts.corpusId,
7257
- opts.excludeGeneration
7258
- ])).rows[0]?.gen ?? null;
7259
- return gen === null ? 0 : Number(gen);
7260
- }
7261
- /**
7262
- * Copy embeddings for chunks whose ENTIRE embed input is unchanged (hash +
7263
- * heading path + node title). Cost ∝ change survives the generational
7264
- * rebuild. Returns rows carried.
7265
- *
7266
- * `modelId` is REQUIRED, never defaulted here: the vendor transport is
7267
- * irrelevant to the space (the same model through two providers is the same
7268
- * space), and a silent module default is exactly how a model bump would
7269
- * carry stale vectors unnoticed.
7270
- */
7271
- async function carryForward(client, opts) {
7272
- if (opts.fromGeneration < 1) return 0;
7273
- return (await client.query(`
7274
- UPDATE chunks new SET embedding = old.embedding, embedding_status = 'embedded',
7275
- embedded_at = old.embedded_at, embedding_model = old.embedding_model
7276
- FROM chunks old, sources os, content_nodes onode, sources ns, content_nodes nnode
7277
- WHERE new.tenant_id = $1 AND new.generation = $2
7278
- AND new.embedding_status = 'pending'
7279
- AND old.tenant_id = new.tenant_id AND old.generation = $3
7280
- AND old.embedding_status = 'embedded'
7281
- -- R-1 gate (oracle review: carry-model-gate-r1): carry ONLY vectors from the CURRENT
7282
- -- embedding model. Without this, a model bump silently carries every old-model vector
7283
- -- forward (pending→0, flip → corpus-wide nonsense cosine vs the new query model, zero
7284
- -- errors). A model change now correctly leaves the old vectors pending → they re-embed.
7285
- AND old.embedding_model = $4
7286
- AND old.source_id = new.source_id
7287
- AND old.chunk_hash = new.chunk_hash
7288
- AND old.heading_path_text IS NOT DISTINCT FROM new.heading_path_text
7289
- AND os.source_id = old.source_id AND os.tenant_id = old.tenant_id
7290
- AND os.generation = old.generation
7291
- AND onode.node_id = os.node_id AND onode.tenant_id = os.tenant_id
7292
- AND ns.source_id = new.source_id AND ns.tenant_id = new.tenant_id
7293
- AND ns.generation = new.generation
7294
- AND nnode.node_id = ns.node_id AND nnode.tenant_id = ns.tenant_id
7295
- AND onode.title = nnode.title
7296
- `, [
7297
- opts.tenantId,
7298
- opts.generation,
7299
- opts.fromGeneration,
7300
- opts.modelId
7301
- ])).rowCount ?? 0;
7302
- }
7303
- /**
7304
- * avg(embedding) per node over servable prose — rows the routing arm reads
7305
- * (never aggregate at query time again). nav/embed/assessment chunks never
7306
- * pollute routing centroids.
7307
- */
7308
- async function materializeCentroids(client, opts) {
7309
- await client.query("DELETE FROM node_centroids WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation]);
7310
- return (await client.query(`
7311
- INSERT INTO node_centroids (tenant_id, generation, node_id, stable_id, chunk_count, embedding)
7312
- SELECT c.tenant_id, c.generation, n.node_id, n.stable_id, count(*), avg(c.embedding)
7313
- FROM chunks c
7314
- JOIN sources s ON s.source_id = c.source_id AND s.tenant_id = c.tenant_id
7315
- AND s.generation = c.generation
7316
- JOIN content_nodes n ON n.node_id = s.node_id AND n.tenant_id = s.tenant_id
7317
- WHERE c.tenant_id = $1 AND c.generation = $2 AND c.embedding_status = 'embedded'
7318
- AND c.labels->>'source_type' = 'prose'
7319
- GROUP BY c.tenant_id, c.generation, n.node_id, n.stable_id
7320
- `, [opts.tenantId, opts.generation])).rowCount ?? 0;
7200
+ out.push(text.slice(i));
7201
+ return out.join("");
7321
7202
  }
7322
- /**
7323
- * The ready gate, factored pure: zero PENDING (the queue drained) + some
7324
- * embedded content + failures within tolerance. The read path serves only
7325
- * `embedded`, so a failed chunk is quarantined, not corrupt.
7326
- */
7327
- function generationReady(health) {
7328
- if (health.pending !== 0 || health.embedded === 0) return false;
7329
- const total = health.embedded + health.failed;
7330
- return health.failed / total <= MAX_FAILED_FRACTION;
7203
+ /** Drop layout tags left BARE by the attribute strip, KEEPING each closer
7204
+ * paired with its opener: a closer carries no attributes, so `</div>` cannot
7205
+ * say whether it belongs to a removed wrapper or to a kept `<div id="x">`.
7206
+ * Track depth; a closer is removed iff its opener was. On any mismatch the tag
7207
+ * is KEPT — a stray tag beats eaten content. `stack` is owned by the CALLER
7208
+ * and persists across every prose segment of one document: a styled wrapper
7209
+ * around a code fence splits at the fence, and its opener/closer land in
7210
+ * different segments (228 of them in the oracle's corpus). */
7211
+ function dropBareLayoutTags(segment, stack) {
7212
+ return segment.replace(LAYOUT_TAG_G, (m0, closing, name, rawAttrs, selfClosing) => {
7213
+ const attrs = pyStrip(rawAttrs);
7214
+ if (closing !== "") {
7215
+ const top = stack[stack.length - 1];
7216
+ if (top !== void 0 && top[0] === name) return stack.pop()[1] ? "" : m0;
7217
+ return m0;
7218
+ }
7219
+ if (selfClosing !== "") return attrs === "" ? "" : m0;
7220
+ stack.push([name, attrs === ""]);
7221
+ return attrs === "" ? "" : m0;
7222
+ });
7331
7223
  }
7332
- async function generationHealth(client, opts) {
7333
- const row = (await client.query("SELECT count(*) FILTER (WHERE embedding_status = 'embedded') AS embedded, count(*) FILTER (WHERE embedding_status = 'pending') AS pending, count(*) FILTER (WHERE embedding_status = 'failed') AS failed FROM chunks WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation])).rows[0];
7334
- return {
7335
- generation: opts.generation,
7336
- embedded: Number(row.embedded),
7337
- pending: Number(row.pending),
7338
- failed: Number(row.failed)
7339
- };
7224
+ /** Restore NUL<n>NUL placeholders — equivalent to the oracle's
7225
+ * re.sub(r"\x00(\d+)\x00", ...): a leftmost scan where a match is a NUL, a
7226
+ * maximal non-empty ASCII digit run, and a closing NUL; anything else stays
7227
+ * verbatim (backtracking cannot produce any other match for this pattern). */
7228
+ function restoreHeld(seg, held) {
7229
+ let out = "";
7230
+ let i = 0;
7231
+ for (;;) {
7232
+ const a = seg.indexOf(NUL, i);
7233
+ if (a < 0) break;
7234
+ let b = a + 1;
7235
+ while (b < seg.length && seg[b] >= "0" && seg[b] <= "9") b += 1;
7236
+ if (b > a + 1 && seg[b] === NUL) {
7237
+ const d = seg.slice(a + 1, b);
7238
+ const h = held[Number(d)];
7239
+ if (h === void 0) throw new Error(`inline-code placeholder NUL${d}NUL has no held span (held ${held.length}): the source document contains a literal NUL-digit-NUL sequence, which collides with the stripper's placeholder scheme. Remove NUL control characters from the file.`);
7240
+ out += seg.slice(i, a) + h;
7241
+ i = b + 1;
7242
+ } else {
7243
+ out += seg.slice(i, a + 1);
7244
+ i = a + 1;
7245
+ }
7246
+ }
7247
+ return out + seg.slice(i);
7340
7248
  }
7341
- function addedSlugs(delta) {
7342
- return [...delta.newSlugs].filter((s) => !delta.priorSlugs.has(s)).sort();
7249
+ /** Strip presentation markup from ONE non-fenced segment. Inline code is
7250
+ * protected first (`<div>` written in prose backticks is a lesson). `stack` is
7251
+ * the document-wide layout-tag stack (see dropBareLayoutTags). */
7252
+ function stripProsePresentation(segment, stack) {
7253
+ const held = [];
7254
+ let seg = segment.replace(INLINE_CODE_G, (m0) => {
7255
+ held.push(m0);
7256
+ return `${NUL}${held.length - 1}${NUL}`;
7257
+ });
7258
+ seg = stripStyleAttr(seg).replace(CLASS_ATTR_G, "");
7259
+ seg = dropBareLayoutTags(seg, stack);
7260
+ seg = seg.replace(BLANK_RUN_G, "\n\n");
7261
+ return restoreHeld(seg, held);
7343
7262
  }
7344
- function removedSlugs(delta) {
7345
- return [...delta.priorSlugs].filter((s) => !delta.newSlugs.has(s)).sort();
7263
+ /** Drop layout markup — `style={{…}}`, `className="…"`, and the `<div>`/
7264
+ * `<span>` wrappers they leave bare — while KEEPING every character of the
7265
+ * text inside them ("The Third Era of AI Tools" is content, `af-hero-eyebrow`
7266
+ * is a CSS hook; the oracle measured a student being served the wrapper).
7267
+ * A layout tag is only stripped once it is BARE: attributes go first, and a
7268
+ * wrapper with nothing left was pure layout by construction, while
7269
+ * `<div id="x">` keeps its tag — no tag allowlist to drift. Deliberately never
7270
+ * touches capitalised components (<Quiz> is curriculum), inline SVG, or
7271
+ * `<details>`/`<summary>` (semantic HTML). FENCE- and inline-code-safe. ONE
7272
+ * layout-tag stack is threaded through every prose segment of the document so
7273
+ * pairing survives a fence split. */
7274
+ function stripPresentationJsx(text) {
7275
+ if (!text.includes("className=") && !text.includes("style={{") && !LAYOUT_TAG_PROBE.test(text)) return text;
7276
+ const out = [];
7277
+ let buf = [];
7278
+ let fence = null;
7279
+ const stack = [];
7280
+ for (const line of pySplitLines(text, true)) {
7281
+ const nxt = fenceStep(line, fence);
7282
+ if (fence === null && nxt === null) buf.push(line);
7283
+ else {
7284
+ if (buf.length > 0) {
7285
+ out.push(stripProsePresentation(buf.join(""), stack));
7286
+ buf = [];
7287
+ }
7288
+ out.push(line);
7289
+ }
7290
+ fence = nxt;
7291
+ }
7292
+ if (buf.length > 0) out.push(stripProsePresentation(buf.join(""), stack));
7293
+ return out.join("");
7346
7294
  }
7347
7295
  /**
7348
- * Net fractional drop in node count vs the prior generation. Zero when the
7349
- * prior generation is empty (a FIRST ingest has nothing to shrink from) so
7350
- * the guard never trips on it.
7296
+ * The body-cleaning pipeline every ingest runs BEFORE the skip-gate hash and
7297
+ * chunking, as ONE ordered unit so the order cannot regress. CRLF→LF is
7298
+ * normalized FIRST — the strippers are \n-anchored (BLANK_RUN_G = /\n{3,}/),
7299
+ * so normalizing AFTER them left a CRLF checkout's blank runs un-collapsed and
7300
+ * every chunk_hash + content_hash diverged from an LF checkout, re-embedding
7301
+ * the whole file while content_hash claimed nothing changed (review,
7302
+ * 2026-08-19). Then style blocks and presentation JSX are stripped so served
7303
+ * chunks reassemble the CLEANED body byte-exact. A bare \r (no following \n)
7304
+ * stays content.
7351
7305
  */
7352
- function shrinkFraction(priorCount, newCount) {
7353
- if (priorCount === 0) return 0;
7354
- return Math.max(0, priorCount - newCount) / priorCount;
7306
+ function cleanBody(rawBody) {
7307
+ return stripPresentationJsx(stripStyleBlocks(rawBody.replaceAll("\r\n", "\n")));
7355
7308
  }
7356
- /**
7357
- * True when the corpus shrank by MORE than the tolerated fraction — the flip
7358
- * should be refused unless the drop is explicitly acknowledged. A first
7359
- * ingest is always safe.
7360
- */
7361
- function shrinkUnsafe(priorCount, newCount, maxShrink) {
7362
- return priorCount > 0 && shrinkFraction(priorCount, newCount) > maxShrink;
7309
+ /** Heading text never counts toward the nav/prose size test — the
7310
+ * "content-only" in the policy name. Note: joins on \n, so exotic line
7311
+ * boundaries are normalized before the length is taken (as in the oracle). */
7312
+ function teachingBody(content) {
7313
+ return pyStrip(pySplitLines(content, false).filter((ln) => !HEADING.test(ln)).join("\n"));
7363
7314
  }
7364
- /** Read the node-slug sets of the active generation and the candidate; the caller decides. */
7365
- async function flipDelta(client, opts) {
7366
- const raw = (await client.query("SELECT active_generation FROM corpora WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId])).rows[0]?.active_generation ?? null;
7367
- const prior = raw === null ? 0 : Number(raw);
7368
- const nodesOf = async (generation) => {
7369
- if (generation < 1) return /* @__PURE__ */ new Set();
7370
- const res = await client.query("SELECT stable_id FROM content_nodes WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, generation]);
7371
- return new Set(res.rows.map((r) => String(r.stable_id)));
7372
- };
7373
- return {
7374
- priorGeneration: prior,
7375
- priorSlugs: await nodesOf(prior),
7376
- newSlugs: await nodesOf(opts.newGeneration)
7315
+ function classify(content, headingPath) {
7316
+ if (JSX_ASSESS.test(content)) return "assessment";
7317
+ const leaf = headingPath.length > 0 ? headingPath[headingPath.length - 1] : "";
7318
+ if (JSX_EMBED.test(content) || content.includes("docs.google.com/presentation") || leaf.includes("Teaching Aid")) return "embed";
7319
+ if (cpLen(teachingBody(content)) < 250) return "nav";
7320
+ return "prose";
7321
+ }
7322
+ /** A segment DOMINATED by a line-leading widget (with < NAV_MAX_CHARS of
7323
+ * teaching body before it) labels EVERY fragment — a char-sliced widget must
7324
+ * not leak as prose. */
7325
+ function segmentMarkerType(span) {
7326
+ for (const [re, label] of [[JSX_ASSESS, "assessment"], [JSX_EMBED, "embed"]]) {
7327
+ const m = re.exec(span);
7328
+ if (m !== null && cpLen(teachingBody(span.slice(0, m.index))) < 250) return label;
7329
+ }
7330
+ return null;
7331
+ }
7332
+ /** Walk lines; headings count only OUTSIDE fences; every line lands in exactly
7333
+ * one segment (byte-exact). H1 records a title but never enters the path. */
7334
+ function segmentText(text) {
7335
+ const segments = [];
7336
+ const titles = /* @__PURE__ */ new Map();
7337
+ const anchors = /* @__PURE__ */ new Map();
7338
+ let buf = [];
7339
+ let curPath = [];
7340
+ let curAnchor = null;
7341
+ let fence = null;
7342
+ const flush = () => {
7343
+ if (buf.length > 0) {
7344
+ segments.push({
7345
+ path: [...curPath],
7346
+ anchor: curAnchor,
7347
+ text: buf.join("")
7348
+ });
7349
+ buf = [];
7350
+ }
7377
7351
  };
7352
+ for (const line of pySplitLines(text, true)) {
7353
+ const m = fence === null ? HEADING.exec(line) : null;
7354
+ if (m !== null) {
7355
+ flush();
7356
+ const level = m[1].length;
7357
+ const rawTitle = m[2];
7358
+ const idM = EXPLICIT_ID.exec(rawTitle);
7359
+ const title = idM !== null ? pyStrip(rawTitle.replace(EXPLICIT_ID, "")) : rawTitle;
7360
+ titles.set(level, title);
7361
+ anchors.set(level, idM !== null ? idM[1] : null);
7362
+ const deeper = [...titles.keys()].filter((lv) => lv > level);
7363
+ for (const lv of deeper) {
7364
+ titles.delete(lv);
7365
+ anchors.delete(lv);
7366
+ }
7367
+ curPath = [...titles.keys()].sort((a, b) => a - b).filter((lv) => lv >= 2 && lv <= level).map((lv) => titles.get(lv));
7368
+ curAnchor = curPath.length > 0 ? anchors.get(level) || slug(title) : null;
7369
+ }
7370
+ buf.push(line);
7371
+ if (m === null) fence = fenceStep(line, fence);
7372
+ }
7373
+ flush();
7374
+ return segments;
7378
7375
  }
7379
- /**
7380
- * Activation + run-state bookkeeping + the ledger row. The health gate is the
7381
- * CALLER's duty never partially activate. Serialized under the tenant
7382
- * advisory lock (re-taken here: the allocate lock died with its own txn) and
7383
- * MONOTONIC: only ever advances.
7384
- */
7385
- async function flip(client, opts) {
7386
- await client.query(LOCK_SQL, [opts.tenantId]);
7387
- if (!(await client.query("UPDATE corpora SET rollback_generation = active_generation, active_generation = $1, updated_at = now() WHERE tenant_id = $2 AND corpus_id = $3 AND active_generation < $1", [
7388
- opts.toGeneration,
7389
- opts.tenantId,
7390
- opts.corpusId
7391
- ])).rowCount) throw new Error(`flip to generation ${opts.toGeneration} refused: active_generation is already >= it (an out-of-order or duplicate flip — refusing to regress the served corpus)`);
7392
- await client.query("UPDATE ingestion_runs SET state = 'retired', finished_at = now() WHERE tenant_id = $1 AND corpus_id = $2 AND state = 'active'", [opts.tenantId, opts.corpusId]);
7393
- await client.query("UPDATE ingestion_runs SET state = 'active', finished_at = COALESCE(finished_at, now()) WHERE tenant_id = $1 AND corpus_id = $2 AND generation = $3", [
7394
- opts.tenantId,
7395
- opts.corpusId,
7396
- opts.toGeneration
7397
- ]);
7398
- await client.query("INSERT INTO retrieval_log (tenant_id, corpus_id, generation, actor, action, detail) VALUES ($1, $2, $3, 'sor-ingest', 'generation_activated', '{}')", [
7399
- opts.tenantId,
7400
- opts.corpusId,
7401
- opts.toGeneration
7402
- ]);
7376
+ /** Split on blank-line runs, greedy-pack, but flush ONLY outside a fence — a
7377
+ * flush never lands between an open fence and its close. The separator is
7378
+ * CAPTURED, so blank-line runs ride along as parts and concatenation is
7379
+ * lossless; nothing is trimmed. */
7380
+ function subsplit(span, maxChars) {
7381
+ if (cpLen(span) <= maxChars) return [span];
7382
+ const parts = span.split(BLANK_SEP);
7383
+ const pieces = [];
7384
+ let buf = "";
7385
+ let bufLen = 0;
7386
+ let fence = null;
7387
+ for (const part of parts) {
7388
+ const partLen = cpLen(part);
7389
+ if (buf !== "" && fence === null && bufLen + partLen > maxChars) {
7390
+ pieces.push(buf);
7391
+ buf = "";
7392
+ bufLen = 0;
7393
+ }
7394
+ buf += part;
7395
+ bufLen += partLen;
7396
+ for (const line of pySplitLines(part, true)) fence = fenceStep(line, fence);
7397
+ }
7398
+ if (buf !== "") pieces.push(buf);
7399
+ const out = [];
7400
+ for (const piece of pieces) out.push(...enforceCeiling(piece));
7401
+ return out;
7403
7402
  }
7404
- /**
7405
- * The §5 algebra: not active, not rollback, past token grace since
7406
- * retirement, ≥2 complete generations REMAIN after collection; abandoned
7407
- * builds reap on heartbeat staleness alone (they were never served, no token
7408
- * can reference them).
7409
- */
7410
- async function collectableGenerations(client, opts) {
7411
- const ts = opts.now ?? /* @__PURE__ */ new Date();
7412
- const pointer = await client.query("SELECT active_generation, rollback_generation FROM corpora WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId]);
7413
- if (pointer.rows.length === 0) return [];
7414
- const active = Number(pointer.rows[0].active_generation);
7415
- const rollbackRaw = pointer.rows[0].rollback_generation;
7416
- const rollbackGen = rollbackRaw === null ? null : Number(rollbackRaw);
7417
- const runs = await client.query("SELECT generation, state, finished_at, heartbeat_at FROM ingestion_runs WHERE tenant_id = $1 AND corpus_id = $2 AND state <> 'reaped' ORDER BY generation", [opts.tenantId, opts.corpusId]);
7418
- const complete = runs.rows.filter((r) => [
7419
- "ready",
7420
- "active",
7421
- "retired"
7422
- ].includes(String(r.state)));
7403
+ /** The ONLY place mid-line/mid-fence slicing can happen (a pathological single
7404
+ * paragraph or giant fence). Slices by CODE POINTS 4000 UTF-16 units would
7405
+ * be a different policy and could split a surrogate pair. */
7406
+ function enforceCeiling(piece) {
7407
+ if (cpLen(piece) <= 4e3) return [piece];
7408
+ const cps = [...piece];
7423
7409
  const out = [];
7424
- let remaining = complete.length;
7425
- for (const row of runs.rows) {
7426
- const gen = Number(row.generation);
7427
- const state = String(row.state);
7428
- const finishedAt = row.finished_at;
7429
- const heartbeatAt = row.heartbeat_at;
7430
- if (state === "building") {
7431
- if (heartbeatAt !== null && ts.getTime() - heartbeatAt.getTime() > 864e5) out.push(gen);
7410
+ for (let i = 0; i < cps.length; i += HARD_MAX_CHARS) out.push(cps.slice(i, i + HARD_MAX_CHARS).join(""));
7411
+ return out;
7412
+ }
7413
+ function chunkText(text, maxChars = MAX_CHARS) {
7414
+ const chunks = [];
7415
+ let prefix = "";
7416
+ const emit = (content, path, anchor, sourceType) => {
7417
+ chunks.push({
7418
+ ordinal: chunks.length,
7419
+ content,
7420
+ chunkHash: sha256(content),
7421
+ headingPath: [...path],
7422
+ anchor,
7423
+ sourceType
7424
+ });
7425
+ };
7426
+ for (const seg of segmentText(text)) {
7427
+ if (pyStrip(seg.text) === "") {
7428
+ if (chunks.length > 0) {
7429
+ const last = chunks[chunks.length - 1];
7430
+ const content = last.content + seg.text;
7431
+ chunks[chunks.length - 1] = {
7432
+ ...last,
7433
+ content,
7434
+ chunkHash: sha256(content)
7435
+ };
7436
+ } else prefix += seg.text;
7432
7437
  continue;
7433
7438
  }
7434
- if (gen === active || rollbackGen !== null && gen === rollbackGen) continue;
7435
- if (finishedAt === null || ts.getTime() - finishedAt.getTime() < GC_GRACE_MS) continue;
7436
- if (remaining - 1 < 2) continue;
7437
- remaining -= 1;
7438
- out.push(gen);
7439
+ const segIsNav = cpLen(teachingBody(seg.text)) < 250;
7440
+ const segMarker = segmentMarkerType(seg.text);
7441
+ for (const piece of subsplit(seg.text, maxChars)) {
7442
+ let sourceType;
7443
+ if (segMarker !== null) sourceType = segMarker;
7444
+ else {
7445
+ sourceType = classify(piece, seg.path);
7446
+ if (sourceType === "nav" && !segIsNav) sourceType = "prose";
7447
+ }
7448
+ const content = prefix !== "" ? prefix + piece : piece;
7449
+ prefix = "";
7450
+ emit(content, seg.path, seg.anchor, sourceType);
7451
+ }
7439
7452
  }
7440
- return out;
7441
- }
7442
- /**
7443
- * Delete one generation's rows (chunks cascade from sources) and mark the run
7444
- * reaped. NEVER touches takedown_denylist or retrieval_log — the ledger and
7445
- * denylist outlive the content they governed (§5).
7446
- */
7447
- async function reap(client, opts) {
7448
- for (const sql of [
7449
- "DELETE FROM node_centroids WHERE tenant_id = $1 AND generation = $2",
7450
- "DELETE FROM slug_aliases WHERE tenant_id = $1 AND generation = $2",
7451
- "DELETE FROM sources WHERE tenant_id = $1 AND generation = $2"
7452
- ]) await client.query(sql, [opts.tenantId, opts.generation]);
7453
- for (;;) if (!(await client.query("DELETE FROM content_nodes n WHERE n.tenant_id = $1 AND n.generation = $2 AND NOT EXISTS (SELECT 1 FROM content_nodes ch WHERE ch.parent_id = n.node_id AND ch.tenant_id = n.tenant_id AND ch.generation = n.generation)", [opts.tenantId, opts.generation])).rowCount) break;
7454
- await client.query("UPDATE ingestion_runs SET state = 'reaped' WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation]);
7453
+ if (prefix !== "") emit(prefix, [], null, "nav");
7454
+ return chunks;
7455
7455
  }
7456
7456
  const FRONTMATTER = /^\uFEFF?---\r?\n([\s\S]*?)\r?\n---[ \t]*\r?\n?/;
7457
7457
  function splitFrontmatter(text) {
@@ -8257,7 +8257,7 @@ async function ingestCommand(args) {
8257
8257
  return await buildGeneration(pool, instance, {
8258
8258
  knowledgeDir: values.knowledge,
8259
8259
  sourceCommit,
8260
- flip: values.flip ?? false,
8260
+ flip: false,
8261
8261
  provider,
8262
8262
  onLog: (line) => process.stdout.write(line + "\n")
8263
8263
  });
@@ -8274,8 +8274,16 @@ async function ingestCommand(args) {
8274
8274
  process.stdout.write(`ingest: generation ${report.generation} — ${report.nodes} nodes, ${report.chunks} chunks; embedded ${report.embedded}, carried ${report.carried}, failed ${report.failed}\n`);
8275
8275
  if (report.refusal !== null) return fail$1(REFUSED, report.refusal);
8276
8276
  const governance = await withPool(dsn, (pool) => assertGovernanceServable(pool, instance, report.generation).then(() => null, (error) => error instanceof Error ? error.message : String(error)));
8277
- if (governance !== null) return fail$1(REFUSED, `generation ${report.generation} was built, but no surface can serve it\n ${governance.split("\n").join("\n ")}`);
8278
- if (!report.flipped) process.stdout.write("ready; flip withheld (pass --flip to activate)\n");
8277
+ if (governance !== null) return fail$1(REFUSED, `generation ${report.generation} was built and NOT activated — no surface could serve it\n ${governance.split("\n").join("\n ")}\n note: generation ${report.generation} is left behind, un-activated; \`ksor gc\` reaps it once the grace window passes. The previously active generation still serves.`);
8278
+ if (values.flip === true && !report.unchanged) {
8279
+ await withPool(dsn, (pool) => runIngest(pool, instance.tenantId, (client) => flip(client, {
8280
+ tenantId: instance.tenantId,
8281
+ corpusId: instance.corpusId,
8282
+ toGeneration: report.generation
8283
+ })));
8284
+ process.stdout.write(`FLIPPED active generation -> ${report.generation}\n`);
8285
+ }
8286
+ if (values.flip !== true) process.stdout.write("ready; flip withheld (pass --flip to activate)\n");
8279
8287
  return 0;
8280
8288
  }
8281
8289
  /**