@panaversity/ksor 0.0.8 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/dist/cli.mjs +1498 -1490
- package/package.json +1 -1
package/dist/cli.mjs
CHANGED
|
@@ -16,7 +16,7 @@ import { bodyLimit } from "hono/body-limit";
|
|
|
16
16
|
import { execFileSync, spawnSync } from "node:child_process";
|
|
17
17
|
import { parseArgs } from "node:util";
|
|
18
18
|
import { readFile, readdir, stat } from "node:fs/promises";
|
|
19
|
-
//#region ../content-gateway/dist/main-
|
|
19
|
+
//#region ../content-gateway/dist/main-tBbDvU1w.mjs
|
|
20
20
|
/**
|
|
21
21
|
* A connection could not be ESTABLISHED in time — retryable.
|
|
22
22
|
*
|
|
@@ -4082,7 +4082,7 @@ async function withPgRetry(op, options = {}) {
|
|
|
4082
4082
|
throw lastError;
|
|
4083
4083
|
}
|
|
4084
4084
|
//#endregion
|
|
4085
|
-
//#region ../content/dist/commands-
|
|
4085
|
+
//#region ../content/dist/commands-1ZBNjWKb.mjs
|
|
4086
4086
|
/**
|
|
4087
4087
|
* EVAL-LOCKED constants, quarried verbatim from the oracle
|
|
4088
4088
|
* (sor-agentfactory @ b554f91, config.py) — changing any of these is a
|
|
@@ -5849,1609 +5849,1609 @@ async function assertGovernanceServable(pool, instance, targetGeneration) {
|
|
|
5849
5849
|
why: an author restricted those documents and nothing would enforce it — this door would serve them in full to every caller, and the frontmatter key saying otherwise would be the only trace. The site refuses to BUILD in this exact state (ksor-visibility-without-audiences); the door must not serve in it
|
|
5850
5850
|
fix: declare the model in instance.md (audiences: least-restricted first, plus default_visibility:), or remove the visibility: keys and re-ingest`);
|
|
5851
5851
|
}
|
|
5852
|
+
/** §5 rule 2: snapshot-token TTL (30 min) + 10 min = 40 min from retirement. */
|
|
5853
|
+
const GC_GRACE_MS = 24e5;
|
|
5852
5854
|
/**
|
|
5853
|
-
*
|
|
5854
|
-
*
|
|
5855
|
-
* is a
|
|
5856
|
-
*
|
|
5855
|
+
* Poison-chunk tolerance (oracle review: poison-chunk-wedge): one
|
|
5856
|
+
* deterministically-failing chunk must not wedge every future flip forever. A
|
|
5857
|
+
* generation is servable if a SMALL fraction failed — the read path already
|
|
5858
|
+
* filters to `embedded`, so a quarantined chunk is simply absent, not
|
|
5859
|
+
* corrupt. Above the fraction, a real ingest break is signalled by
|
|
5860
|
+
* withholding readiness.
|
|
5857
5861
|
*/
|
|
5858
|
-
|
|
5859
|
-
|
|
5860
|
-
|
|
5861
|
-
|
|
5862
|
-
|
|
5863
|
-
|
|
5864
|
-
|
|
5862
|
+
const MAX_FAILED_FRACTION = .02;
|
|
5863
|
+
const LOCK_SQL = "SELECT pg_advisory_xact_lock(hashtextextended('sor-ingest:' || $1, 0))";
|
|
5864
|
+
/**
|
|
5865
|
+
* Take the tenant lock, allocate generation max+1 (monotonic per corpus,
|
|
5866
|
+
* never reused), open the building run. The corpora row seeds at
|
|
5867
|
+
* active_generation=0 (nothing active) on first ingest.
|
|
5868
|
+
*
|
|
5869
|
+
* `manifestSha256` fills `instance_bundle_sha256` — ksor has no bundle
|
|
5870
|
+
* transport (the CLI reads the local repo), so the recorded digest is of the
|
|
5871
|
+
* manifest this build actually consumed: the closest honest provenance.
|
|
5872
|
+
*/
|
|
5873
|
+
async function allocateRun(client, opts) {
|
|
5874
|
+
await client.query(LOCK_SQL, [opts.tenantId]);
|
|
5875
|
+
await client.query("INSERT INTO corpora (tenant_id, corpus_id, active_generation) VALUES ($1, $2, 0) ON CONFLICT (tenant_id, corpus_id) DO NOTHING", [opts.tenantId, opts.corpusId]);
|
|
5876
|
+
const next = await client.query("SELECT COALESCE(max(generation), 0) + 1 AS next FROM ingestion_runs WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId]);
|
|
5877
|
+
const generation = Number(next.rows[0].next);
|
|
5878
|
+
const run = await client.query("INSERT INTO ingestion_runs (tenant_id, corpus_id, generation, state, source_commit, instance_bundle_sha256, schema_version) VALUES ($1, $2, $3, 'building', $4, $5, $6) RETURNING run_id", [
|
|
5879
|
+
opts.tenantId,
|
|
5880
|
+
opts.corpusId,
|
|
5881
|
+
generation,
|
|
5882
|
+
opts.sourceCommit,
|
|
5883
|
+
opts.manifestSha256,
|
|
5884
|
+
schemaVersion()
|
|
5865
5885
|
]);
|
|
5886
|
+
return {
|
|
5887
|
+
runId: Number(run.rows[0].run_id),
|
|
5888
|
+
generation
|
|
5889
|
+
};
|
|
5866
5890
|
}
|
|
5867
5891
|
/**
|
|
5868
|
-
*
|
|
5892
|
+
* The generation to carry embeddings FROM: the newest COMPLETE one holding
|
|
5893
|
+
* embedded chunks.
|
|
5869
5894
|
*
|
|
5870
|
-
*
|
|
5871
|
-
*
|
|
5872
|
-
*
|
|
5895
|
+
* WHY NOT ONLY THE ACTIVE ONE: the eval-before-flip design means a candidate
|
|
5896
|
+
* is often built, measured, and deliberately NOT served; ACTIVE then points
|
|
5897
|
+
* at an OLD generation and the next candidate re-embeds the whole corpus —
|
|
5898
|
+
* measured 2026-08-02: generation 4 re-embedded 5,915 chunks while
|
|
5899
|
+
* generation 3 held near-identical content, because generation 1 was still
|
|
5900
|
+
* active.
|
|
5901
|
+
*
|
|
5902
|
+
* Two constraints a rewrite once dropped (oracle review of PR #420):
|
|
5903
|
+
* CORPUS-SCOPED via the run-table join (chunks carry no corpus_id), and
|
|
5904
|
+
* COMPLETE RUNS ONLY (ready/active/retired) — a crashed `building` queue's
|
|
5905
|
+
* half-drained vectors never qualify.
|
|
5906
|
+
*
|
|
5907
|
+
* Returns 0 when there is no complete embedded generation — the first ingest.
|
|
5873
5908
|
*/
|
|
5874
|
-
async function
|
|
5875
|
-
|
|
5876
|
-
|
|
5877
|
-
|
|
5878
|
-
|
|
5879
|
-
|
|
5880
|
-
|
|
5881
|
-
|
|
5882
|
-
|
|
5883
|
-
|
|
5884
|
-
|
|
5885
|
-
|
|
5886
|
-
|
|
5887
|
-
opts.stableId,
|
|
5888
|
-
opts.scope,
|
|
5889
|
-
opts.reason
|
|
5890
|
-
])).rowCount === 1;
|
|
5891
|
-
await recordAct(client, instance, {
|
|
5892
|
-
stable_id: opts.stableId,
|
|
5893
|
-
scope: opts.scope,
|
|
5894
|
-
reason: opts.reason,
|
|
5895
|
-
change: changed ? "applied" : "unchanged"
|
|
5896
|
-
}, opts.actor);
|
|
5897
|
-
return {
|
|
5898
|
-
stableId: opts.stableId,
|
|
5899
|
-
scope: opts.scope,
|
|
5900
|
-
changed,
|
|
5901
|
-
resolves
|
|
5902
|
-
};
|
|
5903
|
-
});
|
|
5904
|
-
}
|
|
5905
|
-
/** Lift a denial. The ledger keeps the row that recorded imposing it. */
|
|
5906
|
-
async function revokeTakedown(pool, instance, opts) {
|
|
5907
|
-
return runIngest(pool, instance.tenantId, async (client) => {
|
|
5908
|
-
const changed = ((await client.query("DELETE FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2 AND stable_id = $3", [
|
|
5909
|
-
instance.tenantId,
|
|
5910
|
-
instance.corpusId,
|
|
5911
|
-
opts.stableId
|
|
5912
|
-
])).rowCount ?? 0) > 0;
|
|
5913
|
-
await recordAct(client, instance, {
|
|
5914
|
-
stable_id: opts.stableId,
|
|
5915
|
-
change: changed ? "revoked" : "not-denied"
|
|
5916
|
-
}, opts.actor, "takedown_revoked");
|
|
5917
|
-
return {
|
|
5918
|
-
stableId: opts.stableId,
|
|
5919
|
-
scope: "node",
|
|
5920
|
-
changed
|
|
5921
|
-
};
|
|
5922
|
-
});
|
|
5923
|
-
}
|
|
5924
|
-
async function readLedger(pool, instance, limit) {
|
|
5925
|
-
return runAuditRead(pool, instance.tenantId, async (client) => {
|
|
5926
|
-
return (await client.query("SELECT action, actor, generation, detail, created_at FROM retrieval_log WHERE tenant_id = $1 AND corpus_id = $2 ORDER BY created_at DESC, id DESC LIMIT $3", [
|
|
5927
|
-
instance.tenantId,
|
|
5928
|
-
instance.corpusId,
|
|
5929
|
-
limit
|
|
5930
|
-
])).rows.map((row) => ({
|
|
5931
|
-
action: String(row.action),
|
|
5932
|
-
actor: String(row.actor),
|
|
5933
|
-
generation: row.generation === null ? null : Number(row.generation),
|
|
5934
|
-
detail: row.detail ?? {},
|
|
5935
|
-
createdAt: row.created_at
|
|
5936
|
-
}));
|
|
5937
|
-
});
|
|
5909
|
+
async function bestCarrySource(client, opts) {
|
|
5910
|
+
const gen = (await client.query(`
|
|
5911
|
+
SELECT max(c.generation) AS gen FROM chunks c
|
|
5912
|
+
JOIN ingestion_runs r ON r.tenant_id = c.tenant_id AND r.generation = c.generation
|
|
5913
|
+
WHERE c.tenant_id = $1 AND r.corpus_id = $2
|
|
5914
|
+
AND r.state IN ('ready', 'active', 'retired')
|
|
5915
|
+
AND c.generation <> $3 AND c.embedding_status = 'embedded'
|
|
5916
|
+
`, [
|
|
5917
|
+
opts.tenantId,
|
|
5918
|
+
opts.corpusId,
|
|
5919
|
+
opts.excludeGeneration
|
|
5920
|
+
])).rows[0]?.gen ?? null;
|
|
5921
|
+
return gen === null ? 0 : Number(gen);
|
|
5938
5922
|
}
|
|
5939
5923
|
/**
|
|
5940
|
-
*
|
|
5941
|
-
*
|
|
5924
|
+
* Copy embeddings for chunks whose ENTIRE embed input is unchanged (hash +
|
|
5925
|
+
* heading path + node title). Cost ∝ change survives the generational
|
|
5926
|
+
* rebuild. Returns rows carried.
|
|
5942
5927
|
*
|
|
5943
|
-
*
|
|
5944
|
-
*
|
|
5945
|
-
*
|
|
5946
|
-
*
|
|
5947
|
-
* is to resolve the walk where the tree lives and hand over a flat list
|
|
5948
|
-
* (round-2 review of #43).
|
|
5928
|
+
* `modelId` is REQUIRED, never defaulted here: the vendor transport is
|
|
5929
|
+
* irrelevant to the space (the same model through two providers is the same
|
|
5930
|
+
* space), and a silent module default is exactly how a model bump would
|
|
5931
|
+
* carry stale vectors unnoticed.
|
|
5949
5932
|
*/
|
|
5950
|
-
async function
|
|
5951
|
-
|
|
5952
|
-
|
|
5953
|
-
|
|
5954
|
-
|
|
5955
|
-
|
|
5956
|
-
|
|
5957
|
-
|
|
5958
|
-
|
|
5959
|
-
|
|
5960
|
-
|
|
5961
|
-
|
|
5962
|
-
|
|
5963
|
-
|
|
5964
|
-
|
|
5965
|
-
|
|
5966
|
-
|
|
5967
|
-
|
|
5968
|
-
|
|
5969
|
-
|
|
5970
|
-
|
|
5971
|
-
|
|
5972
|
-
|
|
5973
|
-
|
|
5974
|
-
|
|
5975
|
-
|
|
5976
|
-
|
|
5933
|
+
async function carryForward(client, opts) {
|
|
5934
|
+
if (opts.fromGeneration < 1) return 0;
|
|
5935
|
+
return (await client.query(`
|
|
5936
|
+
UPDATE chunks new SET embedding = old.embedding, embedding_status = 'embedded',
|
|
5937
|
+
embedded_at = old.embedded_at, embedding_model = old.embedding_model
|
|
5938
|
+
FROM chunks old, sources os, content_nodes onode, sources ns, content_nodes nnode
|
|
5939
|
+
WHERE new.tenant_id = $1 AND new.generation = $2
|
|
5940
|
+
AND new.embedding_status = 'pending'
|
|
5941
|
+
AND old.tenant_id = new.tenant_id AND old.generation = $3
|
|
5942
|
+
AND old.embedding_status = 'embedded'
|
|
5943
|
+
-- R-1 gate (oracle review: carry-model-gate-r1): carry ONLY vectors from the CURRENT
|
|
5944
|
+
-- embedding model. Without this, a model bump silently carries every old-model vector
|
|
5945
|
+
-- forward (pending→0, flip → corpus-wide nonsense cosine vs the new query model, zero
|
|
5946
|
+
-- errors). A model change now correctly leaves the old vectors pending → they re-embed.
|
|
5947
|
+
AND old.embedding_model = $4
|
|
5948
|
+
AND old.source_id = new.source_id
|
|
5949
|
+
AND old.chunk_hash = new.chunk_hash
|
|
5950
|
+
AND old.heading_path_text IS NOT DISTINCT FROM new.heading_path_text
|
|
5951
|
+
AND os.source_id = old.source_id AND os.tenant_id = old.tenant_id
|
|
5952
|
+
AND os.generation = old.generation
|
|
5953
|
+
AND onode.node_id = os.node_id AND onode.tenant_id = os.tenant_id
|
|
5954
|
+
AND ns.source_id = new.source_id AND ns.tenant_id = new.tenant_id
|
|
5955
|
+
AND ns.generation = new.generation
|
|
5956
|
+
AND nnode.node_id = ns.node_id AND nnode.tenant_id = ns.tenant_id
|
|
5957
|
+
AND onode.title = nnode.title
|
|
5958
|
+
`, [
|
|
5959
|
+
opts.tenantId,
|
|
5960
|
+
opts.generation,
|
|
5961
|
+
opts.fromGeneration,
|
|
5962
|
+
opts.modelId
|
|
5963
|
+
])).rowCount ?? 0;
|
|
5977
5964
|
}
|
|
5978
5965
|
/**
|
|
5979
|
-
*
|
|
5980
|
-
*
|
|
5981
|
-
*
|
|
5982
|
-
* node's own id or path, because neither works:
|
|
5983
|
-
*
|
|
5984
|
-
* a section has no source `knowledge/policies#section` is synthetic — the
|
|
5985
|
-
* tree node for a directory. Joining `sources` on
|
|
5986
|
-
* the denied node itself yields nothing, and a
|
|
5987
|
-
* section is the ordinary target of `--subtree`.
|
|
5988
|
-
* a leaf's directory is not `--subtree` on one document would emit that
|
|
5989
|
-
* its subtree document's directory and deny every sibling.
|
|
5990
|
-
*
|
|
5991
|
-
* So: walk the descendants, take the directory of each one's file, and keep the
|
|
5992
|
-
* SHALLOWEST — a directory that contains another in the set is the subtree
|
|
5993
|
-
* root, and `startsWith` then covers subdirectories added later too. A denial
|
|
5994
|
-
* with no descendants contributes nothing, which is correct: its subtree is
|
|
5995
|
-
* itself, and the flat id list already holds it.
|
|
5996
|
-
*
|
|
5997
|
-
* The seed's OWN file counts when the seed has children, and only then — see
|
|
5998
|
-
* the SQL comment: a container's index.md names its directory, a leaf's file
|
|
5999
|
-
* names its parent's.
|
|
5966
|
+
* avg(embedding) per node over servable prose — rows the routing arm reads
|
|
5967
|
+
* (never aggregate at query time again). nav/embed/assessment chunks never
|
|
5968
|
+
* pollute routing centroids.
|
|
6000
5969
|
*/
|
|
6001
|
-
async function
|
|
6002
|
-
|
|
6003
|
-
|
|
6004
|
-
|
|
6005
|
-
|
|
6006
|
-
|
|
6007
|
-
|
|
6008
|
-
|
|
6009
|
-
|
|
6010
|
-
|
|
6011
|
-
|
|
6012
|
-
|
|
6013
|
-
|
|
6014
|
-
SELECT node_id FROM seed
|
|
6015
|
-
UNION ALL
|
|
6016
|
-
SELECT c.node_id
|
|
6017
|
-
FROM content_nodes c
|
|
6018
|
-
JOIN walk w ON c.parent_id = w.node_id
|
|
6019
|
-
JOIN gen ON c.generation = gen.g
|
|
6020
|
-
WHERE c.tenant_id = $1
|
|
6021
|
-
)
|
|
6022
|
-
SELECT DISTINCT s.origin_path
|
|
6023
|
-
FROM walk w
|
|
6024
|
-
JOIN content_nodes n ON n.node_id = w.node_id
|
|
6025
|
-
JOIN sources s ON s.tenant_id = n.tenant_id AND s.generation = n.generation
|
|
6026
|
-
AND s.node_id = n.node_id
|
|
6027
|
-
-- The seed's own file counts only when the seed HAS CHILDREN.
|
|
6028
|
-
--
|
|
6029
|
-
-- Excluding every seed stopped a LEAF denial emitting its parent
|
|
6030
|
-
-- directory and denying every sibling — right for a leaf, wrong for a
|
|
6031
|
-
-- container. A section's own index.md is the file that names the
|
|
6032
|
-
-- section's DIRECTORY, so a section whose other descendants all live
|
|
6033
|
-
-- one level down contributed only the subdirectory, and a document
|
|
6034
|
-
-- written directly under the withdrawn section published to /docs and
|
|
6035
|
-
-- llms.txt (round-10 review of PR 43).
|
|
6036
|
-
--
|
|
6037
|
-
-- "Has children" is the right test, not "kind = section": it is the
|
|
6038
|
-
-- property that decides whether the node's directory is its subtree
|
|
6039
|
-
-- or its parent's.
|
|
6040
|
-
WHERE w.node_id NOT IN (
|
|
6041
|
-
SELECT s2.node_id FROM seed s2
|
|
6042
|
-
WHERE NOT EXISTS (SELECT 1 FROM content_nodes kid
|
|
6043
|
-
JOIN gen ON kid.generation = gen.g
|
|
6044
|
-
WHERE kid.tenant_id = $1 AND kid.parent_id = s2.node_id)
|
|
6045
|
-
)`, [instance.tenantId, instance.corpusId])).rows.map((r) => String(r.origin_path));
|
|
6046
|
-
});
|
|
6047
|
-
const dirs = /* @__PURE__ */ new Set();
|
|
6048
|
-
for (const raw of paths) {
|
|
6049
|
-
const normalized = raw.replace(/\\/g, "/");
|
|
6050
|
-
const slash = normalized.lastIndexOf("/");
|
|
6051
|
-
dirs.add(slash === -1 ? "/" : `${normalized.slice(0, slash)}/`);
|
|
6052
|
-
}
|
|
6053
|
-
const all = [...dirs];
|
|
6054
|
-
return all.filter((dir) => !all.some((other) => other !== dir && dir.startsWith(other))).sort();
|
|
5970
|
+
async function materializeCentroids(client, opts) {
|
|
5971
|
+
await client.query("DELETE FROM node_centroids WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation]);
|
|
5972
|
+
return (await client.query(`
|
|
5973
|
+
INSERT INTO node_centroids (tenant_id, generation, node_id, stable_id, chunk_count, embedding)
|
|
5974
|
+
SELECT c.tenant_id, c.generation, n.node_id, n.stable_id, count(*), avg(c.embedding)
|
|
5975
|
+
FROM chunks c
|
|
5976
|
+
JOIN sources s ON s.source_id = c.source_id AND s.tenant_id = c.tenant_id
|
|
5977
|
+
AND s.generation = c.generation
|
|
5978
|
+
JOIN content_nodes n ON n.node_id = s.node_id AND n.tenant_id = s.tenant_id
|
|
5979
|
+
WHERE c.tenant_id = $1 AND c.generation = $2 AND c.embedding_status = 'embedded'
|
|
5980
|
+
AND c.labels->>'source_type' = 'prose'
|
|
5981
|
+
GROUP BY c.tenant_id, c.generation, n.node_id, n.stable_id
|
|
5982
|
+
`, [opts.tenantId, opts.generation])).rowCount ?? 0;
|
|
6055
5983
|
}
|
|
6056
|
-
|
|
6057
|
-
|
|
6058
|
-
|
|
6059
|
-
|
|
6060
|
-
|
|
6061
|
-
|
|
6062
|
-
|
|
6063
|
-
|
|
6064
|
-
|
|
5984
|
+
/**
|
|
5985
|
+
* The ready gate, factored pure: zero PENDING (the queue drained) + some
|
|
5986
|
+
* embedded content + failures within tolerance. The read path serves only
|
|
5987
|
+
* `embedded`, so a failed chunk is quarantined, not corrupt.
|
|
5988
|
+
*/
|
|
5989
|
+
function generationReady(health) {
|
|
5990
|
+
if (health.pending !== 0 || health.embedded === 0) return false;
|
|
5991
|
+
const total = health.embedded + health.failed;
|
|
5992
|
+
return health.failed / total <= MAX_FAILED_FRACTION;
|
|
6065
5993
|
}
|
|
6066
|
-
function
|
|
5994
|
+
async function generationHealth(client, opts) {
|
|
5995
|
+
const row = (await client.query("SELECT count(*) FILTER (WHERE embedding_status = 'embedded') AS embedded, count(*) FILTER (WHERE embedding_status = 'pending') AS pending, count(*) FILTER (WHERE embedding_status = 'failed') AS failed FROM chunks WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation])).rows[0];
|
|
6067
5996
|
return {
|
|
6068
|
-
|
|
6069
|
-
|
|
6070
|
-
|
|
6071
|
-
|
|
6072
|
-
exported_at: now.toISOString(),
|
|
6073
|
-
denied: stableIds.map((stable_id) => ({
|
|
6074
|
-
stable_id,
|
|
6075
|
-
scope: "node"
|
|
6076
|
-
}))
|
|
5997
|
+
generation: opts.generation,
|
|
5998
|
+
embedded: Number(row.embedded),
|
|
5999
|
+
pending: Number(row.pending),
|
|
6000
|
+
failed: Number(row.failed)
|
|
6077
6001
|
};
|
|
6078
6002
|
}
|
|
6003
|
+
function addedSlugs(delta) {
|
|
6004
|
+
return [...delta.newSlugs].filter((s) => !delta.priorSlugs.has(s)).sort();
|
|
6005
|
+
}
|
|
6006
|
+
function removedSlugs(delta) {
|
|
6007
|
+
return [...delta.priorSlugs].filter((s) => !delta.newSlugs.has(s)).sort();
|
|
6008
|
+
}
|
|
6079
6009
|
/**
|
|
6080
|
-
*
|
|
6081
|
-
*
|
|
6082
|
-
*
|
|
6083
|
-
* any other adapter's.
|
|
6084
|
-
*
|
|
6085
|
-
* Conventions (deliberately minimal — an operator can satisfy them with a bare
|
|
6086
|
-
* folder):
|
|
6087
|
-
* - directories become `section` nodes; `.md`/`.mdx` files become `document`
|
|
6088
|
-
* nodes;
|
|
6089
|
-
* - `index.md` (or `README.md`) inside a directory is that SECTION's own
|
|
6090
|
-
* content, not a child;
|
|
6091
|
-
* - ordering: frontmatter `position` (or `sidebar_position`) wins, else name
|
|
6092
|
-
* sort;
|
|
6093
|
-
* - titles: frontmatter `title`, else the filename humanized;
|
|
6094
|
-
* - stable ids: frontmatter `sor_id`, else the tree-relative path;
|
|
6095
|
-
* - hidden entries (leading `.` or `_`) and ALL symlinks are skipped LOUDLY
|
|
6096
|
-
* (reported through `onSkip`, console by default — never silent); symlinks
|
|
6097
|
-
* are never followed, so a link cannot walk out of the tree or cycle it;
|
|
6098
|
-
* - a directory carrying MORE than one index-named file (index.md +
|
|
6099
|
-
* README.md …) fails loud: which one is the section's own content is
|
|
6100
|
-
* ambiguous, and silently dropping the loser is exactly the corpus
|
|
6101
|
-
* corruption this adapter must never commit.
|
|
6102
|
-
*
|
|
6103
|
-
* The oracle's `publish_bundle` (deterministic tgz staging) is a separate
|
|
6104
|
-
* slice and is not converted here.
|
|
6010
|
+
* Net fractional drop in node count vs the prior generation. Zero when the
|
|
6011
|
+
* prior generation is empty (a FIRST ingest has nothing to shrink from) so
|
|
6012
|
+
* the guard never trips on it.
|
|
6105
6013
|
*/
|
|
6106
|
-
|
|
6107
|
-
|
|
6108
|
-
|
|
6109
|
-
"README.md"
|
|
6110
|
-
];
|
|
6111
|
-
/** Frontmatter-position fallback for entries that declare none (oracle plain_tree.py:107,114). */
|
|
6112
|
-
const POSITION_FALLBACK = 1e4;
|
|
6113
|
-
/** Walk a directory on disk → manifest + sources. Fail-loud on emptiness and ambiguity. */
|
|
6114
|
-
async function buildManifest(treeRoot, options) {
|
|
6115
|
-
const rootPath = treeRoot.length > 1 ? treeRoot.replace(/\/+$/, "") : treeRoot;
|
|
6116
|
-
let isDir = false;
|
|
6117
|
-
try {
|
|
6118
|
-
isDir = (await stat(rootPath)).isDirectory();
|
|
6119
|
-
} catch {
|
|
6120
|
-
isDir = false;
|
|
6121
|
-
}
|
|
6122
|
-
if (!isDir) throw new ManifestError(`plain-tree root ${rootPath} is not a directory`);
|
|
6123
|
-
return buildManifestFromTree(await readTree(rootPath), {
|
|
6124
|
-
...options,
|
|
6125
|
-
rootPath
|
|
6126
|
-
});
|
|
6014
|
+
function shrinkFraction(priorCount, newCount) {
|
|
6015
|
+
if (priorCount === 0) return 0;
|
|
6016
|
+
return Math.max(0, priorCount - newCount) / priorCount;
|
|
6127
6017
|
}
|
|
6128
6018
|
/**
|
|
6129
|
-
*
|
|
6130
|
-
*
|
|
6131
|
-
*
|
|
6132
|
-
* incidentally followed a symlinked index via `is_file()`, which this port
|
|
6133
|
-
* deliberately does not reproduce). Non-markdown files are invisible to the
|
|
6134
|
-
* walk, exactly as the oracle's suffix filter makes them.
|
|
6019
|
+
* True when the corpus shrank by MORE than the tolerated fraction — the flip
|
|
6020
|
+
* should be refused unless the drop is explicitly acknowledged. A first
|
|
6021
|
+
* ingest is always safe.
|
|
6135
6022
|
*/
|
|
6136
|
-
|
|
6137
|
-
|
|
6138
|
-
|
|
6139
|
-
|
|
6140
|
-
|
|
6141
|
-
|
|
6142
|
-
|
|
6143
|
-
|
|
6144
|
-
|
|
6145
|
-
|
|
6146
|
-
|
|
6147
|
-
|
|
6148
|
-
});
|
|
6023
|
+
function shrinkUnsafe(priorCount, newCount, maxShrink) {
|
|
6024
|
+
return priorCount > 0 && shrinkFraction(priorCount, newCount) > maxShrink;
|
|
6025
|
+
}
|
|
6026
|
+
/** Read the node-slug sets of the active generation and the candidate; the caller decides. */
|
|
6027
|
+
async function flipDelta(client, opts) {
|
|
6028
|
+
const raw = (await client.query("SELECT active_generation FROM corpora WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId])).rows[0]?.active_generation ?? null;
|
|
6029
|
+
const prior = raw === null ? 0 : Number(raw);
|
|
6030
|
+
const nodesOf = async (generation) => {
|
|
6031
|
+
if (generation < 1) return /* @__PURE__ */ new Set();
|
|
6032
|
+
const res = await client.query("SELECT stable_id FROM content_nodes WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, generation]);
|
|
6033
|
+
return new Set(res.rows.map((r) => String(r.stable_id)));
|
|
6034
|
+
};
|
|
6149
6035
|
return {
|
|
6150
|
-
|
|
6151
|
-
|
|
6152
|
-
|
|
6036
|
+
priorGeneration: prior,
|
|
6037
|
+
priorSlugs: await nodesOf(prior),
|
|
6038
|
+
newSlugs: await nodesOf(opts.newGeneration)
|
|
6153
6039
|
};
|
|
6154
6040
|
}
|
|
6155
|
-
/**
|
|
6156
|
-
|
|
6157
|
-
|
|
6158
|
-
|
|
6159
|
-
|
|
6160
|
-
|
|
6161
|
-
|
|
6162
|
-
|
|
6163
|
-
|
|
6164
|
-
|
|
6165
|
-
|
|
6166
|
-
|
|
6167
|
-
|
|
6168
|
-
|
|
6169
|
-
|
|
6170
|
-
|
|
6171
|
-
|
|
6172
|
-
|
|
6173
|
-
|
|
6174
|
-
|
|
6175
|
-
|
|
6176
|
-
|
|
6177
|
-
|
|
6178
|
-
|
|
6179
|
-
else if (e.kind === "file" && isDoc(e.name)) docs.push(e);
|
|
6180
|
-
else if (e.kind === "dir") dirs.push(e);
|
|
6181
|
-
const ordered = [];
|
|
6182
|
-
for (const f of docs) {
|
|
6183
|
-
if (f.name.startsWith(".") || f.name.startsWith("_")) {
|
|
6184
|
-
skipped.push(fullPath(relSegs, f.name));
|
|
6185
|
-
continue;
|
|
6186
|
-
}
|
|
6187
|
-
if (INDEX_NAMES.includes(f.name)) continue;
|
|
6188
|
-
ordered.push({
|
|
6189
|
-
position: positionOf(frontmatterMeta(f.text), POSITION_FALLBACK),
|
|
6190
|
-
nameLower: f.name.toLowerCase(),
|
|
6191
|
-
entry: f
|
|
6192
|
-
});
|
|
6193
|
-
}
|
|
6194
|
-
for (const d of dirs) {
|
|
6195
|
-
if (d.name.startsWith(".") || d.name.startsWith("_")) {
|
|
6196
|
-
skipped.push(fullPath(relSegs, d.name));
|
|
6197
|
-
continue;
|
|
6198
|
-
}
|
|
6199
|
-
const index = indexOf(d, fullPath(relSegs, d.name));
|
|
6200
|
-
const dirMeta = index === null ? {} : frontmatterMeta(index.text);
|
|
6201
|
-
ordered.push({
|
|
6202
|
-
position: positionOf(dirMeta, POSITION_FALLBACK),
|
|
6203
|
-
nameLower: d.name.toLowerCase(),
|
|
6204
|
-
entry: d
|
|
6205
|
-
});
|
|
6206
|
-
}
|
|
6207
|
-
ordered.sort((x, y) => x.position - y.position || codePointCompare(x.nameLower, y.nameLower));
|
|
6208
|
-
let position = 0;
|
|
6209
|
-
for (const { entry } of ordered) {
|
|
6210
|
-
position += 1;
|
|
6211
|
-
if (entry.kind === "dir") {
|
|
6212
|
-
const dirSegs = [...relSegs, entry.name];
|
|
6213
|
-
const index = indexOf(entry, fullPath(relSegs, entry.name));
|
|
6214
|
-
const meta = index === null ? {} : frontmatterMeta(index.text);
|
|
6215
|
-
const sid = index === null ? `${rootName}/${dirSegs.join("/")}#section` : stableIdOf(rootName, [...dirSegs, index.name], meta);
|
|
6216
|
-
nodes.push(manifestNode({
|
|
6217
|
-
stable_id: sid,
|
|
6218
|
-
slug: slugify(entry.name),
|
|
6219
|
-
title: titleOf(meta, entry.name),
|
|
6220
|
-
kind: "section",
|
|
6221
|
-
parent: parentSid,
|
|
6222
|
-
position,
|
|
6223
|
-
governance: index === null ? NO_GOVERNANCE : governanceFromFrontmatter(meta, index.text)
|
|
6224
|
-
}));
|
|
6225
|
-
if (index !== null) addFile(sid, [...dirSegs, index.name]);
|
|
6226
|
-
walk(entry, dirSegs, sid);
|
|
6227
|
-
} else {
|
|
6228
|
-
const meta = frontmatterMeta(entry.text);
|
|
6229
|
-
const stem = stemOf(entry.name);
|
|
6230
|
-
const sid = stableIdOf(rootName, [...relSegs, entry.name], meta);
|
|
6231
|
-
nodes.push(manifestNode({
|
|
6232
|
-
stable_id: sid,
|
|
6233
|
-
slug: slugify(stem),
|
|
6234
|
-
title: titleOf(meta, stem),
|
|
6235
|
-
kind: "document",
|
|
6236
|
-
parent: parentSid,
|
|
6237
|
-
position,
|
|
6238
|
-
governance: governanceFromFrontmatter(meta, entry.text)
|
|
6239
|
-
}));
|
|
6240
|
-
addFile(sid, [...relSegs, entry.name]);
|
|
6241
|
-
}
|
|
6242
|
-
}
|
|
6243
|
-
};
|
|
6244
|
-
const rootIndex = indexOf(root, rootPath);
|
|
6245
|
-
if (rootIndex !== null) {
|
|
6246
|
-
const meta = frontmatterMeta(rootIndex.text);
|
|
6247
|
-
const sid = stableIdOf(rootName, [rootIndex.name], meta);
|
|
6248
|
-
nodes.push(manifestNode({
|
|
6249
|
-
stable_id: sid,
|
|
6250
|
-
slug: slugify(rootName),
|
|
6251
|
-
title: titleOf(meta, rootName),
|
|
6252
|
-
kind: "document",
|
|
6253
|
-
position: 0,
|
|
6254
|
-
governance: governanceFromFrontmatter(meta, rootIndex.text)
|
|
6255
|
-
}));
|
|
6256
|
-
addFile(sid, [rootIndex.name]);
|
|
6257
|
-
}
|
|
6258
|
-
walk(root, [], null);
|
|
6259
|
-
for (const s of skipped) onSkip(`plain-tree: skipped ${s}`);
|
|
6260
|
-
if (files.length === 0) throw new ManifestError(`plain-tree root ${rootPath} contains no Markdown`);
|
|
6261
|
-
const manifest = {
|
|
6262
|
-
format: 1,
|
|
6263
|
-
corpus_id: options.corpusId,
|
|
6264
|
-
source_commit: options.sourceCommit,
|
|
6265
|
-
nodes,
|
|
6266
|
-
files
|
|
6267
|
-
};
|
|
6268
|
-
parseManifest(JSON.stringify(manifestToJson(manifest)));
|
|
6269
|
-
return {
|
|
6270
|
-
manifest,
|
|
6271
|
-
sources
|
|
6272
|
-
};
|
|
6273
|
-
}
|
|
6274
|
-
/** Python `p.suffix in (".md", ".mdx")` parity: a dotfile named exactly ".md" has NO suffix. */
|
|
6275
|
-
function isDoc(name) {
|
|
6276
|
-
const dot = name.lastIndexOf(".");
|
|
6277
|
-
if (dot <= 0) return false;
|
|
6278
|
-
const suffix = name.slice(dot);
|
|
6279
|
-
return suffix === ".md" || suffix === ".mdx";
|
|
6280
|
-
}
|
|
6281
|
-
function indexOf(dir, dirPath) {
|
|
6282
|
-
const present = [];
|
|
6283
|
-
for (const name of INDEX_NAMES) {
|
|
6284
|
-
const hit = dir.entries.find((e) => e.kind === "file" && e.name === name);
|
|
6285
|
-
if (hit !== void 0) present.push(hit);
|
|
6286
|
-
}
|
|
6287
|
-
if (present.length > 1) throw new ManifestError(`ambiguous section index in ${dirPath}: [${present.map((p) => `'${p.name}'`).join(", ")}] — keep exactly one`);
|
|
6288
|
-
return present[0] ?? null;
|
|
6289
|
-
}
|
|
6290
|
-
function stableIdOf(rootName, fileSegs, meta) {
|
|
6291
|
-
const sid = meta["sor_id"];
|
|
6292
|
-
if (typeof sid === "string" && sid.trim() !== "") return sid.trim();
|
|
6293
|
-
return `${rootName}/${withoutSuffix(fileSegs.join("/"))}`;
|
|
6294
|
-
}
|
|
6295
|
-
/** Python Path.with_suffix("") parity: strip the LAST suffix only; a dotfile has none. */
|
|
6296
|
-
function withoutSuffix(rel) {
|
|
6297
|
-
const slash = rel.lastIndexOf("/");
|
|
6298
|
-
const name = rel.slice(slash + 1);
|
|
6299
|
-
const dot = name.lastIndexOf(".");
|
|
6300
|
-
if (dot <= 0) return rel;
|
|
6301
|
-
return rel.slice(0, slash + 1) + name.slice(0, dot);
|
|
6302
|
-
}
|
|
6303
|
-
function stemOf(name) {
|
|
6304
|
-
return withoutSuffix(name);
|
|
6305
|
-
}
|
|
6306
|
-
function slugify(text) {
|
|
6307
|
-
const slug = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
6308
|
-
if (slug !== "") return slug;
|
|
6309
|
-
return "x-" + createHash("sha256").update(text, "utf8").digest("hex").slice(0, 8);
|
|
6041
|
+
/**
|
|
6042
|
+
* Activation + run-state bookkeeping + the ledger row. The health gate is the
|
|
6043
|
+
* CALLER's duty — never partially activate. Serialized under the tenant
|
|
6044
|
+
* advisory lock (re-taken here: the allocate lock died with its own txn) and
|
|
6045
|
+
* MONOTONIC: only ever advances.
|
|
6046
|
+
*/
|
|
6047
|
+
async function flip(client, opts) {
|
|
6048
|
+
await client.query(LOCK_SQL, [opts.tenantId]);
|
|
6049
|
+
if (!(await client.query("UPDATE corpora SET rollback_generation = active_generation, active_generation = $1, updated_at = now() WHERE tenant_id = $2 AND corpus_id = $3 AND active_generation < $1", [
|
|
6050
|
+
opts.toGeneration,
|
|
6051
|
+
opts.tenantId,
|
|
6052
|
+
opts.corpusId
|
|
6053
|
+
])).rowCount) throw new Error(`flip to generation ${opts.toGeneration} refused: active_generation is already >= it (an out-of-order or duplicate flip — refusing to regress the served corpus)`);
|
|
6054
|
+
await client.query("UPDATE ingestion_runs SET state = 'retired', finished_at = now() WHERE tenant_id = $1 AND corpus_id = $2 AND state = 'active'", [opts.tenantId, opts.corpusId]);
|
|
6055
|
+
await client.query("UPDATE ingestion_runs SET state = 'active', finished_at = COALESCE(finished_at, now()) WHERE tenant_id = $1 AND corpus_id = $2 AND generation = $3", [
|
|
6056
|
+
opts.tenantId,
|
|
6057
|
+
opts.corpusId,
|
|
6058
|
+
opts.toGeneration
|
|
6059
|
+
]);
|
|
6060
|
+
await client.query("INSERT INTO retrieval_log (tenant_id, corpus_id, generation, actor, action, detail) VALUES ($1, $2, $3, 'sor-ingest', 'generation_activated', '{}')", [
|
|
6061
|
+
opts.tenantId,
|
|
6062
|
+
opts.corpusId,
|
|
6063
|
+
opts.toGeneration
|
|
6064
|
+
]);
|
|
6310
6065
|
}
|
|
6311
|
-
const CASED = /\p{Cased}/u;
|
|
6312
6066
|
/**
|
|
6313
|
-
*
|
|
6314
|
-
*
|
|
6315
|
-
*
|
|
6316
|
-
*
|
|
6067
|
+
* The §5 algebra: not active, not rollback, past token grace since
|
|
6068
|
+
* retirement, ≥2 complete generations REMAIN after collection; abandoned
|
|
6069
|
+
* builds reap on heartbeat staleness alone (they were never served, no token
|
|
6070
|
+
* can reference them).
|
|
6317
6071
|
*/
|
|
6318
|
-
function
|
|
6319
|
-
const
|
|
6320
|
-
|
|
6321
|
-
|
|
6322
|
-
|
|
6323
|
-
|
|
6324
|
-
|
|
6325
|
-
|
|
6072
|
+
async function collectableGenerations(client, opts) {
|
|
6073
|
+
const ts = opts.now ?? /* @__PURE__ */ new Date();
|
|
6074
|
+
const pointer = await client.query("SELECT active_generation, rollback_generation FROM corpora WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId]);
|
|
6075
|
+
if (pointer.rows.length === 0) return [];
|
|
6076
|
+
const active = Number(pointer.rows[0].active_generation);
|
|
6077
|
+
const rollbackRaw = pointer.rows[0].rollback_generation;
|
|
6078
|
+
const rollbackGen = rollbackRaw === null ? null : Number(rollbackRaw);
|
|
6079
|
+
const runs = await client.query("SELECT generation, state, finished_at, heartbeat_at FROM ingestion_runs WHERE tenant_id = $1 AND corpus_id = $2 AND state <> 'reaped' ORDER BY generation", [opts.tenantId, opts.corpusId]);
|
|
6080
|
+
const complete = runs.rows.filter((r) => [
|
|
6081
|
+
"ready",
|
|
6082
|
+
"active",
|
|
6083
|
+
"retired"
|
|
6084
|
+
].includes(String(r.state)));
|
|
6085
|
+
const out = [];
|
|
6086
|
+
let remaining = complete.length;
|
|
6087
|
+
for (const row of runs.rows) {
|
|
6088
|
+
const gen = Number(row.generation);
|
|
6089
|
+
const state = String(row.state);
|
|
6090
|
+
const finishedAt = row.finished_at;
|
|
6091
|
+
const heartbeatAt = row.heartbeat_at;
|
|
6092
|
+
if (state === "building") {
|
|
6093
|
+
if (heartbeatAt !== null && ts.getTime() - heartbeatAt.getTime() > 864e5) out.push(gen);
|
|
6094
|
+
continue;
|
|
6095
|
+
}
|
|
6096
|
+
if (gen === active || rollbackGen !== null && gen === rollbackGen) continue;
|
|
6097
|
+
if (finishedAt === null || ts.getTime() - finishedAt.getTime() < GC_GRACE_MS) continue;
|
|
6098
|
+
if (remaining - 1 < 2) continue;
|
|
6099
|
+
remaining -= 1;
|
|
6100
|
+
out.push(gen);
|
|
6326
6101
|
}
|
|
6327
6102
|
return out;
|
|
6328
6103
|
}
|
|
6329
|
-
/**
|
|
6330
|
-
|
|
6331
|
-
|
|
6332
|
-
|
|
6333
|
-
|
|
6334
|
-
|
|
6335
|
-
|
|
6336
|
-
|
|
6337
|
-
|
|
6338
|
-
|
|
6339
|
-
|
|
6340
|
-
|
|
6341
|
-
|
|
6342
|
-
/** Python compares strings by code point; JS `<` compares UTF-16 units — they differ on astral names. */
|
|
6343
|
-
function codePointCompare(a, b) {
|
|
6344
|
-
const as = [...a];
|
|
6345
|
-
const bs = [...b];
|
|
6346
|
-
const n = Math.min(as.length, bs.length);
|
|
6347
|
-
for (let i = 0; i < n; i++) {
|
|
6348
|
-
const d = (as[i]?.codePointAt(0) ?? 0) - (bs[i]?.codePointAt(0) ?? 0);
|
|
6349
|
-
if (d !== 0) return d;
|
|
6350
|
-
}
|
|
6351
|
-
return as.length - bs.length;
|
|
6104
|
+
/**
|
|
6105
|
+
* Delete one generation's rows (chunks cascade from sources) and mark the run
|
|
6106
|
+
* reaped. NEVER touches takedown_denylist or retrieval_log — the ledger and
|
|
6107
|
+
* denylist outlive the content they governed (§5).
|
|
6108
|
+
*/
|
|
6109
|
+
async function reap(client, opts) {
|
|
6110
|
+
for (const sql of [
|
|
6111
|
+
"DELETE FROM node_centroids WHERE tenant_id = $1 AND generation = $2",
|
|
6112
|
+
"DELETE FROM slug_aliases WHERE tenant_id = $1 AND generation = $2",
|
|
6113
|
+
"DELETE FROM sources WHERE tenant_id = $1 AND generation = $2"
|
|
6114
|
+
]) await client.query(sql, [opts.tenantId, opts.generation]);
|
|
6115
|
+
for (;;) if (!(await client.query("DELETE FROM content_nodes n WHERE n.tenant_id = $1 AND n.generation = $2 AND NOT EXISTS (SELECT 1 FROM content_nodes ch WHERE ch.parent_id = n.node_id AND ch.tenant_id = n.tenant_id AND ch.generation = n.generation)", [opts.tenantId, opts.generation])).rowCount) break;
|
|
6116
|
+
await client.query("UPDATE ingestion_runs SET state = 'reaped' WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation]);
|
|
6352
6117
|
}
|
|
6353
|
-
/** Re-exported so every reader of a document agrees where its frontmatter ENDS. */
|
|
6354
|
-
const FRONTMATTER$1 = /^\uFEFF?---\r?\n([\s\S]*?)\r?\n---[ \t]*\r?\n?/;
|
|
6355
|
-
const YAML_BOOLS = {
|
|
6356
|
-
yes: true,
|
|
6357
|
-
Yes: true,
|
|
6358
|
-
YES: true,
|
|
6359
|
-
no: false,
|
|
6360
|
-
No: false,
|
|
6361
|
-
NO: false,
|
|
6362
|
-
true: true,
|
|
6363
|
-
True: true,
|
|
6364
|
-
TRUE: true,
|
|
6365
|
-
false: false,
|
|
6366
|
-
False: false,
|
|
6367
|
-
FALSE: false,
|
|
6368
|
-
on: true,
|
|
6369
|
-
On: true,
|
|
6370
|
-
ON: true,
|
|
6371
|
-
off: false,
|
|
6372
|
-
Off: false,
|
|
6373
|
-
OFF: false
|
|
6374
|
-
};
|
|
6375
6118
|
/**
|
|
6376
|
-
*
|
|
6377
|
-
*
|
|
6378
|
-
*
|
|
6379
|
-
*
|
|
6380
|
-
* Scope, deliberately narrow pending a shared markdown module: top-level
|
|
6381
|
-
* `key: scalar` pairs only; nested/indented structure is ignored. Mirroring
|
|
6382
|
-
* the oracle's error path (`parse_frontmatter` catches YAMLError → `{}`), a
|
|
6383
|
-
* document PyYAML would refuse — an UNQUOTED value containing ": ", a block
|
|
6384
|
-
* scalar, an anchor/alias/tag, a non-mapping line — yields an EMPTY meta, so
|
|
6385
|
-
* titles fall back to the humanized filename instead of a half-read mapping.
|
|
6119
|
+
* The §7 row for a governance act, written INSIDE the same transaction as the
|
|
6120
|
+
* act. `logRead` deliberately covers only the four serving actions; a takedown
|
|
6121
|
+
* is a write-plane act, and separating the two writes would allow a denial with
|
|
6122
|
+
* no row proving it happened — the one outcome the ledger exists to prevent.
|
|
6386
6123
|
*/
|
|
6387
|
-
function
|
|
6388
|
-
|
|
6389
|
-
|
|
6390
|
-
|
|
6391
|
-
|
|
6392
|
-
|
|
6393
|
-
|
|
6394
|
-
|
|
6395
|
-
const key = kv?.[1];
|
|
6396
|
-
if (key === void 0) return {};
|
|
6397
|
-
const parsed = scalarValue((kv?.[2] ?? "").trim());
|
|
6398
|
-
if (!parsed.ok) return {};
|
|
6399
|
-
meta[key] = parsed.value;
|
|
6400
|
-
}
|
|
6401
|
-
return meta;
|
|
6402
|
-
}
|
|
6403
|
-
function scalarValue(raw) {
|
|
6404
|
-
if (raw === "") return {
|
|
6405
|
-
ok: true,
|
|
6406
|
-
value: null
|
|
6407
|
-
};
|
|
6408
|
-
const dq = /^"(.*)"$/.exec(raw);
|
|
6409
|
-
if (dq !== null) return {
|
|
6410
|
-
ok: true,
|
|
6411
|
-
value: (dq[1] ?? "").replace(/\\"/g, "\"").replace(/\\\\/g, "\\")
|
|
6412
|
-
};
|
|
6413
|
-
const sq = /^'(.*)'$/.exec(raw);
|
|
6414
|
-
if (sq !== null) return {
|
|
6415
|
-
ok: true,
|
|
6416
|
-
value: (sq[1] ?? "").replace(/''/g, "'")
|
|
6417
|
-
};
|
|
6418
|
-
const plain = raw.replace(/[ \t]+#.*$/, "").trim();
|
|
6419
|
-
if (Object.hasOwn(YAML_BOOLS, plain)) return {
|
|
6420
|
-
ok: true,
|
|
6421
|
-
value: YAML_BOOLS[plain]
|
|
6422
|
-
};
|
|
6423
|
-
if (plain === "~" || /^(?:null|Null|NULL)$/.test(plain)) return {
|
|
6424
|
-
ok: true,
|
|
6425
|
-
value: null
|
|
6426
|
-
};
|
|
6427
|
-
if (/^[-+]?[0-9][0-9_]*$/.test(plain)) return {
|
|
6428
|
-
ok: true,
|
|
6429
|
-
value: Number.parseInt(plain.replaceAll("_", ""), 10)
|
|
6430
|
-
};
|
|
6431
|
-
if (/^[-+]?(?:\.[0-9]+|[0-9][0-9_]*\.[0-9_]*)(?:[eE][-+]?[0-9]+)?$/.test(plain)) return {
|
|
6432
|
-
ok: true,
|
|
6433
|
-
value: Number.parseFloat(plain.replaceAll("_", ""))
|
|
6434
|
-
};
|
|
6435
|
-
if (/:[ \t]/.test(plain) || plain.endsWith(":")) return {
|
|
6436
|
-
ok: false,
|
|
6437
|
-
value: null
|
|
6438
|
-
};
|
|
6439
|
-
if (/^[|>&*!{[]/.test(plain)) return {
|
|
6440
|
-
ok: false,
|
|
6441
|
-
value: null
|
|
6442
|
-
};
|
|
6443
|
-
return {
|
|
6444
|
-
ok: true,
|
|
6445
|
-
value: plain
|
|
6446
|
-
};
|
|
6124
|
+
async function recordAct(client, instance, detail, actor, action = "takedown_applied") {
|
|
6125
|
+
await client.query("INSERT INTO retrieval_log (tenant_id, corpus_id, actor, action, detail) VALUES ($1, $2, $3, $5, $4::jsonb)", [
|
|
6126
|
+
instance.tenantId,
|
|
6127
|
+
instance.corpusId,
|
|
6128
|
+
actor,
|
|
6129
|
+
JSON.stringify(detail),
|
|
6130
|
+
action
|
|
6131
|
+
]);
|
|
6447
6132
|
}
|
|
6448
6133
|
/**
|
|
6449
|
-
*
|
|
6450
|
-
* the record.
|
|
6451
|
-
*
|
|
6452
|
-
* Before this module the ingest adapter kept four frontmatter keys and dropped
|
|
6453
|
-
* the rest, so `visibility`, `status`, `owner` and `provenance` existed only in
|
|
6454
|
-
* markdown — and every surface re-derived them independently. The site enforced
|
|
6455
|
-
* `visibility:`; the MCP door could not, because the record did not carry it.
|
|
6456
|
-
* One reader, one shape, persisted on `content_nodes` (schema 2.2).
|
|
6134
|
+
* Deny a node (or its subtree) and record the act.
|
|
6457
6135
|
*
|
|
6458
|
-
* The
|
|
6459
|
-
*
|
|
6460
|
-
*
|
|
6461
|
-
* door can make the decision with the instance in hand. Refusing unknown values
|
|
6462
|
-
* here would put the audience model in two places again.
|
|
6136
|
+
* The audit row is written in the SAME transaction as the denial: a takedown
|
|
6137
|
+
* that happened without a row proving it happened is exactly the shape the
|
|
6138
|
+
* §7 ledger exists to prevent.
|
|
6463
6139
|
*/
|
|
6464
|
-
|
|
6465
|
-
|
|
6466
|
-
|
|
6467
|
-
|
|
6468
|
-
|
|
6469
|
-
|
|
6470
|
-
|
|
6471
|
-
|
|
6472
|
-
|
|
6473
|
-
|
|
6474
|
-
const
|
|
6475
|
-
|
|
6476
|
-
|
|
6477
|
-
|
|
6478
|
-
|
|
6479
|
-
|
|
6140
|
+
async function applyTakedown(pool, instance, opts) {
|
|
6141
|
+
return runIngest(pool, instance.tenantId, async (client) => {
|
|
6142
|
+
const resolves = ((await client.query(`SELECT 1 FROM content_nodes n
|
|
6143
|
+
JOIN corpora c ON c.tenant_id = n.tenant_id AND c.corpus_id = $2
|
|
6144
|
+
WHERE n.tenant_id = $1 AND n.stable_id = $3 AND n.generation = c.active_generation
|
|
6145
|
+
LIMIT 1`, [
|
|
6146
|
+
instance.tenantId,
|
|
6147
|
+
instance.corpusId,
|
|
6148
|
+
opts.stableId
|
|
6149
|
+
])).rowCount ?? 0) > 0;
|
|
6150
|
+
const changed = (await client.query("INSERT INTO takedown_denylist (tenant_id, corpus_id, stable_id, scope, reason) VALUES ($1, $2, $3, $4, $5) ON CONFLICT (tenant_id, corpus_id, stable_id) DO UPDATE SET scope = EXCLUDED.scope, reason = EXCLUDED.reason WHERE takedown_denylist.scope IS DISTINCT FROM EXCLUDED.scope OR takedown_denylist.reason IS DISTINCT FROM EXCLUDED.reason RETURNING stable_id", [
|
|
6151
|
+
instance.tenantId,
|
|
6152
|
+
instance.corpusId,
|
|
6153
|
+
opts.stableId,
|
|
6154
|
+
opts.scope,
|
|
6155
|
+
opts.reason
|
|
6156
|
+
])).rowCount === 1;
|
|
6157
|
+
await recordAct(client, instance, {
|
|
6158
|
+
stable_id: opts.stableId,
|
|
6159
|
+
scope: opts.scope,
|
|
6160
|
+
reason: opts.reason,
|
|
6161
|
+
change: changed ? "applied" : "unchanged"
|
|
6162
|
+
}, opts.actor);
|
|
6163
|
+
return {
|
|
6164
|
+
stableId: opts.stableId,
|
|
6165
|
+
scope: opts.scope,
|
|
6166
|
+
changed,
|
|
6167
|
+
resolves
|
|
6168
|
+
};
|
|
6169
|
+
});
|
|
6170
|
+
}
|
|
6171
|
+
/** Lift a denial. The ledger keeps the row that recorded imposing it. */
|
|
6172
|
+
async function revokeTakedown(pool, instance, opts) {
|
|
6173
|
+
return runIngest(pool, instance.tenantId, async (client) => {
|
|
6174
|
+
const changed = ((await client.query("DELETE FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2 AND stable_id = $3", [
|
|
6175
|
+
instance.tenantId,
|
|
6176
|
+
instance.corpusId,
|
|
6177
|
+
opts.stableId
|
|
6178
|
+
])).rowCount ?? 0) > 0;
|
|
6179
|
+
await recordAct(client, instance, {
|
|
6180
|
+
stable_id: opts.stableId,
|
|
6181
|
+
change: changed ? "revoked" : "not-denied"
|
|
6182
|
+
}, opts.actor, "takedown_revoked");
|
|
6183
|
+
return {
|
|
6184
|
+
stableId: opts.stableId,
|
|
6185
|
+
scope: "node",
|
|
6186
|
+
changed
|
|
6187
|
+
};
|
|
6188
|
+
});
|
|
6189
|
+
}
|
|
6190
|
+
async function readLedger(pool, instance, limit) {
|
|
6191
|
+
return runAuditRead(pool, instance.tenantId, async (client) => {
|
|
6192
|
+
return (await client.query("SELECT action, actor, generation, detail, created_at FROM retrieval_log WHERE tenant_id = $1 AND corpus_id = $2 ORDER BY created_at DESC, id DESC LIMIT $3", [
|
|
6193
|
+
instance.tenantId,
|
|
6194
|
+
instance.corpusId,
|
|
6195
|
+
limit
|
|
6196
|
+
])).rows.map((row) => ({
|
|
6197
|
+
action: String(row.action),
|
|
6198
|
+
actor: String(row.actor),
|
|
6199
|
+
generation: row.generation === null ? null : Number(row.generation),
|
|
6200
|
+
detail: row.detail ?? {},
|
|
6201
|
+
createdAt: row.created_at
|
|
6202
|
+
}));
|
|
6203
|
+
});
|
|
6480
6204
|
}
|
|
6481
|
-
const BLOCK_LIST = (key) => new RegExp(`^${key}:[ \\t]*\\r?\\n((?:[ \\t]*-[ \\t]+.*\\r?\\n?)+)`, "m");
|
|
6482
6205
|
/**
|
|
6483
|
-
*
|
|
6484
|
-
*
|
|
6485
|
-
*
|
|
6486
|
-
*
|
|
6206
|
+
* Every stable_id a build must not publish, with `subtree` denials EXPANDED to
|
|
6207
|
+
* their actual descendants by the same `parent_id` walk the serving side uses.
|
|
6208
|
+
*
|
|
6209
|
+
* The site cannot do this itself: it has no tree, so it matched a prefix — and
|
|
6210
|
+
* a section's stable_id ends in `/index` (or `#section`), so the prefix never
|
|
6211
|
+
* matched its children and every descendant of a subtree takedown kept
|
|
6212
|
+
* publishing. Decision 14 records exactly why a prefix is wrong here; the fix
|
|
6213
|
+
* is to resolve the walk where the tree lives and hand over a flat list
|
|
6214
|
+
* (round-2 review of #43).
|
|
6487
6215
|
*/
|
|
6488
|
-
function
|
|
6489
|
-
|
|
6490
|
-
|
|
6491
|
-
|
|
6492
|
-
|
|
6493
|
-
|
|
6494
|
-
|
|
6216
|
+
async function deniedStableIds(pool, instance) {
|
|
6217
|
+
return runRead(pool, instance.tenantId, async (client) => {
|
|
6218
|
+
return (await client.query(`WITH RECURSIVE gen AS (
|
|
6219
|
+
SELECT active_generation AS g FROM corpora WHERE tenant_id = $1 AND corpus_id = $2
|
|
6220
|
+
),
|
|
6221
|
+
seed AS (
|
|
6222
|
+
SELECT n.node_id, n.stable_id, d.scope
|
|
6223
|
+
FROM takedown_denylist d
|
|
6224
|
+
JOIN content_nodes n ON n.tenant_id = d.tenant_id AND n.stable_id = d.stable_id
|
|
6225
|
+
JOIN gen ON n.generation = gen.g
|
|
6226
|
+
WHERE d.tenant_id = $1 AND d.corpus_id = $2
|
|
6227
|
+
),
|
|
6228
|
+
walk AS (
|
|
6229
|
+
SELECT node_id, stable_id, scope FROM seed
|
|
6230
|
+
UNION ALL
|
|
6231
|
+
SELECT c.node_id, c.stable_id, w.scope
|
|
6232
|
+
FROM content_nodes c
|
|
6233
|
+
JOIN walk w ON c.parent_id = w.node_id
|
|
6234
|
+
JOIN gen ON c.generation = gen.g
|
|
6235
|
+
WHERE c.tenant_id = $1 AND w.scope = 'subtree'
|
|
6236
|
+
)
|
|
6237
|
+
SELECT DISTINCT stable_id FROM walk
|
|
6238
|
+
UNION
|
|
6239
|
+
-- Denials naming a stable_id no CURRENT generation carries are still
|
|
6240
|
+
-- denied: identity outlives any one generation (decision 14).
|
|
6241
|
+
SELECT stable_id FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2`, [instance.tenantId, instance.corpusId])).rows.map((r) => String(r.stable_id)).sort();
|
|
6242
|
+
});
|
|
6495
6243
|
}
|
|
6496
6244
|
/**
|
|
6497
|
-
*
|
|
6498
|
-
*
|
|
6499
|
-
*
|
|
6500
|
-
*
|
|
6245
|
+
* The knowledge-relative DIRECTORIES that `--subtree` denials govern.
|
|
6246
|
+
*
|
|
6247
|
+
* Derived from the DESCENDANTS' `sources.origin_path`, never from the denied
|
|
6248
|
+
* node's own id or path, because neither works:
|
|
6249
|
+
*
|
|
6250
|
+
* a section has no source `knowledge/policies#section` is synthetic — the
|
|
6251
|
+
* tree node for a directory. Joining `sources` on
|
|
6252
|
+
* the denied node itself yields nothing, and a
|
|
6253
|
+
* section is the ordinary target of `--subtree`.
|
|
6254
|
+
* a leaf's directory is not `--subtree` on one document would emit that
|
|
6255
|
+
* its subtree document's directory and deny every sibling.
|
|
6256
|
+
*
|
|
6257
|
+
* So: walk the descendants, take the directory of each one's file, and keep the
|
|
6258
|
+
* SHALLOWEST — a directory that contains another in the set is the subtree
|
|
6259
|
+
* root, and `startsWith` then covers subdirectories added later too. A denial
|
|
6260
|
+
* with no descendants contributes nothing, which is correct: its subtree is
|
|
6261
|
+
* itself, and the flat id list already holds it.
|
|
6262
|
+
*
|
|
6263
|
+
* The seed's OWN file counts when the seed has children, and only then — see
|
|
6264
|
+
* the SQL comment: a container's index.md names its directory, a leaf's file
|
|
6265
|
+
* names its parent's.
|
|
6501
6266
|
*/
|
|
6502
|
-
|
|
6503
|
-
|
|
6504
|
-
|
|
6505
|
-
|
|
6506
|
-
|
|
6507
|
-
|
|
6508
|
-
|
|
6509
|
-
|
|
6510
|
-
|
|
6511
|
-
|
|
6512
|
-
|
|
6513
|
-
|
|
6514
|
-
|
|
6515
|
-
|
|
6516
|
-
|
|
6517
|
-
|
|
6518
|
-
|
|
6519
|
-
|
|
6520
|
-
|
|
6521
|
-
|
|
6522
|
-
|
|
6523
|
-
|
|
6524
|
-
|
|
6525
|
-
|
|
6526
|
-
|
|
6527
|
-
|
|
6528
|
-
|
|
6529
|
-
|
|
6530
|
-
|
|
6531
|
-
|
|
6532
|
-
|
|
6533
|
-
|
|
6534
|
-
|
|
6535
|
-
|
|
6536
|
-
|
|
6537
|
-
|
|
6538
|
-
|
|
6539
|
-
|
|
6540
|
-
|
|
6541
|
-
|
|
6542
|
-
|
|
6543
|
-
|
|
6544
|
-
|
|
6545
|
-
|
|
6546
|
-
|
|
6547
|
-
|
|
6548
|
-
|
|
6549
|
-
|
|
6550
|
-
|
|
6551
|
-
|
|
6552
|
-
|
|
6553
|
-
let raw;
|
|
6554
|
-
try {
|
|
6555
|
-
raw = JSON.parse(text);
|
|
6556
|
-
} catch (exc) {
|
|
6557
|
-
throw new ManifestError(`manifest.json is not valid JSON: ${exc instanceof Error ? exc.message : String(exc)}`);
|
|
6267
|
+
async function deniedSubtreeDirs(pool, instance) {
|
|
6268
|
+
const paths = await runRead(pool, instance.tenantId, async (client) => {
|
|
6269
|
+
return (await client.query(`WITH RECURSIVE gen AS (
|
|
6270
|
+
SELECT active_generation AS g FROM corpora WHERE tenant_id = $1 AND corpus_id = $2
|
|
6271
|
+
),
|
|
6272
|
+
seed AS (
|
|
6273
|
+
SELECT n.node_id
|
|
6274
|
+
FROM takedown_denylist d
|
|
6275
|
+
JOIN content_nodes n ON n.tenant_id = d.tenant_id AND n.stable_id = d.stable_id
|
|
6276
|
+
JOIN gen ON n.generation = gen.g
|
|
6277
|
+
WHERE d.tenant_id = $1 AND d.corpus_id = $2 AND d.scope = 'subtree'
|
|
6278
|
+
),
|
|
6279
|
+
walk AS (
|
|
6280
|
+
SELECT node_id FROM seed
|
|
6281
|
+
UNION ALL
|
|
6282
|
+
SELECT c.node_id
|
|
6283
|
+
FROM content_nodes c
|
|
6284
|
+
JOIN walk w ON c.parent_id = w.node_id
|
|
6285
|
+
JOIN gen ON c.generation = gen.g
|
|
6286
|
+
WHERE c.tenant_id = $1
|
|
6287
|
+
)
|
|
6288
|
+
SELECT DISTINCT s.origin_path
|
|
6289
|
+
FROM walk w
|
|
6290
|
+
JOIN content_nodes n ON n.node_id = w.node_id
|
|
6291
|
+
JOIN sources s ON s.tenant_id = n.tenant_id AND s.generation = n.generation
|
|
6292
|
+
AND s.node_id = n.node_id
|
|
6293
|
+
-- The seed's own file counts only when the seed HAS CHILDREN.
|
|
6294
|
+
--
|
|
6295
|
+
-- Excluding every seed stopped a LEAF denial emitting its parent
|
|
6296
|
+
-- directory and denying every sibling — right for a leaf, wrong for a
|
|
6297
|
+
-- container. A section's own index.md is the file that names the
|
|
6298
|
+
-- section's DIRECTORY, so a section whose other descendants all live
|
|
6299
|
+
-- one level down contributed only the subdirectory, and a document
|
|
6300
|
+
-- written directly under the withdrawn section published to /docs and
|
|
6301
|
+
-- llms.txt (round-10 review of PR 43).
|
|
6302
|
+
--
|
|
6303
|
+
-- "Has children" is the right test, not "kind = section": it is the
|
|
6304
|
+
-- property that decides whether the node's directory is its subtree
|
|
6305
|
+
-- or its parent's.
|
|
6306
|
+
WHERE w.node_id NOT IN (
|
|
6307
|
+
SELECT s2.node_id FROM seed s2
|
|
6308
|
+
WHERE NOT EXISTS (SELECT 1 FROM content_nodes kid
|
|
6309
|
+
JOIN gen ON kid.generation = gen.g
|
|
6310
|
+
WHERE kid.tenant_id = $1 AND kid.parent_id = s2.node_id)
|
|
6311
|
+
)`, [instance.tenantId, instance.corpusId])).rows.map((r) => String(r.origin_path));
|
|
6312
|
+
});
|
|
6313
|
+
const dirs = /* @__PURE__ */ new Set();
|
|
6314
|
+
for (const raw of paths) {
|
|
6315
|
+
const normalized = raw.replace(/\\/g, "/");
|
|
6316
|
+
const slash = normalized.lastIndexOf("/");
|
|
6317
|
+
dirs.add(slash === -1 ? "/" : `${normalized.slice(0, slash)}/`);
|
|
6558
6318
|
}
|
|
6559
|
-
|
|
6560
|
-
|
|
6561
|
-
|
|
6562
|
-
|
|
6563
|
-
|
|
6564
|
-
|
|
6565
|
-
|
|
6566
|
-
|
|
6567
|
-
|
|
6568
|
-
|
|
6569
|
-
|
|
6570
|
-
|
|
6571
|
-
|
|
6572
|
-
|
|
6573
|
-
keywords: toKeywords(n["keywords"], i),
|
|
6574
|
-
permalink: optString(n["permalink"]),
|
|
6575
|
-
governance: toGovernance(n["governance"], i)
|
|
6576
|
-
}));
|
|
6577
|
-
const files = entriesOf(obj, "files").map((f, i) => manifestFile({
|
|
6578
|
-
path: req(f, "path", i),
|
|
6579
|
-
node: req(f, "node", i),
|
|
6580
|
-
title: optString(f["title"])
|
|
6581
|
-
}));
|
|
6582
|
-
validate(nodes, files);
|
|
6319
|
+
const all = [...dirs];
|
|
6320
|
+
return all.filter((dir) => !all.some((other) => other !== dir && dir.startsWith(other))).sort();
|
|
6321
|
+
}
|
|
6322
|
+
async function listTakedowns(pool, instance) {
|
|
6323
|
+
return runRead(pool, instance.tenantId, async (client) => {
|
|
6324
|
+
return (await client.query("SELECT stable_id, scope, reason, created_at FROM takedown_denylist WHERE tenant_id = $1 AND corpus_id = $2 ORDER BY created_at, stable_id", [instance.tenantId, instance.corpusId])).rows.map((r) => ({
|
|
6325
|
+
stableId: String(r.stable_id),
|
|
6326
|
+
scope: String(r.scope),
|
|
6327
|
+
reason: String(r.reason),
|
|
6328
|
+
createdAt: r.created_at
|
|
6329
|
+
}));
|
|
6330
|
+
});
|
|
6331
|
+
}
|
|
6332
|
+
function denylistManifest(corpusId, stableIds, now, source = "database", deniedSubtrees = []) {
|
|
6583
6333
|
return {
|
|
6584
|
-
format:
|
|
6334
|
+
format: 1,
|
|
6585
6335
|
corpus_id: corpusId,
|
|
6586
|
-
|
|
6587
|
-
|
|
6588
|
-
|
|
6336
|
+
source,
|
|
6337
|
+
denied_subtrees: [...deniedSubtrees].sort(),
|
|
6338
|
+
exported_at: now.toISOString(),
|
|
6339
|
+
denied: stableIds.map((stable_id) => ({
|
|
6340
|
+
stable_id,
|
|
6341
|
+
scope: "node"
|
|
6342
|
+
}))
|
|
6589
6343
|
};
|
|
6590
6344
|
}
|
|
6591
6345
|
/**
|
|
6592
|
-
* The
|
|
6593
|
-
*
|
|
6594
|
-
*
|
|
6595
|
-
*
|
|
6596
|
-
*
|
|
6346
|
+
* The plain-tree corpus adapter — ANY folder of Markdown becomes a corpus.
|
|
6347
|
+
* Converted from the oracle (sor-agentfactory @ b554f91,
|
|
6348
|
+
* ingest/adapters/plain_tree.py); the kernel cannot tell this manifest from
|
|
6349
|
+
* any other adapter's.
|
|
6350
|
+
*
|
|
6351
|
+
* Conventions (deliberately minimal — an operator can satisfy them with a bare
|
|
6352
|
+
* folder):
|
|
6353
|
+
* - directories become `section` nodes; `.md`/`.mdx` files become `document`
|
|
6354
|
+
* nodes;
|
|
6355
|
+
* - `index.md` (or `README.md`) inside a directory is that SECTION's own
|
|
6356
|
+
* content, not a child;
|
|
6357
|
+
* - ordering: frontmatter `position` (or `sidebar_position`) wins, else name
|
|
6358
|
+
* sort;
|
|
6359
|
+
* - titles: frontmatter `title`, else the filename humanized;
|
|
6360
|
+
* - stable ids: frontmatter `sor_id`, else the tree-relative path;
|
|
6361
|
+
* - hidden entries (leading `.` or `_`) and ALL symlinks are skipped LOUDLY
|
|
6362
|
+
* (reported through `onSkip`, console by default — never silent); symlinks
|
|
6363
|
+
* are never followed, so a link cannot walk out of the tree or cycle it;
|
|
6364
|
+
* - a directory carrying MORE than one index-named file (index.md +
|
|
6365
|
+
* README.md …) fails loud: which one is the section's own content is
|
|
6366
|
+
* ambiguous, and silently dropping the loser is exactly the corpus
|
|
6367
|
+
* corruption this adapter must never commit.
|
|
6368
|
+
*
|
|
6369
|
+
* The oracle's `publish_bundle` (deterministic tgz staging) is a separate
|
|
6370
|
+
* slice and is not converted here.
|
|
6597
6371
|
*/
|
|
6598
|
-
|
|
6599
|
-
|
|
6600
|
-
|
|
6601
|
-
|
|
6602
|
-
|
|
6603
|
-
|
|
6604
|
-
|
|
6605
|
-
|
|
6606
|
-
|
|
6607
|
-
|
|
6608
|
-
|
|
6609
|
-
|
|
6610
|
-
|
|
6611
|
-
|
|
6612
|
-
|
|
6372
|
+
const INDEX_NAMES = [
|
|
6373
|
+
"index.md",
|
|
6374
|
+
"index.mdx",
|
|
6375
|
+
"README.md"
|
|
6376
|
+
];
|
|
6377
|
+
/** Frontmatter-position fallback for entries that declare none (oracle plain_tree.py:107,114). */
|
|
6378
|
+
const POSITION_FALLBACK = 1e4;
|
|
6379
|
+
/** Walk a directory on disk → manifest + sources. Fail-loud on emptiness and ambiguity. */
|
|
6380
|
+
async function buildManifest(treeRoot, options) {
|
|
6381
|
+
const rootPath = treeRoot.length > 1 ? treeRoot.replace(/\/+$/, "") : treeRoot;
|
|
6382
|
+
let isDir = false;
|
|
6383
|
+
try {
|
|
6384
|
+
isDir = (await stat(rootPath)).isDirectory();
|
|
6385
|
+
} catch {
|
|
6386
|
+
isDir = false;
|
|
6613
6387
|
}
|
|
6614
|
-
|
|
6615
|
-
|
|
6616
|
-
|
|
6617
|
-
|
|
6618
|
-
provenance,
|
|
6619
|
-
supersededBy: str("superseded_by")
|
|
6620
|
-
};
|
|
6621
|
-
}
|
|
6622
|
-
function topString(obj, key) {
|
|
6623
|
-
const val = obj[key];
|
|
6624
|
-
if (typeof val !== "string" || !val) throw new ManifestError(`manifest.${key} must be a non-empty string`);
|
|
6625
|
-
return val;
|
|
6626
|
-
}
|
|
6627
|
-
function entriesOf(obj, key) {
|
|
6628
|
-
const raw = obj[key] ?? [];
|
|
6629
|
-
if (!Array.isArray(raw)) throw new ManifestError(`manifest.${key} must be an array`);
|
|
6630
|
-
return raw.map((entry, i) => {
|
|
6631
|
-
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) throw new ManifestError(`entry ${i}: must be an object`);
|
|
6632
|
-
return entry;
|
|
6388
|
+
if (!isDir) throw new ManifestError(`plain-tree root ${rootPath} is not a directory`);
|
|
6389
|
+
return buildManifestFromTree(await readTree(rootPath), {
|
|
6390
|
+
...options,
|
|
6391
|
+
rootPath
|
|
6633
6392
|
});
|
|
6634
6393
|
}
|
|
6635
|
-
|
|
6636
|
-
|
|
6637
|
-
|
|
6638
|
-
|
|
6639
|
-
|
|
6640
|
-
|
|
6641
|
-
|
|
6642
|
-
|
|
6643
|
-
|
|
6644
|
-
|
|
6645
|
-
|
|
6646
|
-
|
|
6647
|
-
|
|
6648
|
-
|
|
6649
|
-
|
|
6650
|
-
|
|
6651
|
-
|
|
6652
|
-
|
|
6653
|
-
|
|
6654
|
-
|
|
6655
|
-
if (typeof k !== "string") throw new ManifestError(`entry ${index}: keywords[${j}] must be a string`);
|
|
6656
|
-
return k;
|
|
6394
|
+
/**
|
|
6395
|
+
* Load a directory into an in-memory tree. lstat semantics throughout: a
|
|
6396
|
+
* symlink is represented as a symlink — even one named `index.md` — never
|
|
6397
|
+
* followed, never read (the oracle's docstring contract; its `_index_of`
|
|
6398
|
+
* incidentally followed a symlinked index via `is_file()`, which this port
|
|
6399
|
+
* deliberately does not reproduce). Non-markdown files are invisible to the
|
|
6400
|
+
* walk, exactly as the oracle's suffix filter makes them.
|
|
6401
|
+
*/
|
|
6402
|
+
async function readTree(dirPath) {
|
|
6403
|
+
const dirents = await readdir(dirPath, { withFileTypes: true });
|
|
6404
|
+
const entries = [];
|
|
6405
|
+
for (const d of dirents) if (d.isSymbolicLink()) entries.push({
|
|
6406
|
+
kind: "symlink",
|
|
6407
|
+
name: d.name
|
|
6408
|
+
});
|
|
6409
|
+
else if (d.isDirectory()) entries.push(await readTree(join(dirPath, d.name)));
|
|
6410
|
+
else if (d.isFile() && isDoc(d.name)) entries.push({
|
|
6411
|
+
kind: "file",
|
|
6412
|
+
name: d.name,
|
|
6413
|
+
text: await readFile(join(dirPath, d.name), "utf8")
|
|
6657
6414
|
});
|
|
6415
|
+
return {
|
|
6416
|
+
kind: "dir",
|
|
6417
|
+
name: basename(dirPath),
|
|
6418
|
+
entries
|
|
6419
|
+
};
|
|
6658
6420
|
}
|
|
6659
|
-
|
|
6660
|
-
|
|
6661
|
-
const
|
|
6662
|
-
|
|
6663
|
-
|
|
6664
|
-
|
|
6665
|
-
|
|
6666
|
-
|
|
6667
|
-
|
|
6668
|
-
|
|
6669
|
-
const
|
|
6670
|
-
|
|
6671
|
-
|
|
6672
|
-
|
|
6673
|
-
|
|
6674
|
-
|
|
6675
|
-
|
|
6676
|
-
|
|
6677
|
-
|
|
6678
|
-
|
|
6679
|
-
|
|
6680
|
-
|
|
6681
|
-
|
|
6682
|
-
|
|
6683
|
-
|
|
6684
|
-
|
|
6685
|
-
|
|
6686
|
-
|
|
6687
|
-
|
|
6688
|
-
|
|
6689
|
-
|
|
6690
|
-
|
|
6691
|
-
|
|
6692
|
-
|
|
6693
|
-
|
|
6694
|
-
|
|
6695
|
-
|
|
6696
|
-
|
|
6697
|
-
|
|
6698
|
-
|
|
6421
|
+
/** The pure walk: tree → manifest + {manifest path → source path}. */
|
|
6422
|
+
function buildManifestFromTree(root, options) {
|
|
6423
|
+
const rootName = root.name;
|
|
6424
|
+
const rootPath = options.rootPath ?? rootName;
|
|
6425
|
+
const onSkip = options.onSkip ?? ((line) => console.log(line));
|
|
6426
|
+
const nodes = [];
|
|
6427
|
+
const files = [];
|
|
6428
|
+
const sources = /* @__PURE__ */ new Map();
|
|
6429
|
+
const skipped = [];
|
|
6430
|
+
const fullPath = (relSegs, name) => `${rootPath}/${[...relSegs, name].join("/")}`;
|
|
6431
|
+
const addFile = (nodeSid, fileSegs) => {
|
|
6432
|
+
const rel = fileSegs.join("/");
|
|
6433
|
+
const manifestPath = `${rootName}/${rel}`;
|
|
6434
|
+
files.push(manifestFile({
|
|
6435
|
+
path: manifestPath,
|
|
6436
|
+
node: nodeSid
|
|
6437
|
+
}));
|
|
6438
|
+
sources.set(manifestPath, `${rootPath}/${rel}`);
|
|
6439
|
+
};
|
|
6440
|
+
const walk = (dir, relSegs, parentSid) => {
|
|
6441
|
+
const entries = [...dir.entries].sort((a, b) => codePointCompare(a.name.toLowerCase(), b.name.toLowerCase()));
|
|
6442
|
+
const docs = [];
|
|
6443
|
+
const dirs = [];
|
|
6444
|
+
for (const e of entries) if (e.kind === "symlink") skipped.push(`${fullPath(relSegs, e.name)} (symlink)`);
|
|
6445
|
+
else if (e.kind === "file" && isDoc(e.name)) docs.push(e);
|
|
6446
|
+
else if (e.kind === "dir") dirs.push(e);
|
|
6447
|
+
const ordered = [];
|
|
6448
|
+
for (const f of docs) {
|
|
6449
|
+
if (f.name.startsWith(".") || f.name.startsWith("_")) {
|
|
6450
|
+
skipped.push(fullPath(relSegs, f.name));
|
|
6451
|
+
continue;
|
|
6452
|
+
}
|
|
6453
|
+
if (INDEX_NAMES.includes(f.name)) continue;
|
|
6454
|
+
ordered.push({
|
|
6455
|
+
position: positionOf(frontmatterMeta(f.text), POSITION_FALLBACK),
|
|
6456
|
+
nameLower: f.name.toLowerCase(),
|
|
6457
|
+
entry: f
|
|
6458
|
+
});
|
|
6459
|
+
}
|
|
6460
|
+
for (const d of dirs) {
|
|
6461
|
+
if (d.name.startsWith(".") || d.name.startsWith("_")) {
|
|
6462
|
+
skipped.push(fullPath(relSegs, d.name));
|
|
6463
|
+
continue;
|
|
6464
|
+
}
|
|
6465
|
+
const index = indexOf(d, fullPath(relSegs, d.name));
|
|
6466
|
+
const dirMeta = index === null ? {} : frontmatterMeta(index.text);
|
|
6467
|
+
ordered.push({
|
|
6468
|
+
position: positionOf(dirMeta, POSITION_FALLBACK),
|
|
6469
|
+
nameLower: d.name.toLowerCase(),
|
|
6470
|
+
entry: d
|
|
6471
|
+
});
|
|
6472
|
+
}
|
|
6473
|
+
ordered.sort((x, y) => x.position - y.position || codePointCompare(x.nameLower, y.nameLower));
|
|
6474
|
+
let position = 0;
|
|
6475
|
+
for (const { entry } of ordered) {
|
|
6476
|
+
position += 1;
|
|
6477
|
+
if (entry.kind === "dir") {
|
|
6478
|
+
const dirSegs = [...relSegs, entry.name];
|
|
6479
|
+
const index = indexOf(entry, fullPath(relSegs, entry.name));
|
|
6480
|
+
const meta = index === null ? {} : frontmatterMeta(index.text);
|
|
6481
|
+
const sid = index === null ? `${rootName}/${dirSegs.join("/")}#section` : stableIdOf(rootName, [...dirSegs, index.name], meta);
|
|
6482
|
+
nodes.push(manifestNode({
|
|
6483
|
+
stable_id: sid,
|
|
6484
|
+
slug: slugify(entry.name),
|
|
6485
|
+
title: titleOf(meta, entry.name),
|
|
6486
|
+
kind: "section",
|
|
6487
|
+
parent: parentSid,
|
|
6488
|
+
position,
|
|
6489
|
+
governance: index === null ? NO_GOVERNANCE : governanceFromFrontmatter(meta, index.text)
|
|
6490
|
+
}));
|
|
6491
|
+
if (index !== null) addFile(sid, [...dirSegs, index.name]);
|
|
6492
|
+
walk(entry, dirSegs, sid);
|
|
6493
|
+
} else {
|
|
6494
|
+
const meta = frontmatterMeta(entry.text);
|
|
6495
|
+
const stem = stemOf(entry.name);
|
|
6496
|
+
const sid = stableIdOf(rootName, [...relSegs, entry.name], meta);
|
|
6497
|
+
nodes.push(manifestNode({
|
|
6498
|
+
stable_id: sid,
|
|
6499
|
+
slug: slugify(stem),
|
|
6500
|
+
title: titleOf(meta, stem),
|
|
6501
|
+
kind: "document",
|
|
6502
|
+
parent: parentSid,
|
|
6503
|
+
position,
|
|
6504
|
+
governance: governanceFromFrontmatter(meta, entry.text)
|
|
6505
|
+
}));
|
|
6506
|
+
addFile(sid, [...relSegs, entry.name]);
|
|
6507
|
+
}
|
|
6699
6508
|
}
|
|
6700
|
-
state.set(n.stable_id, 2);
|
|
6701
|
-
out.push(n);
|
|
6702
6509
|
};
|
|
6703
|
-
|
|
6704
|
-
|
|
6705
|
-
|
|
6706
|
-
|
|
6707
|
-
|
|
6708
|
-
|
|
6709
|
-
|
|
6710
|
-
|
|
6711
|
-
|
|
6712
|
-
|
|
6713
|
-
|
|
6714
|
-
|
|
6510
|
+
const rootIndex = indexOf(root, rootPath);
|
|
6511
|
+
if (rootIndex !== null) {
|
|
6512
|
+
const meta = frontmatterMeta(rootIndex.text);
|
|
6513
|
+
const sid = stableIdOf(rootName, [rootIndex.name], meta);
|
|
6514
|
+
nodes.push(manifestNode({
|
|
6515
|
+
stable_id: sid,
|
|
6516
|
+
slug: slugify(rootName),
|
|
6517
|
+
title: titleOf(meta, rootName),
|
|
6518
|
+
kind: "document",
|
|
6519
|
+
position: 0,
|
|
6520
|
+
governance: governanceFromFrontmatter(meta, rootIndex.text)
|
|
6521
|
+
}));
|
|
6522
|
+
addFile(sid, [rootIndex.name]);
|
|
6523
|
+
}
|
|
6524
|
+
walk(root, [], null);
|
|
6525
|
+
for (const s of skipped) onSkip(`plain-tree: skipped ${s}`);
|
|
6526
|
+
if (files.length === 0) throw new ManifestError(`plain-tree root ${rootPath} contains no Markdown`);
|
|
6527
|
+
const manifest = {
|
|
6528
|
+
format: 1,
|
|
6529
|
+
corpus_id: options.corpusId,
|
|
6530
|
+
source_commit: options.sourceCommit,
|
|
6531
|
+
nodes,
|
|
6532
|
+
files
|
|
6533
|
+
};
|
|
6534
|
+
parseManifest(JSON.stringify(manifestToJson(manifest)));
|
|
6715
6535
|
return {
|
|
6716
|
-
|
|
6717
|
-
|
|
6718
|
-
source_commit: m.source_commit,
|
|
6719
|
-
nodes: m.nodes.map(nodeToJson),
|
|
6720
|
-
files: m.files.map((f) => ({
|
|
6721
|
-
path: f.path,
|
|
6722
|
-
node: f.node,
|
|
6723
|
-
...f.title ? { title: f.title } : {}
|
|
6724
|
-
}))
|
|
6536
|
+
manifest,
|
|
6537
|
+
sources
|
|
6725
6538
|
};
|
|
6726
6539
|
}
|
|
6727
|
-
|
|
6728
|
-
|
|
6729
|
-
|
|
6730
|
-
|
|
6731
|
-
|
|
6732
|
-
|
|
6733
|
-
|
|
6734
|
-
|
|
6735
|
-
|
|
6736
|
-
|
|
6737
|
-
|
|
6738
|
-
|
|
6739
|
-
const out = {};
|
|
6740
|
-
for (const [key, val] of fields) {
|
|
6741
|
-
const omit = val === null || val === 0 || Array.isArray(val) && val.length === 0;
|
|
6742
|
-
if (key === "position" || !omit) out[key] = val;
|
|
6540
|
+
/** Python `p.suffix in (".md", ".mdx")` parity: a dotfile named exactly ".md" has NO suffix. */
|
|
6541
|
+
function isDoc(name) {
|
|
6542
|
+
const dot = name.lastIndexOf(".");
|
|
6543
|
+
if (dot <= 0) return false;
|
|
6544
|
+
const suffix = name.slice(dot);
|
|
6545
|
+
return suffix === ".md" || suffix === ".mdx";
|
|
6546
|
+
}
|
|
6547
|
+
function indexOf(dir, dirPath) {
|
|
6548
|
+
const present = [];
|
|
6549
|
+
for (const name of INDEX_NAMES) {
|
|
6550
|
+
const hit = dir.entries.find((e) => e.kind === "file" && e.name === name);
|
|
6551
|
+
if (hit !== void 0) present.push(hit);
|
|
6743
6552
|
}
|
|
6744
|
-
|
|
6745
|
-
|
|
6746
|
-
return out;
|
|
6553
|
+
if (present.length > 1) throw new ManifestError(`ambiguous section index in ${dirPath}: [${present.map((p) => `'${p.name}'`).join(", ")}] — keep exactly one`);
|
|
6554
|
+
return present[0] ?? null;
|
|
6747
6555
|
}
|
|
6748
|
-
function
|
|
6749
|
-
const
|
|
6750
|
-
if (
|
|
6751
|
-
|
|
6752
|
-
|
|
6753
|
-
|
|
6754
|
-
|
|
6755
|
-
|
|
6556
|
+
function stableIdOf(rootName, fileSegs, meta) {
|
|
6557
|
+
const sid = meta["sor_id"];
|
|
6558
|
+
if (typeof sid === "string" && sid.trim() !== "") return sid.trim();
|
|
6559
|
+
return `${rootName}/${withoutSuffix(fileSegs.join("/"))}`;
|
|
6560
|
+
}
|
|
6561
|
+
/** Python Path.with_suffix("") parity: strip the LAST suffix only; a dotfile has none. */
|
|
6562
|
+
function withoutSuffix(rel) {
|
|
6563
|
+
const slash = rel.lastIndexOf("/");
|
|
6564
|
+
const name = rel.slice(slash + 1);
|
|
6565
|
+
const dot = name.lastIndexOf(".");
|
|
6566
|
+
if (dot <= 0) return rel;
|
|
6567
|
+
return rel.slice(0, slash + 1) + name.slice(0, dot);
|
|
6568
|
+
}
|
|
6569
|
+
function stemOf(name) {
|
|
6570
|
+
return withoutSuffix(name);
|
|
6571
|
+
}
|
|
6572
|
+
function slugify(text) {
|
|
6573
|
+
const slug = text.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "");
|
|
6574
|
+
if (slug !== "") return slug;
|
|
6575
|
+
return "x-" + createHash("sha256").update(text, "utf8").digest("hex").slice(0, 8);
|
|
6756
6576
|
}
|
|
6577
|
+
const CASED = /\p{Cased}/u;
|
|
6757
6578
|
/**
|
|
6758
|
-
*
|
|
6759
|
-
*
|
|
6760
|
-
*
|
|
6761
|
-
*
|
|
6762
|
-
* this shape; changing it is a policy decision, never a tidy-up.
|
|
6579
|
+
* Python str.title() parity (the oracle's `_humanize`): a cased character
|
|
6580
|
+
* following an uncased one uppercases, following a cased one lowercases —
|
|
6581
|
+
* apostrophe quirk included ("rock'n'roll" → "Rock'N'Roll"). Node titles are
|
|
6582
|
+
* carry-forward join keys, so the quirk is load-bearing, not cosmetic.
|
|
6763
6583
|
*/
|
|
6764
|
-
function
|
|
6765
|
-
|
|
6584
|
+
function humanize(stem) {
|
|
6585
|
+
const spaced = stem.replace(/[-_]+/g, " ").trim();
|
|
6586
|
+
let out = "";
|
|
6587
|
+
let prevCased = false;
|
|
6588
|
+
for (const ch of spaced) {
|
|
6589
|
+
const cased = CASED.test(ch);
|
|
6590
|
+
out += cased ? prevCased ? ch.toLowerCase() : ch.toUpperCase() : ch;
|
|
6591
|
+
prevCased = cased;
|
|
6592
|
+
}
|
|
6593
|
+
return out;
|
|
6766
6594
|
}
|
|
6767
|
-
/** Python
|
|
6768
|
-
|
|
6769
|
-
|
|
6770
|
-
|
|
6771
|
-
|
|
6772
|
-
for (const _ch of s) n += 1;
|
|
6773
|
-
return n;
|
|
6595
|
+
/** Python `str(meta.get("title") or _humanize(...))` — falsy titles fall back. */
|
|
6596
|
+
function titleOf(meta, fallbackStem) {
|
|
6597
|
+
const t = meta["title"];
|
|
6598
|
+
if (t === void 0 || t === null || t === "" || t === 0 || t === false) return humanize(fallbackStem);
|
|
6599
|
+
return String(t);
|
|
6774
6600
|
}
|
|
6775
|
-
|
|
6776
|
-
|
|
6777
|
-
|
|
6778
|
-
|
|
6779
|
-
const LINE_BOUNDARY = /* @__PURE__ */ new Set([
|
|
6780
|
-
"\n",
|
|
6781
|
-
"\v",
|
|
6782
|
-
"\f",
|
|
6783
|
-
"\r",
|
|
6784
|
-
"",
|
|
6785
|
-
"",
|
|
6786
|
-
"",
|
|
6787
|
-
"
",
|
|
6788
|
-
"\u2028",
|
|
6789
|
-
"\u2029"
|
|
6790
|
-
]);
|
|
6791
|
-
function pySplitLines(text, keepends) {
|
|
6792
|
-
const out = [];
|
|
6793
|
-
let start = 0;
|
|
6794
|
-
let i = 0;
|
|
6795
|
-
while (i < text.length) {
|
|
6796
|
-
const ch = text[i];
|
|
6797
|
-
if (LINE_BOUNDARY.has(ch)) {
|
|
6798
|
-
let end = i + 1;
|
|
6799
|
-
if (ch === "\r" && text[end] === "\n") end += 1;
|
|
6800
|
-
out.push(keepends ? text.slice(start, end) : text.slice(start, i));
|
|
6801
|
-
start = end;
|
|
6802
|
-
i = end;
|
|
6803
|
-
} else i += 1;
|
|
6601
|
+
function positionOf(meta, fallback) {
|
|
6602
|
+
for (const key of ["position", "sidebar_position"]) {
|
|
6603
|
+
const val = meta[key];
|
|
6604
|
+
if (typeof val === "number" && Number.isFinite(val)) return Math.trunc(val);
|
|
6804
6605
|
}
|
|
6805
|
-
|
|
6806
|
-
return out;
|
|
6606
|
+
return fallback;
|
|
6807
6607
|
}
|
|
6808
|
-
/** Python
|
|
6809
|
-
|
|
6810
|
-
|
|
6811
|
-
|
|
6812
|
-
const
|
|
6813
|
-
|
|
6814
|
-
|
|
6815
|
-
|
|
6816
|
-
|
|
6817
|
-
|
|
6818
|
-
return s.slice(a, b);
|
|
6608
|
+
/** Python compares strings by code point; JS `<` compares UTF-16 units — they differ on astral names. */
|
|
6609
|
+
function codePointCompare(a, b) {
|
|
6610
|
+
const as = [...a];
|
|
6611
|
+
const bs = [...b];
|
|
6612
|
+
const n = Math.min(as.length, bs.length);
|
|
6613
|
+
for (let i = 0; i < n; i++) {
|
|
6614
|
+
const d = (as[i]?.codePointAt(0) ?? 0) - (bs[i]?.codePointAt(0) ?? 0);
|
|
6615
|
+
if (d !== 0) return d;
|
|
6616
|
+
}
|
|
6617
|
+
return as.length - bs.length;
|
|
6819
6618
|
}
|
|
6820
|
-
/**
|
|
6821
|
-
const
|
|
6822
|
-
|
|
6823
|
-
|
|
6824
|
-
|
|
6825
|
-
|
|
6826
|
-
|
|
6827
|
-
|
|
6828
|
-
|
|
6829
|
-
|
|
6830
|
-
|
|
6831
|
-
|
|
6832
|
-
|
|
6833
|
-
|
|
6834
|
-
|
|
6835
|
-
|
|
6836
|
-
|
|
6837
|
-
|
|
6838
|
-
|
|
6839
|
-
|
|
6840
|
-
|
|
6841
|
-
|
|
6842
|
-
|
|
6843
|
-
|
|
6844
|
-
|
|
6845
|
-
|
|
6846
|
-
|
|
6619
|
+
/** Re-exported so every reader of a document agrees where its frontmatter ENDS. */
|
|
6620
|
+
const FRONTMATTER$1 = /^\uFEFF?---\r?\n([\s\S]*?)\r?\n---[ \t]*\r?\n?/;
|
|
6621
|
+
const YAML_BOOLS = {
|
|
6622
|
+
yes: true,
|
|
6623
|
+
Yes: true,
|
|
6624
|
+
YES: true,
|
|
6625
|
+
no: false,
|
|
6626
|
+
No: false,
|
|
6627
|
+
NO: false,
|
|
6628
|
+
true: true,
|
|
6629
|
+
True: true,
|
|
6630
|
+
TRUE: true,
|
|
6631
|
+
false: false,
|
|
6632
|
+
False: false,
|
|
6633
|
+
FALSE: false,
|
|
6634
|
+
on: true,
|
|
6635
|
+
On: true,
|
|
6636
|
+
ON: true,
|
|
6637
|
+
off: false,
|
|
6638
|
+
Off: false,
|
|
6639
|
+
OFF: false
|
|
6640
|
+
};
|
|
6641
|
+
/**
|
|
6642
|
+
* Minimal PyYAML-compatible frontmatter reader for the FOUR scalar keys this
|
|
6643
|
+
* adapter consumes (`title`, `position`, `sidebar_position`, `sor_id`) — the
|
|
6644
|
+
* kernel discards every other frontmatter key at build time (taxonomy comes
|
|
6645
|
+
* from the manifest), so a YAML dependency would buy nothing (guard rule 5).
|
|
6646
|
+
* Scope, deliberately narrow pending a shared markdown module: top-level
|
|
6647
|
+
* `key: scalar` pairs only; nested/indented structure is ignored. Mirroring
|
|
6648
|
+
* the oracle's error path (`parse_frontmatter` catches YAMLError → `{}`), a
|
|
6649
|
+
* document PyYAML would refuse — an UNQUOTED value containing ": ", a block
|
|
6650
|
+
* scalar, an anchor/alias/tag, a non-mapping line — yields an EMPTY meta, so
|
|
6651
|
+
* titles fall back to the humanized filename instead of a half-read mapping.
|
|
6652
|
+
*/
|
|
6653
|
+
function frontmatterMeta(text) {
|
|
6654
|
+
const block = FRONTMATTER$1.exec(text)?.[1];
|
|
6655
|
+
if (block === void 0) return {};
|
|
6656
|
+
const meta = {};
|
|
6657
|
+
for (const line of block.split(/\r?\n/)) {
|
|
6658
|
+
if (line.trim() === "" || line.trimStart().startsWith("#")) continue;
|
|
6659
|
+
if (/^[ \t]/.test(line)) continue;
|
|
6660
|
+
const kv = /^([^\s:]+):(?:[ \t]+(.*))?$/.exec(line);
|
|
6661
|
+
const key = kv?.[1];
|
|
6662
|
+
if (key === void 0) return {};
|
|
6663
|
+
const parsed = scalarValue((kv?.[2] ?? "").trim());
|
|
6664
|
+
if (!parsed.ok) return {};
|
|
6665
|
+
meta[key] = parsed.value;
|
|
6666
|
+
}
|
|
6667
|
+
return meta;
|
|
6847
6668
|
}
|
|
6848
|
-
function
|
|
6849
|
-
|
|
6669
|
+
function scalarValue(raw) {
|
|
6670
|
+
if (raw === "") return {
|
|
6671
|
+
ok: true,
|
|
6672
|
+
value: null
|
|
6673
|
+
};
|
|
6674
|
+
const dq = /^"(.*)"$/.exec(raw);
|
|
6675
|
+
if (dq !== null) return {
|
|
6676
|
+
ok: true,
|
|
6677
|
+
value: (dq[1] ?? "").replace(/\\"/g, "\"").replace(/\\\\/g, "\\")
|
|
6678
|
+
};
|
|
6679
|
+
const sq = /^'(.*)'$/.exec(raw);
|
|
6680
|
+
if (sq !== null) return {
|
|
6681
|
+
ok: true,
|
|
6682
|
+
value: (sq[1] ?? "").replace(/''/g, "'")
|
|
6683
|
+
};
|
|
6684
|
+
const plain = raw.replace(/[ \t]+#.*$/, "").trim();
|
|
6685
|
+
if (Object.hasOwn(YAML_BOOLS, plain)) return {
|
|
6686
|
+
ok: true,
|
|
6687
|
+
value: YAML_BOOLS[plain]
|
|
6688
|
+
};
|
|
6689
|
+
if (plain === "~" || /^(?:null|Null|NULL)$/.test(plain)) return {
|
|
6690
|
+
ok: true,
|
|
6691
|
+
value: null
|
|
6692
|
+
};
|
|
6693
|
+
if (/^[-+]?[0-9][0-9_]*$/.test(plain)) return {
|
|
6694
|
+
ok: true,
|
|
6695
|
+
value: Number.parseInt(plain.replaceAll("_", ""), 10)
|
|
6696
|
+
};
|
|
6697
|
+
if (/^[-+]?(?:\.[0-9]+|[0-9][0-9_]*\.[0-9_]*)(?:[eE][-+]?[0-9]+)?$/.test(plain)) return {
|
|
6698
|
+
ok: true,
|
|
6699
|
+
value: Number.parseFloat(plain.replaceAll("_", ""))
|
|
6700
|
+
};
|
|
6701
|
+
if (/:[ \t]/.test(plain) || plain.endsWith(":")) return {
|
|
6702
|
+
ok: false,
|
|
6703
|
+
value: null
|
|
6704
|
+
};
|
|
6705
|
+
if (/^[|>&*!{[]/.test(plain)) return {
|
|
6706
|
+
ok: false,
|
|
6707
|
+
value: null
|
|
6708
|
+
};
|
|
6709
|
+
return {
|
|
6710
|
+
ok: true,
|
|
6711
|
+
value: plain
|
|
6712
|
+
};
|
|
6850
6713
|
}
|
|
6851
|
-
/**
|
|
6852
|
-
*
|
|
6853
|
-
*
|
|
6854
|
-
|
|
6855
|
-
|
|
6856
|
-
|
|
6857
|
-
|
|
6858
|
-
|
|
6859
|
-
|
|
6860
|
-
|
|
6861
|
-
|
|
6862
|
-
|
|
6863
|
-
|
|
6714
|
+
/**
|
|
6715
|
+
* The governance a document declares about itself, read once and carried onto
|
|
6716
|
+
* the record.
|
|
6717
|
+
*
|
|
6718
|
+
* Before this module the ingest adapter kept four frontmatter keys and dropped
|
|
6719
|
+
* the rest, so `visibility`, `status`, `owner` and `provenance` existed only in
|
|
6720
|
+
* markdown — and every surface re-derived them independently. The site enforced
|
|
6721
|
+
* `visibility:`; the MCP door could not, because the record did not carry it.
|
|
6722
|
+
* One reader, one shape, persisted on `content_nodes` (schema 2.2).
|
|
6723
|
+
*
|
|
6724
|
+
* The vocabulary is deliberately NOT closed here. A record that declares an
|
|
6725
|
+
* audience the instance does not know is a corpus error the checker names; the
|
|
6726
|
+
* ingest path's job is to carry what was written, faithfully, so the serving
|
|
6727
|
+
* door can make the decision with the instance in hand. Refusing unknown values
|
|
6728
|
+
* here would put the audience model in two places again.
|
|
6729
|
+
*/
|
|
6730
|
+
const NO_GOVERNANCE = {
|
|
6731
|
+
visibility: null,
|
|
6732
|
+
docStatus: null,
|
|
6733
|
+
owner: null,
|
|
6734
|
+
provenance: null,
|
|
6735
|
+
supersededBy: null
|
|
6736
|
+
};
|
|
6737
|
+
function scalar(meta, key) {
|
|
6738
|
+
const raw = meta[key];
|
|
6739
|
+
if (typeof raw === "string") {
|
|
6740
|
+
const trimmed = raw.trim();
|
|
6741
|
+
return trimmed === "" ? null : trimmed;
|
|
6742
|
+
}
|
|
6743
|
+
if (typeof raw === "boolean") return raw ? "true" : "false";
|
|
6744
|
+
if (typeof raw === "number") return String(raw);
|
|
6745
|
+
return null;
|
|
6746
|
+
}
|
|
6747
|
+
const BLOCK_LIST = (key) => new RegExp(`^${key}:[ \\t]*\\r?\\n((?:[ \\t]*-[ \\t]+.*\\r?\\n?)+)`, "m");
|
|
6748
|
+
/**
|
|
6749
|
+
* Values of a simple `key:` block list, the one nested shape the record's
|
|
6750
|
+
* grammar uses (`provenance:` here, `audiences:` in instance.md). The scalar
|
|
6751
|
+
* reader deliberately ignores indented lines, so without this a provenance list
|
|
6752
|
+
* would vanish silently — the failure mode this whole module exists to end.
|
|
6753
|
+
*/
|
|
6754
|
+
function frontmatterListValues(text, key) {
|
|
6755
|
+
const block = FRONTMATTER$1.exec(text)?.[1];
|
|
6756
|
+
if (block === void 0) return null;
|
|
6757
|
+
const m = BLOCK_LIST(key).exec(block + "\n");
|
|
6758
|
+
if (m === null) return null;
|
|
6759
|
+
const items = (m[1] ?? "").split(/\r?\n/).map((line) => /^[ \t]*-[ \t]+(.*)$/.exec(line)?.[1] ?? "").map((v) => v.trim().replace(/^["']|["']$/g, "").trim()).filter((v) => v !== "");
|
|
6760
|
+
return items.length > 0 ? items : null;
|
|
6761
|
+
}
|
|
6762
|
+
/**
|
|
6763
|
+
* Read the governance keys from an already-parsed scalar map plus the raw
|
|
6764
|
+
* document text (which the list reader needs). Unknown keys are ignored, as
|
|
6765
|
+
* they always were — this module widens what the record carries, it does not
|
|
6766
|
+
* narrow what a document may say.
|
|
6767
|
+
*/
|
|
6768
|
+
var GovernanceParseError = class extends Error {
|
|
6769
|
+
name = "GovernanceParseError";
|
|
6770
|
+
};
|
|
6771
|
+
function governanceFromFrontmatter(meta, text) {
|
|
6772
|
+
if (frontmatterListValues(text, "visibility") !== null) throw new GovernanceParseError("a document declares `visibility:` as a LIST — a document belongs to exactly one tier. Write a single value, e.g. `visibility: internal`.");
|
|
6773
|
+
const declaredInText = /^visibility:[ \t]*(.*)$/m.exec(FRONTMATTER$1.exec(text)?.[1] ?? "");
|
|
6774
|
+
if (declaredInText !== null && scalar(meta, "visibility") === null) {
|
|
6775
|
+
const written = declaredInText[1]?.trim() ?? "";
|
|
6776
|
+
throw new GovernanceParseError(written === "" ? "a document declares `visibility:` with no readable value — an unreadable tier reads as no tier, and no tier is the default tier, which is how a restricted document gets served. Write a single value, e.g. `visibility: internal`." : `a document declares \`visibility: ${written}\` but this reader could not resolve it — usually because ANOTHER key in the same frontmatter is a shape it cannot read (a flow list like \`tags: [a, b]\`, or an unquoted value containing ": "). An unresolved tier would be served at the default. Quote the other value, or write it as a block list.`);
|
|
6777
|
+
}
|
|
6778
|
+
const provenanceScalar = scalar(meta, "provenance");
|
|
6779
|
+
const provenanceList = frontmatterListValues(text, "provenance");
|
|
6780
|
+
return {
|
|
6781
|
+
visibility: scalar(meta, "visibility"),
|
|
6782
|
+
docStatus: scalar(meta, "status"),
|
|
6783
|
+
owner: scalar(meta, "owner"),
|
|
6784
|
+
provenance: provenanceList ?? (provenanceScalar === null ? null : [provenanceScalar]),
|
|
6785
|
+
supersededBy: scalar(meta, "superseded_by")
|
|
6786
|
+
};
|
|
6787
|
+
}
|
|
6788
|
+
const SUPPORTED_FORMATS = [1];
|
|
6789
|
+
/** The manifest is malformed — named precisely; a bad manifest never half-ingests. */
|
|
6790
|
+
var ManifestError = class extends Error {
|
|
6791
|
+
constructor(message) {
|
|
6792
|
+
super(message);
|
|
6793
|
+
this.name = "ManifestError";
|
|
6794
|
+
}
|
|
6795
|
+
};
|
|
6796
|
+
/** Mirrors the oracle dataclass defaults (parent/summary/permalink None, position 0, keywords ()). */
|
|
6797
|
+
function manifestNode(init) {
|
|
6798
|
+
return {
|
|
6799
|
+
stable_id: init.stable_id,
|
|
6800
|
+
slug: init.slug,
|
|
6801
|
+
title: init.title,
|
|
6802
|
+
kind: init.kind,
|
|
6803
|
+
parent: init.parent ?? null,
|
|
6804
|
+
position: init.position ?? 0,
|
|
6805
|
+
summary: init.summary ?? null,
|
|
6806
|
+
keywords: init.keywords ?? [],
|
|
6807
|
+
permalink: init.permalink ?? null,
|
|
6808
|
+
governance: init.governance ?? NO_GOVERNANCE
|
|
6809
|
+
};
|
|
6810
|
+
}
|
|
6811
|
+
function manifestFile(init) {
|
|
6812
|
+
return {
|
|
6813
|
+
path: init.path,
|
|
6814
|
+
node: init.node,
|
|
6815
|
+
title: init.title ?? null
|
|
6816
|
+
};
|
|
6817
|
+
}
|
|
6818
|
+
function parseManifest(text) {
|
|
6819
|
+
let raw;
|
|
6820
|
+
try {
|
|
6821
|
+
raw = JSON.parse(text);
|
|
6822
|
+
} catch (exc) {
|
|
6823
|
+
throw new ManifestError(`manifest.json is not valid JSON: ${exc instanceof Error ? exc.message : String(exc)}`);
|
|
6824
|
+
}
|
|
6825
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw)) throw new ManifestError("manifest.json must be an object");
|
|
6826
|
+
const obj = raw;
|
|
6827
|
+
const fmt = obj["format"];
|
|
6828
|
+
if (typeof fmt !== "number" || !SUPPORTED_FORMATS.includes(fmt)) throw new ManifestError(`manifest format ${JSON.stringify(fmt)} unsupported (supported: ${SUPPORTED_FORMATS.join(", ")})`);
|
|
6829
|
+
const corpusId = topString(obj, "corpus_id");
|
|
6830
|
+
const sourceCommit = topString(obj, "source_commit");
|
|
6831
|
+
const nodes = entriesOf(obj, "nodes").map((n, i) => manifestNode({
|
|
6832
|
+
stable_id: req(n, "stable_id", i),
|
|
6833
|
+
slug: req(n, "slug", i),
|
|
6834
|
+
title: req(n, "title", i),
|
|
6835
|
+
kind: req(n, "kind", i),
|
|
6836
|
+
parent: optString(n["parent"]),
|
|
6837
|
+
position: toPosition(n["position"], i),
|
|
6838
|
+
summary: optString(n["summary"]),
|
|
6839
|
+
keywords: toKeywords(n["keywords"], i),
|
|
6840
|
+
permalink: optString(n["permalink"]),
|
|
6841
|
+
governance: toGovernance(n["governance"], i)
|
|
6842
|
+
}));
|
|
6843
|
+
const files = entriesOf(obj, "files").map((f, i) => manifestFile({
|
|
6844
|
+
path: req(f, "path", i),
|
|
6845
|
+
node: req(f, "node", i),
|
|
6846
|
+
title: optString(f["title"])
|
|
6847
|
+
}));
|
|
6848
|
+
validate(nodes, files);
|
|
6849
|
+
return {
|
|
6850
|
+
format: fmt,
|
|
6851
|
+
corpus_id: corpusId,
|
|
6852
|
+
source_commit: sourceCommit,
|
|
6853
|
+
nodes,
|
|
6854
|
+
files
|
|
6855
|
+
};
|
|
6856
|
+
}
|
|
6857
|
+
/**
|
|
6858
|
+
* The inverse of `governanceToJson`. Absent → NO_GOVERNANCE, which is what a
|
|
6859
|
+
* corpus that declares nothing has always meant. A present-but-wrong shape is
|
|
6860
|
+
* REFUSED rather than silently dropped: dropping it would serve a document at
|
|
6861
|
+
* the instance default, and for a `visibility:` that means serving a restricted
|
|
6862
|
+
* document to everyone.
|
|
6863
|
+
*/
|
|
6864
|
+
function toGovernance(raw, index) {
|
|
6865
|
+
if (raw === void 0 || raw === null) return NO_GOVERNANCE;
|
|
6866
|
+
if (typeof raw !== "object" || Array.isArray(raw)) throw new ManifestError(`entry ${index}: 'governance' must be an object`);
|
|
6867
|
+
const g = raw;
|
|
6868
|
+
const str = (key) => {
|
|
6869
|
+
const val = g[key];
|
|
6870
|
+
if (val === void 0 || val === null) return null;
|
|
6871
|
+
if (typeof val !== "string" || val === "") throw new ManifestError(`entry ${index}: 'governance.${key}' must be a non-empty string`);
|
|
6872
|
+
return val;
|
|
6873
|
+
};
|
|
6874
|
+
const provenanceRaw = g["provenance"];
|
|
6875
|
+
let provenance = null;
|
|
6876
|
+
if (provenanceRaw !== void 0 && provenanceRaw !== null) {
|
|
6877
|
+
if (!Array.isArray(provenanceRaw) || provenanceRaw.some((v) => typeof v !== "string")) throw new ManifestError(`entry ${index}: 'governance.provenance' must be a list of strings`);
|
|
6878
|
+
provenance = provenanceRaw;
|
|
6879
|
+
}
|
|
6880
|
+
return {
|
|
6881
|
+
visibility: str("visibility"),
|
|
6882
|
+
docStatus: str("doc_status"),
|
|
6883
|
+
owner: str("owner"),
|
|
6884
|
+
provenance,
|
|
6885
|
+
supersededBy: str("superseded_by")
|
|
6886
|
+
};
|
|
6887
|
+
}
|
|
6888
|
+
function topString(obj, key) {
|
|
6889
|
+
const val = obj[key];
|
|
6890
|
+
if (typeof val !== "string" || !val) throw new ManifestError(`manifest.${key} must be a non-empty string`);
|
|
6891
|
+
return val;
|
|
6892
|
+
}
|
|
6893
|
+
function entriesOf(obj, key) {
|
|
6894
|
+
const raw = obj[key] ?? [];
|
|
6895
|
+
if (!Array.isArray(raw)) throw new ManifestError(`manifest.${key} must be an array`);
|
|
6896
|
+
return raw.map((entry, i) => {
|
|
6897
|
+
if (typeof entry !== "object" || entry === null || Array.isArray(entry)) throw new ManifestError(`entry ${i}: must be an object`);
|
|
6898
|
+
return entry;
|
|
6899
|
+
});
|
|
6900
|
+
}
|
|
6901
|
+
function req(obj, key, index) {
|
|
6902
|
+
const val = obj[key];
|
|
6903
|
+
if (typeof val !== "string" || !val) throw new ManifestError(`entry ${index}: '${key}' must be a non-empty string`);
|
|
6904
|
+
return val;
|
|
6905
|
+
}
|
|
6906
|
+
function optString(val) {
|
|
6907
|
+
return typeof val === "string" ? val : null;
|
|
6908
|
+
}
|
|
6909
|
+
/** Python `int(...)` parity: truncate finite numbers, parse integer strings, refuse the rest loudly. */
|
|
6910
|
+
function toPosition(val, index) {
|
|
6911
|
+
if (val === void 0) return 0;
|
|
6912
|
+
if (typeof val === "number" && Number.isFinite(val)) return Math.trunc(val);
|
|
6913
|
+
if (typeof val === "boolean") return val ? 1 : 0;
|
|
6914
|
+
if (typeof val === "string" && /^[+-]?\d+$/.test(val.trim())) return Number.parseInt(val.trim(), 10);
|
|
6915
|
+
throw new ManifestError(`entry ${index}: position must be an integer, got ${JSON.stringify(val)}`);
|
|
6916
|
+
}
|
|
6917
|
+
function toKeywords(val, index) {
|
|
6918
|
+
if (val === void 0 || val === null) return [];
|
|
6919
|
+
if (!Array.isArray(val)) throw new ManifestError(`entry ${index}: keywords must be an array of strings`);
|
|
6920
|
+
return val.map((k, j) => {
|
|
6921
|
+
if (typeof k !== "string") throw new ManifestError(`entry ${index}: keywords[${j}] must be a string`);
|
|
6922
|
+
return k;
|
|
6923
|
+
});
|
|
6924
|
+
}
|
|
6925
|
+
function validate(nodes, files) {
|
|
6926
|
+
const seen = /* @__PURE__ */ new Set();
|
|
6927
|
+
const dupes = /* @__PURE__ */ new Set();
|
|
6928
|
+
for (const n of nodes) {
|
|
6929
|
+
if (seen.has(n.stable_id)) dupes.add(n.stable_id);
|
|
6930
|
+
seen.add(n.stable_id);
|
|
6931
|
+
}
|
|
6932
|
+
if (dupes.size > 0) throw new ManifestError(`duplicate node stable_id(s): [${[...dupes].sort().map((d) => `'${d}'`).join(", ")}]`);
|
|
6933
|
+
for (const n of nodes) if (n.parent !== null && !seen.has(n.parent)) throw new ManifestError(`node '${n.stable_id}': unknown parent '${n.parent}'`);
|
|
6934
|
+
for (const f of files) if (!seen.has(f.node)) throw new ManifestError(`file '${f.path}': unknown node '${f.node}'`);
|
|
6935
|
+
const paths = /* @__PURE__ */ new Set();
|
|
6936
|
+
for (const f of files) {
|
|
6937
|
+
if (paths.has(f.path)) throw new ManifestError("duplicate file paths in manifest");
|
|
6938
|
+
paths.add(f.path);
|
|
6864
6939
|
}
|
|
6865
|
-
|
|
6866
|
-
|
|
6867
|
-
|
|
6868
|
-
|
|
6869
|
-
|
|
6870
|
-
|
|
6871
|
-
|
|
6872
|
-
|
|
6873
|
-
* a fence is content and survives. See STYLE_OPEN for the locked fast-path
|
|
6874
|
-
* quirk (^-anchored, no multiline). */
|
|
6875
|
-
function stripStyleBlocks(text) {
|
|
6876
|
-
if (!STYLE_OPEN.test(text)) return text;
|
|
6877
|
-
const out = [];
|
|
6878
|
-
let fence = null;
|
|
6879
|
-
let dropping = false;
|
|
6880
|
-
for (const line of pySplitLines(text, true)) {
|
|
6881
|
-
if (dropping) {
|
|
6882
|
-
if (line.toLowerCase().includes("</style>")) dropping = false;
|
|
6883
|
-
continue;
|
|
6884
|
-
}
|
|
6885
|
-
if (fence === null && STYLE_OPEN.test(line)) {
|
|
6886
|
-
if (!line.toLowerCase().includes("</style>")) dropping = true;
|
|
6887
|
-
continue;
|
|
6888
|
-
}
|
|
6889
|
-
out.push(line);
|
|
6890
|
-
fence = fenceStep(line, fence);
|
|
6940
|
+
const siblingSlugs = /* @__PURE__ */ new Map();
|
|
6941
|
+
for (const n of nodes) {
|
|
6942
|
+
const parent = n.parent ?? "";
|
|
6943
|
+
const bySlug = siblingSlugs.get(parent) ?? /* @__PURE__ */ new Map();
|
|
6944
|
+
const owners = bySlug.get(n.slug) ?? [];
|
|
6945
|
+
owners.push(n.stable_id);
|
|
6946
|
+
bySlug.set(n.slug, owners);
|
|
6947
|
+
siblingSlugs.set(parent, bySlug);
|
|
6891
6948
|
}
|
|
6892
|
-
|
|
6949
|
+
for (const bySlug of siblingSlugs.values()) for (const [slug, owners] of bySlug) if (owners.length > 1) throw new ManifestError(`sibling slug collision: ${owners.map((o) => `'${o}'`).join(", ")} all slug to '${slug}' under the same parent — rename one (a slug is a node's URL segment and must be unique among siblings)`);
|
|
6893
6950
|
}
|
|
6894
|
-
|
|
6895
|
-
|
|
6896
|
-
|
|
6897
|
-
return s.slice(0, b);
|
|
6898
|
-
};
|
|
6899
|
-
/** Remove `style={{ ... }}` attributes by BRACE MATCHING, not regex: the value
|
|
6900
|
-
* is a JS object literal and the oracle measured 168 of them spanning lines.
|
|
6901
|
-
* An unbalanced opener leaves the rest of the text verbatim — a stripper must
|
|
6902
|
-
* never eat the rest of a document to satisfy itself. (The pre-attr rstrip of
|
|
6903
|
-
* spaces/tabs also glues `<div ` + a KEPT unbalanced `style={{` opener into
|
|
6904
|
-
* `<divstyle={{` — an oracle quirk locked by fixture strip-malformed-kept.) */
|
|
6905
|
-
function stripStyleAttr(text) {
|
|
6951
|
+
/** Parents before children (insert order for the FK); a cycle fails loudly. */
|
|
6952
|
+
function topological(nodes) {
|
|
6953
|
+
const byId = new Map(nodes.map((n) => [n.stable_id, n]));
|
|
6906
6954
|
const out = [];
|
|
6907
|
-
|
|
6908
|
-
|
|
6909
|
-
const
|
|
6910
|
-
if (
|
|
6911
|
-
|
|
6912
|
-
|
|
6913
|
-
|
|
6914
|
-
|
|
6915
|
-
|
|
6916
|
-
|
|
6917
|
-
if (ch === "{") depth += 1;
|
|
6918
|
-
else if (ch === "}") {
|
|
6919
|
-
depth -= 1;
|
|
6920
|
-
if (depth === 0) {
|
|
6921
|
-
k += 1;
|
|
6922
|
-
closed = true;
|
|
6923
|
-
break;
|
|
6924
|
-
}
|
|
6925
|
-
}
|
|
6926
|
-
k += 1;
|
|
6927
|
-
}
|
|
6928
|
-
if (!closed) {
|
|
6929
|
-
out.push(text.slice(j));
|
|
6930
|
-
return out.join("");
|
|
6955
|
+
const state = /* @__PURE__ */ new Map();
|
|
6956
|
+
const visit = (n) => {
|
|
6957
|
+
const mark = state.get(n.stable_id) ?? 0;
|
|
6958
|
+
if (mark === 2) return;
|
|
6959
|
+
if (mark === 1) throw new ManifestError(`parent cycle at '${n.stable_id}'`);
|
|
6960
|
+
state.set(n.stable_id, 1);
|
|
6961
|
+
if (n.parent !== null) {
|
|
6962
|
+
const parent = byId.get(n.parent);
|
|
6963
|
+
if (parent === void 0) throw new ManifestError(`node '${n.stable_id}': unknown parent '${n.parent}'`);
|
|
6964
|
+
visit(parent);
|
|
6931
6965
|
}
|
|
6932
|
-
|
|
6933
|
-
|
|
6934
|
-
|
|
6935
|
-
|
|
6966
|
+
state.set(n.stable_id, 2);
|
|
6967
|
+
out.push(n);
|
|
6968
|
+
};
|
|
6969
|
+
for (const n of nodes) visit(n);
|
|
6970
|
+
return out;
|
|
6936
6971
|
}
|
|
6937
|
-
/**
|
|
6938
|
-
*
|
|
6939
|
-
*
|
|
6940
|
-
*
|
|
6941
|
-
*
|
|
6942
|
-
*
|
|
6943
|
-
*
|
|
6944
|
-
|
|
6945
|
-
function
|
|
6946
|
-
return
|
|
6947
|
-
|
|
6948
|
-
|
|
6949
|
-
|
|
6950
|
-
|
|
6951
|
-
|
|
6952
|
-
|
|
6953
|
-
|
|
6954
|
-
|
|
6955
|
-
|
|
6956
|
-
}
|
|
6972
|
+
/**
|
|
6973
|
+
* The one canonical JSON emitter for every adapter (re-homed from the oracle's
|
|
6974
|
+
* `_to_json`, adapters/docusaurus_sidebar.py:410): node keys whose value is
|
|
6975
|
+
* null/empty/zero are omitted EXCEPT `position`, which is always emitted; file
|
|
6976
|
+
* dicts carry `title` only when set. Adapters round-trip the result through
|
|
6977
|
+
* `parseManifest` before writing, so an adapter can never emit what ingest
|
|
6978
|
+
* would refuse.
|
|
6979
|
+
*/
|
|
6980
|
+
function manifestToJson(m) {
|
|
6981
|
+
return {
|
|
6982
|
+
format: m.format,
|
|
6983
|
+
corpus_id: m.corpus_id,
|
|
6984
|
+
source_commit: m.source_commit,
|
|
6985
|
+
nodes: m.nodes.map(nodeToJson),
|
|
6986
|
+
files: m.files.map((f) => ({
|
|
6987
|
+
path: f.path,
|
|
6988
|
+
node: f.node,
|
|
6989
|
+
...f.title ? { title: f.title } : {}
|
|
6990
|
+
}))
|
|
6991
|
+
};
|
|
6957
6992
|
}
|
|
6958
|
-
|
|
6959
|
-
|
|
6960
|
-
|
|
6961
|
-
|
|
6962
|
-
|
|
6963
|
-
|
|
6964
|
-
|
|
6965
|
-
|
|
6966
|
-
|
|
6967
|
-
|
|
6968
|
-
|
|
6969
|
-
|
|
6970
|
-
|
|
6971
|
-
|
|
6972
|
-
|
|
6973
|
-
|
|
6974
|
-
out += seg.slice(i, a) + h;
|
|
6975
|
-
i = b + 1;
|
|
6976
|
-
} else {
|
|
6977
|
-
out += seg.slice(i, a + 1);
|
|
6978
|
-
i = a + 1;
|
|
6979
|
-
}
|
|
6993
|
+
function nodeToJson(n) {
|
|
6994
|
+
const fields = [
|
|
6995
|
+
["stable_id", n.stable_id],
|
|
6996
|
+
["slug", n.slug],
|
|
6997
|
+
["title", n.title],
|
|
6998
|
+
["kind", n.kind],
|
|
6999
|
+
["parent", n.parent],
|
|
7000
|
+
["position", n.position],
|
|
7001
|
+
["summary", n.summary],
|
|
7002
|
+
["keywords", n.keywords],
|
|
7003
|
+
["permalink", n.permalink]
|
|
7004
|
+
];
|
|
7005
|
+
const out = {};
|
|
7006
|
+
for (const [key, val] of fields) {
|
|
7007
|
+
const omit = val === null || val === 0 || Array.isArray(val) && val.length === 0;
|
|
7008
|
+
if (key === "position" || !omit) out[key] = val;
|
|
6980
7009
|
}
|
|
6981
|
-
|
|
7010
|
+
const gov = governanceToJson(n.governance);
|
|
7011
|
+
if (Object.keys(gov).length > 0) out["governance"] = gov;
|
|
7012
|
+
return out;
|
|
6982
7013
|
}
|
|
6983
|
-
|
|
6984
|
-
|
|
6985
|
-
|
|
6986
|
-
|
|
6987
|
-
|
|
6988
|
-
|
|
6989
|
-
|
|
6990
|
-
|
|
6991
|
-
});
|
|
6992
|
-
seg = stripStyleAttr(seg).replace(CLASS_ATTR_G, "");
|
|
6993
|
-
seg = dropBareLayoutTags(seg, stack);
|
|
6994
|
-
seg = seg.replace(BLANK_RUN_G, "\n\n");
|
|
6995
|
-
return restoreHeld(seg, held);
|
|
7014
|
+
function governanceToJson(g) {
|
|
7015
|
+
const out = {};
|
|
7016
|
+
if (g.visibility !== null) out["visibility"] = g.visibility;
|
|
7017
|
+
if (g.docStatus !== null) out["doc_status"] = g.docStatus;
|
|
7018
|
+
if (g.owner !== null) out["owner"] = g.owner;
|
|
7019
|
+
if (g.provenance !== null && g.provenance.length > 0) out["provenance"] = g.provenance;
|
|
7020
|
+
if (g.supersededBy !== null) out["superseded_by"] = g.supersededBy;
|
|
7021
|
+
return out;
|
|
6996
7022
|
}
|
|
6997
|
-
/**
|
|
6998
|
-
*
|
|
6999
|
-
*
|
|
7000
|
-
*
|
|
7001
|
-
*
|
|
7002
|
-
*
|
|
7003
|
-
|
|
7004
|
-
|
|
7005
|
-
|
|
7006
|
-
|
|
7007
|
-
|
|
7008
|
-
|
|
7009
|
-
|
|
7010
|
-
|
|
7011
|
-
let
|
|
7012
|
-
|
|
7013
|
-
|
|
7014
|
-
|
|
7015
|
-
|
|
7016
|
-
|
|
7017
|
-
|
|
7018
|
-
|
|
7019
|
-
|
|
7020
|
-
|
|
7021
|
-
|
|
7022
|
-
|
|
7023
|
-
|
|
7024
|
-
|
|
7023
|
+
/**
|
|
7024
|
+
* Carried verbatim from the oracle's build step (ingest/build.py:109):
|
|
7025
|
+
* `source_id = f.path.removesuffix(".md") + ":prose"`. The ".md"-only strip is
|
|
7026
|
+
* a deliberate quirk — `a/b.md` → `a/b:prose` while `a/b.mdx` KEEPS its suffix
|
|
7027
|
+
* → `a/b.mdx:prose`. Persisted source rows and carry-forward joins already pin
|
|
7028
|
+
* this shape; changing it is a policy decision, never a tidy-up.
|
|
7029
|
+
*/
|
|
7030
|
+
function sourceId(path) {
|
|
7031
|
+
return (path.endsWith(".md") ? path.slice(0, -3) : path) + ":prose";
|
|
7032
|
+
}
|
|
7033
|
+
/** Python len(): Unicode code points, not UTF-16 units. Every limit comparison
|
|
7034
|
+
* and the HARD_MAX_CHARS slice go through code points or the policy silently
|
|
7035
|
+
* changes on astral-plane text (emoji, musical symbols, CJK extensions). */
|
|
7036
|
+
function cpLen(s) {
|
|
7037
|
+
let n = 0;
|
|
7038
|
+
for (const _ch of s) n += 1;
|
|
7039
|
+
return n;
|
|
7040
|
+
}
|
|
7041
|
+
/** Python str.splitlines() boundary set (full code-point scan, 2026-08-19):
|
|
7042
|
+
* \n \v \f \r \x1c \x1d \x1e \x85 \u2028 \u2029, with \r\n as one boundary.
|
|
7043
|
+
* A naive split(/\r?\n/) changes segmentation on \x85, \u2028 etc. All
|
|
7044
|
+
* boundaries are BMP, so a UTF-16 walk cannot land inside a surrogate pair. */
|
|
7045
|
+
const LINE_BOUNDARY = /* @__PURE__ */ new Set([
|
|
7046
|
+
"\n",
|
|
7047
|
+
"\v",
|
|
7048
|
+
"\f",
|
|
7049
|
+
"\r",
|
|
7050
|
+
"",
|
|
7051
|
+
"",
|
|
7052
|
+
"",
|
|
7053
|
+
"
",
|
|
7054
|
+
"\u2028",
|
|
7055
|
+
"\u2029"
|
|
7056
|
+
]);
|
|
7057
|
+
function pySplitLines(text, keepends) {
|
|
7058
|
+
const out = [];
|
|
7059
|
+
let start = 0;
|
|
7060
|
+
let i = 0;
|
|
7061
|
+
while (i < text.length) {
|
|
7062
|
+
const ch = text[i];
|
|
7063
|
+
if (LINE_BOUNDARY.has(ch)) {
|
|
7064
|
+
let end = i + 1;
|
|
7065
|
+
if (ch === "\r" && text[end] === "\n") end += 1;
|
|
7066
|
+
out.push(keepends ? text.slice(start, end) : text.slice(start, i));
|
|
7067
|
+
start = end;
|
|
7068
|
+
i = end;
|
|
7069
|
+
} else i += 1;
|
|
7025
7070
|
}
|
|
7026
|
-
if (
|
|
7027
|
-
return out
|
|
7028
|
-
}
|
|
7029
|
-
/**
|
|
7030
|
-
* The body-cleaning pipeline every ingest runs BEFORE the skip-gate hash and
|
|
7031
|
-
* chunking, as ONE ordered unit so the order cannot regress. CRLF→LF is
|
|
7032
|
-
* normalized FIRST — the strippers are \n-anchored (BLANK_RUN_G = /\n{3,}/),
|
|
7033
|
-
* so normalizing AFTER them left a CRLF checkout's blank runs un-collapsed and
|
|
7034
|
-
* every chunk_hash + content_hash diverged from an LF checkout, re-embedding
|
|
7035
|
-
* the whole file while content_hash claimed nothing changed (review,
|
|
7036
|
-
* 2026-08-19). Then style blocks and presentation JSX are stripped so served
|
|
7037
|
-
* chunks reassemble the CLEANED body byte-exact. A bare \r (no following \n)
|
|
7038
|
-
* stays content.
|
|
7039
|
-
*/
|
|
7040
|
-
function cleanBody(rawBody) {
|
|
7041
|
-
return stripPresentationJsx(stripStyleBlocks(rawBody.replaceAll("\r\n", "\n")));
|
|
7071
|
+
if (start < text.length) out.push(text.slice(start));
|
|
7072
|
+
return out;
|
|
7042
7073
|
}
|
|
7043
|
-
/**
|
|
7044
|
-
*
|
|
7045
|
-
*
|
|
7046
|
-
|
|
7047
|
-
|
|
7074
|
+
/** Python's whitespace set — str.isspace() == str.strip() == re \s for str
|
|
7075
|
+
* patterns (verified identical by full code-point scan, 2026-08-19). Note the
|
|
7076
|
+
* two-way mismatch with JS: \x1c-\x1f and \x85 are whitespace only here;
|
|
7077
|
+
* \ufeff is whitespace to JS trim()/\s but NOT to Python. */
|
|
7078
|
+
const PY_SPACE = /* @__PURE__ */ new Set(" \n\v\f\r
\xA0 \u2028\u2029 ");
|
|
7079
|
+
function pyStrip(s) {
|
|
7080
|
+
let a = 0;
|
|
7081
|
+
let b = s.length;
|
|
7082
|
+
while (a < b && PY_SPACE.has(s[a])) a += 1;
|
|
7083
|
+
while (b > a && PY_SPACE.has(s[b - 1])) b -= 1;
|
|
7084
|
+
return s.slice(a, b);
|
|
7048
7085
|
}
|
|
7049
|
-
|
|
7050
|
-
|
|
7051
|
-
|
|
7052
|
-
|
|
7053
|
-
|
|
7054
|
-
|
|
7086
|
+
/** Character-class text for Python \s (same set as PY_SPACE, for regexes). */
|
|
7087
|
+
const WS = "\\t\\n\\v\\f\\r\\x1c-\\x1f \\x85\\xa0\\u1680\\u2000-\\u200a\\u2028\\u2029\\u202f\\u205f\\u3000";
|
|
7088
|
+
/** Character-class text for Python \w: L* ∪ Nd ∪ Nl ∪ No ∪ {_} — i.e.
|
|
7089
|
+
* str.isalnum() plus underscore (spot-verified: é 中 Ⅰ ½ yes; 😀 and combining
|
|
7090
|
+
* marks no). Used where the oracle wrote \w or \b (JS \w/\b are ASCII-only). */
|
|
7091
|
+
const WORD = "\\p{L}\\p{N}_";
|
|
7092
|
+
const HEADING = new RegExp(`^(#{1,4})[${WS}]+([^\\n]*?)[${WS}]*$`);
|
|
7093
|
+
const FENCE = new RegExp(`^ {0,3}(\`{3,}|~{3,})([^\\n]*?)[${WS}]*$`);
|
|
7094
|
+
const EXPLICIT_ID = new RegExp(`[${WS}]*\\{#([${WORD}-]+)\\}[${WS}]*$`, "u");
|
|
7095
|
+
const SLUG_RUN = /[^a-z0-9]+/g;
|
|
7096
|
+
const JSX_ASSESS = new RegExp(`(?:^|(?<=\\n))[${WS}]*<(?:Quiz|Flashcards)(?![${WORD}])`, "u");
|
|
7097
|
+
const JSX_EMBED = new RegExp(`(?:^|(?<=\\n))[${WS}]*<(?:iframe|AICheck|AICheckField|ProjectCard|CapstoneWorkbook)(?![${WORD}])`, "u");
|
|
7098
|
+
const STYLE_OPEN = new RegExp(`^[${WS}]*<style(?![${WORD}])`, "iu");
|
|
7099
|
+
const CLASS_ATTR_G = new RegExp(`[${WS}]*className=(?:"[^"]*"|'[^']*'|\\{[^{}]*\\})`, "gu");
|
|
7100
|
+
const LAYOUT_NAMES = "div|span|section|figure|article|header|footer|main|aside";
|
|
7101
|
+
const LAYOUT_TAG_G = new RegExp(`<[${WS}]*(\\/?)[${WS}]*(${LAYOUT_NAMES})(?![${WORD}])([^<>]*?)(\\/?)[${WS}]*>`, "gu");
|
|
7102
|
+
const LAYOUT_TAG_PROBE = new RegExp(`<[${WS}]*(\\/?)[${WS}]*(${LAYOUT_NAMES})(?![${WORD}])([^<>]*?)(\\/?)[${WS}]*>`, "u");
|
|
7103
|
+
const INLINE_CODE_G = /* @__PURE__ */ new RegExp("`[^`\\n]+`", "g");
|
|
7104
|
+
const BLANK_RUN_G = /* @__PURE__ */ new RegExp("\\n{3,}", "g");
|
|
7105
|
+
const NUL = "\0";
|
|
7106
|
+
const BLANK_SEP = new RegExp(`(\\n[${WS}]*\\n)`);
|
|
7107
|
+
const sha256 = (s) => createHash("sha256").update(s, "utf8").digest("hex");
|
|
7108
|
+
function slug(title, cap = 60) {
|
|
7109
|
+
const s = title.toLowerCase().replace(SLUG_RUN, "-").replace(/^-+|-+$/g, "");
|
|
7110
|
+
if (s.length <= cap) return s;
|
|
7111
|
+
const cut = s.lastIndexOf("-", cap - 1);
|
|
7112
|
+
return s.slice(0, cut > 0 ? cut : cap).replace(/^-+|-+$/g, "");
|
|
7055
7113
|
}
|
|
7056
|
-
|
|
7057
|
-
|
|
7058
|
-
* not leak as prose. */
|
|
7059
|
-
function segmentMarkerType(span) {
|
|
7060
|
-
for (const [re, label] of [[JSX_ASSESS, "assessment"], [JSX_EMBED, "embed"]]) {
|
|
7061
|
-
const m = re.exec(span);
|
|
7062
|
-
if (m !== null && cpLen(teachingBody(span.slice(0, m.index))) < 250) return label;
|
|
7063
|
-
}
|
|
7064
|
-
return null;
|
|
7114
|
+
function headingPathText(path) {
|
|
7115
|
+
return path.map((p) => slug(p)).filter((s) => s !== "").join("/");
|
|
7065
7116
|
}
|
|
7066
|
-
/**
|
|
7067
|
-
*
|
|
7068
|
-
|
|
7069
|
-
|
|
7070
|
-
const
|
|
7071
|
-
|
|
7072
|
-
let buf = [];
|
|
7073
|
-
let curPath = [];
|
|
7074
|
-
let curAnchor = null;
|
|
7075
|
-
let fence = null;
|
|
7076
|
-
const flush = () => {
|
|
7077
|
-
if (buf.length > 0) {
|
|
7078
|
-
segments.push({
|
|
7079
|
-
path: [...curPath],
|
|
7080
|
-
anchor: curAnchor,
|
|
7081
|
-
text: buf.join("")
|
|
7082
|
-
});
|
|
7083
|
-
buf = [];
|
|
7084
|
-
}
|
|
7085
|
-
};
|
|
7086
|
-
for (const line of pySplitLines(text, true)) {
|
|
7087
|
-
const m = fence === null ? HEADING.exec(line) : null;
|
|
7117
|
+
/** CommonMark fence tracking (v5 — the correctness core). A backtick fence
|
|
7118
|
+
* whose info string contains a backtick is NOT a fence; closing needs the same
|
|
7119
|
+
* char, a run at least as long, and nothing but whitespace after. */
|
|
7120
|
+
function fenceStep(line, fence) {
|
|
7121
|
+
const m = FENCE.exec(line);
|
|
7122
|
+
if (fence === null) {
|
|
7088
7123
|
if (m !== null) {
|
|
7089
|
-
|
|
7090
|
-
const
|
|
7091
|
-
|
|
7092
|
-
|
|
7093
|
-
const title = idM !== null ? pyStrip(rawTitle.replace(EXPLICIT_ID, "")) : rawTitle;
|
|
7094
|
-
titles.set(level, title);
|
|
7095
|
-
anchors.set(level, idM !== null ? idM[1] : null);
|
|
7096
|
-
const deeper = [...titles.keys()].filter((lv) => lv > level);
|
|
7097
|
-
for (const lv of deeper) {
|
|
7098
|
-
titles.delete(lv);
|
|
7099
|
-
anchors.delete(lv);
|
|
7100
|
-
}
|
|
7101
|
-
curPath = [...titles.keys()].sort((a, b) => a - b).filter((lv) => lv >= 2 && lv <= level).map((lv) => titles.get(lv));
|
|
7102
|
-
curAnchor = curPath.length > 0 ? anchors.get(level) || slug(title) : null;
|
|
7124
|
+
const marker = m[1];
|
|
7125
|
+
const info = m[2];
|
|
7126
|
+
if (marker[0] === "`" && info.includes("`")) return null;
|
|
7127
|
+
return [marker[0], marker.length];
|
|
7103
7128
|
}
|
|
7104
|
-
|
|
7105
|
-
if (m === null) fence = fenceStep(line, fence);
|
|
7129
|
+
return null;
|
|
7106
7130
|
}
|
|
7107
|
-
|
|
7108
|
-
return
|
|
7131
|
+
if (m !== null && m[1][0] === fence[0] && m[1].length >= fence[1] && m[2] === "") return null;
|
|
7132
|
+
return fence;
|
|
7109
7133
|
}
|
|
7110
|
-
/**
|
|
7111
|
-
*
|
|
7112
|
-
*
|
|
7113
|
-
*
|
|
7114
|
-
|
|
7115
|
-
|
|
7116
|
-
|
|
7117
|
-
|
|
7118
|
-
|
|
7119
|
-
|
|
7134
|
+
/** Drop prose-level `<style>…</style>` CSS — pure presentation; classify would
|
|
7135
|
+
* size a multi-KB CSS wall as prose and serve it (the oracle's field-test #3
|
|
7136
|
+
* "about" doc opened with ~2KB of it). Runs BEFORE contentHash + chunkText, so
|
|
7137
|
+
* chunks reassemble the CLEANED body byte-exact and only files that actually
|
|
7138
|
+
* held a block re-embed. FENCE-SAFE: a `<style>` shown as example code inside
|
|
7139
|
+
* a fence is content and survives. See STYLE_OPEN for the locked fast-path
|
|
7140
|
+
* quirk (^-anchored, no multiline). */
|
|
7141
|
+
function stripStyleBlocks(text) {
|
|
7142
|
+
if (!STYLE_OPEN.test(text)) return text;
|
|
7143
|
+
const out = [];
|
|
7120
7144
|
let fence = null;
|
|
7121
|
-
|
|
7122
|
-
|
|
7123
|
-
if (
|
|
7124
|
-
|
|
7125
|
-
|
|
7126
|
-
bufLen = 0;
|
|
7145
|
+
let dropping = false;
|
|
7146
|
+
for (const line of pySplitLines(text, true)) {
|
|
7147
|
+
if (dropping) {
|
|
7148
|
+
if (line.toLowerCase().includes("</style>")) dropping = false;
|
|
7149
|
+
continue;
|
|
7127
7150
|
}
|
|
7128
|
-
|
|
7129
|
-
|
|
7130
|
-
|
|
7151
|
+
if (fence === null && STYLE_OPEN.test(line)) {
|
|
7152
|
+
if (!line.toLowerCase().includes("</style>")) dropping = true;
|
|
7153
|
+
continue;
|
|
7154
|
+
}
|
|
7155
|
+
out.push(line);
|
|
7156
|
+
fence = fenceStep(line, fence);
|
|
7131
7157
|
}
|
|
7132
|
-
|
|
7133
|
-
const out = [];
|
|
7134
|
-
for (const piece of pieces) out.push(...enforceCeiling(piece));
|
|
7135
|
-
return out;
|
|
7158
|
+
return out.join("");
|
|
7136
7159
|
}
|
|
7137
|
-
|
|
7138
|
-
|
|
7139
|
-
|
|
7140
|
-
|
|
7141
|
-
|
|
7142
|
-
|
|
7160
|
+
const rstripSpacesTabs = (s) => {
|
|
7161
|
+
let b = s.length;
|
|
7162
|
+
while (b > 0 && (s[b - 1] === " " || s[b - 1] === " ")) b -= 1;
|
|
7163
|
+
return s.slice(0, b);
|
|
7164
|
+
};
|
|
7165
|
+
/** Remove `style={{ ... }}` attributes by BRACE MATCHING, not regex: the value
|
|
7166
|
+
* is a JS object literal and the oracle measured 168 of them spanning lines.
|
|
7167
|
+
* An unbalanced opener leaves the rest of the text verbatim — a stripper must
|
|
7168
|
+
* never eat the rest of a document to satisfy itself. (The pre-attr rstrip of
|
|
7169
|
+
* spaces/tabs also glues `<div ` + a KEPT unbalanced `style={{` opener into
|
|
7170
|
+
* `<divstyle={{` — an oracle quirk locked by fixture strip-malformed-kept.) */
|
|
7171
|
+
function stripStyleAttr(text) {
|
|
7143
7172
|
const out = [];
|
|
7144
|
-
|
|
7145
|
-
|
|
7146
|
-
|
|
7147
|
-
|
|
7148
|
-
|
|
7149
|
-
|
|
7150
|
-
|
|
7151
|
-
|
|
7152
|
-
|
|
7153
|
-
|
|
7154
|
-
|
|
7155
|
-
|
|
7156
|
-
|
|
7157
|
-
|
|
7158
|
-
|
|
7159
|
-
|
|
7160
|
-
|
|
7161
|
-
|
|
7162
|
-
if (chunks.length > 0) {
|
|
7163
|
-
const last = chunks[chunks.length - 1];
|
|
7164
|
-
const content = last.content + seg.text;
|
|
7165
|
-
chunks[chunks.length - 1] = {
|
|
7166
|
-
...last,
|
|
7167
|
-
content,
|
|
7168
|
-
chunkHash: sha256(content)
|
|
7169
|
-
};
|
|
7170
|
-
} else prefix += seg.text;
|
|
7171
|
-
continue;
|
|
7172
|
-
}
|
|
7173
|
-
const segIsNav = cpLen(teachingBody(seg.text)) < 250;
|
|
7174
|
-
const segMarker = segmentMarkerType(seg.text);
|
|
7175
|
-
for (const piece of subsplit(seg.text, maxChars)) {
|
|
7176
|
-
let sourceType;
|
|
7177
|
-
if (segMarker !== null) sourceType = segMarker;
|
|
7178
|
-
else {
|
|
7179
|
-
sourceType = classify(piece, seg.path);
|
|
7180
|
-
if (sourceType === "nav" && !segIsNav) sourceType = "prose";
|
|
7173
|
+
let i = 0;
|
|
7174
|
+
for (;;) {
|
|
7175
|
+
const j = text.indexOf("style={{", i);
|
|
7176
|
+
if (j < 0) break;
|
|
7177
|
+
out.push(rstripSpacesTabs(text.slice(i, j)));
|
|
7178
|
+
let k = j + 6;
|
|
7179
|
+
let depth = 0;
|
|
7180
|
+
let closed = false;
|
|
7181
|
+
while (k < text.length) {
|
|
7182
|
+
const ch = text[k];
|
|
7183
|
+
if (ch === "{") depth += 1;
|
|
7184
|
+
else if (ch === "}") {
|
|
7185
|
+
depth -= 1;
|
|
7186
|
+
if (depth === 0) {
|
|
7187
|
+
k += 1;
|
|
7188
|
+
closed = true;
|
|
7189
|
+
break;
|
|
7190
|
+
}
|
|
7181
7191
|
}
|
|
7182
|
-
|
|
7183
|
-
|
|
7184
|
-
|
|
7192
|
+
k += 1;
|
|
7193
|
+
}
|
|
7194
|
+
if (!closed) {
|
|
7195
|
+
out.push(text.slice(j));
|
|
7196
|
+
return out.join("");
|
|
7185
7197
|
}
|
|
7198
|
+
i = k;
|
|
7186
7199
|
}
|
|
7187
|
-
|
|
7188
|
-
return
|
|
7189
|
-
}
|
|
7190
|
-
/** §5 rule 2: snapshot-token TTL (30 min) + 10 min = 40 min from retirement. */
|
|
7191
|
-
const GC_GRACE_MS = 24e5;
|
|
7192
|
-
/**
|
|
7193
|
-
* Poison-chunk tolerance (oracle review: poison-chunk-wedge): one
|
|
7194
|
-
* deterministically-failing chunk must not wedge every future flip forever. A
|
|
7195
|
-
* generation is servable if a SMALL fraction failed — the read path already
|
|
7196
|
-
* filters to `embedded`, so a quarantined chunk is simply absent, not
|
|
7197
|
-
* corrupt. Above the fraction, a real ingest break is signalled by
|
|
7198
|
-
* withholding readiness.
|
|
7199
|
-
*/
|
|
7200
|
-
const MAX_FAILED_FRACTION = .02;
|
|
7201
|
-
const LOCK_SQL = "SELECT pg_advisory_xact_lock(hashtextextended('sor-ingest:' || $1, 0))";
|
|
7202
|
-
/**
|
|
7203
|
-
* Take the tenant lock, allocate generation max+1 (monotonic per corpus,
|
|
7204
|
-
* never reused), open the building run. The corpora row seeds at
|
|
7205
|
-
* active_generation=0 (nothing active) on first ingest.
|
|
7206
|
-
*
|
|
7207
|
-
* `manifestSha256` fills `instance_bundle_sha256` — ksor has no bundle
|
|
7208
|
-
* transport (the CLI reads the local repo), so the recorded digest is of the
|
|
7209
|
-
* manifest this build actually consumed: the closest honest provenance.
|
|
7210
|
-
*/
|
|
7211
|
-
async function allocateRun(client, opts) {
|
|
7212
|
-
await client.query(LOCK_SQL, [opts.tenantId]);
|
|
7213
|
-
await client.query("INSERT INTO corpora (tenant_id, corpus_id, active_generation) VALUES ($1, $2, 0) ON CONFLICT (tenant_id, corpus_id) DO NOTHING", [opts.tenantId, opts.corpusId]);
|
|
7214
|
-
const next = await client.query("SELECT COALESCE(max(generation), 0) + 1 AS next FROM ingestion_runs WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId]);
|
|
7215
|
-
const generation = Number(next.rows[0].next);
|
|
7216
|
-
const run = await client.query("INSERT INTO ingestion_runs (tenant_id, corpus_id, generation, state, source_commit, instance_bundle_sha256, schema_version) VALUES ($1, $2, $3, 'building', $4, $5, $6) RETURNING run_id", [
|
|
7217
|
-
opts.tenantId,
|
|
7218
|
-
opts.corpusId,
|
|
7219
|
-
generation,
|
|
7220
|
-
opts.sourceCommit,
|
|
7221
|
-
opts.manifestSha256,
|
|
7222
|
-
schemaVersion()
|
|
7223
|
-
]);
|
|
7224
|
-
return {
|
|
7225
|
-
runId: Number(run.rows[0].run_id),
|
|
7226
|
-
generation
|
|
7227
|
-
};
|
|
7228
|
-
}
|
|
7229
|
-
/**
|
|
7230
|
-
* The generation to carry embeddings FROM: the newest COMPLETE one holding
|
|
7231
|
-
* embedded chunks.
|
|
7232
|
-
*
|
|
7233
|
-
* WHY NOT ONLY THE ACTIVE ONE: the eval-before-flip design means a candidate
|
|
7234
|
-
* is often built, measured, and deliberately NOT served; ACTIVE then points
|
|
7235
|
-
* at an OLD generation and the next candidate re-embeds the whole corpus —
|
|
7236
|
-
* measured 2026-08-02: generation 4 re-embedded 5,915 chunks while
|
|
7237
|
-
* generation 3 held near-identical content, because generation 1 was still
|
|
7238
|
-
* active.
|
|
7239
|
-
*
|
|
7240
|
-
* Two constraints a rewrite once dropped (oracle review of PR #420):
|
|
7241
|
-
* CORPUS-SCOPED via the run-table join (chunks carry no corpus_id), and
|
|
7242
|
-
* COMPLETE RUNS ONLY (ready/active/retired) — a crashed `building` queue's
|
|
7243
|
-
* half-drained vectors never qualify.
|
|
7244
|
-
*
|
|
7245
|
-
* Returns 0 when there is no complete embedded generation — the first ingest.
|
|
7246
|
-
*/
|
|
7247
|
-
async function bestCarrySource(client, opts) {
|
|
7248
|
-
const gen = (await client.query(`
|
|
7249
|
-
SELECT max(c.generation) AS gen FROM chunks c
|
|
7250
|
-
JOIN ingestion_runs r ON r.tenant_id = c.tenant_id AND r.generation = c.generation
|
|
7251
|
-
WHERE c.tenant_id = $1 AND r.corpus_id = $2
|
|
7252
|
-
AND r.state IN ('ready', 'active', 'retired')
|
|
7253
|
-
AND c.generation <> $3 AND c.embedding_status = 'embedded'
|
|
7254
|
-
`, [
|
|
7255
|
-
opts.tenantId,
|
|
7256
|
-
opts.corpusId,
|
|
7257
|
-
opts.excludeGeneration
|
|
7258
|
-
])).rows[0]?.gen ?? null;
|
|
7259
|
-
return gen === null ? 0 : Number(gen);
|
|
7260
|
-
}
|
|
7261
|
-
/**
|
|
7262
|
-
* Copy embeddings for chunks whose ENTIRE embed input is unchanged (hash +
|
|
7263
|
-
* heading path + node title). Cost ∝ change survives the generational
|
|
7264
|
-
* rebuild. Returns rows carried.
|
|
7265
|
-
*
|
|
7266
|
-
* `modelId` is REQUIRED, never defaulted here: the vendor transport is
|
|
7267
|
-
* irrelevant to the space (the same model through two providers is the same
|
|
7268
|
-
* space), and a silent module default is exactly how a model bump would
|
|
7269
|
-
* carry stale vectors unnoticed.
|
|
7270
|
-
*/
|
|
7271
|
-
async function carryForward(client, opts) {
|
|
7272
|
-
if (opts.fromGeneration < 1) return 0;
|
|
7273
|
-
return (await client.query(`
|
|
7274
|
-
UPDATE chunks new SET embedding = old.embedding, embedding_status = 'embedded',
|
|
7275
|
-
embedded_at = old.embedded_at, embedding_model = old.embedding_model
|
|
7276
|
-
FROM chunks old, sources os, content_nodes onode, sources ns, content_nodes nnode
|
|
7277
|
-
WHERE new.tenant_id = $1 AND new.generation = $2
|
|
7278
|
-
AND new.embedding_status = 'pending'
|
|
7279
|
-
AND old.tenant_id = new.tenant_id AND old.generation = $3
|
|
7280
|
-
AND old.embedding_status = 'embedded'
|
|
7281
|
-
-- R-1 gate (oracle review: carry-model-gate-r1): carry ONLY vectors from the CURRENT
|
|
7282
|
-
-- embedding model. Without this, a model bump silently carries every old-model vector
|
|
7283
|
-
-- forward (pending→0, flip → corpus-wide nonsense cosine vs the new query model, zero
|
|
7284
|
-
-- errors). A model change now correctly leaves the old vectors pending → they re-embed.
|
|
7285
|
-
AND old.embedding_model = $4
|
|
7286
|
-
AND old.source_id = new.source_id
|
|
7287
|
-
AND old.chunk_hash = new.chunk_hash
|
|
7288
|
-
AND old.heading_path_text IS NOT DISTINCT FROM new.heading_path_text
|
|
7289
|
-
AND os.source_id = old.source_id AND os.tenant_id = old.tenant_id
|
|
7290
|
-
AND os.generation = old.generation
|
|
7291
|
-
AND onode.node_id = os.node_id AND onode.tenant_id = os.tenant_id
|
|
7292
|
-
AND ns.source_id = new.source_id AND ns.tenant_id = new.tenant_id
|
|
7293
|
-
AND ns.generation = new.generation
|
|
7294
|
-
AND nnode.node_id = ns.node_id AND nnode.tenant_id = ns.tenant_id
|
|
7295
|
-
AND onode.title = nnode.title
|
|
7296
|
-
`, [
|
|
7297
|
-
opts.tenantId,
|
|
7298
|
-
opts.generation,
|
|
7299
|
-
opts.fromGeneration,
|
|
7300
|
-
opts.modelId
|
|
7301
|
-
])).rowCount ?? 0;
|
|
7302
|
-
}
|
|
7303
|
-
/**
|
|
7304
|
-
* avg(embedding) per node over servable prose — rows the routing arm reads
|
|
7305
|
-
* (never aggregate at query time again). nav/embed/assessment chunks never
|
|
7306
|
-
* pollute routing centroids.
|
|
7307
|
-
*/
|
|
7308
|
-
async function materializeCentroids(client, opts) {
|
|
7309
|
-
await client.query("DELETE FROM node_centroids WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation]);
|
|
7310
|
-
return (await client.query(`
|
|
7311
|
-
INSERT INTO node_centroids (tenant_id, generation, node_id, stable_id, chunk_count, embedding)
|
|
7312
|
-
SELECT c.tenant_id, c.generation, n.node_id, n.stable_id, count(*), avg(c.embedding)
|
|
7313
|
-
FROM chunks c
|
|
7314
|
-
JOIN sources s ON s.source_id = c.source_id AND s.tenant_id = c.tenant_id
|
|
7315
|
-
AND s.generation = c.generation
|
|
7316
|
-
JOIN content_nodes n ON n.node_id = s.node_id AND n.tenant_id = s.tenant_id
|
|
7317
|
-
WHERE c.tenant_id = $1 AND c.generation = $2 AND c.embedding_status = 'embedded'
|
|
7318
|
-
AND c.labels->>'source_type' = 'prose'
|
|
7319
|
-
GROUP BY c.tenant_id, c.generation, n.node_id, n.stable_id
|
|
7320
|
-
`, [opts.tenantId, opts.generation])).rowCount ?? 0;
|
|
7200
|
+
out.push(text.slice(i));
|
|
7201
|
+
return out.join("");
|
|
7321
7202
|
}
|
|
7322
|
-
/**
|
|
7323
|
-
*
|
|
7324
|
-
*
|
|
7325
|
-
*
|
|
7326
|
-
|
|
7327
|
-
|
|
7328
|
-
|
|
7329
|
-
|
|
7330
|
-
|
|
7203
|
+
/** Drop layout tags left BARE by the attribute strip, KEEPING each closer
|
|
7204
|
+
* paired with its opener: a closer carries no attributes, so `</div>` cannot
|
|
7205
|
+
* say whether it belongs to a removed wrapper or to a kept `<div id="x">`.
|
|
7206
|
+
* Track depth; a closer is removed iff its opener was. On any mismatch the tag
|
|
7207
|
+
* is KEPT — a stray tag beats eaten content. `stack` is owned by the CALLER
|
|
7208
|
+
* and persists across every prose segment of one document: a styled wrapper
|
|
7209
|
+
* around a code fence splits at the fence, and its opener/closer land in
|
|
7210
|
+
* different segments (228 of them in the oracle's corpus). */
|
|
7211
|
+
function dropBareLayoutTags(segment, stack) {
|
|
7212
|
+
return segment.replace(LAYOUT_TAG_G, (m0, closing, name, rawAttrs, selfClosing) => {
|
|
7213
|
+
const attrs = pyStrip(rawAttrs);
|
|
7214
|
+
if (closing !== "") {
|
|
7215
|
+
const top = stack[stack.length - 1];
|
|
7216
|
+
if (top !== void 0 && top[0] === name) return stack.pop()[1] ? "" : m0;
|
|
7217
|
+
return m0;
|
|
7218
|
+
}
|
|
7219
|
+
if (selfClosing !== "") return attrs === "" ? "" : m0;
|
|
7220
|
+
stack.push([name, attrs === ""]);
|
|
7221
|
+
return attrs === "" ? "" : m0;
|
|
7222
|
+
});
|
|
7331
7223
|
}
|
|
7332
|
-
|
|
7333
|
-
|
|
7334
|
-
|
|
7335
|
-
|
|
7336
|
-
|
|
7337
|
-
|
|
7338
|
-
|
|
7339
|
-
|
|
7224
|
+
/** Restore NUL<n>NUL placeholders — equivalent to the oracle's
|
|
7225
|
+
* re.sub(r"\x00(\d+)\x00", ...): a leftmost scan where a match is a NUL, a
|
|
7226
|
+
* maximal non-empty ASCII digit run, and a closing NUL; anything else stays
|
|
7227
|
+
* verbatim (backtracking cannot produce any other match for this pattern). */
|
|
7228
|
+
function restoreHeld(seg, held) {
|
|
7229
|
+
let out = "";
|
|
7230
|
+
let i = 0;
|
|
7231
|
+
for (;;) {
|
|
7232
|
+
const a = seg.indexOf(NUL, i);
|
|
7233
|
+
if (a < 0) break;
|
|
7234
|
+
let b = a + 1;
|
|
7235
|
+
while (b < seg.length && seg[b] >= "0" && seg[b] <= "9") b += 1;
|
|
7236
|
+
if (b > a + 1 && seg[b] === NUL) {
|
|
7237
|
+
const d = seg.slice(a + 1, b);
|
|
7238
|
+
const h = held[Number(d)];
|
|
7239
|
+
if (h === void 0) throw new Error(`inline-code placeholder NUL${d}NUL has no held span (held ${held.length}): the source document contains a literal NUL-digit-NUL sequence, which collides with the stripper's placeholder scheme. Remove NUL control characters from the file.`);
|
|
7240
|
+
out += seg.slice(i, a) + h;
|
|
7241
|
+
i = b + 1;
|
|
7242
|
+
} else {
|
|
7243
|
+
out += seg.slice(i, a + 1);
|
|
7244
|
+
i = a + 1;
|
|
7245
|
+
}
|
|
7246
|
+
}
|
|
7247
|
+
return out + seg.slice(i);
|
|
7340
7248
|
}
|
|
7341
|
-
|
|
7342
|
-
|
|
7249
|
+
/** Strip presentation markup from ONE non-fenced segment. Inline code is
|
|
7250
|
+
* protected first (`<div>` written in prose backticks is a lesson). `stack` is
|
|
7251
|
+
* the document-wide layout-tag stack (see dropBareLayoutTags). */
|
|
7252
|
+
function stripProsePresentation(segment, stack) {
|
|
7253
|
+
const held = [];
|
|
7254
|
+
let seg = segment.replace(INLINE_CODE_G, (m0) => {
|
|
7255
|
+
held.push(m0);
|
|
7256
|
+
return `${NUL}${held.length - 1}${NUL}`;
|
|
7257
|
+
});
|
|
7258
|
+
seg = stripStyleAttr(seg).replace(CLASS_ATTR_G, "");
|
|
7259
|
+
seg = dropBareLayoutTags(seg, stack);
|
|
7260
|
+
seg = seg.replace(BLANK_RUN_G, "\n\n");
|
|
7261
|
+
return restoreHeld(seg, held);
|
|
7343
7262
|
}
|
|
7344
|
-
|
|
7345
|
-
|
|
7263
|
+
/** Drop layout markup — `style={{…}}`, `className="…"`, and the `<div>`/
|
|
7264
|
+
* `<span>` wrappers they leave bare — while KEEPING every character of the
|
|
7265
|
+
* text inside them ("The Third Era of AI Tools" is content, `af-hero-eyebrow`
|
|
7266
|
+
* is a CSS hook; the oracle measured a student being served the wrapper).
|
|
7267
|
+
* A layout tag is only stripped once it is BARE: attributes go first, and a
|
|
7268
|
+
* wrapper with nothing left was pure layout by construction, while
|
|
7269
|
+
* `<div id="x">` keeps its tag — no tag allowlist to drift. Deliberately never
|
|
7270
|
+
* touches capitalised components (<Quiz> is curriculum), inline SVG, or
|
|
7271
|
+
* `<details>`/`<summary>` (semantic HTML). FENCE- and inline-code-safe. ONE
|
|
7272
|
+
* layout-tag stack is threaded through every prose segment of the document so
|
|
7273
|
+
* pairing survives a fence split. */
|
|
7274
|
+
function stripPresentationJsx(text) {
|
|
7275
|
+
if (!text.includes("className=") && !text.includes("style={{") && !LAYOUT_TAG_PROBE.test(text)) return text;
|
|
7276
|
+
const out = [];
|
|
7277
|
+
let buf = [];
|
|
7278
|
+
let fence = null;
|
|
7279
|
+
const stack = [];
|
|
7280
|
+
for (const line of pySplitLines(text, true)) {
|
|
7281
|
+
const nxt = fenceStep(line, fence);
|
|
7282
|
+
if (fence === null && nxt === null) buf.push(line);
|
|
7283
|
+
else {
|
|
7284
|
+
if (buf.length > 0) {
|
|
7285
|
+
out.push(stripProsePresentation(buf.join(""), stack));
|
|
7286
|
+
buf = [];
|
|
7287
|
+
}
|
|
7288
|
+
out.push(line);
|
|
7289
|
+
}
|
|
7290
|
+
fence = nxt;
|
|
7291
|
+
}
|
|
7292
|
+
if (buf.length > 0) out.push(stripProsePresentation(buf.join(""), stack));
|
|
7293
|
+
return out.join("");
|
|
7346
7294
|
}
|
|
7347
7295
|
/**
|
|
7348
|
-
*
|
|
7349
|
-
*
|
|
7350
|
-
* the
|
|
7296
|
+
* The body-cleaning pipeline every ingest runs BEFORE the skip-gate hash and
|
|
7297
|
+
* chunking, as ONE ordered unit so the order cannot regress. CRLF→LF is
|
|
7298
|
+
* normalized FIRST — the strippers are \n-anchored (BLANK_RUN_G = /\n{3,}/),
|
|
7299
|
+
* so normalizing AFTER them left a CRLF checkout's blank runs un-collapsed and
|
|
7300
|
+
* every chunk_hash + content_hash diverged from an LF checkout, re-embedding
|
|
7301
|
+
* the whole file while content_hash claimed nothing changed (review,
|
|
7302
|
+
* 2026-08-19). Then style blocks and presentation JSX are stripped so served
|
|
7303
|
+
* chunks reassemble the CLEANED body byte-exact. A bare \r (no following \n)
|
|
7304
|
+
* stays content.
|
|
7351
7305
|
*/
|
|
7352
|
-
function
|
|
7353
|
-
|
|
7354
|
-
return Math.max(0, priorCount - newCount) / priorCount;
|
|
7306
|
+
function cleanBody(rawBody) {
|
|
7307
|
+
return stripPresentationJsx(stripStyleBlocks(rawBody.replaceAll("\r\n", "\n")));
|
|
7355
7308
|
}
|
|
7356
|
-
/**
|
|
7357
|
-
*
|
|
7358
|
-
*
|
|
7359
|
-
|
|
7360
|
-
|
|
7361
|
-
function shrinkUnsafe(priorCount, newCount, maxShrink) {
|
|
7362
|
-
return priorCount > 0 && shrinkFraction(priorCount, newCount) > maxShrink;
|
|
7309
|
+
/** Heading text never counts toward the nav/prose size test — the
|
|
7310
|
+
* "content-only" in the policy name. Note: joins on \n, so exotic line
|
|
7311
|
+
* boundaries are normalized before the length is taken (as in the oracle). */
|
|
7312
|
+
function teachingBody(content) {
|
|
7313
|
+
return pyStrip(pySplitLines(content, false).filter((ln) => !HEADING.test(ln)).join("\n"));
|
|
7363
7314
|
}
|
|
7364
|
-
|
|
7365
|
-
|
|
7366
|
-
const
|
|
7367
|
-
|
|
7368
|
-
|
|
7369
|
-
|
|
7370
|
-
|
|
7371
|
-
|
|
7372
|
-
|
|
7373
|
-
|
|
7374
|
-
|
|
7375
|
-
|
|
7376
|
-
|
|
7315
|
+
function classify(content, headingPath) {
|
|
7316
|
+
if (JSX_ASSESS.test(content)) return "assessment";
|
|
7317
|
+
const leaf = headingPath.length > 0 ? headingPath[headingPath.length - 1] : "";
|
|
7318
|
+
if (JSX_EMBED.test(content) || content.includes("docs.google.com/presentation") || leaf.includes("Teaching Aid")) return "embed";
|
|
7319
|
+
if (cpLen(teachingBody(content)) < 250) return "nav";
|
|
7320
|
+
return "prose";
|
|
7321
|
+
}
|
|
7322
|
+
/** A segment DOMINATED by a line-leading widget (with < NAV_MAX_CHARS of
|
|
7323
|
+
* teaching body before it) labels EVERY fragment — a char-sliced widget must
|
|
7324
|
+
* not leak as prose. */
|
|
7325
|
+
function segmentMarkerType(span) {
|
|
7326
|
+
for (const [re, label] of [[JSX_ASSESS, "assessment"], [JSX_EMBED, "embed"]]) {
|
|
7327
|
+
const m = re.exec(span);
|
|
7328
|
+
if (m !== null && cpLen(teachingBody(span.slice(0, m.index))) < 250) return label;
|
|
7329
|
+
}
|
|
7330
|
+
return null;
|
|
7331
|
+
}
|
|
7332
|
+
/** Walk lines; headings count only OUTSIDE fences; every line lands in exactly
|
|
7333
|
+
* one segment (byte-exact). H1 records a title but never enters the path. */
|
|
7334
|
+
function segmentText(text) {
|
|
7335
|
+
const segments = [];
|
|
7336
|
+
const titles = /* @__PURE__ */ new Map();
|
|
7337
|
+
const anchors = /* @__PURE__ */ new Map();
|
|
7338
|
+
let buf = [];
|
|
7339
|
+
let curPath = [];
|
|
7340
|
+
let curAnchor = null;
|
|
7341
|
+
let fence = null;
|
|
7342
|
+
const flush = () => {
|
|
7343
|
+
if (buf.length > 0) {
|
|
7344
|
+
segments.push({
|
|
7345
|
+
path: [...curPath],
|
|
7346
|
+
anchor: curAnchor,
|
|
7347
|
+
text: buf.join("")
|
|
7348
|
+
});
|
|
7349
|
+
buf = [];
|
|
7350
|
+
}
|
|
7377
7351
|
};
|
|
7352
|
+
for (const line of pySplitLines(text, true)) {
|
|
7353
|
+
const m = fence === null ? HEADING.exec(line) : null;
|
|
7354
|
+
if (m !== null) {
|
|
7355
|
+
flush();
|
|
7356
|
+
const level = m[1].length;
|
|
7357
|
+
const rawTitle = m[2];
|
|
7358
|
+
const idM = EXPLICIT_ID.exec(rawTitle);
|
|
7359
|
+
const title = idM !== null ? pyStrip(rawTitle.replace(EXPLICIT_ID, "")) : rawTitle;
|
|
7360
|
+
titles.set(level, title);
|
|
7361
|
+
anchors.set(level, idM !== null ? idM[1] : null);
|
|
7362
|
+
const deeper = [...titles.keys()].filter((lv) => lv > level);
|
|
7363
|
+
for (const lv of deeper) {
|
|
7364
|
+
titles.delete(lv);
|
|
7365
|
+
anchors.delete(lv);
|
|
7366
|
+
}
|
|
7367
|
+
curPath = [...titles.keys()].sort((a, b) => a - b).filter((lv) => lv >= 2 && lv <= level).map((lv) => titles.get(lv));
|
|
7368
|
+
curAnchor = curPath.length > 0 ? anchors.get(level) || slug(title) : null;
|
|
7369
|
+
}
|
|
7370
|
+
buf.push(line);
|
|
7371
|
+
if (m === null) fence = fenceStep(line, fence);
|
|
7372
|
+
}
|
|
7373
|
+
flush();
|
|
7374
|
+
return segments;
|
|
7378
7375
|
}
|
|
7379
|
-
/**
|
|
7380
|
-
*
|
|
7381
|
-
*
|
|
7382
|
-
*
|
|
7383
|
-
|
|
7384
|
-
|
|
7385
|
-
|
|
7386
|
-
|
|
7387
|
-
|
|
7388
|
-
|
|
7389
|
-
|
|
7390
|
-
|
|
7391
|
-
|
|
7392
|
-
|
|
7393
|
-
|
|
7394
|
-
|
|
7395
|
-
|
|
7396
|
-
|
|
7397
|
-
|
|
7398
|
-
|
|
7399
|
-
|
|
7400
|
-
|
|
7401
|
-
|
|
7402
|
-
]
|
|
7376
|
+
/** Split on blank-line runs, greedy-pack, but flush ONLY outside a fence — a
|
|
7377
|
+
* flush never lands between an open fence and its close. The separator is
|
|
7378
|
+
* CAPTURED, so blank-line runs ride along as parts and concatenation is
|
|
7379
|
+
* lossless; nothing is trimmed. */
|
|
7380
|
+
function subsplit(span, maxChars) {
|
|
7381
|
+
if (cpLen(span) <= maxChars) return [span];
|
|
7382
|
+
const parts = span.split(BLANK_SEP);
|
|
7383
|
+
const pieces = [];
|
|
7384
|
+
let buf = "";
|
|
7385
|
+
let bufLen = 0;
|
|
7386
|
+
let fence = null;
|
|
7387
|
+
for (const part of parts) {
|
|
7388
|
+
const partLen = cpLen(part);
|
|
7389
|
+
if (buf !== "" && fence === null && bufLen + partLen > maxChars) {
|
|
7390
|
+
pieces.push(buf);
|
|
7391
|
+
buf = "";
|
|
7392
|
+
bufLen = 0;
|
|
7393
|
+
}
|
|
7394
|
+
buf += part;
|
|
7395
|
+
bufLen += partLen;
|
|
7396
|
+
for (const line of pySplitLines(part, true)) fence = fenceStep(line, fence);
|
|
7397
|
+
}
|
|
7398
|
+
if (buf !== "") pieces.push(buf);
|
|
7399
|
+
const out = [];
|
|
7400
|
+
for (const piece of pieces) out.push(...enforceCeiling(piece));
|
|
7401
|
+
return out;
|
|
7403
7402
|
}
|
|
7404
|
-
/**
|
|
7405
|
-
*
|
|
7406
|
-
*
|
|
7407
|
-
|
|
7408
|
-
|
|
7409
|
-
|
|
7410
|
-
async function collectableGenerations(client, opts) {
|
|
7411
|
-
const ts = opts.now ?? /* @__PURE__ */ new Date();
|
|
7412
|
-
const pointer = await client.query("SELECT active_generation, rollback_generation FROM corpora WHERE tenant_id = $1 AND corpus_id = $2", [opts.tenantId, opts.corpusId]);
|
|
7413
|
-
if (pointer.rows.length === 0) return [];
|
|
7414
|
-
const active = Number(pointer.rows[0].active_generation);
|
|
7415
|
-
const rollbackRaw = pointer.rows[0].rollback_generation;
|
|
7416
|
-
const rollbackGen = rollbackRaw === null ? null : Number(rollbackRaw);
|
|
7417
|
-
const runs = await client.query("SELECT generation, state, finished_at, heartbeat_at FROM ingestion_runs WHERE tenant_id = $1 AND corpus_id = $2 AND state <> 'reaped' ORDER BY generation", [opts.tenantId, opts.corpusId]);
|
|
7418
|
-
const complete = runs.rows.filter((r) => [
|
|
7419
|
-
"ready",
|
|
7420
|
-
"active",
|
|
7421
|
-
"retired"
|
|
7422
|
-
].includes(String(r.state)));
|
|
7403
|
+
/** The ONLY place mid-line/mid-fence slicing can happen (a pathological single
|
|
7404
|
+
* paragraph or giant fence). Slices by CODE POINTS — 4000 UTF-16 units would
|
|
7405
|
+
* be a different policy and could split a surrogate pair. */
|
|
7406
|
+
function enforceCeiling(piece) {
|
|
7407
|
+
if (cpLen(piece) <= 4e3) return [piece];
|
|
7408
|
+
const cps = [...piece];
|
|
7423
7409
|
const out = [];
|
|
7424
|
-
let
|
|
7425
|
-
|
|
7426
|
-
|
|
7427
|
-
|
|
7428
|
-
|
|
7429
|
-
|
|
7430
|
-
|
|
7431
|
-
|
|
7410
|
+
for (let i = 0; i < cps.length; i += HARD_MAX_CHARS) out.push(cps.slice(i, i + HARD_MAX_CHARS).join(""));
|
|
7411
|
+
return out;
|
|
7412
|
+
}
|
|
7413
|
+
function chunkText(text, maxChars = MAX_CHARS) {
|
|
7414
|
+
const chunks = [];
|
|
7415
|
+
let prefix = "";
|
|
7416
|
+
const emit = (content, path, anchor, sourceType) => {
|
|
7417
|
+
chunks.push({
|
|
7418
|
+
ordinal: chunks.length,
|
|
7419
|
+
content,
|
|
7420
|
+
chunkHash: sha256(content),
|
|
7421
|
+
headingPath: [...path],
|
|
7422
|
+
anchor,
|
|
7423
|
+
sourceType
|
|
7424
|
+
});
|
|
7425
|
+
};
|
|
7426
|
+
for (const seg of segmentText(text)) {
|
|
7427
|
+
if (pyStrip(seg.text) === "") {
|
|
7428
|
+
if (chunks.length > 0) {
|
|
7429
|
+
const last = chunks[chunks.length - 1];
|
|
7430
|
+
const content = last.content + seg.text;
|
|
7431
|
+
chunks[chunks.length - 1] = {
|
|
7432
|
+
...last,
|
|
7433
|
+
content,
|
|
7434
|
+
chunkHash: sha256(content)
|
|
7435
|
+
};
|
|
7436
|
+
} else prefix += seg.text;
|
|
7432
7437
|
continue;
|
|
7433
7438
|
}
|
|
7434
|
-
|
|
7435
|
-
|
|
7436
|
-
|
|
7437
|
-
|
|
7438
|
-
|
|
7439
|
+
const segIsNav = cpLen(teachingBody(seg.text)) < 250;
|
|
7440
|
+
const segMarker = segmentMarkerType(seg.text);
|
|
7441
|
+
for (const piece of subsplit(seg.text, maxChars)) {
|
|
7442
|
+
let sourceType;
|
|
7443
|
+
if (segMarker !== null) sourceType = segMarker;
|
|
7444
|
+
else {
|
|
7445
|
+
sourceType = classify(piece, seg.path);
|
|
7446
|
+
if (sourceType === "nav" && !segIsNav) sourceType = "prose";
|
|
7447
|
+
}
|
|
7448
|
+
const content = prefix !== "" ? prefix + piece : piece;
|
|
7449
|
+
prefix = "";
|
|
7450
|
+
emit(content, seg.path, seg.anchor, sourceType);
|
|
7451
|
+
}
|
|
7439
7452
|
}
|
|
7440
|
-
|
|
7441
|
-
|
|
7442
|
-
/**
|
|
7443
|
-
* Delete one generation's rows (chunks cascade from sources) and mark the run
|
|
7444
|
-
* reaped. NEVER touches takedown_denylist or retrieval_log — the ledger and
|
|
7445
|
-
* denylist outlive the content they governed (§5).
|
|
7446
|
-
*/
|
|
7447
|
-
async function reap(client, opts) {
|
|
7448
|
-
for (const sql of [
|
|
7449
|
-
"DELETE FROM node_centroids WHERE tenant_id = $1 AND generation = $2",
|
|
7450
|
-
"DELETE FROM slug_aliases WHERE tenant_id = $1 AND generation = $2",
|
|
7451
|
-
"DELETE FROM sources WHERE tenant_id = $1 AND generation = $2"
|
|
7452
|
-
]) await client.query(sql, [opts.tenantId, opts.generation]);
|
|
7453
|
-
for (;;) if (!(await client.query("DELETE FROM content_nodes n WHERE n.tenant_id = $1 AND n.generation = $2 AND NOT EXISTS (SELECT 1 FROM content_nodes ch WHERE ch.parent_id = n.node_id AND ch.tenant_id = n.tenant_id AND ch.generation = n.generation)", [opts.tenantId, opts.generation])).rowCount) break;
|
|
7454
|
-
await client.query("UPDATE ingestion_runs SET state = 'reaped' WHERE tenant_id = $1 AND generation = $2", [opts.tenantId, opts.generation]);
|
|
7453
|
+
if (prefix !== "") emit(prefix, [], null, "nav");
|
|
7454
|
+
return chunks;
|
|
7455
7455
|
}
|
|
7456
7456
|
const FRONTMATTER = /^\uFEFF?---\r?\n([\s\S]*?)\r?\n---[ \t]*\r?\n?/;
|
|
7457
7457
|
function splitFrontmatter(text) {
|
|
@@ -8257,7 +8257,7 @@ async function ingestCommand(args) {
|
|
|
8257
8257
|
return await buildGeneration(pool, instance, {
|
|
8258
8258
|
knowledgeDir: values.knowledge,
|
|
8259
8259
|
sourceCommit,
|
|
8260
|
-
flip:
|
|
8260
|
+
flip: false,
|
|
8261
8261
|
provider,
|
|
8262
8262
|
onLog: (line) => process.stdout.write(line + "\n")
|
|
8263
8263
|
});
|
|
@@ -8274,8 +8274,16 @@ async function ingestCommand(args) {
|
|
|
8274
8274
|
process.stdout.write(`ingest: generation ${report.generation} — ${report.nodes} nodes, ${report.chunks} chunks; embedded ${report.embedded}, carried ${report.carried}, failed ${report.failed}\n`);
|
|
8275
8275
|
if (report.refusal !== null) return fail$1(REFUSED, report.refusal);
|
|
8276
8276
|
const governance = await withPool(dsn, (pool) => assertGovernanceServable(pool, instance, report.generation).then(() => null, (error) => error instanceof Error ? error.message : String(error)));
|
|
8277
|
-
if (governance !== null) return fail$1(REFUSED, `generation ${report.generation} was built
|
|
8278
|
-
if (
|
|
8277
|
+
if (governance !== null) return fail$1(REFUSED, `generation ${report.generation} was built and NOT activated — no surface could serve it\n ${governance.split("\n").join("\n ")}\n note: generation ${report.generation} is left behind, un-activated; \`ksor gc\` reaps it once the grace window passes. The previously active generation still serves.`);
|
|
8278
|
+
if (values.flip === true && !report.unchanged) {
|
|
8279
|
+
await withPool(dsn, (pool) => runIngest(pool, instance.tenantId, (client) => flip(client, {
|
|
8280
|
+
tenantId: instance.tenantId,
|
|
8281
|
+
corpusId: instance.corpusId,
|
|
8282
|
+
toGeneration: report.generation
|
|
8283
|
+
})));
|
|
8284
|
+
process.stdout.write(`FLIPPED active generation -> ${report.generation}\n`);
|
|
8285
|
+
}
|
|
8286
|
+
if (values.flip !== true) process.stdout.write("ready; flip withheld (pass --flip to activate)\n");
|
|
8279
8287
|
return 0;
|
|
8280
8288
|
}
|
|
8281
8289
|
/**
|