@stravica-ai/digital-operator-pa 0.1.8 → 0.1.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/engineering-pa-cli/src/knowledge/cli.mjs +4 -0
- package/dist/engineering-pa-cli/src/knowledge/graph.mjs +57 -4
- package/dist/engineering-pa-cli/src/knowledge/rebuild.mjs +118 -0
- package/dist/engineering-pa-core/docs/knowledge-scaffold.md +29 -2
- package/dist/engineering-pa-core/src/distill/prompt.mjs +26 -9
- package/dist/engineering-pa-core/src/distill/run.mjs +9 -3
- package/dist/engineering-pa-core/src/knowledge/retrieval/backlink-indexer.mjs +102 -93
- package/dist/engineering-pa-core/src/knowledge/retrieval/graph-builder.mjs +181 -107
- package/dist/engineering-pa-core/src/knowledge/retrieval/links.mjs +445 -0
- package/dist/engineering-pa-core/src/knowledge/retrieval/write-atomic.mjs +36 -0
- package/package.json +1 -1
|
@@ -63,6 +63,10 @@ Graph location: <repo-root>/operator/knowledge/.graph.json.
|
|
|
63
63
|
Repo-root resolution: \$DAVE_REPO_ROOT / \$PA_REPO_ROOT, else walk upward
|
|
64
64
|
from cwd looking for a .git entry.
|
|
65
65
|
|
|
66
|
+
The graph is a per-device artefact: untracked, and rebuilt locally by
|
|
67
|
+
library-sync-cycle's graph-rebuild leg. If it is absent, the first query
|
|
68
|
+
rebuilds it on demand (one stderr line, then the answer) rather than failing.
|
|
69
|
+
|
|
66
70
|
JSON output shape (stable-by-convention — consumed by dispatch-briefer):
|
|
67
71
|
{ "verb": "<name>", ...verb-specific fields }
|
|
68
72
|
|
|
@@ -31,12 +31,65 @@ const GRAPH_RELATIVE_PATH = 'operator/knowledge/.graph.json';
|
|
|
31
31
|
|
|
32
32
|
// --- loader ---------------------------------------------------------------
|
|
33
33
|
|
|
34
|
-
|
|
34
|
+
// SELF-HEAL ON A MISSING GRAPH (w-2026-07-29-007)
|
|
35
|
+
//
|
|
36
|
+
// Since the estate moved to per-device retrieval artefacts (w-2026-07-29-002),
|
|
37
|
+
// `.graph.json` is untracked and rebuilt locally by `library-sync-cycle`'s
|
|
38
|
+
// graph-rebuild leg. Any host that pulls the untracking commit before it runs a
|
|
39
|
+
// sync cycle -- an unmigrated laptop, a host that has been off during a move --
|
|
40
|
+
// therefore has a knowledge tree on disk and no graph over it, and every
|
|
41
|
+
// `pa knowledge` verb used to exit 3. Refusing to answer when the source data
|
|
42
|
+
// is right there is the wrong default for a retrieval tool.
|
|
43
|
+
//
|
|
44
|
+
// So a missing artefact triggers an on-demand build (same two builders, same
|
|
45
|
+
// order, same root as the scheduled leg) and the query is served. The artefact
|
|
46
|
+
// is a pure function of the tree, so this cannot produce a graph the scheduled
|
|
47
|
+
// leg would not have produced -- only a less well-stamped one, which the
|
|
48
|
+
// `coverage.builtBy` marker makes visible.
|
|
49
|
+
//
|
|
50
|
+
// GATE. Self-heal fires only when the caller passed a `repoRoot` and did NOT
|
|
51
|
+
// pass an explicit `graphPath`. An explicit `graphPath` means the caller is
|
|
52
|
+
// pointing at a specific artefact (a fixture, a snapshot, an inspection copy);
|
|
53
|
+
// the builders write to `<root>/operator/knowledge/.graph.json` and nowhere
|
|
54
|
+
// else, so building would silently answer a different question than the one
|
|
55
|
+
// asked. Those callers keep the exit-3 refusal.
|
|
56
|
+
//
|
|
57
|
+
// Set `autoBuild: false` to opt out entirely.
|
|
58
|
+
export async function loadGraph({ repoRoot, graphPath, autoBuild = true, stderr = process.stderr } = {}) {
|
|
35
59
|
const path = graphPath ?? join(repoRoot, GRAPH_RELATIVE_PATH);
|
|
36
60
|
if (!existsSync(path)) {
|
|
37
|
-
const
|
|
38
|
-
|
|
39
|
-
|
|
61
|
+
const canBuild = autoBuild && Boolean(repoRoot) && graphPath == null;
|
|
62
|
+
if (!canBuild) {
|
|
63
|
+
const err = new Error(`knowledge graph not found at ${path}`);
|
|
64
|
+
err.exitCode = 3;
|
|
65
|
+
throw err;
|
|
66
|
+
}
|
|
67
|
+
try {
|
|
68
|
+
const { rebuildRetrievalArtefacts } = await import('./rebuild.mjs');
|
|
69
|
+
const built = await rebuildRetrievalArtefacts({ root: repoRoot });
|
|
70
|
+
stderr.write(
|
|
71
|
+
`[pa knowledge] no graph at ${path} -- built on demand ` +
|
|
72
|
+
`(${built.nodeCount} nodes, ${built.edgeCount} edges, ${built.durationMs}ms); ` +
|
|
73
|
+
'retrieval artefacts are per-device and normally rebuilt by library-sync-cycle\n',
|
|
74
|
+
);
|
|
75
|
+
} catch (e) {
|
|
76
|
+
// Both errors, in one message: "graph missing" alone sends the reader
|
|
77
|
+
// looking for the artefact, when what they need to fix is why it could
|
|
78
|
+
// not be built.
|
|
79
|
+
const err = new Error(
|
|
80
|
+
`knowledge graph not found at ${path}, and the on-demand rebuild failed: ${e.message}`,
|
|
81
|
+
);
|
|
82
|
+
err.exitCode = 3;
|
|
83
|
+
err.cause = e;
|
|
84
|
+
throw err;
|
|
85
|
+
}
|
|
86
|
+
if (!existsSync(path)) {
|
|
87
|
+
const err = new Error(
|
|
88
|
+
`knowledge graph not found at ${path}, and the on-demand rebuild did not produce it`,
|
|
89
|
+
);
|
|
90
|
+
err.exitCode = 3;
|
|
91
|
+
throw err;
|
|
92
|
+
}
|
|
40
93
|
}
|
|
41
94
|
let raw;
|
|
42
95
|
try {
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
// rebuild.mjs — in-process rebuild of the retrieval artefacts.
|
|
2
|
+
//
|
|
3
|
+
// WHY THIS EXISTS
|
|
4
|
+
// ---------------
|
|
5
|
+
// As of 2026-07-29 (estate ruling w-2026-07-29-002) `operator/knowledge/.graph.json`
|
|
6
|
+
// and `.backlinks.json` are PER-DEVICE artefacts: untracked by the library
|
|
7
|
+
// repos, and rebuilt locally by `library-sync-cycle`'s `graph-rebuild` leg
|
|
8
|
+
// immediately after the down-path materialises the tree they describe. Each
|
|
9
|
+
// device's graph then describes exactly the tree that device can see, instead
|
|
10
|
+
// of one build host's partial view being shipped to everyone.
|
|
11
|
+
//
|
|
12
|
+
// The migration has a gap. A host that pulls the untracking commit but has not
|
|
13
|
+
// yet run a sync cycle -- or whose cycle is not installed at all -- ends up with
|
|
14
|
+
// no artefact on disk, and every `pa knowledge` verb hard-failed with exit 3
|
|
15
|
+
// ("knowledge graph not found"). The retrieval tool going dark is a bad failure
|
|
16
|
+
// for a host whose whole job is to answer questions from the library, and the
|
|
17
|
+
// data needed to fix it is sitting right there on disk.
|
|
18
|
+
//
|
|
19
|
+
// So `loadGraph` self-heals: on a missing artefact it calls this module, which
|
|
20
|
+
// runs the same two builders the graph-rebuild leg runs, in the same order,
|
|
21
|
+
// against the same workspace root -- then serves the query. The artefact is a
|
|
22
|
+
// pure function of the tree, so building it on demand cannot produce anything
|
|
23
|
+
// the scheduled leg would not have produced.
|
|
24
|
+
//
|
|
25
|
+
// IN-PROCESS, NOT SPAWNED
|
|
26
|
+
// -----------------------
|
|
27
|
+
// The builders ship in the same package as this module, so we import and call
|
|
28
|
+
// them rather than spawning `node <script> --root <root>`. That removes a
|
|
29
|
+
// process launch (~50 ms of the total) and, more importantly, removes the
|
|
30
|
+
// `--root`-resolution hazard entirely: a spawned builder without `--root`
|
|
31
|
+
// resolves its default four levels up from its own script path, which for a
|
|
32
|
+
// package install lands inside the versioned `dist/` tree rather than the
|
|
33
|
+
// workspace (the "versioned-dist-path trap"). An imported function takes the
|
|
34
|
+
// root as an argument and cannot get this wrong.
|
|
35
|
+
//
|
|
36
|
+
// PROVENANCE
|
|
37
|
+
// ----------
|
|
38
|
+
// Both artefacts carry a `coverage` stamp naming which path built them. The
|
|
39
|
+
// scheduled leg writes `builtBy: 'library-sync-cycle:graph-rebuild'` plus the
|
|
40
|
+
// device's entitlement picture; this path writes `builtBy: 'loadGraph:on-demand'`
|
|
41
|
+
// and honestly reports `complete: null`, because a CLI invocation has no
|
|
42
|
+
// registry context and therefore cannot know which knowledge mounts this device
|
|
43
|
+
// is not entitled to. An on-demand graph is a fallback, not a substitute for
|
|
44
|
+
// the stamped one -- and it says so in its own output.
|
|
45
|
+
|
|
46
|
+
import { buildBacklinkIndex } from '../../../engineering-pa-core/src/knowledge/retrieval/backlink-indexer.mjs';
|
|
47
|
+
import { buildKnowledgeGraph } from '../../../engineering-pa-core/src/knowledge/retrieval/graph-builder.mjs';
|
|
48
|
+
|
|
49
|
+
/** Marks artefacts produced by the self-heal path rather than the scheduled leg. */
|
|
50
|
+
export const ON_DEMAND_BUILT_BY = 'loadGraph:on-demand';
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* The coverage stamp this path writes.
|
|
54
|
+
*
|
|
55
|
+
* Shape-compatible with the one `library-sync-cycle:graph-rebuild` writes so a
|
|
56
|
+
* reader can branch on the same fields, but honest about what it does not know:
|
|
57
|
+
* `complete: null` means "this build could not determine its own coverage",
|
|
58
|
+
* which is different from `false` ("it has a named hole"). `registryError`
|
|
59
|
+
* carries the reason in the same slot the scheduled leg uses, so the estate's
|
|
60
|
+
* `formatCoverageSummary` renders a truthful "coverage UNKNOWN (...)" line
|
|
61
|
+
* without special-casing.
|
|
62
|
+
*
|
|
63
|
+
* `rebuildDurationMs` stays null on this path: the stamp has to exist before the
|
|
64
|
+
* artefacts are serialised, so the elapsed time is not known yet. The caller
|
|
65
|
+
* gets it as a return value for its stderr line instead of the artefact
|
|
66
|
+
* carrying a number it would have to be rewritten to hold.
|
|
67
|
+
*
|
|
68
|
+
* @param {{workspace:string, builtOn?:string}} args
|
|
69
|
+
* @returns {object}
|
|
70
|
+
*/
|
|
71
|
+
export function buildOnDemandCoverage({
|
|
72
|
+
workspace,
|
|
73
|
+
builtOn = new Date().toISOString(),
|
|
74
|
+
}) {
|
|
75
|
+
return {
|
|
76
|
+
builtOn,
|
|
77
|
+
deviceId: null,
|
|
78
|
+
builtBy: ON_DEMAND_BUILT_BY,
|
|
79
|
+
workspace,
|
|
80
|
+
sourceMarkdownCount: null,
|
|
81
|
+
rebuildDurationMs: null,
|
|
82
|
+
registryDir: null,
|
|
83
|
+
entitledRepos: [],
|
|
84
|
+
unentitledRepos: [],
|
|
85
|
+
unentitledKnowledgeMountTargets: [],
|
|
86
|
+
complete: null,
|
|
87
|
+
registryError: 'built on demand by pa knowledge -- no registry context, coverage not determined',
|
|
88
|
+
};
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Rebuild `.backlinks.json` then `.graph.json` over `root`.
|
|
93
|
+
*
|
|
94
|
+
* Order is a correctness property, not a preference: `graph-builder` reads the
|
|
95
|
+
* file `backlink-indexer` writes, so running them the other way round builds a
|
|
96
|
+
* graph from the previous run's backlinks.
|
|
97
|
+
*
|
|
98
|
+
* Throws whatever the builders throw (a missing `operator/knowledge/` yields
|
|
99
|
+
* ENOENT on the first write) -- the caller decides how loud to be.
|
|
100
|
+
*
|
|
101
|
+
* @param {{root:string, coverage?:object|null}} args
|
|
102
|
+
* @returns {Promise<{nodeCount:number, edgeCount:number, targetCount:number, durationMs:number, graphPath:string, backlinksPath:string}>}
|
|
103
|
+
*/
|
|
104
|
+
export async function rebuildRetrievalArtefacts({ root, coverage }) {
|
|
105
|
+
const started = Date.now();
|
|
106
|
+
const stamp = coverage ?? buildOnDemandCoverage({ workspace: root });
|
|
107
|
+
const backlinks = await buildBacklinkIndex({ root, coverage: stamp });
|
|
108
|
+
const graph = await buildKnowledgeGraph({ root, coverage: stamp });
|
|
109
|
+
const durationMs = Date.now() - started;
|
|
110
|
+
return {
|
|
111
|
+
nodeCount: graph.nodeCount,
|
|
112
|
+
edgeCount: graph.edgeCount,
|
|
113
|
+
targetCount: backlinks.targetCount,
|
|
114
|
+
durationMs,
|
|
115
|
+
graphPath: graph.outPath,
|
|
116
|
+
backlinksPath: backlinks.outPath,
|
|
117
|
+
};
|
|
118
|
+
}
|
|
@@ -56,7 +56,13 @@ Use double-bracket syntax to link to other documents:
|
|
|
56
56
|
See [[fermentation]] for details on lacto-fermentation.
|
|
57
57
|
```
|
|
58
58
|
|
|
59
|
-
Wiki-links are resolved
|
|
59
|
+
Wiki-links are resolved against the real file set, in this order:
|
|
60
|
+
|
|
61
|
+
1. A target containing a `/` (`[[../wespa/spi-no-staging]]`, `[[engineering/standards/secrets]]`) is tried relative to the linking document first, then repo-root-relative, then relative to each knowledge root. An over-deep `../` chain is salvaged when exactly one file can be meant.
|
|
62
|
+
2. A bare target (`[[fermentation]]`) matches on filename across all knowledge domains -- exact case first, case-insensitive second.
|
|
63
|
+
3. If several files share the name, the one in the linking document's own directory wins, then the one in its own domain. If neither disambiguates, the link is reported as ambiguous and produces no edge -- the builder does not guess, because a guess silently changes what `related`, `path` and `orphans` answer.
|
|
64
|
+
|
|
65
|
+
A wiki-link that resolves to nothing is reported as dangling and produces no edge. It is never turned into a node: a graph that invents a document for every broken reference inflates itself and corrupts its own orphan statistics.
|
|
60
66
|
|
|
61
67
|
### Standard markdown links
|
|
62
68
|
|
|
@@ -66,6 +72,26 @@ Use standard markdown links for explicit paths:
|
|
|
66
72
|
See [fermentation](techniques/fermentation.md) for details.
|
|
67
73
|
```
|
|
68
74
|
|
|
75
|
+
Markdown links resolve relative to the linking document. Unlike wiki-links, a markdown link to a file that is not present still produces a node -- that is deliberate, and is how a document the build host is not entitled to stays visible in the graph.
|
|
76
|
+
|
|
77
|
+
### Machine-readable hubs (`index.json`)
|
|
78
|
+
|
|
79
|
+
A directory can carry an `index.json` listing its documents with curated descriptions:
|
|
80
|
+
|
|
81
|
+
```json
|
|
82
|
+
{
|
|
83
|
+
"version": "1.0",
|
|
84
|
+
"patterns": [
|
|
85
|
+
{ "id": "some-pattern", "path": "knowledge/engineering/patterns/some-pattern.md",
|
|
86
|
+
"description": "What it is and when it bites" }
|
|
87
|
+
]
|
|
88
|
+
}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
Every `.md` path in such a file becomes an edge from the hub to the document, so a document listed in its domain's index is indexed for retrieval without also needing a prose link. Entry paths may be canonical-repo-relative (`knowledge/...`), repo-root-relative, or relative to the index file. A `path` naming a directory resolves to that directory's `README.md`. Hubs nest: each `index.json` is linked from the nearest `index.json` above it.
|
|
92
|
+
|
|
93
|
+
`index.json` files are hand-maintained source, not build output -- no builder writes them.
|
|
94
|
+
|
|
69
95
|
|
|
70
96
|
## Backlink indexing
|
|
71
97
|
|
|
@@ -89,7 +115,8 @@ The retrieval system builds a graph of document relationships based on links. Th
|
|
|
89
115
|
|
|
90
116
|
The retrieval tools at `core/knowledge/retrieval/` provide:
|
|
91
117
|
|
|
92
|
-
- **index-builder.mjs** -- Rebuilds
|
|
118
|
+
- **index-builder.mjs** -- Rebuilds the domain registry (`operator/knowledge/domains.json`)
|
|
119
|
+
- **backlink-indexer.mjs** / **graph-builder.mjs** -- Rebuild the per-device retrieval artefacts (`.backlinks.json`, `.graph.json`). These two are untracked and device-local: each host builds its own over the tree it can actually see, and `pa knowledge` rebuilds them on demand if they are missing. Do not commit them.
|
|
93
120
|
- **search.mjs** -- Full-text search across all knowledge
|
|
94
121
|
- **graph.mjs** -- Graph queries (neighbours, paths, clusters)
|
|
95
122
|
|
|
@@ -14,6 +14,11 @@ const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
|
14
14
|
// Absolute paths to the bundled retrieval tools. From this module
|
|
15
15
|
// (`packages/core/src/distill/prompt.mjs`) the retrieval tools live at
|
|
16
16
|
// `../knowledge/retrieval/`.
|
|
17
|
+
//
|
|
18
|
+
// Only the index builder is named in the prompt. The backlink indexer and the
|
|
19
|
+
// graph builder are still exported here -- they are part of this module's
|
|
20
|
+
// public surface and callers outside the sweep use them -- but a distillation
|
|
21
|
+
// sweep must NOT run them; see step 4 for why.
|
|
17
22
|
export const RETRIEVAL_INDEX_BUILDER = resolve(__dirname, '..', 'knowledge', 'retrieval', 'index-builder.mjs');
|
|
18
23
|
export const RETRIEVAL_BACKLINK_INDEXER = resolve(__dirname, '..', 'knowledge', 'retrieval', 'backlink-indexer.mjs');
|
|
19
24
|
export const RETRIEVAL_GRAPH_BUILDER = resolve(__dirname, '..', 'knowledge', 'retrieval', 'graph-builder.mjs');
|
|
@@ -22,11 +27,19 @@ export const RETRIEVAL_GRAPH_BUILDER = resolve(__dirname, '..', 'knowledge', 're
|
|
|
22
27
|
* Build the distillation prompt text. Absolute paths are interpolated for the
|
|
23
28
|
* retrieval-tool invocations so Claude doesn't have to guess at install
|
|
24
29
|
* locations.
|
|
30
|
+
*
|
|
31
|
+
* `workspaceRoot` is interpolated as the builder's `--root`, and it is
|
|
32
|
+
* load-bearing. Without `--root` the builder falls back to `$DAVE_ROOT`, then to
|
|
33
|
+
* a path resolved four levels up from its own script -- which for a
|
|
34
|
+
* package-installed platform lands inside the versioned `dist/` tree, so the
|
|
35
|
+
* sweep writes its index INTO THE PACKAGE instead of the workspace (the
|
|
36
|
+
* "versioned-dist-path trap"). `runDistill` passes the repo root it has already
|
|
37
|
+
* resolved from prereqs; the default is this process's cwd, which is correct for
|
|
38
|
+
* the cron/bin invocations that construct the prompt with no arguments.
|
|
25
39
|
*/
|
|
26
40
|
export function buildDistillPrompt({
|
|
27
41
|
indexBuilder = RETRIEVAL_INDEX_BUILDER,
|
|
28
|
-
|
|
29
|
-
graphBuilder = RETRIEVAL_GRAPH_BUILDER,
|
|
42
|
+
workspaceRoot = process.cwd(),
|
|
30
43
|
} = {}) {
|
|
31
44
|
return `You are operating as the Librarian. Run a full distillation sweep now.
|
|
32
45
|
|
|
@@ -41,10 +54,12 @@ export function buildDistillPrompt({
|
|
|
41
54
|
|
|
42
55
|
3. **Synthesise the operator profile narrative.** After updating operator/profile/operator.json, regenerate operator/profile/profile.md as a human-readable markdown narrative derived from the JSON plus the body of accumulated observations. This is the file the workspace and PA read first; it must always exist and reflect current state. Replace it in full each run — it is a derived artefact, not an append log.
|
|
43
56
|
|
|
44
|
-
4. **Rebuild
|
|
45
|
-
- node ${indexBuilder}
|
|
46
|
-
|
|
47
|
-
|
|
57
|
+
4. **Rebuild the domain index.** After all content changes, run exactly this one command:
|
|
58
|
+
- node ${indexBuilder} --root ${workspaceRoot}
|
|
59
|
+
|
|
60
|
+
The \`--root\` argument is required: without it the builder resolves a default root inside the installed package and writes the index there instead of into the workspace.
|
|
61
|
+
|
|
62
|
+
Do NOT rebuild, and do NOT commit, \`operator/knowledge/.backlinks.json\` or \`operator/knowledge/.graph.json\`. Since w-2026-07-29-002 those two artefacts are per-device and untracked -- every host rebuilds its own from the tree it can actually see, via library-sync-cycle's graph-rebuild leg -- so a sweep that rebuilds them commits one host's partial view of the library over everybody else's.
|
|
48
63
|
|
|
49
64
|
5. **Library hygiene check.** Per your role definition — verify JSON camelCase, cross-reference integrity, index completeness, and lastUpdated freshness. Fix violations in place.
|
|
50
65
|
|
|
@@ -54,9 +69,11 @@ export function buildDistillPrompt({
|
|
|
54
69
|
}
|
|
55
70
|
|
|
56
71
|
/**
|
|
57
|
-
* Back-compat constant: the prompt assembled with default (bundled) paths
|
|
58
|
-
*
|
|
59
|
-
* should use buildDistillPrompt() with
|
|
72
|
+
* Back-compat constant: the prompt assembled with default (bundled) paths and
|
|
73
|
+
* this process's cwd as the workspace root. Existing consumers that import
|
|
74
|
+
* DISTILL_PROMPT keep working; new callers should use buildDistillPrompt() with
|
|
75
|
+
* an explicit `workspaceRoot`, as `runDistill` does -- the resolved repo root is
|
|
76
|
+
* more trustworthy than cwd at module-load time.
|
|
60
77
|
*/
|
|
61
78
|
export const DISTILL_PROMPT = buildDistillPrompt();
|
|
62
79
|
|
|
@@ -37,7 +37,7 @@ import { basename, join } from 'node:path';
|
|
|
37
37
|
|
|
38
38
|
import { resolvePrereqs } from './prereqs.mjs';
|
|
39
39
|
import { countStagedObservations } from './staging.mjs';
|
|
40
|
-
import {
|
|
40
|
+
import { buildDistillPrompt, claudeArgs } from './prompt.mjs';
|
|
41
41
|
|
|
42
42
|
function log(line, { stderr = process.stderr } = {}) {
|
|
43
43
|
stderr.write(`[distill] ${line}\n`);
|
|
@@ -116,6 +116,12 @@ export async function runDistill({
|
|
|
116
116
|
}
|
|
117
117
|
log(`found ${stagingCount} staged observation(s)`, { stderr });
|
|
118
118
|
|
|
119
|
+
// Assemble the sweep prompt against THIS host's resolved repo root. The
|
|
120
|
+
// index-builder invocation in step 4 carries it as `--root`; the module-level
|
|
121
|
+
// DISTILL_PROMPT default (cwd) is a fallback for callers that have not
|
|
122
|
+
// resolved a root, and a cron sweep always has.
|
|
123
|
+
const prompt = buildDistillPrompt({ workspaceRoot: repoRoot });
|
|
124
|
+
|
|
119
125
|
// Step 3 — assemble & invoke.
|
|
120
126
|
//
|
|
121
127
|
// Two timestamp shapes for two jobs:
|
|
@@ -165,7 +171,7 @@ export async function runDistill({
|
|
|
165
171
|
log(` appended system: ${librarianDef} (${librarianBase.length} bytes)`, { stderr });
|
|
166
172
|
log(` operator supplement: ${supplementPlanLine}`, { stderr });
|
|
167
173
|
log(` composed total: ${librarianIdentity.length} bytes`, { stderr });
|
|
168
|
-
log(` prompt: <inline> (${
|
|
174
|
+
log(` prompt: <inline> (${prompt.length} bytes, --root ${repoRoot})`, { stderr });
|
|
169
175
|
log(' command: claude --print --permission-mode bypassPermissions \\', { stderr });
|
|
170
176
|
log(` --no-session-persistence --name ${sessionName} \\`, { stderr });
|
|
171
177
|
log(' --append-system-prompt <librarian definition> \\', { stderr });
|
|
@@ -177,7 +183,7 @@ export async function runDistill({
|
|
|
177
183
|
log(`starting distillation sweep — log: ${runLogPath}`, { stderr });
|
|
178
184
|
const start = Date.now();
|
|
179
185
|
|
|
180
|
-
const claudeArgv = claudeArgs({ sessionName, librarianDef: librarianIdentity });
|
|
186
|
+
const claudeArgv = claudeArgs({ sessionName, librarianDef: librarianIdentity, prompt });
|
|
181
187
|
const claudeResult = await runner.spawn('claude', claudeArgv, { cwd: repoRoot });
|
|
182
188
|
|
|
183
189
|
// Persist claude stdout/stderr to the per-run log file.
|
|
@@ -7,10 +7,28 @@
|
|
|
7
7
|
// Output: operator/knowledge/.backlinks.json
|
|
8
8
|
//
|
|
9
9
|
// CLI: node backlink-indexer.mjs [--root /path/to/repo]
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
10
|
+
//
|
|
11
|
+
// Extraction and resolution live in ./links.mjs, shared with graph-builder.mjs
|
|
12
|
+
// — graph-builder reverses every entry in this file into a graph edge, so the
|
|
13
|
+
// two must agree on what a link points at. See that file for the contract.
|
|
14
|
+
//
|
|
15
|
+
// index.json hub references are deliberately NOT indexed here: this file's
|
|
16
|
+
// contract is "which markdown documents link to this target". Hub ingestion is
|
|
17
|
+
// graph-builder's job.
|
|
18
|
+
//
|
|
19
|
+
// LIBRARY + CLI. The work lives in the exported `buildBacklinkIndex()` so an
|
|
20
|
+
// in-process caller can rebuild the artefact without paying for a child
|
|
21
|
+
// process -- `pa knowledge`'s on-demand self-heal is the live consumer. The
|
|
22
|
+
// script keeps its CLI face unchanged (argv parsing, console reporting,
|
|
23
|
+
// `process.exit`), now behind a directly-invoked guard so importing the module
|
|
24
|
+
// does not run a sweep and exit the host process.
|
|
25
|
+
|
|
26
|
+
import { readFile } from 'node:fs/promises';
|
|
27
|
+
import { realpathSync } from 'node:fs';
|
|
28
|
+
import { join, resolve, dirname, relative } from 'node:path';
|
|
13
29
|
import { fileURLToPath } from 'node:url';
|
|
30
|
+
import { buildFileIndex, extractLinks, scanDirsFor, toPosix, walkFiles } from './links.mjs';
|
|
31
|
+
import { writeJsonAtomic } from './write-atomic.mjs';
|
|
14
32
|
|
|
15
33
|
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
16
34
|
const DEFAULT_ROOT = resolve(__dirname, '../../../..');
|
|
@@ -23,77 +41,79 @@ Usage:
|
|
|
23
41
|
|
|
24
42
|
Options:
|
|
25
43
|
--root <path> Repository root (default: 4 levels up from this script)
|
|
44
|
+
--quiet Suppress the unresolved-link report
|
|
26
45
|
--help, -h Show this help
|
|
27
46
|
|
|
28
47
|
Scans all .md files under operator/knowledge/ and core/knowledge/.
|
|
29
48
|
Finds:
|
|
30
|
-
- Wiki-style links: [[target]]
|
|
49
|
+
- Wiki-style links: [[target]] (resolved against the real file set)
|
|
31
50
|
- Markdown links: [text](relative/path.md)
|
|
32
51
|
|
|
33
52
|
Output: operator/knowledge/.backlinks.json
|
|
34
53
|
`.trim();
|
|
35
54
|
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
55
|
+
/**
|
|
56
|
+
* Build the backlink index over `root` and write
|
|
57
|
+
* `operator/knowledge/.backlinks.json`.
|
|
58
|
+
*
|
|
59
|
+
* @param {object} args
|
|
60
|
+
* @param {string} args.root workspace root the knowledge tree hangs off
|
|
61
|
+
* @param {object|null} [args.coverage] optional provenance stamp merged into
|
|
62
|
+
* the artefact as `coverage`. The estate's `library-sync-cycle:graph-rebuild`
|
|
63
|
+
* leg writes a device/entitlement stamp; `pa knowledge`'s on-demand build
|
|
64
|
+
* writes a minimal one. Readers treat it as optional either way.
|
|
65
|
+
* @returns {Promise<{outPath:string, targetCount:number, unresolved:object[]}>}
|
|
66
|
+
*/
|
|
67
|
+
export async function buildBacklinkIndex({ root, coverage = null }) {
|
|
68
|
+
// Pass 1 — enumerate. `[[slug]]` cannot be resolved without knowing which
|
|
69
|
+
// files exist, so the whole tree is collected before any link is read.
|
|
70
|
+
const allFiles = [];
|
|
71
|
+
for (const dir of scanDirsFor(root)) {
|
|
72
|
+
allFiles.push(...await walkFiles(dir));
|
|
46
73
|
}
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
74
|
+
const fileIds = allFiles.map((f) => toPosix(relative(root, f)));
|
|
75
|
+
const fileIndex = buildFileIndex(fileIds);
|
|
76
|
+
|
|
77
|
+
// Pass 2 — build backlinks map: target -> [sources]
|
|
78
|
+
const backlinks = {};
|
|
79
|
+
const unresolved = [];
|
|
80
|
+
|
|
81
|
+
for (let i = 0; i < allFiles.length; i++) {
|
|
82
|
+
const content = await readFile(allFiles[i], 'utf8');
|
|
83
|
+
const relSource = fileIds[i];
|
|
84
|
+
const result = extractLinks(content, relSource, fileIndex);
|
|
85
|
+
unresolved.push(...result.unresolved);
|
|
86
|
+
|
|
87
|
+
for (const target of result.links) {
|
|
88
|
+
if (!backlinks[target]) backlinks[target] = [];
|
|
89
|
+
if (!backlinks[target].includes(relSource)) {
|
|
90
|
+
backlinks[target].push(relSource);
|
|
91
|
+
}
|
|
54
92
|
}
|
|
55
93
|
}
|
|
56
|
-
return results;
|
|
57
|
-
}
|
|
58
94
|
|
|
59
|
-
|
|
60
|
-
const
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
// Wiki-style: [[target]]
|
|
64
|
-
const wikiRe = /\[\[([^\]]+)\]\]/g;
|
|
65
|
-
let m;
|
|
66
|
-
while ((m = wikiRe.exec(content)) !== null) {
|
|
67
|
-
let target = m[1].trim();
|
|
68
|
-
// Resolve as relative to repo root
|
|
69
|
-
if (!target.endsWith('.md')) target += '.md';
|
|
70
|
-
const resolved = normalize(target);
|
|
71
|
-
links.push(resolved);
|
|
95
|
+
// Sort keys and source arrays for deterministic output.
|
|
96
|
+
const sortedBacklinks = {};
|
|
97
|
+
for (const target of Object.keys(backlinks).sort()) {
|
|
98
|
+
sortedBacklinks[target] = backlinks[target].sort();
|
|
72
99
|
}
|
|
73
100
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
const path = href.split('#')[0];
|
|
82
|
-
if (!path) continue;
|
|
83
|
-
if (!path.endsWith('.md')) continue;
|
|
84
|
-
// Resolve relative to source file's directory
|
|
85
|
-
const abs = resolve(sourceDir, path);
|
|
86
|
-
const rel = relative(repoRoot, abs);
|
|
87
|
-
links.push(normalize(rel));
|
|
88
|
-
}
|
|
101
|
+
const output = {
|
|
102
|
+
version: '1.0',
|
|
103
|
+
generatedAt: new Date().toISOString(),
|
|
104
|
+
unresolvedLinkCount: unresolved.length,
|
|
105
|
+
backlinks: sortedBacklinks,
|
|
106
|
+
};
|
|
107
|
+
if (coverage) output.coverage = coverage;
|
|
89
108
|
|
|
90
|
-
|
|
109
|
+
const outPath = join(root, 'operator', 'knowledge', '.backlinks.json');
|
|
110
|
+
await writeJsonAtomic(outPath, output);
|
|
111
|
+
return { outPath, targetCount: Object.keys(sortedBacklinks).length, unresolved };
|
|
91
112
|
}
|
|
92
113
|
|
|
93
|
-
// ── Main ────────────────────────────────────────────────────────────
|
|
94
|
-
|
|
95
114
|
async function main(args) {
|
|
96
115
|
let root = DEFAULT_ROOT;
|
|
116
|
+
let quiet = false;
|
|
97
117
|
|
|
98
118
|
for (let i = 0; i < args.length; i++) {
|
|
99
119
|
const a = args[i];
|
|
@@ -101,54 +121,43 @@ async function main(args) {
|
|
|
101
121
|
console.log(USAGE);
|
|
102
122
|
return 0;
|
|
103
123
|
}
|
|
124
|
+
if (a === '--quiet') quiet = true;
|
|
104
125
|
if (a === '--root') {
|
|
105
126
|
root = resolve(args[++i]);
|
|
106
127
|
}
|
|
107
128
|
}
|
|
108
129
|
|
|
109
|
-
const
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
const backlinks = {};
|
|
122
|
-
|
|
123
|
-
for (const filePath of allFiles) {
|
|
124
|
-
const content = await readFile(filePath, 'utf8');
|
|
125
|
-
const relSource = relative(root, filePath);
|
|
126
|
-
const links = extractLinks(content, filePath, root);
|
|
127
|
-
|
|
128
|
-
for (const target of links) {
|
|
129
|
-
if (!backlinks[target]) backlinks[target] = [];
|
|
130
|
-
if (!backlinks[target].includes(relSource)) {
|
|
131
|
-
backlinks[target].push(relSource);
|
|
130
|
+
const { outPath, targetCount, unresolved } = await buildBacklinkIndex({ root });
|
|
131
|
+
|
|
132
|
+
console.log(`Wrote ${outPath} (${targetCount} targets indexed)`);
|
|
133
|
+
if (unresolved.length > 0) {
|
|
134
|
+
const byReason = new Map();
|
|
135
|
+
for (const e of unresolved) byReason.set(e.reason, (byReason.get(e.reason) ?? 0) + 1);
|
|
136
|
+
const summary = [...byReason.entries()].map(([r, n]) => `${r}=${n}`).join(' ');
|
|
137
|
+
console.warn(`${unresolved.length} wiki links unresolved (${summary})`);
|
|
138
|
+
if (!quiet) {
|
|
139
|
+
for (const e of unresolved) {
|
|
140
|
+
const extra = e.candidates ? ` [candidates: ${e.candidates.join(', ')}]` : '';
|
|
141
|
+
console.warn(` ${e.reason}: ${e.source} -> ${e.target}${extra}`);
|
|
132
142
|
}
|
|
133
143
|
}
|
|
134
144
|
}
|
|
145
|
+
return 0;
|
|
146
|
+
}
|
|
135
147
|
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
148
|
+
// Realpath both sides: under pnpm, argv[1] is frequently a symlink into a
|
|
149
|
+
// store path, and a raw string comparison silently no-ops. Same shape as
|
|
150
|
+
// core/src/scheduler/*.mjs.
|
|
151
|
+
function isDirectlyInvoked() {
|
|
152
|
+
if (!process.argv[1]) return false;
|
|
153
|
+
try {
|
|
154
|
+
return fileURLToPath(import.meta.url) === realpathSync(resolve(process.argv[1]));
|
|
155
|
+
} catch {
|
|
156
|
+
return false;
|
|
139
157
|
}
|
|
140
|
-
|
|
141
|
-
const output = {
|
|
142
|
-
version: '1.0',
|
|
143
|
-
generatedAt: new Date().toISOString(),
|
|
144
|
-
backlinks,
|
|
145
|
-
};
|
|
146
|
-
|
|
147
|
-
const outPath = join(root, 'operator', 'knowledge', '.backlinks.json');
|
|
148
|
-
await writeFile(outPath, JSON.stringify(output, null, 2) + '\n', 'utf8');
|
|
149
|
-
console.log(`Wrote ${outPath} (${Object.keys(backlinks).length} targets indexed)`);
|
|
150
|
-
return 0;
|
|
151
158
|
}
|
|
152
159
|
|
|
153
|
-
|
|
154
|
-
process.
|
|
160
|
+
if (isDirectlyInvoked()) {
|
|
161
|
+
const exitCode = await main(process.argv.slice(2));
|
|
162
|
+
process.exit(exitCode ?? 0);
|
|
163
|
+
}
|