melchizedek-agents 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENT_SETUP.md +182 -0
- package/DOCUMENTATION.md +614 -0
- package/LICENSE +21 -0
- package/QUICKSTART.md +164 -0
- package/README.md +151 -0
- package/config/agents/README.md +27 -0
- package/config/agents/examples/ares.yaml +74 -0
- package/config/agents/examples/cartographers.yaml +115 -0
- package/config/agents/examples/claude.yaml +17 -0
- package/config/agents/examples/council.yaml +65 -0
- package/config/agents/examples/critic.yaml +74 -0
- package/config/agents/examples/delegation.yaml +29 -0
- package/config/agents/examples/hierarchical.yaml +26 -0
- package/config/agents/examples/image_production.yaml +112 -0
- package/config/agents/examples/librarian.yaml +75 -0
- package/config/agents/examples/model_zoo.yaml +81 -0
- package/config/agents/examples/patient_advocate.yaml +107 -0
- package/config/agents/examples/scriptorium.yaml +96 -0
- package/config/agents/examples/style_council.yaml +83 -0
- package/config/agents/examples/syndicate.yaml +34 -0
- package/config/agents/examples/tutor.yaml +66 -0
- package/config/agents/syndicateSchema.yaml +291 -0
- package/dist/lib/config.d.ts +108 -0
- package/dist/lib/config.js +110 -0
- package/dist/lib/dispatch.d.ts +133 -0
- package/dist/lib/dispatch.js +182 -0
- package/dist/lib/grounding.d.ts +28 -0
- package/dist/lib/grounding.js +61 -0
- package/dist/lib/index.d.ts +14 -0
- package/dist/lib/index.js +13 -0
- package/dist/lib/loadEnv.d.ts +21 -0
- package/dist/lib/loadEnv.js +58 -0
- package/dist/lib/loadSyndicate.d.ts +177 -0
- package/dist/lib/loadSyndicate.js +264 -0
- package/dist/lib/memory/supabaseMemoryService.d.ts +118 -0
- package/dist/lib/memory/supabaseMemoryService.js +499 -0
- package/dist/lib/models/claudeLlm.d.ts +93 -0
- package/dist/lib/models/claudeLlm.js +332 -0
- package/dist/lib/models/gptLlm.d.ts +110 -0
- package/dist/lib/models/gptLlm.js +497 -0
- package/dist/lib/models/grokLlm.d.ts +64 -0
- package/dist/lib/models/grokLlm.js +86 -0
- package/dist/lib/models/ollamaLlm.d.ts +105 -0
- package/dist/lib/models/ollamaLlm.js +134 -0
- package/dist/lib/models/openAiCompatibleLlm.d.ts +125 -0
- package/dist/lib/models/openAiCompatibleLlm.js +331 -0
- package/dist/lib/models/providerMap.d.ts +28 -0
- package/dist/lib/models/providerMap.js +45 -0
- package/dist/lib/models/registry.d.ts +71 -0
- package/dist/lib/models/registry.js +159 -0
- package/dist/lib/models/schemaNormalize.d.ts +22 -0
- package/dist/lib/models/schemaNormalize.js +58 -0
- package/dist/lib/observability/supabaseSpanExporter.d.ts +35 -0
- package/dist/lib/observability/supabaseSpanExporter.js +115 -0
- package/dist/lib/observability/tracer.d.ts +44 -0
- package/dist/lib/observability/tracer.js +242 -0
- package/dist/lib/persistence/supabaseProvider.d.ts +44 -0
- package/dist/lib/persistence/supabaseProvider.js +88 -0
- package/dist/lib/session/supabaseSessionService.d.ts +26 -0
- package/dist/lib/session/supabaseSessionService.js +193 -0
- package/dist/lib/session/transcript.d.ts +193 -0
- package/dist/lib/session/transcript.js +309 -0
- package/dist/lib/toolRegistry.d.ts +15 -0
- package/dist/lib/toolRegistry.js +58 -0
- package/dist/lib/tools/collectionsSearchTool.d.ts +55 -0
- package/dist/lib/tools/collectionsSearchTool.js +86 -0
- package/dist/lib/tools/generateImageTool.d.ts +16 -0
- package/dist/lib/tools/generateImageTool.js +119 -0
- package/dist/lib/tools/inspectImageTool.d.ts +21 -0
- package/dist/lib/tools/inspectImageTool.js +97 -0
- package/dist/lib/tools/mcpToolFactory.d.ts +2 -0
- package/dist/lib/tools/mcpToolFactory.js +116 -0
- package/dist/lib/tools/toolContract.d.ts +78 -0
- package/dist/lib/tools/toolContract.js +129 -0
- package/dist/lib/tools/webExtractTool.d.ts +70 -0
- package/dist/lib/tools/webExtractTool.js +461 -0
- package/dist/lib/tools/webSearchTool.d.ts +53 -0
- package/dist/lib/tools/webSearchTool.js +126 -0
- package/dist/lib/tools/wikiTools.d.ts +112 -0
- package/dist/lib/tools/wikiTools.js +697 -0
- package/dist/lib/tools/xSearchTool.d.ts +61 -0
- package/dist/lib/tools/xSearchTool.js +125 -0
- package/dist/lib/wiki/agentRun.d.ts +39 -0
- package/dist/lib/wiki/agentRun.js +79 -0
- package/dist/lib/wiki/builder.d.ts +71 -0
- package/dist/lib/wiki/builder.js +165 -0
- package/dist/lib/wiki/dive.d.ts +56 -0
- package/dist/lib/wiki/dive.js +208 -0
- package/dist/lib/wiki/entities.d.ts +220 -0
- package/dist/lib/wiki/entities.js +661 -0
- package/dist/lib/wiki/extract.d.ts +97 -0
- package/dist/lib/wiki/extract.js +305 -0
- package/dist/lib/wiki/fill.d.ts +37 -0
- package/dist/lib/wiki/fill.js +175 -0
- package/dist/lib/wiki/format.d.ts +132 -0
- package/dist/lib/wiki/format.js +177 -0
- package/dist/lib/wiki/graph.d.ts +80 -0
- package/dist/lib/wiki/graph.js +147 -0
- package/dist/lib/wiki/lint.d.ts +58 -0
- package/dist/lib/wiki/lint.js +267 -0
- package/dist/lib/wiki/markdown.d.ts +127 -0
- package/dist/lib/wiki/markdown.js +336 -0
- package/dist/lib/wiki/pipeline.d.ts +69 -0
- package/dist/lib/wiki/pipeline.js +119 -0
- package/dist/lib/wiki/vault.d.ts +84 -0
- package/dist/lib/wiki/vault.js +194 -0
- package/dist/scripts/a2a_server.d.ts +2 -0
- package/dist/scripts/a2a_server.js +997 -0
- package/dist/scripts/syndicate_chat.d.ts +2 -0
- package/dist/scripts/syndicate_chat.js +485 -0
- package/package.json +96 -0
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/wiki/extract.ts — zero-dependency scanners that read structure out of source.
|
|
3
|
+
*
|
|
4
|
+
* WHY this file exists:
|
|
5
|
+
* The entity graph (lib/wiki/entities.ts) is only worth having if its
|
|
6
|
+
* extracted tier is cheap and honest. These are the readers that make it
|
|
7
|
+
* so: import edges, environment reads, table definitions and mentions,
|
|
8
|
+
* pulled straight out of file text with regexes and no parser
|
|
9
|
+
* dependency — the same judgment as the markdown engine
|
|
10
|
+
* (wiki/decisions/0002): at this scale a careful scanner beats a
|
|
11
|
+
* toolchain, and rebuild-from-source beats a cached index.
|
|
12
|
+
*
|
|
13
|
+
* They are deliberately CONSERVATIVE. A regex cannot see a dynamic
|
|
14
|
+
* import built from a variable, or a table name assembled at runtime;
|
|
15
|
+
* what it does report, it read verbatim. Nothing here guesses, so an
|
|
16
|
+
* extracted edge always has a literal line of source behind it.
|
|
17
|
+
*
|
|
18
|
+
* Bundle-agnostic and repo-agnostic: callers hand in text and paths.
|
|
19
|
+
*/
|
|
20
|
+
export interface WalkOptions {
|
|
21
|
+
/** Only files whose name ends with one of these. */
|
|
22
|
+
extensions?: string[];
|
|
23
|
+
/** Directory names skipped wherever they appear. */
|
|
24
|
+
skipDirs?: string[];
|
|
25
|
+
}
|
|
26
|
+
/** Repo-relative POSIX paths under `dir`, sorted, dotfiles skipped. */
|
|
27
|
+
export declare function walkFiles(root: string, dir: string, options?: WalkOptions): string[];
|
|
28
|
+
export interface ModuleScan {
|
|
29
|
+
/** Import specifiers exactly as written. */
|
|
30
|
+
specifiers: string[];
|
|
31
|
+
/** Repo-relative paths of the local modules imported. */
|
|
32
|
+
localImports: string[];
|
|
33
|
+
/** Bare package names imported (first path segment, scope-aware). */
|
|
34
|
+
packages: string[];
|
|
35
|
+
/** Environment variables the module reads. */
|
|
36
|
+
envVars: string[];
|
|
37
|
+
}
|
|
38
|
+
/** Resolve a relative specifier against the importing file's repo path. */
|
|
39
|
+
export declare function resolveLocalImport(fromRelPath: string, specifier: string): string | null;
|
|
40
|
+
export declare function scanModule(text: string, relPath: string): ModuleScan;
|
|
41
|
+
export interface SqlScan {
|
|
42
|
+
/** Tables this DDL creates. */
|
|
43
|
+
defined: string[];
|
|
44
|
+
/** Tables it alters or references without creating. */
|
|
45
|
+
referenced: string[];
|
|
46
|
+
}
|
|
47
|
+
export declare function scanSql(sql: string): SqlScan;
|
|
48
|
+
/**
|
|
49
|
+
* Which known tables a source file ADDRESSES — matched only as a quoted
|
|
50
|
+
* literal, which is how every client library names a table:
|
|
51
|
+
* `.from(<quoted name>)`. A bare word is deliberately not enough, because
|
|
52
|
+
* prose in a comment mentions table names constantly, and an edge saying
|
|
53
|
+
* "this module touches that table" must mean it.
|
|
54
|
+
*/
|
|
55
|
+
export declare function tableMentions(text: string, tables: readonly string[]): string[];
|
|
56
|
+
export interface ScriptScan {
|
|
57
|
+
name: string;
|
|
58
|
+
command: string;
|
|
59
|
+
/** Repo-relative source file the script executes, when one is named. */
|
|
60
|
+
entry: string | null;
|
|
61
|
+
/** Long-form flag values, e.g. { syndicate: 'critic' }. */
|
|
62
|
+
flags: Record<string, string>;
|
|
63
|
+
}
|
|
64
|
+
export declare function scanNpmScripts(scripts: Record<string, string>): ScriptScan[];
|
|
65
|
+
/**
|
|
66
|
+
* Keys of a named top-level object literal, read from source text —
|
|
67
|
+
* `const TOOL_MAP: Record<string, unknown> = { web_search: …, … }` yields
|
|
68
|
+
* ['web_search', …]. Spread entries are skipped: they name no key here.
|
|
69
|
+
* Reading the registry as text keeps a build offline that importing it
|
|
70
|
+
* would not (module side effects, provider clients, credentials).
|
|
71
|
+
*/
|
|
72
|
+
export declare function scanObjectKeys(text: string, declName: string): string[];
|
|
73
|
+
export interface PythonScan {
|
|
74
|
+
/** Repo-relative paths of local modules this file imports. */
|
|
75
|
+
localImports: string[];
|
|
76
|
+
/** Third-party / stdlib top-level package names. */
|
|
77
|
+
packages: string[];
|
|
78
|
+
/** Environment variables the module reads. */
|
|
79
|
+
envVars: string[];
|
|
80
|
+
}
|
|
81
|
+
/**
|
|
82
|
+
* Resolve a Python module path against the files a repo actually has.
|
|
83
|
+
* `services.market_data` → `services/market_data.py`; a leading dot is
|
|
84
|
+
* package-relative to the importing file's directory. Anything that does
|
|
85
|
+
* not land on a known file is treated as a third-party package, which is
|
|
86
|
+
* the conservative reading — a wrong local edge is worse than a missing one.
|
|
87
|
+
*/
|
|
88
|
+
export declare function resolvePythonImport(fromRelPath: string, moduleName: string, known: ReadonlySet<string>): string | null;
|
|
89
|
+
export declare function scanPython(text: string, relPath: string, knownFiles: ReadonlySet<string>): PythonScan;
|
|
90
|
+
export interface ProcfileEntry {
|
|
91
|
+
/** Process type — the dyno name. */
|
|
92
|
+
name: string;
|
|
93
|
+
command: string;
|
|
94
|
+
/** The script the process runs, when the command names one. */
|
|
95
|
+
entry: string | null;
|
|
96
|
+
}
|
|
97
|
+
export declare function scanProcfile(text: string): ProcfileEntry[];
|
|
@@ -0,0 +1,305 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/wiki/extract.ts — zero-dependency scanners that read structure out of source.
|
|
3
|
+
*
|
|
4
|
+
* WHY this file exists:
|
|
5
|
+
* The entity graph (lib/wiki/entities.ts) is only worth having if its
|
|
6
|
+
* extracted tier is cheap and honest. These are the readers that make it
|
|
7
|
+
* so: import edges, environment reads, table definitions and mentions,
|
|
8
|
+
* pulled straight out of file text with regexes and no parser
|
|
9
|
+
* dependency — the same judgment as the markdown engine
|
|
10
|
+
* (wiki/decisions/0002): at this scale a careful scanner beats a
|
|
11
|
+
* toolchain, and rebuild-from-source beats a cached index.
|
|
12
|
+
*
|
|
13
|
+
* They are deliberately CONSERVATIVE. A regex cannot see a dynamic
|
|
14
|
+
* import built from a variable, or a table name assembled at runtime;
|
|
15
|
+
* what it does report, it read verbatim. Nothing here guesses, so an
|
|
16
|
+
* extracted edge always has a literal line of source behind it.
|
|
17
|
+
*
|
|
18
|
+
* Bundle-agnostic and repo-agnostic: callers hand in text and paths.
|
|
19
|
+
*/
|
|
20
|
+
import { readdirSync, statSync } from 'node:fs';
|
|
21
|
+
import { join, posix, relative, sep } from 'node:path';
|
|
22
|
+
const DEFAULT_SKIP = ['node_modules', 'dist', 'build', 'coverage', 'outputs'];
|
|
23
|
+
/** Repo-relative POSIX paths under `dir`, sorted, dotfiles skipped. */
|
|
24
|
+
export function walkFiles(root, dir, options = {}) {
|
|
25
|
+
const skip = new Set([...DEFAULT_SKIP, ...(options.skipDirs ?? [])]);
|
|
26
|
+
const out = [];
|
|
27
|
+
const visit = (absDir) => {
|
|
28
|
+
let entries;
|
|
29
|
+
try {
|
|
30
|
+
entries = readdirSync(absDir, { withFileTypes: true });
|
|
31
|
+
}
|
|
32
|
+
catch {
|
|
33
|
+
return;
|
|
34
|
+
}
|
|
35
|
+
for (const entry of entries) {
|
|
36
|
+
if (entry.name.startsWith('.'))
|
|
37
|
+
continue;
|
|
38
|
+
if (entry.isDirectory()) {
|
|
39
|
+
if (skip.has(entry.name))
|
|
40
|
+
continue;
|
|
41
|
+
visit(join(absDir, entry.name));
|
|
42
|
+
}
|
|
43
|
+
else if (entry.isFile()) {
|
|
44
|
+
if (options.extensions &&
|
|
45
|
+
!options.extensions.some((ext) => entry.name.endsWith(ext))) {
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
out.push(relative(root, join(absDir, entry.name)).split(sep).join('/'));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
};
|
|
52
|
+
try {
|
|
53
|
+
if (statSync(dir).isDirectory())
|
|
54
|
+
visit(dir);
|
|
55
|
+
}
|
|
56
|
+
catch {
|
|
57
|
+
return [];
|
|
58
|
+
}
|
|
59
|
+
return out.sort();
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* `from '<specifier>'` covers every static import and re-export, however
|
|
63
|
+
* many lines the binding list spans; a bare side-effect import and a
|
|
64
|
+
* literal dynamic import are matched separately. Deliberately literal: a
|
|
65
|
+
* specifier assembled from a variable is invisible here, and stays so.
|
|
66
|
+
*/
|
|
67
|
+
const FROM_RE = /\bfrom\s+['"]([^'"\n]+)['"]/g;
|
|
68
|
+
const BARE_IMPORT_RE = /(?:^|\n)\s*import\s+['"]([^'"\n]+)['"]/g;
|
|
69
|
+
const DYNAMIC_IMPORT_RE = /\bimport\(\s*['"]([^'"\n]+)['"]\s*\)/g;
|
|
70
|
+
const ENV_DOT_RE = /process\.env\.([A-Za-z_][A-Za-z0-9_]*)/g;
|
|
71
|
+
const ENV_INDEX_RE = /process\.env\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]/g;
|
|
72
|
+
/** Resolve a relative specifier against the importing file's repo path. */
|
|
73
|
+
export function resolveLocalImport(fromRelPath, specifier) {
|
|
74
|
+
if (!specifier.startsWith('.'))
|
|
75
|
+
return null;
|
|
76
|
+
const resolved = posix.normalize(posix.join(posix.dirname(fromRelPath), specifier));
|
|
77
|
+
return resolved.startsWith('..') ? null : resolved;
|
|
78
|
+
}
|
|
79
|
+
function packageName(specifier) {
|
|
80
|
+
if (specifier.startsWith('.') || specifier.startsWith('/'))
|
|
81
|
+
return null;
|
|
82
|
+
if (specifier.startsWith('node:'))
|
|
83
|
+
return specifier;
|
|
84
|
+
const parts = specifier.split('/');
|
|
85
|
+
return specifier.startsWith('@') ? parts.slice(0, 2).join('/') : parts[0];
|
|
86
|
+
}
|
|
87
|
+
export function scanModule(text, relPath) {
|
|
88
|
+
const specifiers = new Set();
|
|
89
|
+
for (const re of [FROM_RE, BARE_IMPORT_RE, DYNAMIC_IMPORT_RE]) {
|
|
90
|
+
re.lastIndex = 0;
|
|
91
|
+
let match;
|
|
92
|
+
while ((match = re.exec(text)) !== null)
|
|
93
|
+
specifiers.add(match[1]);
|
|
94
|
+
}
|
|
95
|
+
const envVars = new Set();
|
|
96
|
+
for (const re of [ENV_DOT_RE, ENV_INDEX_RE]) {
|
|
97
|
+
re.lastIndex = 0;
|
|
98
|
+
let match;
|
|
99
|
+
while ((match = re.exec(text)) !== null)
|
|
100
|
+
envVars.add(match[1]);
|
|
101
|
+
}
|
|
102
|
+
const localImports = new Set();
|
|
103
|
+
const packages = new Set();
|
|
104
|
+
for (const specifier of specifiers) {
|
|
105
|
+
const local = resolveLocalImport(relPath, specifier);
|
|
106
|
+
if (local) {
|
|
107
|
+
// A file cannot import itself; a match on its own path came from a
|
|
108
|
+
// usage example in a comment, which is documentation, not an edge.
|
|
109
|
+
if (local !== relPath)
|
|
110
|
+
localImports.add(local);
|
|
111
|
+
continue;
|
|
112
|
+
}
|
|
113
|
+
const pkg = packageName(specifier);
|
|
114
|
+
if (pkg)
|
|
115
|
+
packages.add(pkg);
|
|
116
|
+
}
|
|
117
|
+
return {
|
|
118
|
+
specifiers: [...specifiers].sort(),
|
|
119
|
+
localImports: [...localImports].sort(),
|
|
120
|
+
packages: [...packages].sort(),
|
|
121
|
+
envVars: [...envVars].sort(),
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
const CREATE_TABLE_RE = /create\s+table\s+(?:if\s+not\s+exists\s+)?(?:(?:"[^"]+"|[a-z0-9_]+)\.)?("?[a-z0-9_]+"?)/gi;
|
|
125
|
+
const ALTER_TABLE_RE = /alter\s+table\s+(?:if\s+exists\s+)?(?:(?:"[^"]+"|[a-z0-9_]+)\.)?("?[a-z0-9_]+"?)/gi;
|
|
126
|
+
const REFERENCES_RE = /references\s+(?:(?:"[^"]+"|[a-z0-9_]+)\.)?("?[a-z0-9_]+"?)/gi;
|
|
127
|
+
function unquote(name) {
|
|
128
|
+
return name.replace(/^"|"$/g, '');
|
|
129
|
+
}
|
|
130
|
+
export function scanSql(sql) {
|
|
131
|
+
const defined = new Set();
|
|
132
|
+
const referenced = new Set();
|
|
133
|
+
const collect = (re, into) => {
|
|
134
|
+
re.lastIndex = 0;
|
|
135
|
+
let match;
|
|
136
|
+
while ((match = re.exec(sql)) !== null)
|
|
137
|
+
into.add(unquote(match[1]));
|
|
138
|
+
};
|
|
139
|
+
collect(CREATE_TABLE_RE, defined);
|
|
140
|
+
collect(ALTER_TABLE_RE, referenced);
|
|
141
|
+
collect(REFERENCES_RE, referenced);
|
|
142
|
+
for (const name of defined)
|
|
143
|
+
referenced.delete(name);
|
|
144
|
+
return { defined: [...defined].sort(), referenced: [...referenced].sort() };
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Which known tables a source file ADDRESSES — matched only as a quoted
|
|
148
|
+
* literal, which is how every client library names a table:
|
|
149
|
+
* `.from(<quoted name>)`. A bare word is deliberately not enough, because
|
|
150
|
+
* prose in a comment mentions table names constantly, and an edge saying
|
|
151
|
+
* "this module touches that table" must mean it.
|
|
152
|
+
*/
|
|
153
|
+
export function tableMentions(text, tables) {
|
|
154
|
+
const found = new Set();
|
|
155
|
+
for (const table of tables) {
|
|
156
|
+
if (new RegExp(`['"\`]${table}['"\`]`).test(text))
|
|
157
|
+
found.add(table);
|
|
158
|
+
}
|
|
159
|
+
return [...found].sort();
|
|
160
|
+
}
|
|
161
|
+
const ENTRY_RE = /(?:^|\s)((?:scripts|lib|tests|demo)\/[\w./-]+\.(?:ts|mjs|js))/;
|
|
162
|
+
const FLAG_RE = /--([a-z][\w-]*)\s+([^\s]+)/g;
|
|
163
|
+
export function scanNpmScripts(scripts) {
|
|
164
|
+
return Object.entries(scripts).map(([name, command]) => {
|
|
165
|
+
const entry = ENTRY_RE.exec(command)?.[1] ?? null;
|
|
166
|
+
const flags = {};
|
|
167
|
+
FLAG_RE.lastIndex = 0;
|
|
168
|
+
let match;
|
|
169
|
+
while ((match = FLAG_RE.exec(command)) !== null) {
|
|
170
|
+
if (!match[2].startsWith('-'))
|
|
171
|
+
flags[match[1]] = match[2];
|
|
172
|
+
}
|
|
173
|
+
return { name, command, entry, flags };
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
// ── Object-literal keys ──────────────────────────────────────────────────────
|
|
177
|
+
/**
|
|
178
|
+
* Keys of a named top-level object literal, read from source text —
|
|
179
|
+
* `const TOOL_MAP: Record<string, unknown> = { web_search: …, … }` yields
|
|
180
|
+
* ['web_search', …]. Spread entries are skipped: they name no key here.
|
|
181
|
+
* Reading the registry as text keeps a build offline that importing it
|
|
182
|
+
* would not (module side effects, provider clients, credentials).
|
|
183
|
+
*/
|
|
184
|
+
export function scanObjectKeys(text, declName) {
|
|
185
|
+
const start = new RegExp(`\\b(?:const|let|var)\\s+${declName}\\b[^=]*=\\s*\\{`).exec(text);
|
|
186
|
+
if (!start)
|
|
187
|
+
return [];
|
|
188
|
+
let depth = 0;
|
|
189
|
+
let end = -1;
|
|
190
|
+
const open = start.index + start[0].length - 1;
|
|
191
|
+
for (let i = open; i < text.length; i++) {
|
|
192
|
+
const ch = text[i];
|
|
193
|
+
if (ch === '{')
|
|
194
|
+
depth++;
|
|
195
|
+
else if (ch === '}') {
|
|
196
|
+
depth--;
|
|
197
|
+
if (depth === 0) {
|
|
198
|
+
end = i;
|
|
199
|
+
break;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
if (end === -1)
|
|
204
|
+
return [];
|
|
205
|
+
const body = text.slice(open + 1, end);
|
|
206
|
+
const keys = new Set();
|
|
207
|
+
let nesting = 0;
|
|
208
|
+
for (const line of body.split('\n')) {
|
|
209
|
+
const trimmed = line.trim();
|
|
210
|
+
if (nesting === 0) {
|
|
211
|
+
const match = /^(?:'([^']+)'|"([^"]+)"|([A-Za-z_$][\w$]*))\s*:/.exec(trimmed);
|
|
212
|
+
if (match)
|
|
213
|
+
keys.add(match[1] ?? match[2] ?? match[3]);
|
|
214
|
+
}
|
|
215
|
+
for (const ch of line) {
|
|
216
|
+
if (ch === '{' || ch === '[' || ch === '(')
|
|
217
|
+
nesting++;
|
|
218
|
+
else if (ch === '}' || ch === ']' || ch === ')')
|
|
219
|
+
nesting--;
|
|
220
|
+
}
|
|
221
|
+
if (nesting < 0)
|
|
222
|
+
nesting = 0;
|
|
223
|
+
}
|
|
224
|
+
return [...keys].sort();
|
|
225
|
+
}
|
|
226
|
+
const PY_IMPORT_RE = /(?:^|\n)\s*import\s+([A-Za-z_][\w.]*(?:\s*,\s*[A-Za-z_][\w.]*)*)/g;
|
|
227
|
+
const PY_FROM_RE = /(?:^|\n)\s*from\s+(\.*[A-Za-z_][\w.]*|\.+)\s+import\s+/g;
|
|
228
|
+
const PY_ENV_RE = /os\.environ(?:\.get)?\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]|os\.getenv\(\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]|os\.environ\[\s*['"]([A-Za-z_][A-Za-z0-9_]*)['"]\s*\]/g;
|
|
229
|
+
/**
|
|
230
|
+
* Resolve a Python module path against the files a repo actually has.
|
|
231
|
+
* `services.market_data` → `services/market_data.py`; a leading dot is
|
|
232
|
+
* package-relative to the importing file's directory. Anything that does
|
|
233
|
+
* not land on a known file is treated as a third-party package, which is
|
|
234
|
+
* the conservative reading — a wrong local edge is worse than a missing one.
|
|
235
|
+
*/
|
|
236
|
+
export function resolvePythonImport(fromRelPath, moduleName, known) {
|
|
237
|
+
const dots = /^\.*/.exec(moduleName)?.[0].length ?? 0;
|
|
238
|
+
const bare = moduleName.slice(dots);
|
|
239
|
+
let baseDir = posix.dirname(fromRelPath);
|
|
240
|
+
for (let i = 1; i < dots; i++)
|
|
241
|
+
baseDir = posix.dirname(baseDir);
|
|
242
|
+
const segments = bare === '' ? [] : bare.split('.');
|
|
243
|
+
const candidates = [];
|
|
244
|
+
if (dots > 0) {
|
|
245
|
+
const joined = [baseDir, ...segments].filter((p) => p && p !== '.').join('/');
|
|
246
|
+
candidates.push(`${joined}.py`, `${joined}/__init__.py`);
|
|
247
|
+
}
|
|
248
|
+
else {
|
|
249
|
+
const joined = segments.join('/');
|
|
250
|
+
candidates.push(`${joined}.py`, `${joined}/__init__.py`);
|
|
251
|
+
// A module imported by bare name from the repo root, e.g. `import advisor`.
|
|
252
|
+
if (segments.length === 1)
|
|
253
|
+
candidates.push(`${segments[0]}.py`);
|
|
254
|
+
}
|
|
255
|
+
return candidates.find((c) => known.has(c)) ?? null;
|
|
256
|
+
}
|
|
257
|
+
export function scanPython(text, relPath, knownFiles) {
|
|
258
|
+
const modules = new Set();
|
|
259
|
+
PY_IMPORT_RE.lastIndex = 0;
|
|
260
|
+
let match;
|
|
261
|
+
while ((match = PY_IMPORT_RE.exec(text)) !== null) {
|
|
262
|
+
for (const name of match[1].split(','))
|
|
263
|
+
modules.add(name.trim());
|
|
264
|
+
}
|
|
265
|
+
PY_FROM_RE.lastIndex = 0;
|
|
266
|
+
while ((match = PY_FROM_RE.exec(text)) !== null)
|
|
267
|
+
modules.add(match[1].trim());
|
|
268
|
+
const localImports = new Set();
|
|
269
|
+
const packages = new Set();
|
|
270
|
+
for (const moduleName of modules) {
|
|
271
|
+
const local = resolvePythonImport(relPath, moduleName, knownFiles);
|
|
272
|
+
if (local && local !== relPath)
|
|
273
|
+
localImports.add(local);
|
|
274
|
+
else if (!moduleName.startsWith('.'))
|
|
275
|
+
packages.add(moduleName.split('.')[0]);
|
|
276
|
+
}
|
|
277
|
+
const envVars = new Set();
|
|
278
|
+
PY_ENV_RE.lastIndex = 0;
|
|
279
|
+
while ((match = PY_ENV_RE.exec(text)) !== null) {
|
|
280
|
+
envVars.add(match[1] ?? match[2] ?? match[3]);
|
|
281
|
+
}
|
|
282
|
+
return {
|
|
283
|
+
localImports: [...localImports].sort(),
|
|
284
|
+
packages: [...packages].sort(),
|
|
285
|
+
envVars: [...envVars].sort(),
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
const PROC_ENTRY_RE = /(?:^|\s)([\w./-]+\.(?:py|ts|js|mjs|sh))(?:\s|$)/;
|
|
289
|
+
export function scanProcfile(text) {
|
|
290
|
+
const entries = [];
|
|
291
|
+
for (const line of text.split('\n')) {
|
|
292
|
+
const trimmed = line.trim();
|
|
293
|
+
if (trimmed === '' || trimmed.startsWith('#'))
|
|
294
|
+
continue;
|
|
295
|
+
const colon = trimmed.indexOf(':');
|
|
296
|
+
if (colon === -1)
|
|
297
|
+
continue;
|
|
298
|
+
const name = trimmed.slice(0, colon).trim();
|
|
299
|
+
const command = trimmed.slice(colon + 1).trim();
|
|
300
|
+
if (name === '' || command === '')
|
|
301
|
+
continue;
|
|
302
|
+
entries.push({ name, command, entry: PROC_ENTRY_RE.exec(command)?.[1] ?? null });
|
|
303
|
+
}
|
|
304
|
+
return entries;
|
|
305
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/wiki/fill.ts — the LLM pass: write the prose a structural build can't.
|
|
3
|
+
*
|
|
4
|
+
* WHY this file exists:
|
|
5
|
+
* A structural build produces true-but-mute documents: tables, listings,
|
|
6
|
+
* links, and empty wiki:fill slots. This pass walks the bundle for slots
|
|
7
|
+
* still holding the TODO sentinel and asks a model to write ONLY that
|
|
8
|
+
* missing prose — one slot at a time, grounded in the document itself,
|
|
9
|
+
* its graph neighbors (the only links it may use), and excerpts of the
|
|
10
|
+
* repo files named in `sources`. The model never touches structure: the
|
|
11
|
+
* slot interior is the entire blast radius, enforced by the marker
|
|
12
|
+
* splice, and lint gates the result like any other write.
|
|
13
|
+
*
|
|
14
|
+
* Provenance is honest: a filled document's `generated.by` becomes
|
|
15
|
+
* `melchizedek/<model>` — the §5.3 trust derivation then reports it as
|
|
16
|
+
* machine-confirmed at best, never human-reviewed, until a person adds
|
|
17
|
+
* a `verified` entry. No key configured → the pass reports itself
|
|
18
|
+
* skipped and the skeleton keeps its TODO sentinels; nothing breaks.
|
|
19
|
+
*/
|
|
20
|
+
import { type Vault } from './vault.ts';
|
|
21
|
+
export interface FillOptions {
|
|
22
|
+
/** Model string routed via lib/models/registry.ts. */
|
|
23
|
+
model: string;
|
|
24
|
+
/** Cap on documents touched in one pass. */
|
|
25
|
+
maxDocs?: number;
|
|
26
|
+
/** YYYY-MM-DD stamped into generated.at. */
|
|
27
|
+
date: string;
|
|
28
|
+
log?: (message: string) => void;
|
|
29
|
+
}
|
|
30
|
+
export interface FillOutcome {
|
|
31
|
+
docsTouched: number;
|
|
32
|
+
slotsFilled: number;
|
|
33
|
+
slotsFailed: number;
|
|
34
|
+
/** Why nothing ran, when nothing ran. */
|
|
35
|
+
skippedReason?: string;
|
|
36
|
+
}
|
|
37
|
+
export declare function fillVault(vault: Vault, options: FillOptions): Promise<FillOutcome>;
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* lib/wiki/fill.ts — the LLM pass: write the prose a structural build can't.
|
|
3
|
+
*
|
|
4
|
+
* WHY this file exists:
|
|
5
|
+
* A structural build produces true-but-mute documents: tables, listings,
|
|
6
|
+
* links, and empty wiki:fill slots. This pass walks the bundle for slots
|
|
7
|
+
* still holding the TODO sentinel and asks a model to write ONLY that
|
|
8
|
+
* missing prose — one slot at a time, grounded in the document itself,
|
|
9
|
+
* its graph neighbors (the only links it may use), and excerpts of the
|
|
10
|
+
* repo files named in `sources`. The model never touches structure: the
|
|
11
|
+
* slot interior is the entire blast radius, enforced by the marker
|
|
12
|
+
* splice, and lint gates the result like any other write.
|
|
13
|
+
*
|
|
14
|
+
* Provenance is honest: a filled document's `generated.by` becomes
|
|
15
|
+
* `melchizedek/<model>` — the §5.3 trust derivation then reports it as
|
|
16
|
+
* machine-confirmed at best, never human-reviewed, until a person adds
|
|
17
|
+
* a `verified` entry. No key configured → the pass reports itself
|
|
18
|
+
* skipped and the skeleton keeps its TODO sentinels; nothing breaks.
|
|
19
|
+
*/
|
|
20
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
21
|
+
import { resolve } from 'node:path';
|
|
22
|
+
import { runWikiAgent, modelAvailable } from "./agentRun.js";
|
|
23
|
+
import { llmActor } from "./format.js";
|
|
24
|
+
import { buildGraph } from "./graph.js";
|
|
25
|
+
import { emitFrontmatter, parseDoc, setSlotContent, slotIsUnfilled, TODO_SENTINEL, } from "./markdown.js";
|
|
26
|
+
import { loadDoc, writeDocFile } from "./vault.js";
|
|
27
|
+
// ── Prompt assembly ──────────────────────────────────────────────────────────
|
|
28
|
+
const WRITER_INSTRUCTION = `You write missing prose for knowledge-base documents about a software system.
|
|
29
|
+
You receive one document, the id of one empty slot in it, a hint describing what belongs there, link candidates, and source excerpts.
|
|
30
|
+
Rules:
|
|
31
|
+
- Write ONLY the slot's prose: no preamble, no code fences around the reply, no headings unless the hint asks.
|
|
32
|
+
- 60–140 words unless the hint says otherwise. Plain, concrete, present tense. No marketing language, no hedging.
|
|
33
|
+
- Every claim must come from the provided document, excerpts, or link candidates. If the context does not support a claim, leave it out.
|
|
34
|
+
- Markdown links are allowed ONLY to the exact paths listed under LINK CANDIDATES. Never invent a path or URL.
|
|
35
|
+
- If the context is insufficient to write anything true, reply with exactly: INSUFFICIENT_CONTEXT`;
|
|
36
|
+
function neighborCandidates(vault, graph, doc) {
|
|
37
|
+
const paths = new Set();
|
|
38
|
+
for (const e of graph.outbound.get(doc.bundlePath) ?? []) {
|
|
39
|
+
if (e.resolved)
|
|
40
|
+
paths.add(e.target);
|
|
41
|
+
}
|
|
42
|
+
for (const e of graph.inbound.get(doc.bundlePath) ?? [])
|
|
43
|
+
paths.add(e.source);
|
|
44
|
+
const lines = [];
|
|
45
|
+
for (const path of [...paths].slice(0, 14)) {
|
|
46
|
+
const neighbor = vault.docs.get(path);
|
|
47
|
+
if (!neighbor)
|
|
48
|
+
continue;
|
|
49
|
+
const desc = neighbor.fm?.description ?? '';
|
|
50
|
+
lines.push(`- ${path} — ${neighbor.title}${desc ? `: ${desc}` : ''}`);
|
|
51
|
+
}
|
|
52
|
+
return lines.length > 0 ? lines.join('\n') : '(none)';
|
|
53
|
+
}
|
|
54
|
+
function sourceExcerpts(vault, doc) {
|
|
55
|
+
const repoRoot = resolve(vault.root, '..');
|
|
56
|
+
const chunks = [];
|
|
57
|
+
for (const source of doc.fm?.sources ?? []) {
|
|
58
|
+
const resource = source.resource;
|
|
59
|
+
if (/^[a-z][a-z0-9+.-]*:/i.test(resource))
|
|
60
|
+
continue; // URLs: not readable here
|
|
61
|
+
const abs = resource.startsWith('/')
|
|
62
|
+
? resolve(vault.root, `.${resource}`)
|
|
63
|
+
: resolve(repoRoot, resource);
|
|
64
|
+
if (!abs.startsWith(repoRoot) || !existsSync(abs))
|
|
65
|
+
continue;
|
|
66
|
+
try {
|
|
67
|
+
const text = readFileSync(abs, 'utf-8').split('\n').slice(0, 120).join('\n');
|
|
68
|
+
chunks.push(`--- ${resource} (first 120 lines) ---\n${text}`);
|
|
69
|
+
}
|
|
70
|
+
catch {
|
|
71
|
+
// unreadable source: the slot simply gets less context
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
return chunks.length > 0 ? chunks.join('\n\n') : '(none)';
|
|
75
|
+
}
|
|
76
|
+
function buildUserText(vault, graph, doc, slotId, hint) {
|
|
77
|
+
return [
|
|
78
|
+
`DOCUMENT (${doc.bundlePath}):`,
|
|
79
|
+
doc.parsed.raw,
|
|
80
|
+
`\nSLOT TO FILL: "${slotId}"`,
|
|
81
|
+
`HINT: ${hint}`,
|
|
82
|
+
`\nLINK CANDIDATES (the only linkable paths):`,
|
|
83
|
+
neighborCandidates(vault, graph, doc),
|
|
84
|
+
`\nSOURCE EXCERPTS:`,
|
|
85
|
+
sourceExcerpts(vault, doc),
|
|
86
|
+
].join('\n');
|
|
87
|
+
}
|
|
88
|
+
function slotHint(slot) {
|
|
89
|
+
const text = slot.content.trim();
|
|
90
|
+
if (text.startsWith(TODO_SENTINEL)) {
|
|
91
|
+
return text.slice(TODO_SENTINEL.length).replace(/_$/, '').trim();
|
|
92
|
+
}
|
|
93
|
+
return slot.id;
|
|
94
|
+
}
|
|
95
|
+
/** Strip a whole-reply code fence if the model added one despite the rules. */
|
|
96
|
+
function cleanReply(reply) {
|
|
97
|
+
const trimmed = reply.trim();
|
|
98
|
+
const fenced = trimmed.match(/^```[a-z]*\n([\s\S]*?)\n```$/);
|
|
99
|
+
return (fenced ? fenced[1] : trimmed).trim();
|
|
100
|
+
}
|
|
101
|
+
// ── The pass ─────────────────────────────────────────────────────────────────
|
|
102
|
+
export async function fillVault(vault, options) {
|
|
103
|
+
const log = options.log ?? (() => { });
|
|
104
|
+
const availability = modelAvailable(options.model);
|
|
105
|
+
if (!availability.ok) {
|
|
106
|
+
return {
|
|
107
|
+
docsTouched: 0,
|
|
108
|
+
slotsFilled: 0,
|
|
109
|
+
slotsFailed: 0,
|
|
110
|
+
skippedReason: `fill skipped: ${availability.reason}`,
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
const graph = buildGraph(vault);
|
|
114
|
+
const candidates = [...vault.docs.values()].filter((doc) => doc.kind !== 'log' && doc.parsed.slots.some(slotIsUnfilled));
|
|
115
|
+
const limit = options.maxDocs ?? 50;
|
|
116
|
+
let docsTouched = 0;
|
|
117
|
+
let slotsFilled = 0;
|
|
118
|
+
let slotsFailed = 0;
|
|
119
|
+
for (const doc of candidates.slice(0, limit)) {
|
|
120
|
+
let raw = doc.parsed.raw;
|
|
121
|
+
let docChanged = false;
|
|
122
|
+
for (const slot of doc.parsed.slots.filter(slotIsUnfilled)) {
|
|
123
|
+
const hint = slotHint(slot);
|
|
124
|
+
log(`fill ${doc.bundlePath} · slot "${slot.id}"`);
|
|
125
|
+
const result = await runWikiAgent({
|
|
126
|
+
name: 'wiki_scribe',
|
|
127
|
+
description: 'Writes one missing prose passage for a wiki document.',
|
|
128
|
+
model: options.model,
|
|
129
|
+
instruction: WRITER_INSTRUCTION,
|
|
130
|
+
userText: buildUserText(vault, graph, doc, slot.id, hint),
|
|
131
|
+
temperature: 0.3,
|
|
132
|
+
// Reasoning models spend thought tokens from the same budget; a
|
|
133
|
+
// tight cap truncates the visible prose mid-sentence.
|
|
134
|
+
maxOutputTokens: 4096,
|
|
135
|
+
});
|
|
136
|
+
if (result.error || result.text === '' || result.text === 'INSUFFICIENT_CONTEXT') {
|
|
137
|
+
slotsFailed++;
|
|
138
|
+
log(` ✗ ${slot.id}: ${result.error ?? (result.text || 'empty reply')}`);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
const prose = cleanReply(result.text);
|
|
142
|
+
const truncated = !/[.!?)`\]]$/.test(prose.trimEnd());
|
|
143
|
+
if (prose.includes(TODO_SENTINEL) || prose === '' || truncated) {
|
|
144
|
+
if (truncated)
|
|
145
|
+
log(` ✗ ${slot.id}: reply looks truncated, slot left unfilled`);
|
|
146
|
+
slotsFailed += truncated || prose === '' ? 1 : 0;
|
|
147
|
+
continue;
|
|
148
|
+
}
|
|
149
|
+
const next = setSlotContent(raw, slot.id, prose);
|
|
150
|
+
if (next) {
|
|
151
|
+
raw = next;
|
|
152
|
+
docChanged = true;
|
|
153
|
+
slotsFilled++;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
if (docChanged) {
|
|
157
|
+
// Stamp provenance: the model made the last meaningful change.
|
|
158
|
+
const parsed = parseDoc(raw);
|
|
159
|
+
if (parsed.frontmatter) {
|
|
160
|
+
const fm = {
|
|
161
|
+
...parsed.frontmatter,
|
|
162
|
+
generated: { by: llmActor(options.model), at: options.date },
|
|
163
|
+
};
|
|
164
|
+
const lines = raw.split('\n');
|
|
165
|
+
const body = lines.slice(parsed.bodyStartLine - 1).join('\n');
|
|
166
|
+
raw = `${emitFrontmatter(fm)}\n${body.replace(/^\n+/, '')}`;
|
|
167
|
+
}
|
|
168
|
+
writeDocFile(vault.root, doc.bundlePath, raw);
|
|
169
|
+
// Refresh the in-memory doc so later passes see current content.
|
|
170
|
+
vault.docs.set(doc.bundlePath, loadDoc(vault.root, doc.absPath));
|
|
171
|
+
docsTouched++;
|
|
172
|
+
}
|
|
173
|
+
}
|
|
174
|
+
return { docsTouched, slotsFilled, slotsFailed };
|
|
175
|
+
}
|