akm-cli 0.7.4 → 0.8.0-rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/{CHANGELOG.md → .github/CHANGELOG.md} +34 -1
- package/.github/LICENSE +374 -0
- package/dist/cli/parse-args.js +43 -0
- package/dist/cli.js +1007 -593
- package/dist/commands/agent-dispatch.js +102 -0
- package/dist/commands/agent-support.js +62 -0
- package/dist/commands/config-cli.js +68 -84
- package/dist/commands/consolidate.js +823 -0
- package/dist/commands/curate.js +1 -0
- package/dist/commands/distill-promotion-policy.js +658 -0
- package/dist/commands/distill.js +250 -48
- package/dist/commands/eval-cases.js +40 -0
- package/dist/commands/events.js +12 -24
- package/dist/commands/graph.js +222 -0
- package/dist/commands/health.js +376 -0
- package/dist/commands/help/help-accept.md +9 -0
- package/dist/commands/help/help-improve.md +53 -0
- package/dist/commands/help/help-proposals.md +15 -0
- package/dist/commands/help/help-propose.md +17 -0
- package/dist/commands/help/help-reject.md +8 -0
- package/dist/commands/history.js +3 -30
- package/dist/commands/improve.js +1170 -0
- package/dist/commands/info.js +2 -2
- package/dist/commands/init.js +2 -2
- package/dist/commands/install-audit.js +5 -1
- package/dist/commands/installed-stashes.js +118 -138
- package/dist/commands/knowledge.js +133 -0
- package/dist/commands/lint/agent-linter.js +46 -0
- package/dist/commands/lint/base-linter.js +251 -0
- package/dist/commands/lint/command-linter.js +46 -0
- package/dist/commands/lint/default-linter.js +13 -0
- package/dist/commands/lint/index.js +107 -0
- package/dist/commands/lint/knowledge-linter.js +13 -0
- package/dist/commands/lint/memory-linter.js +58 -0
- package/dist/commands/lint/registry.js +33 -0
- package/dist/commands/lint/skill-linter.js +42 -0
- package/dist/commands/lint/task-linter.js +47 -0
- package/dist/commands/lint/types.js +1 -0
- package/dist/commands/lint/workflow-linter.js +53 -0
- package/dist/commands/lint.js +1 -0
- package/dist/commands/migration-help.js +2 -2
- package/dist/commands/proposal.js +8 -7
- package/dist/commands/propose.js +113 -43
- package/dist/commands/reflect.js +175 -41
- package/dist/commands/registry-search.js +2 -2
- package/dist/commands/remember.js +55 -1
- package/dist/commands/schema-repair.js +130 -0
- package/dist/commands/search.js +21 -5
- package/dist/commands/show.js +131 -52
- package/dist/commands/source-add.js +10 -10
- package/dist/commands/source-manage.js +11 -19
- package/dist/commands/tasks.js +385 -0
- package/dist/commands/url-checker.js +39 -0
- package/dist/commands/vault.js +7 -33
- package/dist/core/action-contributors.js +25 -0
- package/dist/core/asset-registry.js +5 -17
- package/dist/core/asset-spec.js +11 -1
- package/dist/core/common.js +94 -0
- package/dist/core/concurrent.js +22 -0
- package/dist/core/config.js +229 -122
- package/dist/core/events.js +87 -123
- package/dist/core/frontmatter.js +3 -1
- package/dist/core/markdown.js +17 -0
- package/dist/core/memory-improve.js +678 -0
- package/dist/core/parse.js +155 -0
- package/dist/core/paths.js +101 -3
- package/dist/core/proposal-validators.js +61 -0
- package/dist/core/proposals.js +49 -38
- package/dist/core/state-db.js +775 -0
- package/dist/core/time.js +51 -0
- package/dist/core/warn.js +59 -1
- package/dist/indexer/db-search.js +86 -472
- package/dist/indexer/db.js +392 -6
- package/dist/indexer/ensure-index.js +133 -0
- package/dist/indexer/graph-boost.js +247 -94
- package/dist/indexer/graph-db.js +201 -0
- package/dist/indexer/graph-dedup.js +99 -0
- package/dist/indexer/graph-extraction.js +417 -74
- package/dist/indexer/index-context.js +10 -0
- package/dist/indexer/indexer.js +466 -298
- package/dist/indexer/llm-cache.js +47 -0
- package/dist/indexer/match-contributors.js +141 -0
- package/dist/indexer/matchers.js +24 -190
- package/dist/indexer/memory-inference.js +63 -29
- package/dist/indexer/metadata-contributors.js +26 -0
- package/dist/indexer/metadata.js +188 -175
- package/dist/indexer/path-resolver.js +89 -0
- package/dist/indexer/ranking-contributors.js +204 -0
- package/dist/indexer/ranking.js +74 -0
- package/dist/indexer/search-hit-enrichers.js +22 -0
- package/dist/indexer/search-source.js +24 -9
- package/dist/indexer/semantic-status.js +2 -16
- package/dist/indexer/walker.js +25 -0
- package/dist/integrations/agent/config.js +175 -3
- package/dist/integrations/agent/index.js +3 -1
- package/dist/integrations/agent/pipeline.js +39 -0
- package/dist/integrations/agent/profiles.js +67 -5
- package/dist/integrations/agent/prompts.js +114 -29
- package/dist/integrations/agent/runners.js +31 -0
- package/dist/integrations/agent/sdk-runner.js +120 -0
- package/dist/integrations/agent/spawn.js +136 -28
- package/dist/integrations/lockfile.js +10 -18
- package/dist/integrations/session-logs/index.js +65 -0
- package/dist/integrations/session-logs/providers/claude-code.js +56 -0
- package/dist/integrations/session-logs/providers/opencode.js +52 -0
- package/dist/integrations/session-logs/types.js +1 -0
- package/dist/llm/call-ai.js +74 -0
- package/dist/llm/client.js +63 -86
- package/dist/llm/feature-gate.js +27 -16
- package/dist/llm/graph-extract.js +297 -64
- package/dist/llm/memory-infer.js +52 -71
- package/dist/llm/metadata-enhance.js +39 -22
- package/dist/llm/prompts/graph-extract-user-prompt.md +12 -0
- package/dist/output/cli-hints-full.md +277 -0
- package/dist/output/cli-hints-short.md +65 -0
- package/dist/output/cli-hints.js +2 -309
- package/dist/output/renderers.js +196 -124
- package/dist/output/shapes.js +41 -3
- package/dist/output/text.js +257 -21
- package/dist/registry/providers/skills-sh.js +61 -49
- package/dist/registry/providers/static-index.js +44 -48
- package/dist/setup/setup.js +510 -11
- package/dist/sources/provider-factory.js +2 -1
- package/dist/sources/providers/git.js +44 -2
- package/dist/sources/website-ingest.js +4 -0
- package/dist/tasks/backends/cron.js +200 -0
- package/dist/tasks/backends/exec-utils.js +25 -0
- package/dist/tasks/backends/index.js +32 -0
- package/dist/tasks/backends/launchd-template.xml +19 -0
- package/dist/tasks/backends/launchd.js +184 -0
- package/dist/tasks/backends/schtasks-template.xml +29 -0
- package/dist/tasks/backends/schtasks.js +212 -0
- package/dist/tasks/parser.js +198 -0
- package/dist/tasks/resolveAkmBin.js +84 -0
- package/dist/tasks/runner.js +432 -0
- package/dist/tasks/schedule.js +208 -0
- package/dist/tasks/schema.js +13 -0
- package/dist/tasks/validator.js +59 -0
- package/dist/wiki/index-template.md +12 -0
- package/dist/wiki/ingest-workflow-template.md +54 -0
- package/dist/wiki/log-template.md +8 -0
- package/dist/wiki/schema-template.md +61 -0
- package/dist/wiki/wiki-templates.js +12 -0
- package/dist/wiki/wiki.js +10 -61
- package/dist/workflows/authoring.js +5 -25
- package/dist/workflows/db.js +9 -0
- package/dist/workflows/renderer.js +8 -3
- package/dist/workflows/runs.js +73 -88
- package/dist/workflows/scope-key.js +76 -0
- package/dist/workflows/validator.js +1 -1
- package/dist/workflows/workflow-template.md +24 -0
- package/docs/README.md +3 -0
- package/docs/migration/release-notes/0.7.0.md +1 -1
- package/docs/migration/release-notes/0.7.4.md +1 -1
- package/docs/migration/release-notes/0.7.5.md +20 -0
- package/docs/migration/release-notes/0.8.0.md +43 -0
- package/package.json +4 -3
- package/dist/templates/wiki-templates.js +0 -100
package/dist/llm/client.js
CHANGED
|
@@ -8,6 +8,10 @@
|
|
|
8
8
|
* `llm.ts` re-exports everything from this module for backward compatibility.
|
|
9
9
|
*/
|
|
10
10
|
import { fetchWithTimeout } from "../core/common";
|
|
11
|
+
import { escapeJsonStringControls, parseJsonResponse, stripCodeFences, stripThinkBlocks } from "../core/parse";
|
|
12
|
+
// Re-export shared parse utilities so existing importers of `client.ts` continue
|
|
13
|
+
// to resolve `parseJsonResponse` and `parseEmbeddedJsonResponse` from this module.
|
|
14
|
+
export { escapeJsonStringControls, parseEmbeddedJsonResponse, parseJsonResponse, stripCodeFences, stripThinkBlocks, } from "../core/parse";
|
|
11
15
|
/** Maximum length of an LLM error response body included in thrown errors. */
|
|
12
16
|
const ERROR_BODY_MAX_LEN = 200;
|
|
13
17
|
/**
|
|
@@ -38,105 +42,78 @@ export function redactErrorBody(input) {
|
|
|
38
42
|
}
|
|
39
43
|
return out;
|
|
40
44
|
}
|
|
45
|
+
export class LlmCallError extends Error {
|
|
46
|
+
code;
|
|
47
|
+
statusCode;
|
|
48
|
+
constructor(message, code, statusCode) {
|
|
49
|
+
super(message);
|
|
50
|
+
this.code = code;
|
|
51
|
+
this.statusCode = statusCode;
|
|
52
|
+
this.name = "LlmCallError";
|
|
53
|
+
}
|
|
54
|
+
}
|
|
41
55
|
export async function chatCompletion(config, messages, options) {
|
|
56
|
+
const timeoutMs = options?.timeoutMs ?? config.timeoutMs ?? 120_000;
|
|
42
57
|
const headers = { "Content-Type": "application/json" };
|
|
43
58
|
if (config.apiKey) {
|
|
44
59
|
headers.Authorization = `Bearer ${config.apiKey}`;
|
|
45
60
|
}
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
61
|
+
// Only include max_tokens when explicitly set. The model/API knows its own
|
|
62
|
+
// limits; a hardcoded default creates silent truncation failures when the
|
|
63
|
+
// guess is wrong. Users who need a cap can set llm.maxTokens in config.
|
|
64
|
+
const resolvedMaxTokens = options?.maxTokens ?? config.maxTokens;
|
|
65
|
+
let response;
|
|
66
|
+
try {
|
|
67
|
+
response = await fetchWithTimeout(config.endpoint, {
|
|
68
|
+
method: "POST",
|
|
69
|
+
headers,
|
|
70
|
+
body: JSON.stringify({
|
|
71
|
+
model: config.model,
|
|
72
|
+
messages,
|
|
73
|
+
temperature: options?.temperature ?? config.temperature ?? 0.3,
|
|
74
|
+
...(resolvedMaxTokens !== undefined ? { max_tokens: resolvedMaxTokens } : {}),
|
|
75
|
+
...config.extraParams,
|
|
76
|
+
}),
|
|
77
|
+
}, timeoutMs, options?.signal);
|
|
78
|
+
}
|
|
79
|
+
catch (err) {
|
|
80
|
+
// fetchWithTimeout throws a plain Error with a message containing
|
|
81
|
+
// "timed out" for AbortController-driven timeouts, or "aborted" for
|
|
82
|
+
// caller-driven cancellations. Map both to typed LlmCallError.
|
|
83
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
84
|
+
if (err instanceof DOMException && err.name === "AbortError") {
|
|
85
|
+
throw new LlmCallError(`Request timed out after ${timeoutMs}ms`, "timeout");
|
|
86
|
+
}
|
|
87
|
+
if (msg.includes("timed out")) {
|
|
88
|
+
throw new LlmCallError(`Request timed out after ${timeoutMs}ms`, "timeout");
|
|
89
|
+
}
|
|
90
|
+
throw new LlmCallError(`Network error: ${msg}`, "network_error");
|
|
91
|
+
}
|
|
57
92
|
if (!response.ok) {
|
|
58
93
|
const rawBody = await response.text().catch(() => "");
|
|
59
94
|
const safeBody = redactErrorBody(rawBody);
|
|
60
|
-
|
|
95
|
+
const status = response.status;
|
|
96
|
+
if (status === 429) {
|
|
97
|
+
throw new LlmCallError(`LLM request rate limited (429) ${config.endpoint}: ${safeBody}`, "rate_limited", status);
|
|
98
|
+
}
|
|
99
|
+
if (status >= 500) {
|
|
100
|
+
throw new LlmCallError(`LLM provider error (${status}) ${config.endpoint}: ${safeBody}`, "provider_error", status);
|
|
101
|
+
}
|
|
102
|
+
throw new LlmCallError(`LLM request failed (${status}) ${config.endpoint}: ${safeBody}`, "provider_error", status);
|
|
61
103
|
}
|
|
62
104
|
const json = (await response.json());
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
export function stripJsonFences(raw) {
|
|
67
|
-
return raw
|
|
68
|
-
.trim()
|
|
69
|
-
.replace(/<think>[\s\S]*?<\/think>/gi, "")
|
|
70
|
-
.replace(/^```(?:json)?\s*\n?/i, "")
|
|
71
|
-
.replace(/\n?```\s*$/i, "")
|
|
72
|
-
.trim();
|
|
73
|
-
}
|
|
74
|
-
/** Parse a possibly-fenced JSON response. Returns undefined if invalid. */
|
|
75
|
-
export function parseJsonResponse(raw) {
|
|
76
|
-
try {
|
|
77
|
-
return JSON.parse(stripJsonFences(raw));
|
|
78
|
-
}
|
|
79
|
-
catch {
|
|
80
|
-
return undefined;
|
|
81
|
-
}
|
|
105
|
+
const content = (json.choices?.[0]?.message?.content ?? "").trim();
|
|
106
|
+
const reasoning = (json.choices?.[0]?.message?.reasoning_content ?? "").trim();
|
|
107
|
+
return content || reasoning;
|
|
82
108
|
}
|
|
83
109
|
/**
|
|
84
|
-
*
|
|
85
|
-
*
|
|
110
|
+
* Strip `<think>` blocks, code fences, and escape control characters in JSON
|
|
111
|
+
* strings. Thin wrapper kept for backward compatibility with call sites that
|
|
112
|
+
* import `stripJsonFences` from this module. New code should prefer the
|
|
113
|
+
* granular helpers from `../core/parse`.
|
|
86
114
|
*/
|
|
87
|
-
export function
|
|
88
|
-
|
|
89
|
-
if (direct !== undefined)
|
|
90
|
-
return direct;
|
|
91
|
-
const text = stripJsonFences(raw);
|
|
92
|
-
let arrayFallback;
|
|
93
|
-
for (let start = 0; start < text.length; start++) {
|
|
94
|
-
const opener = text[start];
|
|
95
|
-
if (opener !== "{" && opener !== "[")
|
|
96
|
-
continue;
|
|
97
|
-
const closer = opener === "{" ? "}" : "]";
|
|
98
|
-
let depth = 0;
|
|
99
|
-
let inString = false;
|
|
100
|
-
let escaped = false;
|
|
101
|
-
for (let i = start; i < text.length; i++) {
|
|
102
|
-
const ch = text[i];
|
|
103
|
-
if (inString) {
|
|
104
|
-
if (escaped) {
|
|
105
|
-
escaped = false;
|
|
106
|
-
}
|
|
107
|
-
else if (ch === "\\") {
|
|
108
|
-
escaped = true;
|
|
109
|
-
}
|
|
110
|
-
else if (ch === '"') {
|
|
111
|
-
inString = false;
|
|
112
|
-
}
|
|
113
|
-
continue;
|
|
114
|
-
}
|
|
115
|
-
if (ch === '"') {
|
|
116
|
-
inString = true;
|
|
117
|
-
continue;
|
|
118
|
-
}
|
|
119
|
-
if (ch === opener)
|
|
120
|
-
depth += 1;
|
|
121
|
-
if (ch === closer) {
|
|
122
|
-
depth -= 1;
|
|
123
|
-
if (depth === 0) {
|
|
124
|
-
try {
|
|
125
|
-
const parsed = JSON.parse(text.slice(start, i + 1));
|
|
126
|
-
if (!Array.isArray(parsed)) {
|
|
127
|
-
return parsed;
|
|
128
|
-
}
|
|
129
|
-
arrayFallback ??= parsed;
|
|
130
|
-
break;
|
|
131
|
-
}
|
|
132
|
-
catch {
|
|
133
|
-
break;
|
|
134
|
-
}
|
|
135
|
-
}
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
}
|
|
139
|
-
return arrayFallback;
|
|
115
|
+
export function stripJsonFences(raw) {
|
|
116
|
+
return escapeJsonStringControls(stripCodeFences(stripThinkBlocks(raw)));
|
|
140
117
|
}
|
|
141
118
|
// ── Availability check ──────────────────────────────────────────────────────
|
|
142
119
|
/**
|
package/dist/llm/feature-gate.js
CHANGED
|
@@ -8,16 +8,15 @@
|
|
|
8
8
|
* The seam is intentionally tiny:
|
|
9
9
|
*
|
|
10
10
|
* - `isLlmFeatureEnabled(config, feature)` — pure predicate, no side
|
|
11
|
-
* effects, no I/O. Returns `true`
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
* opts in.
|
|
11
|
+
* effects, no I/O. Returns `true` when the feature flag is explicitly
|
|
12
|
+
* `true`, or when the feature has a non-false default (currently
|
|
13
|
+
* `graph_extraction`).
|
|
15
14
|
* - `tryLlmFeature(feature, config, fn, fallback, opts?)` — single-call
|
|
16
15
|
* wrapper that runs `fn()` only when the gate is open, enforces a hard
|
|
17
|
-
* timeout (default
|
|
18
|
-
* on disablement, throw, or timeout. The wrapper
|
|
19
|
-
* transparent for any given (gate-state, fn-result)
|
|
20
|
-
* state is mutated.
|
|
16
|
+
* timeout (default 600s — overridable per call via `opts.timeoutMs`),
|
|
17
|
+
* and returns `fallback` on disablement, throw, or timeout. The wrapper
|
|
18
|
+
* is referentially transparent for any given (gate-state, fn-result)
|
|
19
|
+
* pair: no module state is mutated.
|
|
21
20
|
*
|
|
22
21
|
* Statelessness invariant (v1 spec §14.4): nothing in this module holds
|
|
23
22
|
* state across calls. There are no caches, no module-level singletons, no
|
|
@@ -29,18 +28,30 @@
|
|
|
29
28
|
/**
|
|
30
29
|
* Pure predicate: is the named feature gate explicitly enabled in `config`?
|
|
31
30
|
*
|
|
32
|
-
* Returns `false` when
|
|
33
|
-
*
|
|
34
|
-
* - the `features` block is missing,
|
|
35
|
-
* - the key is absent (defaults are `false`),
|
|
36
|
-
* - the key is set to `false`.
|
|
31
|
+
* Returns `false` only when the key is explicitly set to `false`, or when
|
|
32
|
+
* the key is absent and its default is `false`.
|
|
37
33
|
*/
|
|
34
|
+
const FEATURE_DEFAULTS = {
|
|
35
|
+
memory_inference: true,
|
|
36
|
+
graph_extraction: true,
|
|
37
|
+
};
|
|
38
38
|
export function isLlmFeatureEnabled(config, feature) {
|
|
39
|
-
|
|
39
|
+
const configured = config?.llm?.features?.[feature];
|
|
40
|
+
if (configured === true)
|
|
41
|
+
return true;
|
|
42
|
+
if (configured === false)
|
|
40
43
|
return false;
|
|
41
|
-
return
|
|
44
|
+
return FEATURE_DEFAULTS[feature] === true;
|
|
42
45
|
}
|
|
43
|
-
|
|
46
|
+
/**
|
|
47
|
+
* Default hard timeout for every bounded in-tree LLM call. Set to 10 minutes
|
|
48
|
+
* (600 000 ms) — generous enough for a slow local model on a single-threaded
|
|
49
|
+
* server. Override per-call via `TryLlmFeatureOptions.timeoutMs`.
|
|
50
|
+
*
|
|
51
|
+
* Do NOT reduce this default without a documented user-facing reason — local
|
|
52
|
+
* model users need the headroom.
|
|
53
|
+
*/
|
|
54
|
+
const DEFAULT_TIMEOUT_MS = 600_000;
|
|
44
55
|
/**
|
|
45
56
|
* Run `fn()` only if `isLlmFeatureEnabled(config, feature)` is `true`. On
|
|
46
57
|
* disablement, throw, or timeout, return `fallback` (or — if it is a
|
|
@@ -5,8 +5,8 @@
|
|
|
5
5
|
* asks the configured LLM to surface the entities mentioned in it and the
|
|
6
6
|
* relations between them. The pass itself
|
|
7
7
|
* (`src/indexer/graph-extraction.ts`) is responsible for deciding which
|
|
8
|
-
* files to extract, persisting the resulting nodes/edges to
|
|
9
|
-
*
|
|
8
|
+
* files to extract, persisting the resulting nodes/edges to the index DB,
|
|
9
|
+
* and feeding the graph data into the FTS5+boosts
|
|
10
10
|
* search pipeline as a single boost component.
|
|
11
11
|
*
|
|
12
12
|
* This module is intentionally tiny and stateless so tests can stub it via
|
|
@@ -20,88 +20,321 @@
|
|
|
20
20
|
import { toErrorMessage } from "../core/common";
|
|
21
21
|
import { warn } from "../core/warn";
|
|
22
22
|
import { chatCompletion, parseEmbeddedJsonResponse } from "./client";
|
|
23
|
+
import { tryLlmFeature } from "./feature-gate";
|
|
24
|
+
import userPromptTemplate from "./prompts/graph-extract-user-prompt.md" with { type: "text" };
|
|
25
|
+
/**
|
|
26
|
+
* Separator token used between assets in a batch prompt.
|
|
27
|
+
* Chosen to be visually clear and unlikely to appear verbatim in asset bodies.
|
|
28
|
+
*/
|
|
29
|
+
const BATCH_ASSET_SEPARATOR = "=== ASSET";
|
|
23
30
|
/** Hard cap on body chars sent to the model. */
|
|
24
31
|
const MAX_BODY_CHARS = 4000;
|
|
25
32
|
/** Hard cap on entities returned per asset — guards against runaway LLM output. */
|
|
26
33
|
const MAX_ENTITIES_PER_ASSET = 32;
|
|
27
34
|
/** Hard cap on relations returned per asset. */
|
|
28
35
|
const MAX_RELATIONS_PER_ASSET = 32;
|
|
29
|
-
|
|
30
|
-
const
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
36
|
+
const SYSTEM_PROMPT = "You extract a knowledge graph from developer notes. Return ONLY valid JSON — no prose, no markdown fences, no preamble.";
|
|
37
|
+
const USER_PROMPT_PREFIX = userPromptTemplate
|
|
38
|
+
.replace("{{MAX_ENTITIES}}", String(MAX_ENTITIES_PER_ASSET))
|
|
39
|
+
.replace("{{MAX_RELATIONS}}", String(MAX_RELATIONS_PER_ASSET));
|
|
40
|
+
function parseConfidence(raw) {
|
|
41
|
+
if (typeof raw !== "number" || !Number.isFinite(raw))
|
|
42
|
+
return undefined;
|
|
43
|
+
return Math.max(0, Math.min(1, raw));
|
|
44
|
+
}
|
|
45
|
+
function normalizeEntityName(raw) {
|
|
46
|
+
return raw
|
|
47
|
+
.trim()
|
|
48
|
+
.replace(/^[`"']+|[`"']+$/g, "")
|
|
49
|
+
.replace(/\s+/g, " ")
|
|
50
|
+
.replace(/[;,!?]+$/g, "")
|
|
51
|
+
.trim();
|
|
52
|
+
}
|
|
53
|
+
function normalizeRelationType(raw) {
|
|
54
|
+
const normalized = raw
|
|
55
|
+
.trim()
|
|
56
|
+
.toLowerCase()
|
|
57
|
+
.replace(/^[`"']+|[`"']+$/g, "")
|
|
58
|
+
.replace(/\s+/g, " ")
|
|
59
|
+
.replace(/[.;,!?]+$/g, "")
|
|
60
|
+
.trim();
|
|
61
|
+
if (!normalized)
|
|
62
|
+
return undefined;
|
|
63
|
+
if (normalized === "use" || normalized === "utilizes")
|
|
64
|
+
return "uses";
|
|
65
|
+
if (normalized === "depend on" || normalized === "depends")
|
|
66
|
+
return "depends on";
|
|
67
|
+
if (normalized === "integrates" || normalized === "integration with")
|
|
68
|
+
return "integrates with";
|
|
69
|
+
return normalized;
|
|
70
|
+
}
|
|
71
|
+
function parseGraphExtraction(raw) {
|
|
72
|
+
const empty = { entities: [], relations: [] };
|
|
73
|
+
if (typeof raw !== "object" || raw === null || Array.isArray(raw))
|
|
74
|
+
return empty;
|
|
75
|
+
const item = raw;
|
|
76
|
+
const entityCanonical = new Map();
|
|
77
|
+
if (Array.isArray(item.entities)) {
|
|
78
|
+
for (const value of item.entities) {
|
|
79
|
+
if (typeof value !== "string")
|
|
80
|
+
continue;
|
|
81
|
+
const normalized = normalizeEntityName(value);
|
|
82
|
+
if (!normalized)
|
|
83
|
+
continue;
|
|
84
|
+
const key = normalized.toLowerCase();
|
|
85
|
+
if (!entityCanonical.has(key))
|
|
86
|
+
entityCanonical.set(key, normalized);
|
|
87
|
+
if (entityCanonical.size >= MAX_ENTITIES_PER_ASSET)
|
|
88
|
+
break;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
const entities = Array.from(entityCanonical.values());
|
|
92
|
+
const relations = [];
|
|
93
|
+
if (Array.isArray(item.relations)) {
|
|
94
|
+
for (const relation of item.relations) {
|
|
95
|
+
if (typeof relation !== "object" || relation === null || Array.isArray(relation))
|
|
96
|
+
continue;
|
|
97
|
+
const rel = relation;
|
|
98
|
+
const fromRaw = typeof rel.from === "string" ? normalizeEntityName(rel.from) : "";
|
|
99
|
+
const toRaw = typeof rel.to === "string" ? normalizeEntityName(rel.to) : "";
|
|
100
|
+
if (!fromRaw || !toRaw)
|
|
101
|
+
continue;
|
|
102
|
+
const from = entityCanonical.get(fromRaw.toLowerCase());
|
|
103
|
+
const to = entityCanonical.get(toRaw.toLowerCase());
|
|
104
|
+
if (!from || !to)
|
|
105
|
+
continue;
|
|
106
|
+
const type = typeof rel.type === "string" ? normalizeRelationType(rel.type) : undefined;
|
|
107
|
+
const confidence = parseConfidence(rel.confidence);
|
|
108
|
+
relations.push({
|
|
109
|
+
from,
|
|
110
|
+
to,
|
|
111
|
+
...(type ? { type } : {}),
|
|
112
|
+
...(confidence !== undefined ? { confidence } : {}),
|
|
113
|
+
});
|
|
114
|
+
if (relations.length >= MAX_RELATIONS_PER_ASSET)
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
const confidence = parseConfidence(item.confidence);
|
|
119
|
+
return {
|
|
120
|
+
entities,
|
|
121
|
+
relations,
|
|
122
|
+
...(confidence !== undefined ? { confidence } : {}),
|
|
123
|
+
};
|
|
124
|
+
}
|
|
125
|
+
/**
|
|
126
|
+
* Build the system prompt for a batched graph-extraction call.
|
|
127
|
+
*
|
|
128
|
+
* The prompt instructs the model to return a JSON array of exactly `count`
|
|
129
|
+
* objects, one per asset, in input order. Index alignment is the critical
|
|
130
|
+
* invariant — if the model drops an asset it still must emit an empty
|
|
131
|
+
* placeholder `{"entities":[],"relations":[]}` at that position.
|
|
132
|
+
*
|
|
133
|
+
* Worked example (3 assets, abbreviated):
|
|
134
|
+
*
|
|
135
|
+
* Input user message:
|
|
136
|
+
* Extract entities and relations from the N=3 assets below.
|
|
137
|
+
* ...rules...
|
|
138
|
+
* === ASSET 1 ===
|
|
139
|
+
* ServiceA integrates with ServiceB.
|
|
140
|
+
* === ASSET 2 ===
|
|
141
|
+
* Terraform provisions the Prod cluster.
|
|
142
|
+
* === ASSET 3 ===
|
|
143
|
+
* No extractable graph content here.
|
|
144
|
+
*
|
|
145
|
+
* Expected model output (valid JSON array, no prose):
|
|
146
|
+
* [
|
|
147
|
+
* {"entities":["ServiceA","ServiceB"],"relations":[{"from":"ServiceA","to":"ServiceB","type":"integrates with"}]},
|
|
148
|
+
* {"entities":["Terraform","Prod cluster"],"relations":[{"from":"Terraform","to":"Prod cluster","type":"provisions"}]},
|
|
149
|
+
* {"entities":[],"relations":[]}
|
|
150
|
+
* ]
|
|
151
|
+
*
|
|
152
|
+
* If the model returns fewer than 3 items (partial failure), the caller
|
|
153
|
+
* (`extractGraphFromBodies`) falls back to individual calls for missing indices.
|
|
154
|
+
*/
|
|
155
|
+
function buildBatchSystemPrompt() {
|
|
156
|
+
return ("You extract knowledge graphs from developer notes. " +
|
|
157
|
+
"Return ONLY a valid JSON array — no prose, no markdown fences, no preamble. " +
|
|
158
|
+
"Each element of the array corresponds to one input asset, in order. " +
|
|
159
|
+
"The array length MUST equal the number of assets provided. " +
|
|
160
|
+
'Use {"entities":[],"relations":[]} for assets with no extractable graph content.');
|
|
161
|
+
}
|
|
162
|
+
function buildBatchUserPrompt(bodies) {
|
|
163
|
+
const count = bodies.length;
|
|
164
|
+
const assetBlocks = bodies
|
|
165
|
+
.map((body, i) => `${BATCH_ASSET_SEPARATOR} ${i + 1} ===\n${body.trim().slice(0, MAX_BODY_CHARS)}`)
|
|
166
|
+
.join("\n\n");
|
|
167
|
+
return (`Extract entities and relations from the N=${count} assets below.\n\n` +
|
|
168
|
+
`Rules:\n` +
|
|
169
|
+
`- Output ONLY a JSON array of exactly ${count} objects, one per asset, preserving input order.\n` +
|
|
170
|
+
`- Each object: {"entities": ["Entity One", ...], "relations": [{"from": "A", "to": "B", "type": "uses"}, ...]}\n` +
|
|
171
|
+
`- Entities are short, canonical noun phrases (project names, services, tools, people, file/dir names, technical concepts).\n` +
|
|
172
|
+
`- Relations connect two entities that both appear in that asset's entities array.\n` +
|
|
173
|
+
`- "type" is a short verb phrase (e.g. "uses", "depends on", "owns"). Optional; omit when unsure.\n` +
|
|
174
|
+
`- Drop pleasantries, meta-commentary, and timestamps.\n` +
|
|
175
|
+
`- Limit to at most ${MAX_ENTITIES_PER_ASSET} entities and ${MAX_RELATIONS_PER_ASSET} relations per asset.\n` +
|
|
176
|
+
`- Use {"entities":[],"relations":[]} for assets with no extractable graph content.\n` +
|
|
177
|
+
`- The array MUST have exactly ${count} elements — one placeholder per asset even if empty.\n\n` +
|
|
178
|
+
assetBlocks);
|
|
179
|
+
}
|
|
180
|
+
/**
|
|
181
|
+
* Parse and validate a single item from the batch response array.
|
|
182
|
+
* Mirrors the validation logic in `extractGraphFromBody`.
|
|
183
|
+
*/
|
|
184
|
+
function parseBatchItem(raw) {
|
|
185
|
+
return parseGraphExtraction(raw);
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Extract entities and relations from multiple asset bodies in a single LLM
|
|
189
|
+
* call (batched graph extraction).
|
|
190
|
+
*
|
|
191
|
+
* Sends all `bodies` as a single prompt with `=== ASSET N ===` separators
|
|
192
|
+
* and expects a JSON array where element `i` corresponds to `bodies[i]`.
|
|
193
|
+
*
|
|
194
|
+
* **Partial-failure handling**: if the model returns fewer elements than
|
|
195
|
+
* `bodies.length`, missing indices are filled by falling back to individual
|
|
196
|
+
* `extractGraphFromBody` calls — ensuring every input always has a result.
|
|
197
|
+
*
|
|
198
|
+
* Returns an array of the same length as `bodies` (never shorter).
|
|
199
|
+
* Individual elements default to `{entities:[], relations:[]}` on failure.
|
|
200
|
+
*
|
|
201
|
+
* Routes through `tryLlmFeature("graph_extraction", ...)` so the feature gate
|
|
202
|
+
* and onFallback hook are honoured uniformly.
|
|
203
|
+
*
|
|
204
|
+
* @param llmConfig - LLM connection configuration.
|
|
205
|
+
* @param bodies - Asset body strings to process in one batch.
|
|
206
|
+
* @param signal - Optional AbortSignal for cancellation.
|
|
207
|
+
* @param akmConfig - Full AKM config (for feature-gate checks).
|
|
208
|
+
* @param onFallback - Optional fallback event sink.
|
|
209
|
+
*/
|
|
210
|
+
export async function extractGraphFromBodies(llmConfig, bodies, signal, akmConfig, onFallback) {
|
|
211
|
+
const empty = () => ({ entities: [], relations: [] });
|
|
212
|
+
// Degenerate case: no bodies → empty array (not an error).
|
|
213
|
+
if (bodies.length === 0)
|
|
214
|
+
return [];
|
|
215
|
+
// Single body: delegate to the single-asset path for identical behaviour.
|
|
216
|
+
if (bodies.length === 1) {
|
|
217
|
+
const result = await extractGraphFromBody(llmConfig, bodies[0] ?? "", signal, akmConfig, onFallback);
|
|
218
|
+
return [result];
|
|
219
|
+
}
|
|
220
|
+
// Filter out bodies that are empty so we don't waste tokens, but keep
|
|
221
|
+
// index correspondence by tracking which indices were non-empty.
|
|
222
|
+
const results = bodies.map(empty);
|
|
223
|
+
const nonEmptyIndices = [];
|
|
224
|
+
const nonEmptyBodies = [];
|
|
225
|
+
for (let i = 0; i < bodies.length; i++) {
|
|
226
|
+
const trimmed = (bodies[i] ?? "").trim();
|
|
227
|
+
if (trimmed) {
|
|
228
|
+
nonEmptyIndices.push(i);
|
|
229
|
+
nonEmptyBodies.push(trimmed);
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
if (nonEmptyBodies.length === 0)
|
|
233
|
+
return results;
|
|
234
|
+
const systemPrompt = buildBatchSystemPrompt();
|
|
235
|
+
const userPrompt = buildBatchUserPrompt(nonEmptyBodies);
|
|
236
|
+
const batchResult = await tryLlmFeature("graph_extraction", akmConfig, async () => {
|
|
237
|
+
try {
|
|
238
|
+
const raw = await chatCompletion(llmConfig, [
|
|
239
|
+
{ role: "system", content: systemPrompt },
|
|
240
|
+
{ role: "user", content: userPrompt },
|
|
241
|
+
], {
|
|
242
|
+
temperature: 0.1,
|
|
243
|
+
timeoutMs: llmConfig.timeoutMs,
|
|
244
|
+
signal,
|
|
245
|
+
});
|
|
246
|
+
if (!raw)
|
|
247
|
+
return null;
|
|
248
|
+
const parsed = parseEmbeddedJsonResponse(raw);
|
|
249
|
+
if (!Array.isArray(parsed)) {
|
|
250
|
+
warn("graph extraction (batch): LLM response was not a JSON array; will fall back per-asset.");
|
|
251
|
+
return null;
|
|
252
|
+
}
|
|
253
|
+
return parsed;
|
|
254
|
+
}
|
|
255
|
+
catch (err) {
|
|
256
|
+
warn(`graph extraction (batch) failed: ${toErrorMessage(err)}`);
|
|
257
|
+
return null;
|
|
258
|
+
}
|
|
259
|
+
}, null, {
|
|
260
|
+
timeoutMs: llmConfig.timeoutMs,
|
|
261
|
+
onFallback,
|
|
262
|
+
});
|
|
263
|
+
// Map successful batch results back to their original indices.
|
|
264
|
+
if (batchResult !== null) {
|
|
265
|
+
for (let j = 0; j < nonEmptyBodies.length; j++) {
|
|
266
|
+
const originalIndex = nonEmptyIndices[j];
|
|
267
|
+
if (originalIndex === undefined)
|
|
268
|
+
continue;
|
|
269
|
+
if (j < batchResult.length) {
|
|
270
|
+
results[originalIndex] = parseBatchItem(batchResult[j]);
|
|
271
|
+
}
|
|
272
|
+
// j >= batchResult.length → partial failure; handled below.
|
|
273
|
+
}
|
|
274
|
+
}
|
|
275
|
+
// Partial-failure fallback: any non-empty body whose result is still the
|
|
276
|
+
// empty placeholder (either because batchResult was null or the array was
|
|
277
|
+
// shorter than expected) gets an individual retry.
|
|
278
|
+
const fallbackIndices = nonEmptyIndices.filter((_origIdx, j) => {
|
|
279
|
+
// Result is still empty → needs a fallback call.
|
|
280
|
+
if (batchResult === null)
|
|
281
|
+
return true;
|
|
282
|
+
// batchResult was shorter than the number of non-empty bodies.
|
|
283
|
+
return j >= batchResult.length;
|
|
284
|
+
});
|
|
285
|
+
if (fallbackIndices.length > 0) {
|
|
286
|
+
if (batchResult !== null) {
|
|
287
|
+
// Only warn on partial failure (not when the whole batch failed, which
|
|
288
|
+
// already emitted a warn above).
|
|
289
|
+
warn(`graph extraction (batch): response had ${batchResult.length} items for ${nonEmptyBodies.length} assets; ` +
|
|
290
|
+
`falling back to individual calls for ${fallbackIndices.length} missing asset(s).`);
|
|
291
|
+
}
|
|
292
|
+
await Promise.all(fallbackIndices.map(async (origIdx) => {
|
|
293
|
+
const body = bodies[origIdx] ?? "";
|
|
294
|
+
results[origIdx] = await extractGraphFromBody(llmConfig, body, signal, akmConfig, onFallback);
|
|
295
|
+
}));
|
|
296
|
+
}
|
|
297
|
+
return results;
|
|
298
|
+
}
|
|
45
299
|
/**
|
|
46
300
|
* Extract entities and relations from a single asset body via the configured LLM.
|
|
47
301
|
*
|
|
48
302
|
* Returns `{entities: [], relations: []}` on any failure (timeout, invalid
|
|
49
303
|
* JSON, empty response). Errors are logged via `warn()` but never thrown — a
|
|
50
304
|
* failed extraction for one asset must not abort the rest of the index pass.
|
|
305
|
+
*
|
|
306
|
+
* Routes through `tryLlmFeature("graph_extraction", ...)` so the feature gate
|
|
307
|
+
* and onFallback hook are honoured uniformly (Fix C5).
|
|
51
308
|
*/
|
|
52
|
-
export async function extractGraphFromBody(llmConfig, body, signal) {
|
|
309
|
+
export async function extractGraphFromBody(llmConfig, body, signal, akmConfig, onFallback) {
|
|
53
310
|
const empty = { entities: [], relations: [] };
|
|
54
311
|
const trimmedBody = body.trim();
|
|
55
312
|
if (!trimmedBody)
|
|
56
313
|
return empty;
|
|
57
314
|
const userPrompt = `${USER_PROMPT_PREFIX}${trimmedBody.slice(0, MAX_BODY_CHARS)}`;
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
chatCompletion(llmConfig, [
|
|
315
|
+
return tryLlmFeature("graph_extraction", akmConfig, async () => {
|
|
316
|
+
try {
|
|
317
|
+
const raw = await chatCompletion(llmConfig, [
|
|
62
318
|
{ role: "system", content: SYSTEM_PROMPT },
|
|
63
319
|
{ role: "user", content: userPrompt },
|
|
64
|
-
], {
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
320
|
+
], { temperature: 0.1, timeoutMs: llmConfig.timeoutMs, signal });
|
|
321
|
+
if (!raw)
|
|
322
|
+
return empty;
|
|
323
|
+
const parsed = parseEmbeddedJsonResponse(raw);
|
|
324
|
+
if (!parsed) {
|
|
325
|
+
warn("graph extraction: invalid JSON response from LLM; skipping asset.");
|
|
326
|
+
return empty;
|
|
327
|
+
}
|
|
328
|
+
return parseGraphExtraction(parsed);
|
|
329
|
+
}
|
|
330
|
+
catch (err) {
|
|
331
|
+
warn(`graph extraction failed: ${toErrorMessage(err)}`);
|
|
74
332
|
return empty;
|
|
75
333
|
}
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
.filter((e) => e.length > 0)
|
|
81
|
-
.slice(0, MAX_ENTITIES_PER_ASSET)
|
|
82
|
-
: [];
|
|
83
|
-
const entitySet = new Set(entities);
|
|
84
|
-
const relations = Array.isArray(parsed.relations)
|
|
85
|
-
? parsed.relations
|
|
86
|
-
.filter((r) => typeof r === "object" && r !== null && !Array.isArray(r))
|
|
87
|
-
.map((r) => ({
|
|
88
|
-
from: typeof r.from === "string" ? r.from.trim() : "",
|
|
89
|
-
to: typeof r.to === "string" ? r.to.trim() : "",
|
|
90
|
-
type: typeof r.type === "string" && r.type.trim() ? r.type.trim() : undefined,
|
|
91
|
-
}))
|
|
92
|
-
// Both endpoints must be non-empty AND mentioned in entities[];
|
|
93
|
-
// dangling relations are noise and inflate the boost component.
|
|
94
|
-
.filter((r) => r.from && r.to && entitySet.has(r.from) && entitySet.has(r.to))
|
|
95
|
-
.slice(0, MAX_RELATIONS_PER_ASSET)
|
|
96
|
-
: [];
|
|
97
|
-
return { entities, relations };
|
|
98
|
-
}
|
|
99
|
-
catch (err) {
|
|
100
|
-
warn(`graph extraction failed: ${toErrorMessage(err)}`);
|
|
101
|
-
return empty;
|
|
102
|
-
}
|
|
103
|
-
finally {
|
|
104
|
-
if (timeoutHandle !== undefined)
|
|
105
|
-
clearTimeout(timeoutHandle);
|
|
106
|
-
}
|
|
334
|
+
}, empty, {
|
|
335
|
+
timeoutMs: llmConfig.timeoutMs,
|
|
336
|
+
onFallback,
|
|
337
|
+
});
|
|
107
338
|
}
|
|
339
|
+
// deduplicateGraph moved to src/indexer/graph-dedup.ts (pure utility, no LLM calls).
|
|
340
|
+
export { deduplicateGraph } from "../indexer/graph-dedup";
|