akm-cli 0.9.0-beta.26 → 0.9.0-beta.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/dist/commands/improve/improve.js +281 -146
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +1 -0
- package/dist/commands/lint/registry.js +2 -0
- package/dist/commands/read/curate.js +50 -1
- package/dist/commands/read/show.js +67 -2
- package/dist/core/asset/asset-registry.js +2 -0
- package/dist/core/asset/asset-spec.js +14 -0
- package/dist/core/config/config-schema.js +14 -2
- package/dist/indexer/db/db.js +17 -0
- package/dist/indexer/db/graph-db.js +48 -0
- package/dist/indexer/graph/graph-extraction.js +203 -3
- package/dist/indexer/search/ranking-contributors.js +22 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/integrations/agent/prompts.js +1 -0
- package/dist/output/renderers.js +73 -1
- package/dist/scripts/migrate-storage.js +88 -4
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +30 -0
- package/package.json +6 -5
|
@@ -13,12 +13,18 @@
|
|
|
13
13
|
* helpers (`curateSearchResults`, `deriveCurateFallbackQueries`,
|
|
14
14
|
* `mergeCurateSearchResponses`) by importing them directly.
|
|
15
15
|
*/
|
|
16
|
+
import fs from "node:fs";
|
|
16
17
|
import { parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
18
|
+
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
19
|
+
import { getIndexPassConfig, loadConfig } from "../../core/config/config.js";
|
|
17
20
|
import { rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
18
21
|
import { appendEvent } from "../../core/events.js";
|
|
19
|
-
import { closeDatabase, openExistingDatabase } from "../../indexer/db/db.js";
|
|
22
|
+
import { closeDatabase, computeBodyHash, openExistingDatabase } from "../../indexer/db/db.js";
|
|
23
|
+
import { enqueueGraphExtraction, hasGraphData } from "../../indexer/db/graph-db.js";
|
|
24
|
+
import { findSourceForPath, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
20
25
|
import { insertUsageEvent } from "../../indexer/usage/usage-events.js";
|
|
21
26
|
import { truncateDescription } from "../../output/shapes.js";
|
|
27
|
+
import { withIndexDb } from "../../storage/repositories/index-db.js";
|
|
22
28
|
import { akmSearch, parseSearchSource } from "./search.js";
|
|
23
29
|
import { akmShowUnified } from "./show.js";
|
|
24
30
|
const CURATE_FALLBACK_FILTER_WORDS = new Set([
|
|
@@ -145,6 +151,11 @@ async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs) {
|
|
|
145
151
|
catch {
|
|
146
152
|
shown = undefined;
|
|
147
153
|
}
|
|
154
|
+
// #624-P3: when lazy graph extraction is opted in, enqueue an ungraphed
|
|
155
|
+
// asset for a later pass to extract. Fire-and-forget, non-blocking, NO inline
|
|
156
|
+
// extraction and NO LLM call here. Default-off (flag unset) = byte-identical.
|
|
157
|
+
if (shown?.path)
|
|
158
|
+
maybeEnqueueLazyGraph(shown.path);
|
|
148
159
|
const description = shown?.description ?? hit.description;
|
|
149
160
|
const preview = buildCuratedPreview(shown, hit);
|
|
150
161
|
const mergedSupportRefs = mergeCurateSupportRefs(supportRefs, shown?.related?.hits, selectedRefs, hit.ref);
|
|
@@ -164,6 +175,44 @@ async function enrichCuratedStashHit(query, hit, supportRefs, selectedRefs) {
|
|
|
164
175
|
...(hit.score !== undefined ? { score: hit.score } : {}),
|
|
165
176
|
};
|
|
166
177
|
}
|
|
178
|
+
/**
|
|
179
|
+
* #624-P3 — enqueue an ungraphed asset for lazy graph extraction when the
|
|
180
|
+
* `index.graph.lazyGraphExtraction` flag is on. Pure side-effect, fully
|
|
181
|
+
* best-effort: any failure (config, fs, db) is swallowed so curate never fails
|
|
182
|
+
* on it. NO LLM call and NO inline extraction — only a cheap queue insert.
|
|
183
|
+
* Default-off (flag unset) returns immediately = byte-identical behavior.
|
|
184
|
+
*/
|
|
185
|
+
function maybeEnqueueLazyGraph(assetPath) {
|
|
186
|
+
try {
|
|
187
|
+
const config = loadConfig();
|
|
188
|
+
if (getIndexPassConfig(config.index, "graph")?.lazyGraphExtraction !== true)
|
|
189
|
+
return;
|
|
190
|
+
const sources = resolveSourceEntries();
|
|
191
|
+
const source = findSourceForPath(assetPath, sources);
|
|
192
|
+
const stashRoot = source?.path;
|
|
193
|
+
if (!stashRoot)
|
|
194
|
+
return;
|
|
195
|
+
let raw;
|
|
196
|
+
try {
|
|
197
|
+
raw = fs.readFileSync(assetPath, "utf8");
|
|
198
|
+
}
|
|
199
|
+
catch {
|
|
200
|
+
return;
|
|
201
|
+
}
|
|
202
|
+
const body = parseFrontmatter(raw).content.trim();
|
|
203
|
+
if (!body)
|
|
204
|
+
return;
|
|
205
|
+
const bodyHash = computeBodyHash(body);
|
|
206
|
+
withIndexDb((db) => {
|
|
207
|
+
if (!hasGraphData(db, stashRoot, assetPath)) {
|
|
208
|
+
enqueueGraphExtraction(db, stashRoot, assetPath, bodyHash, 0);
|
|
209
|
+
}
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
catch (err) {
|
|
213
|
+
rethrowIfTestIsolationError(err);
|
|
214
|
+
}
|
|
215
|
+
}
|
|
167
216
|
function buildCuratedRegistryItem(query, hit) {
|
|
168
217
|
return {
|
|
169
218
|
source: "registry",
|
|
@@ -22,18 +22,22 @@ import { parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
|
22
22
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
23
23
|
import { META_DIR, parseMetaRef, resolveMetaFilePath } from "../../core/asset/stash-meta.js";
|
|
24
24
|
import { asNonEmptyString } from "../../core/common.js";
|
|
25
|
-
import { loadConfig } from "../../core/config/config.js";
|
|
25
|
+
import { getIndexPassConfig, loadConfig } from "../../core/config/config.js";
|
|
26
26
|
import { NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
27
27
|
import { appendEvent, readEvents } from "../../core/events.js";
|
|
28
|
-
import { findEntryIdByRef } from "../../indexer/db/db.js";
|
|
28
|
+
import { closeDatabase, computeBodyHash, findEntryIdByRef, openExistingDatabase } from "../../indexer/db/db.js";
|
|
29
|
+
import { hasGraphData } from "../../indexer/db/graph-db.js";
|
|
29
30
|
import { ensureIndex } from "../../indexer/ensure-index.js";
|
|
30
31
|
import { listRelatedPathsForFile } from "../../indexer/graph/graph-boost.js";
|
|
32
|
+
import { extractGraphForSingleFile } from "../../indexer/graph/graph-extraction.js";
|
|
31
33
|
import { lookup } from "../../indexer/indexer.js";
|
|
32
34
|
import { buildEditHint, findSourceForPath, isEditable, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
33
35
|
import { insertUsageEvent } from "../../indexer/usage/usage-events.js";
|
|
34
36
|
import { buildFileContext, buildRenderContext, getRenderer, runMatchers } from "../../indexer/walk/file-context.js";
|
|
35
37
|
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
38
|
+
import { resolveIndexPassLLM } from "../../llm/index-passes.js";
|
|
36
39
|
import { resolveSourcesForOrigin } from "../../registry/origin-resolve.js";
|
|
40
|
+
import { resolveStorageLocations } from "../../storage/locations.js";
|
|
37
41
|
import { withIndexDb } from "../../storage/repositories/index-db.js";
|
|
38
42
|
// Eagerly import source providers to trigger self-registration.
|
|
39
43
|
import "../../sources/providers/index.js";
|
|
@@ -383,6 +387,15 @@ export async function showLocal(input) {
|
|
|
383
387
|
if (activeRun) {
|
|
384
388
|
fullResponse.activeRun = activeRun;
|
|
385
389
|
}
|
|
390
|
+
// #624-P3: opt-in inline graph extraction. Default OFF — when the flag is
|
|
391
|
+
// unset this whole block is skipped (no hasGraphData check, no LLM call), so
|
|
392
|
+
// behavior is byte-identical to today. When ON, it extracts graph data for an
|
|
393
|
+
// ungraphed asset, but ONLY when a model is configured (model-available
|
|
394
|
+
// guard) and ALWAYS bounded by a 30s timeout so `show` can never hang. Any
|
|
395
|
+
// timeout/model-unavailable/error path returns the response unchanged.
|
|
396
|
+
if (getIndexPassConfig(config.index, "graph")?.lazyGraphExtraction === true) {
|
|
397
|
+
await maybeExtractGraphInline(config, sourceStashDir, assetPath);
|
|
398
|
+
}
|
|
386
399
|
if (input.detail === "brief") {
|
|
387
400
|
return buildBriefResponse(fullResponse, assetPath);
|
|
388
401
|
}
|
|
@@ -391,6 +404,58 @@ export async function showLocal(input) {
|
|
|
391
404
|
}
|
|
392
405
|
return fullResponse;
|
|
393
406
|
}
|
|
407
|
+
/**
|
|
408
|
+
* #624-P3 — opt-in inline graph extraction for `akm show`. Best-effort and
|
|
409
|
+
* timeout-bounded: never throws, never hangs, never mutates the response.
|
|
410
|
+
*
|
|
411
|
+
* Preconditions (caller already checked the flag): a model must be configured
|
|
412
|
+
* (model-available guard via {@link resolveIndexPassLLM}) and the asset must be
|
|
413
|
+
* ungraphed ({@link hasGraphData}). Extraction races a 30s timeout so `show`
|
|
414
|
+
* cannot block on a slow provider; any timeout/error/missing-model path is
|
|
415
|
+
* swallowed and `show` returns its already-assembled response unchanged.
|
|
416
|
+
*/
|
|
417
|
+
async function maybeExtractGraphInline(config, sourceStashDir, assetPath) {
|
|
418
|
+
try {
|
|
419
|
+
// Model-available guard — no provider configured ⇒ silent skip, no LLM call.
|
|
420
|
+
if (!resolveIndexPassLLM("graph", config))
|
|
421
|
+
return;
|
|
422
|
+
let alreadyGraphed = false;
|
|
423
|
+
let bodyHash;
|
|
424
|
+
try {
|
|
425
|
+
const raw = fs.readFileSync(assetPath, "utf8");
|
|
426
|
+
bodyHash = computeBodyHash(parseFrontmatter(raw).content.trim());
|
|
427
|
+
}
|
|
428
|
+
catch {
|
|
429
|
+
return; // file gone/unreadable ⇒ nothing to extract
|
|
430
|
+
}
|
|
431
|
+
withIndexDb((db) => {
|
|
432
|
+
alreadyGraphed = hasGraphData(db, sourceStashDir, assetPath);
|
|
433
|
+
});
|
|
434
|
+
if (alreadyGraphed)
|
|
435
|
+
return;
|
|
436
|
+
// Open the db for the async extraction ourselves: `withIndexDb` is
|
|
437
|
+
// synchronous and would close the connection the instant the async fn
|
|
438
|
+
// returns its Promise (before extraction completes). Close it explicitly
|
|
439
|
+
// after the race settles instead.
|
|
440
|
+
const db = openExistingDatabase(resolveStorageLocations().indexDb);
|
|
441
|
+
let timer;
|
|
442
|
+
const timeout = new Promise((resolve) => {
|
|
443
|
+
timer = setTimeout(resolve, 30_000);
|
|
444
|
+
});
|
|
445
|
+
try {
|
|
446
|
+
await Promise.race([extractGraphForSingleFile(db, sourceStashDir, assetPath, bodyHash, { config }), timeout]);
|
|
447
|
+
}
|
|
448
|
+
finally {
|
|
449
|
+
if (timer)
|
|
450
|
+
clearTimeout(timer);
|
|
451
|
+
closeDatabase(db);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
catch (err) {
|
|
455
|
+
rethrowIfTestIsolationError(err);
|
|
456
|
+
// Any other failure: silently return the unchanged show response.
|
|
457
|
+
}
|
|
458
|
+
}
|
|
394
459
|
/**
|
|
395
460
|
* Minimal `show`: ref → indexer lookup → file contents. Used by callers that
|
|
396
461
|
* just need the raw file (e.g. clone, write-source) and don't want the full
|
|
@@ -29,6 +29,7 @@ export const TYPE_TO_RENDERER = {
|
|
|
29
29
|
wiki: "wiki-md",
|
|
30
30
|
task: "task-yaml",
|
|
31
31
|
session: "session-md",
|
|
32
|
+
fact: "fact-md",
|
|
32
33
|
};
|
|
33
34
|
/** Map asset types to action builder functions for search results. */
|
|
34
35
|
export const ACTION_BUILDERS = {
|
|
@@ -45,6 +46,7 @@ export const ACTION_BUILDERS = {
|
|
|
45
46
|
wiki: (ref) => `akm show ${ref} -> read the wiki page`,
|
|
46
47
|
task: (ref) => `akm tasks show ${ref.replace(/^task:/, "")} -> inspect; akm tasks run <id> -> run now; akm tasks remove <id> -> unschedule`,
|
|
47
48
|
session: (ref) => `akm show ${ref} -> read the session summary; follow the \`access\` frontmatter to open the raw log at \`log_path\``,
|
|
49
|
+
fact: (ref) => `akm show ${ref} -> read the stash fact and apply it as durable context`,
|
|
48
50
|
};
|
|
49
51
|
/**
|
|
50
52
|
* Register a type-to-renderer mapping.
|
|
@@ -154,6 +154,20 @@ const ASSET_SPECS_INTERNAL = {
|
|
|
154
154
|
rendererName: "session-md",
|
|
155
155
|
actionBuilder: (ref) => `akm show ${ref} -> read the session summary; follow the \`access\` frontmatter to open the raw log at \`log_path\``,
|
|
156
156
|
},
|
|
157
|
+
// Durable stash-level semantic knowledge — facts about the user, team, or
|
|
158
|
+
// project (personal details, team tool stacks, coding conventions /
|
|
159
|
+
// "constitution", and stash-meta like naming conventions or the active
|
|
160
|
+
// projects list). Unlike `memory` (episodic, recency-decayed) these are
|
|
161
|
+
// mostly-static declarations meant to be reliably surfaced as context. A
|
|
162
|
+
// plain markdown spec; `category` frontmatter scopes the fact and
|
|
163
|
+
// `pinned: true` marks the small always-injected core. See
|
|
164
|
+
// docs/design/fact-asset-type.md.
|
|
165
|
+
fact: {
|
|
166
|
+
stashDir: "facts",
|
|
167
|
+
...markdownSpec,
|
|
168
|
+
rendererName: "fact-md",
|
|
169
|
+
actionBuilder: (ref) => `akm show ${ref} -> read the stash fact and apply it as durable context`,
|
|
170
|
+
},
|
|
157
171
|
};
|
|
158
172
|
export const ASSET_SPECS = ASSET_SPECS_INTERNAL;
|
|
159
173
|
/**
|
|
@@ -190,6 +190,11 @@ export const ImproveProcessConfigSchema = z
|
|
|
190
190
|
// proactiveMaintenance process: top-N bound per run (default 25). Alias for
|
|
191
191
|
// `limit`; `maxPerRun` wins when both are set.
|
|
192
192
|
maxPerRun: positiveInt.optional(),
|
|
193
|
+
// graphExtraction process (#624 P2): when set, rank eligible files by
|
|
194
|
+
// utility_scores DESC and process only the top-N per run (incremental
|
|
195
|
+
// high-signal-first sweep). Unset = process all eligible (current
|
|
196
|
+
// behavior). Only meaningful on `graphExtraction`.
|
|
197
|
+
topN: positiveInt.optional(),
|
|
193
198
|
// MemoryInference process: minimum pending memory count to run the pass.
|
|
194
199
|
minPendingCount: z.number().int().min(0).optional(),
|
|
195
200
|
// Extract process: minimum number of new (unseen, in-window) candidate
|
|
@@ -386,6 +391,10 @@ export const ImproveProfileConfigSchema = z
|
|
|
386
391
|
processes: ImproveProfileProcessesSchema.optional(),
|
|
387
392
|
autoAccept: nonNegativeNumber.optional(),
|
|
388
393
|
limit: positiveInt.optional(),
|
|
394
|
+
// #616 — bounded multi-cycle phasing. Number of prep->loop->post-loop
|
|
395
|
+
// cycles per run. positiveInt forbids 0/negative. DEFAULT 1 => byte-identical
|
|
396
|
+
// single-pass behavior.
|
|
397
|
+
maxCycles: positiveInt.optional(),
|
|
389
398
|
// #614 — symmetric valence weighting in the eligibility sort. When true,
|
|
390
399
|
// the attention term becomes |valence| MAGNITUDE so BOTH strong positive
|
|
391
400
|
// and strong negative feedback drive attention (utility stays dominant) and
|
|
@@ -608,6 +617,7 @@ const GRAPH_EXTRACTION_INCLUDE_TYPES_ALLOWED = [
|
|
|
608
617
|
"lesson",
|
|
609
618
|
"task",
|
|
610
619
|
"wiki",
|
|
620
|
+
"fact",
|
|
611
621
|
];
|
|
612
622
|
const INDEX_PASS_PROVIDER_KEYS = new Set([
|
|
613
623
|
"endpoint",
|
|
@@ -624,6 +634,7 @@ const INDEX_PASS_KNOWN_KEYS = new Set([
|
|
|
624
634
|
"graphExtractionBatchSize",
|
|
625
635
|
"graphExtractionIncludeTypes",
|
|
626
636
|
"memoryInferenceBatchSize",
|
|
637
|
+
"lazyGraphExtraction",
|
|
627
638
|
]);
|
|
628
639
|
/**
|
|
629
640
|
* Per-pass `index.<pass>` entry. Uses preprocess + manual validation so we can
|
|
@@ -650,8 +661,8 @@ export const IndexPassConfigSchema = z.preprocess((raw, ctx) => {
|
|
|
650
661
|
ctx.addIssue({
|
|
651
662
|
code: z.ZodIssueCode.custom,
|
|
652
663
|
message: `Unknown key \`${[...(ctx.path ?? []), key].join(".")}\`. Per-pass entries support \`llm\` ` +
|
|
653
|
-
"(boolean opt-out), `graphExtractionBatchSize`, `graphExtractionIncludeTypes`,
|
|
654
|
-
"`memoryInferenceBatchSize`.",
|
|
664
|
+
"(boolean opt-out), `graphExtractionBatchSize`, `graphExtractionIncludeTypes`, " +
|
|
665
|
+
"`memoryInferenceBatchSize`, and `lazyGraphExtraction`.",
|
|
655
666
|
});
|
|
656
667
|
return raw;
|
|
657
668
|
}
|
|
@@ -670,6 +681,7 @@ export const IndexPassConfigSchema = z.preprocess((raw, ctx) => {
|
|
|
670
681
|
graphExtractionBatchSize: positiveInt.optional(),
|
|
671
682
|
graphExtractionIncludeTypes: z.array(z.enum(GRAPH_EXTRACTION_INCLUDE_TYPES_ALLOWED)).nonempty().optional(),
|
|
672
683
|
memoryInferenceBatchSize: positiveInt.optional(),
|
|
684
|
+
lazyGraphExtraction: z.boolean().optional(),
|
|
673
685
|
})
|
|
674
686
|
.passthrough());
|
|
675
687
|
const MetadataEnhanceSchema = z.object({ enabled: z.boolean().optional() }).strict();
|
package/dist/indexer/db/db.js
CHANGED
|
@@ -397,6 +397,22 @@ function ensureSchema(db, embeddingDim, options) {
|
|
|
397
397
|
FOREIGN KEY (stash_root, file_path, body_hash)
|
|
398
398
|
REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
|
|
399
399
|
);
|
|
400
|
+
|
|
401
|
+
-- #624-P3: lazy graph-extraction queue. Standalone table (NO FK to
|
|
402
|
+
-- graph_files — a queued file by definition has no graph row yet).
|
|
403
|
+
-- Idempotent on (stash_root, file_path); drained highest-priority-first.
|
|
404
|
+
-- CREATE TABLE IF NOT EXISTS is the forward migration (no DB_VERSION bump).
|
|
405
|
+
CREATE TABLE IF NOT EXISTS graph_extraction_queue (
|
|
406
|
+
stash_root TEXT NOT NULL,
|
|
407
|
+
file_path TEXT NOT NULL,
|
|
408
|
+
body_hash TEXT NOT NULL,
|
|
409
|
+
queued_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
410
|
+
priority INTEGER NOT NULL DEFAULT 0,
|
|
411
|
+
PRIMARY KEY (stash_root, file_path)
|
|
412
|
+
);
|
|
413
|
+
|
|
414
|
+
CREATE INDEX IF NOT EXISTS idx_graph_extraction_queue_drain
|
|
415
|
+
ON graph_extraction_queue(stash_root, priority DESC, queued_at);
|
|
400
416
|
`);
|
|
401
417
|
// #624-P1 migration step 2: copy any renamed-aside legacy graph data into the
|
|
402
418
|
// new-shape tables (just created above), then drop the legacy tables. No-op
|
|
@@ -516,6 +532,7 @@ function handleVersionUpgrade(db) {
|
|
|
516
532
|
db.exec("DROP TABLE IF EXISTS index_dir_state");
|
|
517
533
|
db.exec("DROP TABLE IF EXISTS llm_enrichment_cache");
|
|
518
534
|
db.exec("DROP INDEX IF EXISTS idx_llm_cache_updated");
|
|
535
|
+
db.exec("DROP TABLE IF EXISTS graph_extraction_queue");
|
|
519
536
|
db.exec("DROP TABLE IF EXISTS graph_file_relations");
|
|
520
537
|
db.exec("DROP TABLE IF EXISTS graph_file_entities");
|
|
521
538
|
db.exec("DROP TABLE IF EXISTS graph_files");
|
|
@@ -167,6 +167,54 @@ export function hasGraphData(db, stashRoot, filePath) {
|
|
|
167
167
|
return false;
|
|
168
168
|
}
|
|
169
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* #624-P3 — enqueue a file for lazy graph extraction. Idempotent on the
|
|
172
|
+
* (stash_root, file_path) PK: a second enqueue refreshes body_hash + queued_at
|
|
173
|
+
* and keeps the HIGHER priority. Non-blocking, no LLM call — the queued row is
|
|
174
|
+
* drained later by the graph-extraction pass. Tolerant of a missing table /
|
|
175
|
+
* db error (best-effort), but never masks the bun-test isolation guard.
|
|
176
|
+
*/
|
|
177
|
+
export function enqueueGraphExtraction(db, stashRoot, filePath, bodyHash, priority = 0) {
|
|
178
|
+
try {
|
|
179
|
+
db.prepare(`INSERT INTO graph_extraction_queue (stash_root, file_path, body_hash, priority)
|
|
180
|
+
VALUES (?, ?, ?, ?)
|
|
181
|
+
ON CONFLICT(stash_root, file_path) DO UPDATE SET
|
|
182
|
+
body_hash = excluded.body_hash,
|
|
183
|
+
priority = MAX(graph_extraction_queue.priority, excluded.priority),
|
|
184
|
+
queued_at = datetime('now')`).run(stashRoot, filePath, bodyHash, priority);
|
|
185
|
+
}
|
|
186
|
+
catch (err) {
|
|
187
|
+
rethrowIfTestIsolationError(err);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* #624-P3 — drain up to `limit` queued files for a stash, highest-priority
|
|
192
|
+
* first (then oldest queued_at). The returned rows are DELETED from the queue
|
|
193
|
+
* in the SAME transaction (SELECT-then-DELETE-by-PK), so a drain is exactly
|
|
194
|
+
* once. Tolerant of a missing table / db error (returns []), but never masks
|
|
195
|
+
* the bun-test isolation guard.
|
|
196
|
+
*/
|
|
197
|
+
export function drainExtractionQueue(db, stashRoot, limit) {
|
|
198
|
+
try {
|
|
199
|
+
return db.transaction(() => {
|
|
200
|
+
const rows = db
|
|
201
|
+
.prepare(`SELECT file_path, body_hash, priority
|
|
202
|
+
FROM graph_extraction_queue
|
|
203
|
+
WHERE stash_root = ?
|
|
204
|
+
ORDER BY priority DESC, queued_at ASC
|
|
205
|
+
LIMIT ?`)
|
|
206
|
+
.all(stashRoot, limit);
|
|
207
|
+
const del = db.prepare("DELETE FROM graph_extraction_queue WHERE stash_root = ? AND file_path = ?");
|
|
208
|
+
for (const row of rows)
|
|
209
|
+
del.run(stashRoot, row.file_path);
|
|
210
|
+
return rows.map((row) => ({ filePath: row.file_path, bodyHash: row.body_hash, priority: row.priority }));
|
|
211
|
+
})();
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
rethrowIfTestIsolationError(err);
|
|
215
|
+
return [];
|
|
216
|
+
}
|
|
217
|
+
}
|
|
170
218
|
/**
|
|
171
219
|
* Scoped loader — only the graph_meta row for a stash. Used by callers that
|
|
172
220
|
* only need summary numbers (e.g. `akm graph summary`).
|
|
@@ -40,13 +40,14 @@ import path from "node:path";
|
|
|
40
40
|
import { TYPE_DIRS } from "../../core/asset/asset-spec.js";
|
|
41
41
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
42
42
|
import { concurrentMap } from "../../core/concurrent.js";
|
|
43
|
-
import { getIndexPassConfig, resolveBatchSize } from "../../core/config/config.js";
|
|
43
|
+
import { getIndexPassConfig, loadConfig, resolveBatchSize } from "../../core/config/config.js";
|
|
44
|
+
import { rethrowIfTestIsolationError } from "../../core/errors.js";
|
|
44
45
|
import { warn, warnVerbose } from "../../core/warn.js";
|
|
45
46
|
import { isProcessEnabled } from "../../llm/feature-gate.js";
|
|
46
47
|
import * as graphExtract from "../../llm/graph-extract.js";
|
|
47
48
|
import { resolveIndexPassLLM } from "../../llm/index-passes.js";
|
|
48
49
|
import { computeBodyHash, GRAPH_SCHEMA_VERSION, getLlmCacheEntriesByRefs, getLlmCacheEntry, upsertLlmCacheEntry, } from "../db/db.js";
|
|
49
|
-
import { loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
|
|
50
|
+
import { drainExtractionQueue, loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
|
|
50
51
|
import { walkMarkdownFiles } from "../walk/walker.js";
|
|
51
52
|
import { deduplicateGraph } from "./graph-dedup.js";
|
|
52
53
|
/** Schema version for the persisted artifact — bumps trigger a full rebuild. */
|
|
@@ -92,6 +93,12 @@ function computeGraphQualityTelemetry(consideredFiles, extractedFiles, entityCou
|
|
|
92
93
|
};
|
|
93
94
|
}
|
|
94
95
|
export const DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES = ["memory", "knowledge"];
|
|
96
|
+
/**
|
|
97
|
+
* Max number of lazy-extraction queue rows drained per pass (#624-P3). Bounds
|
|
98
|
+
* per-run work so a large backlog is spread across runs rather than processed
|
|
99
|
+
* all at once. Generous default — the queue is normally near-empty.
|
|
100
|
+
*/
|
|
101
|
+
const GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT = 100;
|
|
95
102
|
const SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES = new Set([
|
|
96
103
|
"memory",
|
|
97
104
|
"knowledge",
|
|
@@ -291,8 +298,29 @@ export async function runGraphExtractionPass(ctx) {
|
|
|
291
298
|
warnVerbose("graph extraction: skipped because no primary stash source is available.");
|
|
292
299
|
return { ...EMPTY_RESULT };
|
|
293
300
|
}
|
|
301
|
+
// #624-P3: drain the lazy-extraction queue BEFORE the ranked sweep, highest
|
|
302
|
+
// priority first. Queued paths are extracted individually (per-file merge,
|
|
303
|
+
// other files untouched) so they are processed even when they fall outside
|
|
304
|
+
// the normal candidate set. Default (empty queue) is a byte-identical no-op:
|
|
305
|
+
// drainExtractionQueue returns [] and the loop body never runs.
|
|
306
|
+
if (db) {
|
|
307
|
+
const drained = drainExtractionQueue(db, primary.path, GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT);
|
|
308
|
+
for (const queued of drained) {
|
|
309
|
+
if (signal?.aborted)
|
|
310
|
+
break;
|
|
311
|
+
await extractGraphForSingleFile(db, primary.path, queued.filePath, queued.bodyHash, { config, signal });
|
|
312
|
+
}
|
|
313
|
+
}
|
|
294
314
|
const includeTypes = getGraphExtractionIncludeTypes(config);
|
|
295
|
-
|
|
315
|
+
let eligible = collectEligibleFiles(primary.path, includeTypes).filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
|
|
316
|
+
// P2 (#624): when topN is set and a DB is available, rank the (already
|
|
317
|
+
// candidate-filtered) eligible set by utility_scores DESC and keep only the
|
|
318
|
+
// top-N. Default (topN unset) is byte-identical to today — no ranking query
|
|
319
|
+
// is issued and the eligible set is untouched. Ranking composes WITH the
|
|
320
|
+
// candidatePaths filter: scoped-then-ranked-then-sliced.
|
|
321
|
+
if (db && options.topN != null && options.topN >= 0) {
|
|
322
|
+
eligible = rankCandidatesByUtility(db, eligible, primary.path).slice(0, options.topN);
|
|
323
|
+
}
|
|
296
324
|
const considered = eligible.length;
|
|
297
325
|
if (considered === 0) {
|
|
298
326
|
const scoped = options.candidatePaths ? ` matching ${options.candidatePaths.size} candidate path(s)` : "";
|
|
@@ -628,6 +656,178 @@ export async function runGraphExtractionPass(ctx) {
|
|
|
628
656
|
warnings,
|
|
629
657
|
};
|
|
630
658
|
}
|
|
659
|
+
/**
|
|
660
|
+
* Infer the asset type (`memory`, `knowledge`, …) for a path from the stash
|
|
661
|
+
* directory segment it lives under. Returns the matching include-type, or
|
|
662
|
+
* `undefined` when the path is not under a known graph-eligible type dir.
|
|
663
|
+
*/
|
|
664
|
+
function inferGraphTypeForPath(stashRoot, absPath) {
|
|
665
|
+
const rel = path.relative(stashRoot, absPath);
|
|
666
|
+
const firstSeg = rel.split(path.sep)[0];
|
|
667
|
+
if (!firstSeg)
|
|
668
|
+
return undefined;
|
|
669
|
+
for (const type of SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES) {
|
|
670
|
+
if (TYPE_DIRS[type] === firstSeg)
|
|
671
|
+
return type;
|
|
672
|
+
}
|
|
673
|
+
return undefined;
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* #624-P3 — extract graph data for a SINGLE file and merge it into the stored
|
|
677
|
+
* graph WITHOUT clobbering other files' rows.
|
|
678
|
+
*
|
|
679
|
+
* Re-reads the body from disk at call time (the queued body_hash is NOT trusted
|
|
680
|
+
* blindly — the file may have been deleted or changed since enqueue) and skips
|
|
681
|
+
* silently (returns `false`) when the file is gone or empty. Resolves the LLM
|
|
682
|
+
* via {@link resolveIndexPassLLM} (model-available guard: returns `false` when
|
|
683
|
+
* no provider is configured) UNLESS `opts.llmOverride` is supplied, in which
|
|
684
|
+
* case the override is the extractor seam (used by tests and by callers that
|
|
685
|
+
* already hold a resolved model).
|
|
686
|
+
*
|
|
687
|
+
* Returns `true` when a graph row was written for the file, `false` on any
|
|
688
|
+
* skip (missing file, empty body, unknown type, no model, or extraction error).
|
|
689
|
+
*/
|
|
690
|
+
export async function extractGraphForSingleFile(db, stashRoot, filePath, bodyHash, opts) {
|
|
691
|
+
try {
|
|
692
|
+
// Re-read from disk — never trust a stale queued body.
|
|
693
|
+
let raw;
|
|
694
|
+
try {
|
|
695
|
+
raw = fs.readFileSync(filePath, "utf8");
|
|
696
|
+
}
|
|
697
|
+
catch {
|
|
698
|
+
return false; // file gone / unreadable → silent skip
|
|
699
|
+
}
|
|
700
|
+
const parsed = parseFrontmatter(raw);
|
|
701
|
+
const body = parsed.content.trim();
|
|
702
|
+
if (!body)
|
|
703
|
+
return false;
|
|
704
|
+
const type = inferGraphTypeForPath(stashRoot, filePath) ?? "memory";
|
|
705
|
+
const effectiveHash = bodyHash ?? computeBodyHash(body);
|
|
706
|
+
// Extract — via the injected seam, or the real per-asset path.
|
|
707
|
+
let extraction;
|
|
708
|
+
if (opts?.llmOverride) {
|
|
709
|
+
const out = await opts.llmOverride(body);
|
|
710
|
+
extraction = {
|
|
711
|
+
entities: out.entities,
|
|
712
|
+
relations: out.relations,
|
|
713
|
+
...(out.confidence !== undefined ? { confidence: out.confidence } : {}),
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
else {
|
|
717
|
+
const config = opts?.config ?? loadConfig();
|
|
718
|
+
if (!isProcessEnabled("index", "graph_extraction", config))
|
|
719
|
+
return false;
|
|
720
|
+
const llmConfig = resolveIndexPassLLM("graph", config);
|
|
721
|
+
if (!llmConfig)
|
|
722
|
+
return false; // model-available guard
|
|
723
|
+
const result = await graphExtract.extractGraphFromBody(llmConfig, body, opts?.signal, config);
|
|
724
|
+
extraction = {
|
|
725
|
+
entities: result.entities,
|
|
726
|
+
relations: result.relations,
|
|
727
|
+
...(result.confidence !== undefined ? { confidence: result.confidence } : {}),
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
const entities = [...new Set(extraction.entities.map((e) => e.trim()).filter(Boolean))];
|
|
731
|
+
const relations = extraction.relations
|
|
732
|
+
.map((r) => ({
|
|
733
|
+
from: r.from.trim(),
|
|
734
|
+
to: r.to.trim(),
|
|
735
|
+
...(r.type ? { type: r.type.trim() } : {}),
|
|
736
|
+
...(normalizeConfidence(r.confidence) !== undefined ? { confidence: normalizeConfidence(r.confidence) } : {}),
|
|
737
|
+
}))
|
|
738
|
+
.filter((r) => r.from && r.to);
|
|
739
|
+
const node = {
|
|
740
|
+
path: filePath,
|
|
741
|
+
type,
|
|
742
|
+
bodyHash: effectiveHash,
|
|
743
|
+
entities,
|
|
744
|
+
relations,
|
|
745
|
+
...(normalizeConfidence(extraction.confidence) !== undefined
|
|
746
|
+
? { confidence: normalizeConfidence(extraction.confidence) }
|
|
747
|
+
: {}),
|
|
748
|
+
status: entities.length > 0 ? "extracted" : "empty",
|
|
749
|
+
reason: entities.length > 0 ? "none" : "no_graph_content",
|
|
750
|
+
extractionRunId: crypto.randomUUID(),
|
|
751
|
+
};
|
|
752
|
+
// Merge with the previously-stored nodes, scoping the refresh to JUST this
|
|
753
|
+
// path so other files' rows are preserved (and graph_meta counts refresh).
|
|
754
|
+
const previousGraph = loadGraphFile(stashRoot, db);
|
|
755
|
+
const candidatePaths = new Set([filePath]);
|
|
756
|
+
const mergedNodes = mergeGraphNodes(previousGraph.files, [node], candidatePaths);
|
|
757
|
+
const assetRefs = mergedNodes.map((n) => n.path);
|
|
758
|
+
const deduped = deduplicateGraph(mergedNodes.map((n) => ({ entities: n.entities, relations: n.relations })), assetRefs);
|
|
759
|
+
const qualityExtracted = mergedNodes.filter((n) => n.status === "extracted" && n.entities.length > 0).length;
|
|
760
|
+
const quality = computeGraphQualityTelemetry(mergedNodes.length, qualityExtracted, deduped.entities.length, deduped.relations.length);
|
|
761
|
+
const graph = {
|
|
762
|
+
schemaVersion: GRAPH_FILE_SCHEMA_VERSION,
|
|
763
|
+
generatedAt: new Date().toISOString(),
|
|
764
|
+
stashRoot,
|
|
765
|
+
files: mergedNodes,
|
|
766
|
+
entities: deduped.entities,
|
|
767
|
+
relations: deduped.relations,
|
|
768
|
+
quality,
|
|
769
|
+
...(previousGraph.telemetry ? { telemetry: previousGraph.telemetry } : {}),
|
|
770
|
+
};
|
|
771
|
+
return writeGraphFile(stashRoot, graph, db);
|
|
772
|
+
}
|
|
773
|
+
catch (err) {
|
|
774
|
+
rethrowIfTestIsolationError(err);
|
|
775
|
+
return false;
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
// ── Eligible-file detection ─────────────────────────────────────────────────
|
|
779
|
+
/**
|
|
780
|
+
* Rank eligible graph-extraction candidates by their entry `utility_scores`,
|
|
781
|
+
* highest first, for the incremental high-signal-first sweep (P2 of #624).
|
|
782
|
+
*
|
|
783
|
+
* The join is READ-ONLY (`entries.file_path = candidate.absPath`, then
|
|
784
|
+
* `entries.id -> utility_scores.entry_id`) and does NOT re-couple the graph
|
|
785
|
+
* rows to `entries`. It reads the GLOBAL `utility_scores` table (not the
|
|
786
|
+
* per-scope `utility_scores_scoped`), so ranking is corpus-wide; `stashRoot`
|
|
787
|
+
* is accepted for call-site symmetry/future scoping but is not used to filter
|
|
788
|
+
* (the global table has no `stash_root` column).
|
|
789
|
+
*
|
|
790
|
+
* Candidates with no matching `entries` row, or an entry with no
|
|
791
|
+
* `utility_scores` row, get an effective utility of 0 (LEFT JOIN + COALESCE)
|
|
792
|
+
* and sort LAST — they are deprioritized, never dropped, so a `topN >= total`
|
|
793
|
+
* slice still includes them and they remain reachable on later runs.
|
|
794
|
+
*
|
|
795
|
+
* Ties (equal utility) break by `file_path` ASC for deterministic output.
|
|
796
|
+
* Returns a NEW array; the input is not mutated. SQLite's ~999 bound-parameter
|
|
797
|
+
* cap is respected by chunking the `IN (...)` lookup at 500.
|
|
798
|
+
*
|
|
799
|
+
* Exported for direct unit testing.
|
|
800
|
+
*/
|
|
801
|
+
export function rankCandidatesByUtility(db, candidates, _stashRoot) {
|
|
802
|
+
// Cannot rank without a DB → return the input unranked rather than throw.
|
|
803
|
+
// Keeps the DB-less code path (reuse-from-memory) working when topN is set.
|
|
804
|
+
if (!db || candidates.length === 0)
|
|
805
|
+
return candidates;
|
|
806
|
+
const utilityByPath = new Map();
|
|
807
|
+
const CHUNK = 500;
|
|
808
|
+
for (let start = 0; start < candidates.length; start += CHUNK) {
|
|
809
|
+
const chunk = candidates.slice(start, start + CHUNK);
|
|
810
|
+
const paths = chunk.map((c) => c.absPath);
|
|
811
|
+
const placeholders = paths.map(() => "?").join(", ");
|
|
812
|
+
const rows = db
|
|
813
|
+
.prepare(`SELECT e.file_path AS file_path, COALESCE(MAX(u.utility), 0) AS utility
|
|
814
|
+
FROM entries e
|
|
815
|
+
LEFT JOIN utility_scores u ON u.entry_id = e.id
|
|
816
|
+
WHERE e.file_path IN (${placeholders})
|
|
817
|
+
GROUP BY e.file_path`)
|
|
818
|
+
.all(...paths);
|
|
819
|
+
for (const row of rows) {
|
|
820
|
+
utilityByPath.set(row.file_path, row.utility ?? 0);
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
return [...candidates].sort((a, b) => {
|
|
824
|
+
const ua = utilityByPath.get(a.absPath) ?? 0;
|
|
825
|
+
const ub = utilityByPath.get(b.absPath) ?? 0;
|
|
826
|
+
if (ub !== ua)
|
|
827
|
+
return ub - ua; // utility DESC
|
|
828
|
+
return a.absPath < b.absPath ? -1 : a.absPath > b.absPath ? 1 : 0; // tie-break: path ASC
|
|
829
|
+
});
|
|
830
|
+
}
|
|
631
831
|
/**
|
|
632
832
|
* Scan the primary stash for `memory:` and `knowledge:` markdown files
|
|
633
833
|
* suitable for graph extraction. The directory layout convention is the
|
|
@@ -9,6 +9,9 @@ const TYPE_BOOST = {
|
|
|
9
9
|
agent: 0.3,
|
|
10
10
|
script: 0.2,
|
|
11
11
|
knowledge: 0.22,
|
|
12
|
+
// Facts are authoritative, durable declarations about the stash — rank them
|
|
13
|
+
// alongside knowledge so they surface reliably when relevant.
|
|
14
|
+
fact: 0.22,
|
|
12
15
|
memory: -0.02,
|
|
13
16
|
};
|
|
14
17
|
const MAX_BOOST_SUM = 3.0;
|
|
@@ -206,6 +209,24 @@ const lessonStrengthContributor = {
|
|
|
206
209
|
return Math.min(0.3, 0.06 * strength);
|
|
207
210
|
},
|
|
208
211
|
};
|
|
212
|
+
/**
|
|
213
|
+
* Pinned-fact boost.
|
|
214
|
+
*
|
|
215
|
+
* Facts marked `pinned: true` form the small always-injected "core context"
|
|
216
|
+
* (see docs/design/fact-asset-type.md). The fact metadata contributor records
|
|
217
|
+
* a `pinned` search hint; here we give those facts a modest additive boost so
|
|
218
|
+
* the core outranks ordinary facts on otherwise-equal queries. Capped small so
|
|
219
|
+
* it cannot overpower an exact-name match.
|
|
220
|
+
*/
|
|
221
|
+
const pinnedFactRankingContributor = {
|
|
222
|
+
name: "pinned-fact-ranking",
|
|
223
|
+
appliesTo(item) {
|
|
224
|
+
return item.entry.type === "fact" && (item.entry.searchHints?.includes("pinned") ?? false);
|
|
225
|
+
},
|
|
226
|
+
adjust() {
|
|
227
|
+
return 0.15;
|
|
228
|
+
},
|
|
229
|
+
};
|
|
209
230
|
/**
|
|
210
231
|
* Blend ratio for scoped vs. global utility signals.
|
|
211
232
|
*
|
|
@@ -310,6 +331,7 @@ export const defaultRankingContributors = [
|
|
|
310
331
|
graphRankingContributor,
|
|
311
332
|
captureModeRankingContributor,
|
|
312
333
|
lessonStrengthContributor,
|
|
334
|
+
pinnedFactRankingContributor,
|
|
313
335
|
projectContextRankingContributor,
|
|
314
336
|
];
|
|
315
337
|
export const defaultUtilityRankingContributors = [utilityRankingContributor];
|
|
@@ -78,6 +78,15 @@ const DIR_TYPE_MAP = [
|
|
|
78
78
|
type: "session",
|
|
79
79
|
test: (ext) => ext === ".md",
|
|
80
80
|
},
|
|
81
|
+
{
|
|
82
|
+
// Durable stash-level facts live under `facts/<category>/<name>.md`.
|
|
83
|
+
// classifyByDirectory walks every ancestor dir, so nested category
|
|
84
|
+
// subdirs still match. Without this entry a fact file would fall through
|
|
85
|
+
// to classifyBySmartMd and be mistyped as `knowledge`.
|
|
86
|
+
dir: "facts",
|
|
87
|
+
type: "fact",
|
|
88
|
+
test: (ext) => ext === ".md",
|
|
89
|
+
},
|
|
81
90
|
];
|
|
82
91
|
const COMMAND_PLACEHOLDER_RE = /\$ARGUMENTS|\$[123]\b/;
|
|
83
92
|
// Files that should never be treated as the typed asset for the surrounding
|