akm-cli 0.9.0-beta.26 → 0.9.0-beta.28
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +64 -0
- package/dist/cli.js +2 -0
- package/dist/commands/agent/contribute-cli.js +20 -0
- package/dist/commands/fact/fact-cli.js +155 -0
- package/dist/commands/fact/fact-context.js +149 -0
- package/dist/commands/improve/improve.js +281 -146
- package/dist/commands/lint/fact-linter.js +39 -0
- package/dist/commands/lint/index.js +1 -0
- package/dist/commands/lint/registry.js +2 -0
- package/dist/commands/read/curate.js +50 -1
- package/dist/commands/read/knowledge.js +7 -4
- package/dist/commands/read/show.js +67 -2
- package/dist/core/asset/asset-registry.js +2 -0
- package/dist/core/asset/asset-spec.js +14 -0
- package/dist/core/config/config-schema.js +14 -2
- package/dist/indexer/db/db.js +17 -0
- package/dist/indexer/db/graph-db.js +48 -0
- package/dist/indexer/graph/graph-extraction.js +203 -3
- package/dist/indexer/search/ranking-contributors.js +22 -0
- package/dist/indexer/walk/matchers.js +9 -0
- package/dist/integrations/agent/prompts.js +1 -0
- package/dist/output/renderers.js +73 -1
- package/dist/output/shapes/passthrough.js +3 -0
- package/dist/scripts/migrate-storage.js +88 -4
- package/dist/scripts/migrations/import-fs-improve-runs-to-db.js +30 -0
- package/package.json +6 -5
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
import fs from "node:fs";
|
|
12
12
|
import path from "node:path";
|
|
13
13
|
import { assertFlatAssetName, combineCreatePath, normalizeCreateSubPath } from "../../core/asset/asset-create.js";
|
|
14
|
-
import { resolveAssetPathFromName } from "../../core/asset/asset-spec.js";
|
|
14
|
+
import { resolveAssetPathFromName, TYPE_DIRS } from "../../core/asset/asset-spec.js";
|
|
15
15
|
import { isHttpUrl, isWithin, tryReadStdinText } from "../../core/common.js";
|
|
16
16
|
import { loadConfig } from "../../core/config/config.js";
|
|
17
17
|
import { UsageError } from "../../core/errors.js";
|
|
@@ -109,7 +109,9 @@ export async function readKnowledgeInput(source) {
|
|
|
109
109
|
}
|
|
110
110
|
// ── Asset writing ────────────────────────────────────────────────────────────
|
|
111
111
|
/**
|
|
112
|
-
* Write a markdown asset (knowledge or
|
|
112
|
+
* Write a markdown asset (knowledge, memory, or fact) to the resolved write
|
|
113
|
+
* target. The type's stash subdirectory is resolved from the asset registry
|
|
114
|
+
* (`TYPE_DIRS`), so adding a markdown type here needs no further changes.
|
|
113
115
|
*
|
|
114
116
|
* Resolves the write target via the v1 precedence chain (`--target` →
|
|
115
117
|
* `defaultWriteTarget` → working stash), validates the path is within the
|
|
@@ -120,7 +122,7 @@ export async function writeMarkdownAsset(options) {
|
|
|
120
122
|
const cfg = loadConfig();
|
|
121
123
|
const target = resolveWriteTarget(cfg, options.target);
|
|
122
124
|
const { source, config } = target;
|
|
123
|
-
const typeRoot = path.join(source.path, options.type
|
|
125
|
+
const typeRoot = path.join(source.path, TYPE_DIRS[options.type] ?? options.type);
|
|
124
126
|
// `--name` is the flat asset name; `--path` is the subdirectory under the
|
|
125
127
|
// type root. Combine them into the nested name the path resolver expects.
|
|
126
128
|
const subPath = normalizeCreateSubPath(options.path);
|
|
@@ -134,7 +136,8 @@ export async function writeMarkdownAsset(options) {
|
|
|
134
136
|
throw new UsageError(`Resolved ${options.type} path escapes the stash: "${normalizedName}"`);
|
|
135
137
|
}
|
|
136
138
|
if (fs.existsSync(assetPath) && !options.force) {
|
|
137
|
-
|
|
139
|
+
const label = `${options.type.charAt(0).toUpperCase()}${options.type.slice(1)}`;
|
|
140
|
+
throw new UsageError(`${label} "${normalizedName}" already exists. Re-run with --force to overwrite it.`, "RESOURCE_ALREADY_EXISTS");
|
|
138
141
|
}
|
|
139
142
|
const ref = { type: options.type, name: normalizedName };
|
|
140
143
|
const result = await writeAssetToSource(source, config, ref, options.content);
|
|
@@ -22,18 +22,22 @@ import { parseAssetRef } from "../../core/asset/asset-ref.js";
|
|
|
22
22
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
23
23
|
import { META_DIR, parseMetaRef, resolveMetaFilePath } from "../../core/asset/stash-meta.js";
|
|
24
24
|
import { asNonEmptyString } from "../../core/common.js";
|
|
25
|
-
import { loadConfig } from "../../core/config/config.js";
|
|
25
|
+
import { getIndexPassConfig, loadConfig } from "../../core/config/config.js";
|
|
26
26
|
import { NotFoundError, rethrowIfTestIsolationError, UsageError } from "../../core/errors.js";
|
|
27
27
|
import { appendEvent, readEvents } from "../../core/events.js";
|
|
28
|
-
import { findEntryIdByRef } from "../../indexer/db/db.js";
|
|
28
|
+
import { closeDatabase, computeBodyHash, findEntryIdByRef, openExistingDatabase } from "../../indexer/db/db.js";
|
|
29
|
+
import { hasGraphData } from "../../indexer/db/graph-db.js";
|
|
29
30
|
import { ensureIndex } from "../../indexer/ensure-index.js";
|
|
30
31
|
import { listRelatedPathsForFile } from "../../indexer/graph/graph-boost.js";
|
|
32
|
+
import { extractGraphForSingleFile } from "../../indexer/graph/graph-extraction.js";
|
|
31
33
|
import { lookup } from "../../indexer/indexer.js";
|
|
32
34
|
import { buildEditHint, findSourceForPath, isEditable, resolveSourceEntries } from "../../indexer/search/search-source.js";
|
|
33
35
|
import { insertUsageEvent } from "../../indexer/usage/usage-events.js";
|
|
34
36
|
import { buildFileContext, buildRenderContext, getRenderer, runMatchers } from "../../indexer/walk/file-context.js";
|
|
35
37
|
import { resolveAssetPath } from "../../indexer/walk/path-resolver.js";
|
|
38
|
+
import { resolveIndexPassLLM } from "../../llm/index-passes.js";
|
|
36
39
|
import { resolveSourcesForOrigin } from "../../registry/origin-resolve.js";
|
|
40
|
+
import { resolveStorageLocations } from "../../storage/locations.js";
|
|
37
41
|
import { withIndexDb } from "../../storage/repositories/index-db.js";
|
|
38
42
|
// Eagerly import source providers to trigger self-registration.
|
|
39
43
|
import "../../sources/providers/index.js";
|
|
@@ -383,6 +387,15 @@ export async function showLocal(input) {
|
|
|
383
387
|
if (activeRun) {
|
|
384
388
|
fullResponse.activeRun = activeRun;
|
|
385
389
|
}
|
|
390
|
+
// #624-P3: opt-in inline graph extraction. Default OFF — when the flag is
|
|
391
|
+
// unset this whole block is skipped (no hasGraphData check, no LLM call), so
|
|
392
|
+
// behavior is byte-identical to today. When ON, it extracts graph data for an
|
|
393
|
+
// ungraphed asset, but ONLY when a model is configured (model-available
|
|
394
|
+
// guard) and ALWAYS bounded by a 30s timeout so `show` can never hang. Any
|
|
395
|
+
// timeout/model-unavailable/error path returns the response unchanged.
|
|
396
|
+
if (getIndexPassConfig(config.index, "graph")?.lazyGraphExtraction === true) {
|
|
397
|
+
await maybeExtractGraphInline(config, sourceStashDir, assetPath);
|
|
398
|
+
}
|
|
386
399
|
if (input.detail === "brief") {
|
|
387
400
|
return buildBriefResponse(fullResponse, assetPath);
|
|
388
401
|
}
|
|
@@ -391,6 +404,58 @@ export async function showLocal(input) {
|
|
|
391
404
|
}
|
|
392
405
|
return fullResponse;
|
|
393
406
|
}
|
|
407
|
+
/**
|
|
408
|
+
* #624-P3 — opt-in inline graph extraction for `akm show`. Best-effort and
|
|
409
|
+
* timeout-bounded: never throws, never hangs, never mutates the response.
|
|
410
|
+
*
|
|
411
|
+
* Preconditions (caller already checked the flag): a model must be configured
|
|
412
|
+
* (model-available guard via {@link resolveIndexPassLLM}) and the asset must be
|
|
413
|
+
* ungraphed ({@link hasGraphData}). Extraction races a 30s timeout so `show`
|
|
414
|
+
* cannot block on a slow provider; any timeout/error/missing-model path is
|
|
415
|
+
* swallowed and `show` returns its already-assembled response unchanged.
|
|
416
|
+
*/
|
|
417
|
+
async function maybeExtractGraphInline(config, sourceStashDir, assetPath) {
|
|
418
|
+
try {
|
|
419
|
+
// Model-available guard — no provider configured ⇒ silent skip, no LLM call.
|
|
420
|
+
if (!resolveIndexPassLLM("graph", config))
|
|
421
|
+
return;
|
|
422
|
+
let alreadyGraphed = false;
|
|
423
|
+
let bodyHash;
|
|
424
|
+
try {
|
|
425
|
+
const raw = fs.readFileSync(assetPath, "utf8");
|
|
426
|
+
bodyHash = computeBodyHash(parseFrontmatter(raw).content.trim());
|
|
427
|
+
}
|
|
428
|
+
catch {
|
|
429
|
+
return; // file gone/unreadable ⇒ nothing to extract
|
|
430
|
+
}
|
|
431
|
+
withIndexDb((db) => {
|
|
432
|
+
alreadyGraphed = hasGraphData(db, sourceStashDir, assetPath);
|
|
433
|
+
});
|
|
434
|
+
if (alreadyGraphed)
|
|
435
|
+
return;
|
|
436
|
+
// Open the db for the async extraction ourselves: `withIndexDb` is
|
|
437
|
+
// synchronous and would close the connection the instant the async fn
|
|
438
|
+
// returns its Promise (before extraction completes). Close it explicitly
|
|
439
|
+
// after the race settles instead.
|
|
440
|
+
const db = openExistingDatabase(resolveStorageLocations().indexDb);
|
|
441
|
+
let timer;
|
|
442
|
+
const timeout = new Promise((resolve) => {
|
|
443
|
+
timer = setTimeout(resolve, 30_000);
|
|
444
|
+
});
|
|
445
|
+
try {
|
|
446
|
+
await Promise.race([extractGraphForSingleFile(db, sourceStashDir, assetPath, bodyHash, { config }), timeout]);
|
|
447
|
+
}
|
|
448
|
+
finally {
|
|
449
|
+
if (timer)
|
|
450
|
+
clearTimeout(timer);
|
|
451
|
+
closeDatabase(db);
|
|
452
|
+
}
|
|
453
|
+
}
|
|
454
|
+
catch (err) {
|
|
455
|
+
rethrowIfTestIsolationError(err);
|
|
456
|
+
// Any other failure: silently return the unchanged show response.
|
|
457
|
+
}
|
|
458
|
+
}
|
|
394
459
|
/**
|
|
395
460
|
* Minimal `show`: ref → indexer lookup → file contents. Used by callers that
|
|
396
461
|
* just need the raw file (e.g. clone, write-source) and don't want the full
|
|
@@ -29,6 +29,7 @@ export const TYPE_TO_RENDERER = {
|
|
|
29
29
|
wiki: "wiki-md",
|
|
30
30
|
task: "task-yaml",
|
|
31
31
|
session: "session-md",
|
|
32
|
+
fact: "fact-md",
|
|
32
33
|
};
|
|
33
34
|
/** Map asset types to action builder functions for search results. */
|
|
34
35
|
export const ACTION_BUILDERS = {
|
|
@@ -45,6 +46,7 @@ export const ACTION_BUILDERS = {
|
|
|
45
46
|
wiki: (ref) => `akm show ${ref} -> read the wiki page`,
|
|
46
47
|
task: (ref) => `akm tasks show ${ref.replace(/^task:/, "")} -> inspect; akm tasks run <id> -> run now; akm tasks remove <id> -> unschedule`,
|
|
47
48
|
session: (ref) => `akm show ${ref} -> read the session summary; follow the \`access\` frontmatter to open the raw log at \`log_path\``,
|
|
49
|
+
fact: (ref) => `akm show ${ref} -> read the stash fact and apply it as durable context`,
|
|
48
50
|
};
|
|
49
51
|
/**
|
|
50
52
|
* Register a type-to-renderer mapping.
|
|
@@ -154,6 +154,20 @@ const ASSET_SPECS_INTERNAL = {
|
|
|
154
154
|
rendererName: "session-md",
|
|
155
155
|
actionBuilder: (ref) => `akm show ${ref} -> read the session summary; follow the \`access\` frontmatter to open the raw log at \`log_path\``,
|
|
156
156
|
},
|
|
157
|
+
// Durable stash-level semantic knowledge — facts about the user, team, or
|
|
158
|
+
// project (personal details, team tool stacks, coding conventions /
|
|
159
|
+
// "constitution", and stash-meta like naming conventions or the active
|
|
160
|
+
// projects list). Unlike `memory` (episodic, recency-decayed) these are
|
|
161
|
+
// mostly-static declarations meant to be reliably surfaced as context. A
|
|
162
|
+
// plain markdown spec; `category` frontmatter scopes the fact and
|
|
163
|
+
// `pinned: true` marks the small always-injected core. See
|
|
164
|
+
// docs/design/fact-asset-type.md.
|
|
165
|
+
fact: {
|
|
166
|
+
stashDir: "facts",
|
|
167
|
+
...markdownSpec,
|
|
168
|
+
rendererName: "fact-md",
|
|
169
|
+
actionBuilder: (ref) => `akm show ${ref} -> read the stash fact and apply it as durable context`,
|
|
170
|
+
},
|
|
157
171
|
};
|
|
158
172
|
export const ASSET_SPECS = ASSET_SPECS_INTERNAL;
|
|
159
173
|
/**
|
|
@@ -190,6 +190,11 @@ export const ImproveProcessConfigSchema = z
|
|
|
190
190
|
// proactiveMaintenance process: top-N bound per run (default 25). Alias for
|
|
191
191
|
// `limit`; `maxPerRun` wins when both are set.
|
|
192
192
|
maxPerRun: positiveInt.optional(),
|
|
193
|
+
// graphExtraction process (#624 P2): when set, rank eligible files by
|
|
194
|
+
// utility_scores DESC and process only the top-N per run (incremental
|
|
195
|
+
// high-signal-first sweep). Unset = process all eligible (current
|
|
196
|
+
// behavior). Only meaningful on `graphExtraction`.
|
|
197
|
+
topN: positiveInt.optional(),
|
|
193
198
|
// MemoryInference process: minimum pending memory count to run the pass.
|
|
194
199
|
minPendingCount: z.number().int().min(0).optional(),
|
|
195
200
|
// Extract process: minimum number of new (unseen, in-window) candidate
|
|
@@ -386,6 +391,10 @@ export const ImproveProfileConfigSchema = z
|
|
|
386
391
|
processes: ImproveProfileProcessesSchema.optional(),
|
|
387
392
|
autoAccept: nonNegativeNumber.optional(),
|
|
388
393
|
limit: positiveInt.optional(),
|
|
394
|
+
// #616 — bounded multi-cycle phasing. Number of prep->loop->post-loop
|
|
395
|
+
// cycles per run. positiveInt forbids 0/negative. DEFAULT 1 => byte-identical
|
|
396
|
+
// single-pass behavior.
|
|
397
|
+
maxCycles: positiveInt.optional(),
|
|
389
398
|
// #614 — symmetric valence weighting in the eligibility sort. When true,
|
|
390
399
|
// the attention term becomes |valence| MAGNITUDE so BOTH strong positive
|
|
391
400
|
// and strong negative feedback drive attention (utility stays dominant) and
|
|
@@ -608,6 +617,7 @@ const GRAPH_EXTRACTION_INCLUDE_TYPES_ALLOWED = [
|
|
|
608
617
|
"lesson",
|
|
609
618
|
"task",
|
|
610
619
|
"wiki",
|
|
620
|
+
"fact",
|
|
611
621
|
];
|
|
612
622
|
const INDEX_PASS_PROVIDER_KEYS = new Set([
|
|
613
623
|
"endpoint",
|
|
@@ -624,6 +634,7 @@ const INDEX_PASS_KNOWN_KEYS = new Set([
|
|
|
624
634
|
"graphExtractionBatchSize",
|
|
625
635
|
"graphExtractionIncludeTypes",
|
|
626
636
|
"memoryInferenceBatchSize",
|
|
637
|
+
"lazyGraphExtraction",
|
|
627
638
|
]);
|
|
628
639
|
/**
|
|
629
640
|
* Per-pass `index.<pass>` entry. Uses preprocess + manual validation so we can
|
|
@@ -650,8 +661,8 @@ export const IndexPassConfigSchema = z.preprocess((raw, ctx) => {
|
|
|
650
661
|
ctx.addIssue({
|
|
651
662
|
code: z.ZodIssueCode.custom,
|
|
652
663
|
message: `Unknown key \`${[...(ctx.path ?? []), key].join(".")}\`. Per-pass entries support \`llm\` ` +
|
|
653
|
-
"(boolean opt-out), `graphExtractionBatchSize`, `graphExtractionIncludeTypes`,
|
|
654
|
-
"`memoryInferenceBatchSize`.",
|
|
664
|
+
"(boolean opt-out), `graphExtractionBatchSize`, `graphExtractionIncludeTypes`, " +
|
|
665
|
+
"`memoryInferenceBatchSize`, and `lazyGraphExtraction`.",
|
|
655
666
|
});
|
|
656
667
|
return raw;
|
|
657
668
|
}
|
|
@@ -670,6 +681,7 @@ export const IndexPassConfigSchema = z.preprocess((raw, ctx) => {
|
|
|
670
681
|
graphExtractionBatchSize: positiveInt.optional(),
|
|
671
682
|
graphExtractionIncludeTypes: z.array(z.enum(GRAPH_EXTRACTION_INCLUDE_TYPES_ALLOWED)).nonempty().optional(),
|
|
672
683
|
memoryInferenceBatchSize: positiveInt.optional(),
|
|
684
|
+
lazyGraphExtraction: z.boolean().optional(),
|
|
673
685
|
})
|
|
674
686
|
.passthrough());
|
|
675
687
|
const MetadataEnhanceSchema = z.object({ enabled: z.boolean().optional() }).strict();
|
package/dist/indexer/db/db.js
CHANGED
|
@@ -397,6 +397,22 @@ function ensureSchema(db, embeddingDim, options) {
|
|
|
397
397
|
FOREIGN KEY (stash_root, file_path, body_hash)
|
|
398
398
|
REFERENCES graph_files(stash_root, file_path, body_hash) ON DELETE CASCADE
|
|
399
399
|
);
|
|
400
|
+
|
|
401
|
+
-- #624-P3: lazy graph-extraction queue. Standalone table (NO FK to
|
|
402
|
+
-- graph_files — a queued file by definition has no graph row yet).
|
|
403
|
+
-- Idempotent on (stash_root, file_path); drained highest-priority-first.
|
|
404
|
+
-- CREATE TABLE IF NOT EXISTS is the forward migration (no DB_VERSION bump).
|
|
405
|
+
CREATE TABLE IF NOT EXISTS graph_extraction_queue (
|
|
406
|
+
stash_root TEXT NOT NULL,
|
|
407
|
+
file_path TEXT NOT NULL,
|
|
408
|
+
body_hash TEXT NOT NULL,
|
|
409
|
+
queued_at TEXT NOT NULL DEFAULT (datetime('now')),
|
|
410
|
+
priority INTEGER NOT NULL DEFAULT 0,
|
|
411
|
+
PRIMARY KEY (stash_root, file_path)
|
|
412
|
+
);
|
|
413
|
+
|
|
414
|
+
CREATE INDEX IF NOT EXISTS idx_graph_extraction_queue_drain
|
|
415
|
+
ON graph_extraction_queue(stash_root, priority DESC, queued_at);
|
|
400
416
|
`);
|
|
401
417
|
// #624-P1 migration step 2: copy any renamed-aside legacy graph data into the
|
|
402
418
|
// new-shape tables (just created above), then drop the legacy tables. No-op
|
|
@@ -516,6 +532,7 @@ function handleVersionUpgrade(db) {
|
|
|
516
532
|
db.exec("DROP TABLE IF EXISTS index_dir_state");
|
|
517
533
|
db.exec("DROP TABLE IF EXISTS llm_enrichment_cache");
|
|
518
534
|
db.exec("DROP INDEX IF EXISTS idx_llm_cache_updated");
|
|
535
|
+
db.exec("DROP TABLE IF EXISTS graph_extraction_queue");
|
|
519
536
|
db.exec("DROP TABLE IF EXISTS graph_file_relations");
|
|
520
537
|
db.exec("DROP TABLE IF EXISTS graph_file_entities");
|
|
521
538
|
db.exec("DROP TABLE IF EXISTS graph_files");
|
|
@@ -167,6 +167,54 @@ export function hasGraphData(db, stashRoot, filePath) {
|
|
|
167
167
|
return false;
|
|
168
168
|
}
|
|
169
169
|
}
|
|
170
|
+
/**
|
|
171
|
+
* #624-P3 — enqueue a file for lazy graph extraction. Idempotent on the
|
|
172
|
+
* (stash_root, file_path) PK: a second enqueue refreshes body_hash + queued_at
|
|
173
|
+
* and keeps the HIGHER priority. Non-blocking, no LLM call — the queued row is
|
|
174
|
+
* drained later by the graph-extraction pass. Tolerant of a missing table /
|
|
175
|
+
* db error (best-effort), but never masks the bun-test isolation guard.
|
|
176
|
+
*/
|
|
177
|
+
export function enqueueGraphExtraction(db, stashRoot, filePath, bodyHash, priority = 0) {
|
|
178
|
+
try {
|
|
179
|
+
db.prepare(`INSERT INTO graph_extraction_queue (stash_root, file_path, body_hash, priority)
|
|
180
|
+
VALUES (?, ?, ?, ?)
|
|
181
|
+
ON CONFLICT(stash_root, file_path) DO UPDATE SET
|
|
182
|
+
body_hash = excluded.body_hash,
|
|
183
|
+
priority = MAX(graph_extraction_queue.priority, excluded.priority),
|
|
184
|
+
queued_at = datetime('now')`).run(stashRoot, filePath, bodyHash, priority);
|
|
185
|
+
}
|
|
186
|
+
catch (err) {
|
|
187
|
+
rethrowIfTestIsolationError(err);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
/**
|
|
191
|
+
* #624-P3 — drain up to `limit` queued files for a stash, highest-priority
|
|
192
|
+
* first (then oldest queued_at). The returned rows are DELETED from the queue
|
|
193
|
+
* in the SAME transaction (SELECT-then-DELETE-by-PK), so a drain is exactly
|
|
194
|
+
* once. Tolerant of a missing table / db error (returns []), but never masks
|
|
195
|
+
* the bun-test isolation guard.
|
|
196
|
+
*/
|
|
197
|
+
export function drainExtractionQueue(db, stashRoot, limit) {
|
|
198
|
+
try {
|
|
199
|
+
return db.transaction(() => {
|
|
200
|
+
const rows = db
|
|
201
|
+
.prepare(`SELECT file_path, body_hash, priority
|
|
202
|
+
FROM graph_extraction_queue
|
|
203
|
+
WHERE stash_root = ?
|
|
204
|
+
ORDER BY priority DESC, queued_at ASC
|
|
205
|
+
LIMIT ?`)
|
|
206
|
+
.all(stashRoot, limit);
|
|
207
|
+
const del = db.prepare("DELETE FROM graph_extraction_queue WHERE stash_root = ? AND file_path = ?");
|
|
208
|
+
for (const row of rows)
|
|
209
|
+
del.run(stashRoot, row.file_path);
|
|
210
|
+
return rows.map((row) => ({ filePath: row.file_path, bodyHash: row.body_hash, priority: row.priority }));
|
|
211
|
+
})();
|
|
212
|
+
}
|
|
213
|
+
catch (err) {
|
|
214
|
+
rethrowIfTestIsolationError(err);
|
|
215
|
+
return [];
|
|
216
|
+
}
|
|
217
|
+
}
|
|
170
218
|
/**
|
|
171
219
|
* Scoped loader — only the graph_meta row for a stash. Used by callers that
|
|
172
220
|
* only need summary numbers (e.g. `akm graph summary`).
|
|
@@ -40,13 +40,14 @@ import path from "node:path";
|
|
|
40
40
|
import { TYPE_DIRS } from "../../core/asset/asset-spec.js";
|
|
41
41
|
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
42
42
|
import { concurrentMap } from "../../core/concurrent.js";
|
|
43
|
-
import { getIndexPassConfig, resolveBatchSize } from "../../core/config/config.js";
|
|
43
|
+
import { getIndexPassConfig, loadConfig, resolveBatchSize } from "../../core/config/config.js";
|
|
44
|
+
import { rethrowIfTestIsolationError } from "../../core/errors.js";
|
|
44
45
|
import { warn, warnVerbose } from "../../core/warn.js";
|
|
45
46
|
import { isProcessEnabled } from "../../llm/feature-gate.js";
|
|
46
47
|
import * as graphExtract from "../../llm/graph-extract.js";
|
|
47
48
|
import { resolveIndexPassLLM } from "../../llm/index-passes.js";
|
|
48
49
|
import { computeBodyHash, GRAPH_SCHEMA_VERSION, getLlmCacheEntriesByRefs, getLlmCacheEntry, upsertLlmCacheEntry, } from "../db/db.js";
|
|
49
|
-
import { loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
|
|
50
|
+
import { drainExtractionQueue, loadStoredGraphSnapshot, replaceStoredGraph } from "../db/graph-db.js";
|
|
50
51
|
import { walkMarkdownFiles } from "../walk/walker.js";
|
|
51
52
|
import { deduplicateGraph } from "./graph-dedup.js";
|
|
52
53
|
/** Schema version for the persisted artifact — bumps trigger a full rebuild. */
|
|
@@ -92,6 +93,12 @@ function computeGraphQualityTelemetry(consideredFiles, extractedFiles, entityCou
|
|
|
92
93
|
};
|
|
93
94
|
}
|
|
94
95
|
export const DEFAULT_GRAPH_EXTRACTION_INCLUDE_TYPES = ["memory", "knowledge"];
|
|
96
|
+
/**
|
|
97
|
+
* Max number of lazy-extraction queue rows drained per pass (#624-P3). Bounds
|
|
98
|
+
* per-run work so a large backlog is spread across runs rather than processed
|
|
99
|
+
* all at once. Generous default — the queue is normally near-empty.
|
|
100
|
+
*/
|
|
101
|
+
const GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT = 100;
|
|
95
102
|
const SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES = new Set([
|
|
96
103
|
"memory",
|
|
97
104
|
"knowledge",
|
|
@@ -291,8 +298,29 @@ export async function runGraphExtractionPass(ctx) {
|
|
|
291
298
|
warnVerbose("graph extraction: skipped because no primary stash source is available.");
|
|
292
299
|
return { ...EMPTY_RESULT };
|
|
293
300
|
}
|
|
301
|
+
// #624-P3: drain the lazy-extraction queue BEFORE the ranked sweep, highest
|
|
302
|
+
// priority first. Queued paths are extracted individually (per-file merge,
|
|
303
|
+
// other files untouched) so they are processed even when they fall outside
|
|
304
|
+
// the normal candidate set. Default (empty queue) is a byte-identical no-op:
|
|
305
|
+
// drainExtractionQueue returns [] and the loop body never runs.
|
|
306
|
+
if (db) {
|
|
307
|
+
const drained = drainExtractionQueue(db, primary.path, GRAPH_EXTRACTION_QUEUE_DRAIN_LIMIT);
|
|
308
|
+
for (const queued of drained) {
|
|
309
|
+
if (signal?.aborted)
|
|
310
|
+
break;
|
|
311
|
+
await extractGraphForSingleFile(db, primary.path, queued.filePath, queued.bodyHash, { config, signal });
|
|
312
|
+
}
|
|
313
|
+
}
|
|
294
314
|
const includeTypes = getGraphExtractionIncludeTypes(config);
|
|
295
|
-
|
|
315
|
+
let eligible = collectEligibleFiles(primary.path, includeTypes).filter((candidate) => !options.candidatePaths || options.candidatePaths.has(candidate.absPath));
|
|
316
|
+
// P2 (#624): when topN is set and a DB is available, rank the (already
|
|
317
|
+
// candidate-filtered) eligible set by utility_scores DESC and keep only the
|
|
318
|
+
// top-N. Default (topN unset) is byte-identical to today — no ranking query
|
|
319
|
+
// is issued and the eligible set is untouched. Ranking composes WITH the
|
|
320
|
+
// candidatePaths filter: scoped-then-ranked-then-sliced.
|
|
321
|
+
if (db && options.topN != null && options.topN >= 0) {
|
|
322
|
+
eligible = rankCandidatesByUtility(db, eligible, primary.path).slice(0, options.topN);
|
|
323
|
+
}
|
|
296
324
|
const considered = eligible.length;
|
|
297
325
|
if (considered === 0) {
|
|
298
326
|
const scoped = options.candidatePaths ? ` matching ${options.candidatePaths.size} candidate path(s)` : "";
|
|
@@ -628,6 +656,178 @@ export async function runGraphExtractionPass(ctx) {
|
|
|
628
656
|
warnings,
|
|
629
657
|
};
|
|
630
658
|
}
|
|
659
|
+
/**
|
|
660
|
+
* Infer the asset type (`memory`, `knowledge`, …) for a path from the stash
|
|
661
|
+
* directory segment it lives under. Returns the matching include-type, or
|
|
662
|
+
* `undefined` when the path is not under a known graph-eligible type dir.
|
|
663
|
+
*/
|
|
664
|
+
function inferGraphTypeForPath(stashRoot, absPath) {
|
|
665
|
+
const rel = path.relative(stashRoot, absPath);
|
|
666
|
+
const firstSeg = rel.split(path.sep)[0];
|
|
667
|
+
if (!firstSeg)
|
|
668
|
+
return undefined;
|
|
669
|
+
for (const type of SUPPORTED_GRAPH_EXTRACTION_INCLUDE_TYPES) {
|
|
670
|
+
if (TYPE_DIRS[type] === firstSeg)
|
|
671
|
+
return type;
|
|
672
|
+
}
|
|
673
|
+
return undefined;
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* #624-P3 — extract graph data for a SINGLE file and merge it into the stored
|
|
677
|
+
* graph WITHOUT clobbering other files' rows.
|
|
678
|
+
*
|
|
679
|
+
* Re-reads the body from disk at call time (the queued body_hash is NOT trusted
|
|
680
|
+
* blindly — the file may have been deleted or changed since enqueue) and skips
|
|
681
|
+
* silently (returns `false`) when the file is gone or empty. Resolves the LLM
|
|
682
|
+
* via {@link resolveIndexPassLLM} (model-available guard: returns `false` when
|
|
683
|
+
* no provider is configured) UNLESS `opts.llmOverride` is supplied, in which
|
|
684
|
+
* case the override is the extractor seam (used by tests and by callers that
|
|
685
|
+
* already hold a resolved model).
|
|
686
|
+
*
|
|
687
|
+
* Returns `true` when a graph row was written for the file, `false` on any
|
|
688
|
+
* skip (missing file, empty body, unknown type, no model, or extraction error).
|
|
689
|
+
*/
|
|
690
|
+
export async function extractGraphForSingleFile(db, stashRoot, filePath, bodyHash, opts) {
|
|
691
|
+
try {
|
|
692
|
+
// Re-read from disk — never trust a stale queued body.
|
|
693
|
+
let raw;
|
|
694
|
+
try {
|
|
695
|
+
raw = fs.readFileSync(filePath, "utf8");
|
|
696
|
+
}
|
|
697
|
+
catch {
|
|
698
|
+
return false; // file gone / unreadable → silent skip
|
|
699
|
+
}
|
|
700
|
+
const parsed = parseFrontmatter(raw);
|
|
701
|
+
const body = parsed.content.trim();
|
|
702
|
+
if (!body)
|
|
703
|
+
return false;
|
|
704
|
+
const type = inferGraphTypeForPath(stashRoot, filePath) ?? "memory";
|
|
705
|
+
const effectiveHash = bodyHash ?? computeBodyHash(body);
|
|
706
|
+
// Extract — via the injected seam, or the real per-asset path.
|
|
707
|
+
let extraction;
|
|
708
|
+
if (opts?.llmOverride) {
|
|
709
|
+
const out = await opts.llmOverride(body);
|
|
710
|
+
extraction = {
|
|
711
|
+
entities: out.entities,
|
|
712
|
+
relations: out.relations,
|
|
713
|
+
...(out.confidence !== undefined ? { confidence: out.confidence } : {}),
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
else {
|
|
717
|
+
const config = opts?.config ?? loadConfig();
|
|
718
|
+
if (!isProcessEnabled("index", "graph_extraction", config))
|
|
719
|
+
return false;
|
|
720
|
+
const llmConfig = resolveIndexPassLLM("graph", config);
|
|
721
|
+
if (!llmConfig)
|
|
722
|
+
return false; // model-available guard
|
|
723
|
+
const result = await graphExtract.extractGraphFromBody(llmConfig, body, opts?.signal, config);
|
|
724
|
+
extraction = {
|
|
725
|
+
entities: result.entities,
|
|
726
|
+
relations: result.relations,
|
|
727
|
+
...(result.confidence !== undefined ? { confidence: result.confidence } : {}),
|
|
728
|
+
};
|
|
729
|
+
}
|
|
730
|
+
const entities = [...new Set(extraction.entities.map((e) => e.trim()).filter(Boolean))];
|
|
731
|
+
const relations = extraction.relations
|
|
732
|
+
.map((r) => ({
|
|
733
|
+
from: r.from.trim(),
|
|
734
|
+
to: r.to.trim(),
|
|
735
|
+
...(r.type ? { type: r.type.trim() } : {}),
|
|
736
|
+
...(normalizeConfidence(r.confidence) !== undefined ? { confidence: normalizeConfidence(r.confidence) } : {}),
|
|
737
|
+
}))
|
|
738
|
+
.filter((r) => r.from && r.to);
|
|
739
|
+
const node = {
|
|
740
|
+
path: filePath,
|
|
741
|
+
type,
|
|
742
|
+
bodyHash: effectiveHash,
|
|
743
|
+
entities,
|
|
744
|
+
relations,
|
|
745
|
+
...(normalizeConfidence(extraction.confidence) !== undefined
|
|
746
|
+
? { confidence: normalizeConfidence(extraction.confidence) }
|
|
747
|
+
: {}),
|
|
748
|
+
status: entities.length > 0 ? "extracted" : "empty",
|
|
749
|
+
reason: entities.length > 0 ? "none" : "no_graph_content",
|
|
750
|
+
extractionRunId: crypto.randomUUID(),
|
|
751
|
+
};
|
|
752
|
+
// Merge with the previously-stored nodes, scoping the refresh to JUST this
|
|
753
|
+
// path so other files' rows are preserved (and graph_meta counts refresh).
|
|
754
|
+
const previousGraph = loadGraphFile(stashRoot, db);
|
|
755
|
+
const candidatePaths = new Set([filePath]);
|
|
756
|
+
const mergedNodes = mergeGraphNodes(previousGraph.files, [node], candidatePaths);
|
|
757
|
+
const assetRefs = mergedNodes.map((n) => n.path);
|
|
758
|
+
const deduped = deduplicateGraph(mergedNodes.map((n) => ({ entities: n.entities, relations: n.relations })), assetRefs);
|
|
759
|
+
const qualityExtracted = mergedNodes.filter((n) => n.status === "extracted" && n.entities.length > 0).length;
|
|
760
|
+
const quality = computeGraphQualityTelemetry(mergedNodes.length, qualityExtracted, deduped.entities.length, deduped.relations.length);
|
|
761
|
+
const graph = {
|
|
762
|
+
schemaVersion: GRAPH_FILE_SCHEMA_VERSION,
|
|
763
|
+
generatedAt: new Date().toISOString(),
|
|
764
|
+
stashRoot,
|
|
765
|
+
files: mergedNodes,
|
|
766
|
+
entities: deduped.entities,
|
|
767
|
+
relations: deduped.relations,
|
|
768
|
+
quality,
|
|
769
|
+
...(previousGraph.telemetry ? { telemetry: previousGraph.telemetry } : {}),
|
|
770
|
+
};
|
|
771
|
+
return writeGraphFile(stashRoot, graph, db);
|
|
772
|
+
}
|
|
773
|
+
catch (err) {
|
|
774
|
+
rethrowIfTestIsolationError(err);
|
|
775
|
+
return false;
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
// ── Eligible-file detection ─────────────────────────────────────────────────
|
|
779
|
+
/**
|
|
780
|
+
* Rank eligible graph-extraction candidates by their entry `utility_scores`,
|
|
781
|
+
* highest first, for the incremental high-signal-first sweep (P2 of #624).
|
|
782
|
+
*
|
|
783
|
+
* The join is READ-ONLY (`entries.file_path = candidate.absPath`, then
|
|
784
|
+
* `entries.id -> utility_scores.entry_id`) and does NOT re-couple the graph
|
|
785
|
+
* rows to `entries`. It reads the GLOBAL `utility_scores` table (not the
|
|
786
|
+
* per-scope `utility_scores_scoped`), so ranking is corpus-wide; `stashRoot`
|
|
787
|
+
* is accepted for call-site symmetry/future scoping but is not used to filter
|
|
788
|
+
* (the global table has no `stash_root` column).
|
|
789
|
+
*
|
|
790
|
+
* Candidates with no matching `entries` row, or an entry with no
|
|
791
|
+
* `utility_scores` row, get an effective utility of 0 (LEFT JOIN + COALESCE)
|
|
792
|
+
* and sort LAST — they are deprioritized, never dropped, so a `topN >= total`
|
|
793
|
+
* slice still includes them and they remain reachable on later runs.
|
|
794
|
+
*
|
|
795
|
+
* Ties (equal utility) break by `file_path` ASC for deterministic output.
|
|
796
|
+
* Returns a NEW array; the input is not mutated. SQLite's ~999 bound-parameter
|
|
797
|
+
* cap is respected by chunking the `IN (...)` lookup at 500.
|
|
798
|
+
*
|
|
799
|
+
* Exported for direct unit testing.
|
|
800
|
+
*/
|
|
801
|
+
export function rankCandidatesByUtility(db, candidates, _stashRoot) {
|
|
802
|
+
// Cannot rank without a DB → return the input unranked rather than throw.
|
|
803
|
+
// Keeps the DB-less code path (reuse-from-memory) working when topN is set.
|
|
804
|
+
if (!db || candidates.length === 0)
|
|
805
|
+
return candidates;
|
|
806
|
+
const utilityByPath = new Map();
|
|
807
|
+
const CHUNK = 500;
|
|
808
|
+
for (let start = 0; start < candidates.length; start += CHUNK) {
|
|
809
|
+
const chunk = candidates.slice(start, start + CHUNK);
|
|
810
|
+
const paths = chunk.map((c) => c.absPath);
|
|
811
|
+
const placeholders = paths.map(() => "?").join(", ");
|
|
812
|
+
const rows = db
|
|
813
|
+
.prepare(`SELECT e.file_path AS file_path, COALESCE(MAX(u.utility), 0) AS utility
|
|
814
|
+
FROM entries e
|
|
815
|
+
LEFT JOIN utility_scores u ON u.entry_id = e.id
|
|
816
|
+
WHERE e.file_path IN (${placeholders})
|
|
817
|
+
GROUP BY e.file_path`)
|
|
818
|
+
.all(...paths);
|
|
819
|
+
for (const row of rows) {
|
|
820
|
+
utilityByPath.set(row.file_path, row.utility ?? 0);
|
|
821
|
+
}
|
|
822
|
+
}
|
|
823
|
+
return [...candidates].sort((a, b) => {
|
|
824
|
+
const ua = utilityByPath.get(a.absPath) ?? 0;
|
|
825
|
+
const ub = utilityByPath.get(b.absPath) ?? 0;
|
|
826
|
+
if (ub !== ua)
|
|
827
|
+
return ub - ua; // utility DESC
|
|
828
|
+
return a.absPath < b.absPath ? -1 : a.absPath > b.absPath ? 1 : 0; // tie-break: path ASC
|
|
829
|
+
});
|
|
830
|
+
}
|
|
631
831
|
/**
|
|
632
832
|
* Scan the primary stash for `memory:` and `knowledge:` markdown files
|
|
633
833
|
* suitable for graph extraction. The directory layout convention is the
|
|
@@ -9,6 +9,9 @@ const TYPE_BOOST = {
|
|
|
9
9
|
agent: 0.3,
|
|
10
10
|
script: 0.2,
|
|
11
11
|
knowledge: 0.22,
|
|
12
|
+
// Facts are authoritative, durable declarations about the stash — rank them
|
|
13
|
+
// alongside knowledge so they surface reliably when relevant.
|
|
14
|
+
fact: 0.22,
|
|
12
15
|
memory: -0.02,
|
|
13
16
|
};
|
|
14
17
|
const MAX_BOOST_SUM = 3.0;
|
|
@@ -206,6 +209,24 @@ const lessonStrengthContributor = {
|
|
|
206
209
|
return Math.min(0.3, 0.06 * strength);
|
|
207
210
|
},
|
|
208
211
|
};
|
|
212
|
+
/**
|
|
213
|
+
* Pinned-fact boost.
|
|
214
|
+
*
|
|
215
|
+
* Facts marked `pinned: true` form the small always-injected "core context"
|
|
216
|
+
* (see docs/design/fact-asset-type.md). The fact metadata contributor records
|
|
217
|
+
* a `pinned` search hint; here we give those facts a modest additive boost so
|
|
218
|
+
* the core outranks ordinary facts on otherwise-equal queries. Capped small so
|
|
219
|
+
* it cannot overpower an exact-name match.
|
|
220
|
+
*/
|
|
221
|
+
const pinnedFactRankingContributor = {
|
|
222
|
+
name: "pinned-fact-ranking",
|
|
223
|
+
appliesTo(item) {
|
|
224
|
+
return item.entry.type === "fact" && (item.entry.searchHints?.includes("pinned") ?? false);
|
|
225
|
+
},
|
|
226
|
+
adjust() {
|
|
227
|
+
return 0.15;
|
|
228
|
+
},
|
|
229
|
+
};
|
|
209
230
|
/**
|
|
210
231
|
* Blend ratio for scoped vs. global utility signals.
|
|
211
232
|
*
|
|
@@ -310,6 +331,7 @@ export const defaultRankingContributors = [
|
|
|
310
331
|
graphRankingContributor,
|
|
311
332
|
captureModeRankingContributor,
|
|
312
333
|
lessonStrengthContributor,
|
|
334
|
+
pinnedFactRankingContributor,
|
|
313
335
|
projectContextRankingContributor,
|
|
314
336
|
];
|
|
315
337
|
export const defaultUtilityRankingContributors = [utilityRankingContributor];
|
|
@@ -78,6 +78,15 @@ const DIR_TYPE_MAP = [
|
|
|
78
78
|
type: "session",
|
|
79
79
|
test: (ext) => ext === ".md",
|
|
80
80
|
},
|
|
81
|
+
{
|
|
82
|
+
// Durable stash-level facts live under `facts/<category>/<name>.md`.
|
|
83
|
+
// classifyByDirectory walks every ancestor dir, so nested category
|
|
84
|
+
// subdirs still match. Without this entry a fact file would fall through
|
|
85
|
+
// to classifyBySmartMd and be mistyped as `knowledge`.
|
|
86
|
+
dir: "facts",
|
|
87
|
+
type: "fact",
|
|
88
|
+
test: (ext) => ext === ".md",
|
|
89
|
+
},
|
|
81
90
|
];
|
|
82
91
|
const COMMAND_PLACEHOLDER_RE = /\$ARGUMENTS|\$[123]\b/;
|
|
83
92
|
// Files that should never be treated as the typed asset for the surrounding
|
|
@@ -43,6 +43,7 @@ const TYPE_HINTS = {
|
|
|
43
43
|
script: "script assets are executable text files. Include a shebang and minimal usage comment.",
|
|
44
44
|
env: "env assets are `.env` files holding a group of related CONFIGURATION for an app/service (KEY=VALUE pairs, `#` comments) — URLs, flags, and any credentials it needs. Values may or may not be sensitive; all are protected (key names discoverable, values stay on disk). Inject with `akm env run env:<name> -- <cmd>` (the safe path — values never reach stdout/your context); do NOT run `akm env export` and read its output, as that prints values. For a single sensitive value used on its own for authentication (token, key, cert) use a `secret` instead. Never echo values back to the user.",
|
|
45
45
|
wiki: "wiki assets are markdown reference pages with `# Title` and structured headings.",
|
|
46
|
+
fact: "fact assets are durable stash-level facts (personal/team/project details, coding conventions, stash-meta). Frontmatter SHOULD include `description` and a `category` (personal|team|project|convention|meta); set `pinned: true` only for the small always-injected core. Keep each fact short, high-signal, and self-contained — it is durable context, not an episodic note.",
|
|
46
47
|
};
|
|
47
48
|
function hintForType(type) {
|
|
48
49
|
return TYPE_HINTS[type] ?? `assets of type "${type}" — produce sensible markdown with optional frontmatter.`;
|