akm-cli 0.9.7 → 0.9.8-beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +310 -0
- package/dist/commands/env/env.js +2 -21
- package/dist/commands/health/checks.js +0 -25
- package/dist/commands/health/data-dir-usage.js +165 -0
- package/dist/commands/health/improve-metrics.js +8 -34
- package/dist/commands/health/windows.js +0 -4
- package/dist/commands/health.js +18 -37
- package/dist/commands/improve/consolidate/eligibility.js +11 -5
- package/dist/commands/improve/extract.js +36 -32
- package/dist/commands/improve/memory/memory-belief.js +15 -5
- package/dist/commands/improve/memory/memory-contradiction-detect.js +16 -25
- package/dist/commands/improve/memory/memory-improve.js +7 -19
- package/dist/commands/improve/preparation.js +52 -36
- package/dist/commands/improve/reflect.js +13 -17
- package/dist/commands/lint/base-linter.js +42 -2
- package/dist/commands/lint/index.js +4 -5
- package/dist/commands/migrate/config-extra-params.js +61 -0
- package/dist/commands/migrate/dead-residue.js +113 -0
- package/dist/commands/migrate/stale-txn.js +49 -0
- package/dist/commands/migrate-cli.js +42 -1
- package/dist/commands/proposal/proposal.js +1 -21
- package/dist/commands/proposal/repository.js +0 -4
- package/dist/commands/proposal/validators/proposal-quality-validators.js +0 -6
- package/dist/commands/read/curate.js +4 -18
- package/dist/commands/read/show.js +2 -1
- package/dist/commands/sources/installed-stashes.js +5 -1
- package/dist/commands/sources/self-update.js +38 -1
- package/dist/commands/sources/sources-cli.js +17 -1
- package/dist/commands/tasks/tasks.js +0 -8
- package/dist/commands/url-checker.js +20 -28
- package/dist/commands/workflow-cli.js +11 -61
- package/dist/core/adapter/adapters/agent-skills-adapter.js +2 -6
- package/dist/core/adapter/adapters/akm-adapter.js +2 -3
- package/dist/core/adapter/adapters/akm-lint.js +2 -20
- package/dist/core/adapter/adapters/akm-task-adapter.js +2 -6
- package/dist/core/adapter/adapters/akm-workflow-adapter.js +2 -6
- package/dist/core/adapter/adapters/dotenv-adapter.js +2 -22
- package/dist/core/adapter/adapters/generic-files-adapter.js +3 -9
- package/dist/core/adapter/adapters/llm-wiki-adapter.js +2 -6
- package/dist/core/adapter/adapters/okf-adapter.js +7 -18
- package/dist/core/adapter/adapters/shared.js +2 -0
- package/dist/core/adapter/adapters/tool-dir-shared.js +3 -9
- package/dist/core/adapter/adapters/website-snapshot-adapter.js +2 -6
- package/dist/core/adapter/validate-context.js +1 -3
- package/dist/core/asset/asset-placement.js +1 -3
- package/dist/core/asset/frontmatter.js +96 -0
- package/dist/core/common.js +66 -2
- package/dist/core/config/config-io.js +21 -8
- package/dist/core/config/config-walker.js +6 -10
- package/dist/core/config/config.js +11 -8
- package/dist/core/extra-params.js +17 -13
- package/dist/core/improve-result.js +1 -3
- package/dist/core/json-schema.js +9 -11
- package/dist/core/state/migrations.js +74 -2
- package/dist/core/state-db.js +2 -1
- package/dist/core/warn.js +15 -0
- package/dist/execution/executable-identity.js +1 -3
- package/dist/execution/guarded-source.js +1 -6
- package/dist/indexer/bundle-identity-guard.js +6 -1
- package/dist/indexer/db/graph-db.js +139 -154
- package/dist/indexer/ensure-index.js +11 -19
- package/dist/indexer/graph/graph-boost.js +23 -34
- package/dist/indexer/graph/graph-extraction.js +12 -2
- package/dist/indexer/indexer.js +43 -22
- package/dist/indexer/lookup/adapter-concept-owner.js +12 -9
- package/dist/indexer/passes/dir-staleness.js +30 -9
- package/dist/indexer/passes/memory-inference.js +7 -2
- package/dist/indexer/scan/drain-dir.js +2 -4
- package/dist/indexer/search/db-search.js +3 -3
- package/dist/indexer/search/fts-query.js +10 -15
- package/dist/indexer/search/search-source.js +0 -13
- package/dist/indexer/usage/usage-events.js +9 -1
- package/dist/indexer/walk/walker.js +11 -6
- package/dist/integrations/harnesses/claude/result-extractor.js +30 -15
- package/dist/integrations/harnesses/codex/result-extractor.js +43 -6
- package/dist/integrations/harnesses/copilot/result-extractor.js +39 -12
- package/dist/integrations/harnesses/gemini/result-extractor.js +40 -12
- package/dist/integrations/harnesses/index.js +0 -4
- package/dist/integrations/harnesses/openhands/result-extractor.js +51 -19
- package/dist/integrations/harnesses/pi/result-extractor.js +45 -12
- package/dist/integrations/lockfile.js +0 -14
- package/dist/integrations/session-logs/index.js +0 -81
- package/dist/llm/client.js +0 -12
- package/dist/llm/memory-infer.js +1 -3
- package/dist/llm/usage-telemetry.js +1 -3
- package/dist/registry/create-provider-registry.js +4 -0
- package/dist/registry/factory.js +4 -0
- package/dist/registry/resolve.js +23 -8
- package/dist/runtime.js +0 -12
- package/dist/scripts/akm-migrate-node.js +179 -248
- package/dist/scripts/akm-migrate.js +179 -248
- package/dist/setup/setup.js +6 -12
- package/dist/sources/providers/git-install.js +7 -2
- package/dist/sources/providers/tar-utils.js +1 -7
- package/dist/sources/snapshot-fetchers/content-extract.js +0 -17
- package/dist/storage/managed-db.js +20 -7
- package/dist/storage/repositories/events-repository.js +0 -81
- package/dist/storage/repositories/index-entries-repository.js +24 -28
- package/dist/storage/repositories/index-fts-repository.js +24 -30
- package/dist/storage/repositories/index-meta-repository.js +6 -4
- package/dist/storage/repositories/index-schema.js +16 -1
- package/dist/storage/repositories/index-utility-repository.js +0 -57
- package/dist/storage/repositories/index-vec-repository.js +25 -27
- package/dist/storage/repositories/proposals-repository.js +4 -1
- package/dist/storage/repositories/task-history-repository.js +9 -3
- package/dist/tasks/backends/launchd.js +8 -18
- package/dist/tasks/resolve-akm-bin.js +15 -0
- package/dist/tasks/run/task-history.js +21 -31
- package/dist/tasks/scheduler-sync.js +1 -6
- package/dist/tasks/source/bounded-document.js +1 -14
- package/dist/tasks/source/task-to-v3.js +10 -2
- package/dist/workflows/exec/child-workflow.js +1 -1
- package/dist/workflows/exec/native-executor.js +2 -2
- package/dist/workflows/exec/step-work.js +5 -17
- package/dist/workflows/exec/worktree.js +40 -6
- package/dist/workflows/freeze/task-bindings.js +2 -4
- package/dist/workflows/ir/compile.js +3 -14
- package/dist/workflows/ir/schema-v4.js +4 -6
- package/dist/workflows/ir/schema.js +2 -5
- package/dist/workflows/parser.js +23 -26
- package/dist/workflows/source-files.js +8 -13
- package/dist/workflows/source-ir/schema.js +1 -19
- package/docs/migration/v0.9.0-troubleshooting.md +12 -2
- package/docs/migration/v0.9.1-to-v0.9.2.md +18 -4
- package/docs/reference/cli.md +3 -1
- package/docs/reference/data-and-telemetry.md +1 -0
- package/docs/reference/tasks.md +11 -0
- package/docs/reference/workflow-schema.md +24 -0
- package/package.json +2 -9
- package/dist/workflows/source-ir/compare.js +0 -17
|
@@ -259,25 +259,20 @@ export function acknowledgeExtractionQueueEntry(db, stashRoot, filePath, bodyHas
|
|
|
259
259
|
export function loadGraphFilesOnly(stashPath, db) {
|
|
260
260
|
try {
|
|
261
261
|
return withReadableGraphDb(db, (readDb) => {
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
}));
|
|
277
|
-
}
|
|
278
|
-
catch {
|
|
279
|
-
return [];
|
|
280
|
-
}
|
|
262
|
+
const rows = readDb
|
|
263
|
+
.prepare(`SELECT file_path, file_type, body_hash, confidence, status, reason
|
|
264
|
+
FROM graph_files
|
|
265
|
+
WHERE stash_root = ?
|
|
266
|
+
ORDER BY file_order`)
|
|
267
|
+
.all(stashPath);
|
|
268
|
+
return rows.map((row) => ({
|
|
269
|
+
path: row.file_path,
|
|
270
|
+
type: row.file_type,
|
|
271
|
+
bodyHash: row.body_hash,
|
|
272
|
+
...(typeof row.confidence === "number" ? { confidence: row.confidence } : {}),
|
|
273
|
+
...(row.status ? { status: row.status } : {}),
|
|
274
|
+
...(row.reason ? { reason: row.reason } : {}),
|
|
275
|
+
}));
|
|
281
276
|
});
|
|
282
277
|
}
|
|
283
278
|
catch (err) {
|
|
@@ -292,66 +287,61 @@ export function loadGraphFilesOnly(stashPath, db) {
|
|
|
292
287
|
export function loadStoredGraphMeta(stashPath, db) {
|
|
293
288
|
try {
|
|
294
289
|
return withReadableGraphDb(db, (readDb) => {
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
if (!row)
|
|
320
|
-
return null;
|
|
321
|
-
return {
|
|
322
|
-
stashPath: row.stash_root,
|
|
323
|
-
graphPath: getDbPath(),
|
|
324
|
-
schemaVersion: row.schema_version,
|
|
325
|
-
generatedAt: row.generated_at,
|
|
326
|
-
quality: {
|
|
327
|
-
consideredFiles: row.considered_files,
|
|
328
|
-
extractedFiles: row.extracted_files,
|
|
329
|
-
entityCount: row.entity_count,
|
|
330
|
-
relationCount: row.relation_count,
|
|
331
|
-
extractionCoverage: row.extraction_coverage,
|
|
332
|
-
density: row.density,
|
|
333
|
-
},
|
|
334
|
-
telemetry: {
|
|
335
|
-
...(row.extractor_id ? { extractorId: row.extractor_id } : {}),
|
|
336
|
-
...(row.extraction_run_id ? { extractionRunId: row.extraction_run_id } : {}),
|
|
337
|
-
...(row.model ? { model: row.model } : {}),
|
|
338
|
-
...(row.prompt_version ? { promptVersion: row.prompt_version } : {}),
|
|
339
|
-
...(typeof row.batch_size === "number" ? { batchSize: row.batch_size } : {}),
|
|
340
|
-
cacheHits: row.cache_hits,
|
|
341
|
-
cacheMisses: row.cache_misses,
|
|
342
|
-
truncationCount: row.truncation_count,
|
|
343
|
-
failureCount: row.failure_count,
|
|
344
|
-
// `retry_attempts` is not persisted to the graph-meta table (it is
|
|
345
|
-
// surfaced from the run's emitted telemetry into `akm health`, not
|
|
346
|
-
// from the reuse cache). Default to 0 so the loaded shape satisfies
|
|
347
|
-
// GraphExtractionTelemetry.
|
|
348
|
-
retryAttempts: 0,
|
|
349
|
-
},
|
|
350
|
-
};
|
|
351
|
-
}
|
|
352
|
-
catch {
|
|
290
|
+
const row = readDb
|
|
291
|
+
.prepare(`SELECT
|
|
292
|
+
stash_root,
|
|
293
|
+
schema_version,
|
|
294
|
+
generated_at,
|
|
295
|
+
considered_files,
|
|
296
|
+
extracted_files,
|
|
297
|
+
entity_count,
|
|
298
|
+
relation_count,
|
|
299
|
+
extraction_coverage,
|
|
300
|
+
density,
|
|
301
|
+
extractor_id,
|
|
302
|
+
extraction_run_id,
|
|
303
|
+
model,
|
|
304
|
+
prompt_version,
|
|
305
|
+
batch_size,
|
|
306
|
+
cache_hits,
|
|
307
|
+
cache_misses,
|
|
308
|
+
truncation_count,
|
|
309
|
+
failure_count
|
|
310
|
+
FROM graph_meta
|
|
311
|
+
WHERE stash_root = ?`)
|
|
312
|
+
.get(stashPath);
|
|
313
|
+
if (!row)
|
|
353
314
|
return null;
|
|
354
|
-
|
|
315
|
+
return {
|
|
316
|
+
stashPath: row.stash_root,
|
|
317
|
+
graphPath: getDbPath(),
|
|
318
|
+
schemaVersion: row.schema_version,
|
|
319
|
+
generatedAt: row.generated_at,
|
|
320
|
+
quality: {
|
|
321
|
+
consideredFiles: row.considered_files,
|
|
322
|
+
extractedFiles: row.extracted_files,
|
|
323
|
+
entityCount: row.entity_count,
|
|
324
|
+
relationCount: row.relation_count,
|
|
325
|
+
extractionCoverage: row.extraction_coverage,
|
|
326
|
+
density: row.density,
|
|
327
|
+
},
|
|
328
|
+
telemetry: {
|
|
329
|
+
...(row.extractor_id ? { extractorId: row.extractor_id } : {}),
|
|
330
|
+
...(row.extraction_run_id ? { extractionRunId: row.extraction_run_id } : {}),
|
|
331
|
+
...(row.model ? { model: row.model } : {}),
|
|
332
|
+
...(row.prompt_version ? { promptVersion: row.prompt_version } : {}),
|
|
333
|
+
...(typeof row.batch_size === "number" ? { batchSize: row.batch_size } : {}),
|
|
334
|
+
cacheHits: row.cache_hits,
|
|
335
|
+
cacheMisses: row.cache_misses,
|
|
336
|
+
truncationCount: row.truncation_count,
|
|
337
|
+
failureCount: row.failure_count,
|
|
338
|
+
// `retry_attempts` is not persisted to the graph-meta table (it is
|
|
339
|
+
// surfaced from the run's emitted telemetry into `akm health`, not
|
|
340
|
+
// from the reuse cache). Default to 0 so the loaded shape satisfies
|
|
341
|
+
// GraphExtractionTelemetry.
|
|
342
|
+
retryAttempts: 0,
|
|
343
|
+
},
|
|
344
|
+
};
|
|
355
345
|
});
|
|
356
346
|
}
|
|
357
347
|
catch (err) {
|
|
@@ -368,85 +358,80 @@ export function loadStoredGraphSnapshot(stashPath, db) {
|
|
|
368
358
|
const meta = loadStoredGraphMeta(stashPath, readDb);
|
|
369
359
|
if (!meta)
|
|
370
360
|
return null;
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
...(row.relation_type ? { type: row.relation_type } : {}),
|
|
416
|
-
...(typeof row.confidence === "number" ? { confidence: row.confidence } : {}),
|
|
417
|
-
};
|
|
418
|
-
const bucket = relationsByPath.get(row.file_path);
|
|
419
|
-
if (bucket)
|
|
420
|
-
bucket.push(relation);
|
|
421
|
-
else
|
|
422
|
-
relationsByPath.set(row.file_path, [relation]);
|
|
423
|
-
}
|
|
424
|
-
const files = fileRows.map((row) => ({
|
|
425
|
-
path: row.file_path,
|
|
426
|
-
type: row.file_type,
|
|
427
|
-
...(row.body_hash ? { bodyHash: row.body_hash } : {}),
|
|
428
|
-
entities: entitiesByPath.get(row.file_path) ?? [],
|
|
429
|
-
relations: relationsByPath.get(row.file_path) ?? [],
|
|
361
|
+
const fileRows = readDb
|
|
362
|
+
.prepare(`SELECT file_path, file_type, body_hash, confidence, status, reason, extraction_run_id
|
|
363
|
+
FROM graph_files
|
|
364
|
+
WHERE stash_root = ?
|
|
365
|
+
ORDER BY file_order`)
|
|
366
|
+
.all(stashPath);
|
|
367
|
+
const entityRows = readDb
|
|
368
|
+
.prepare(`SELECT gf.file_path AS file_path, gfe.entity AS entity
|
|
369
|
+
FROM graph_file_entities gfe
|
|
370
|
+
JOIN graph_files gf
|
|
371
|
+
ON gf.stash_root = gfe.stash_root
|
|
372
|
+
AND gf.file_path = gfe.file_path
|
|
373
|
+
AND gf.body_hash = gfe.body_hash
|
|
374
|
+
WHERE gf.stash_root = ?
|
|
375
|
+
ORDER BY gf.file_order, gfe.entity_order`)
|
|
376
|
+
.all(stashPath);
|
|
377
|
+
const relationRows = readDb
|
|
378
|
+
.prepare(`SELECT gf.file_path AS file_path,
|
|
379
|
+
gfr.from_entity AS from_entity,
|
|
380
|
+
gfr.to_entity AS to_entity,
|
|
381
|
+
gfr.relation_type AS relation_type,
|
|
382
|
+
gfr.confidence AS confidence
|
|
383
|
+
FROM graph_file_relations gfr
|
|
384
|
+
JOIN graph_files gf
|
|
385
|
+
ON gf.stash_root = gfr.stash_root
|
|
386
|
+
AND gf.file_path = gfr.file_path
|
|
387
|
+
AND gf.body_hash = gfr.body_hash
|
|
388
|
+
WHERE gf.stash_root = ?
|
|
389
|
+
ORDER BY gf.file_order, gfr.relation_order`)
|
|
390
|
+
.all(stashPath);
|
|
391
|
+
const entitiesByPath = new Map();
|
|
392
|
+
for (const row of entityRows) {
|
|
393
|
+
const bucket = entitiesByPath.get(row.file_path);
|
|
394
|
+
if (bucket)
|
|
395
|
+
bucket.push(row.entity);
|
|
396
|
+
else
|
|
397
|
+
entitiesByPath.set(row.file_path, [row.entity]);
|
|
398
|
+
}
|
|
399
|
+
const relationsByPath = new Map();
|
|
400
|
+
for (const row of relationRows) {
|
|
401
|
+
const relation = {
|
|
402
|
+
from: row.from_entity,
|
|
403
|
+
to: row.to_entity,
|
|
404
|
+
...(row.relation_type ? { type: row.relation_type } : {}),
|
|
430
405
|
...(typeof row.confidence === "number" ? { confidence: row.confidence } : {}),
|
|
431
|
-
...(row.status ? { status: row.status } : {}),
|
|
432
|
-
...(row.reason ? { reason: row.reason } : {}),
|
|
433
|
-
...(row.extraction_run_id ? { extractionRunId: row.extraction_run_id } : {}),
|
|
434
|
-
}));
|
|
435
|
-
return {
|
|
436
|
-
stashPath: meta.stashPath,
|
|
437
|
-
graphPath: meta.graphPath,
|
|
438
|
-
schemaVersion: meta.schemaVersion,
|
|
439
|
-
generatedAt: meta.generatedAt,
|
|
440
|
-
...(meta.quality ? { quality: meta.quality } : {}),
|
|
441
|
-
...(meta.telemetry ? { telemetry: meta.telemetry } : {}),
|
|
442
|
-
files,
|
|
443
|
-
entities: uniqueSorted(files.flatMap((file) => file.entities)),
|
|
444
|
-
relations: files.flatMap((file) => file.relations),
|
|
445
406
|
};
|
|
407
|
+
const bucket = relationsByPath.get(row.file_path);
|
|
408
|
+
if (bucket)
|
|
409
|
+
bucket.push(relation);
|
|
410
|
+
else
|
|
411
|
+
relationsByPath.set(row.file_path, [relation]);
|
|
446
412
|
}
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
413
|
+
const files = fileRows.map((row) => ({
|
|
414
|
+
path: row.file_path,
|
|
415
|
+
type: row.file_type,
|
|
416
|
+
...(row.body_hash ? { bodyHash: row.body_hash } : {}),
|
|
417
|
+
entities: entitiesByPath.get(row.file_path) ?? [],
|
|
418
|
+
relations: relationsByPath.get(row.file_path) ?? [],
|
|
419
|
+
...(typeof row.confidence === "number" ? { confidence: row.confidence } : {}),
|
|
420
|
+
...(row.status ? { status: row.status } : {}),
|
|
421
|
+
...(row.reason ? { reason: row.reason } : {}),
|
|
422
|
+
...(row.extraction_run_id ? { extractionRunId: row.extraction_run_id } : {}),
|
|
423
|
+
}));
|
|
424
|
+
return {
|
|
425
|
+
stashPath: meta.stashPath,
|
|
426
|
+
graphPath: meta.graphPath,
|
|
427
|
+
schemaVersion: meta.schemaVersion,
|
|
428
|
+
generatedAt: meta.generatedAt,
|
|
429
|
+
...(meta.quality ? { quality: meta.quality } : {}),
|
|
430
|
+
...(meta.telemetry ? { telemetry: meta.telemetry } : {}),
|
|
431
|
+
files,
|
|
432
|
+
entities: uniqueSorted(files.flatMap((file) => file.entities)),
|
|
433
|
+
relations: files.flatMap((file) => file.relations),
|
|
434
|
+
};
|
|
450
435
|
});
|
|
451
436
|
}
|
|
452
437
|
catch (err) {
|
|
@@ -24,7 +24,6 @@ import path from "node:path";
|
|
|
24
24
|
import { placementSpecList } from "../core/asset/asset-placement.js";
|
|
25
25
|
import { classifyPathAccess } from "../core/path-access.js";
|
|
26
26
|
import { getDbPath } from "../core/paths.js";
|
|
27
|
-
import { warn } from "../core/warn.js";
|
|
28
27
|
import { assertIndexPathReadable, closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
29
28
|
import { getEntryCount, getIndexedFilePaths } from "../storage/repositories/index-entries-repository.js";
|
|
30
29
|
import { getMeta } from "../storage/repositories/index-meta-repository.js";
|
|
@@ -188,22 +187,14 @@ function indexCanServeStash(stashDir) {
|
|
|
188
187
|
}
|
|
189
188
|
}
|
|
190
189
|
async function runInlineReindex(stashDir, options = {}) {
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
return true;
|
|
200
|
-
}
|
|
201
|
-
catch (error) {
|
|
202
|
-
if (options.signal?.aborted)
|
|
203
|
-
throw error;
|
|
204
|
-
warn("Auto-index failed, proceeding with existing index:", error instanceof Error ? error.message : String(error));
|
|
205
|
-
return false;
|
|
206
|
-
}
|
|
190
|
+
const { akmIndex } = await import("./indexer.js");
|
|
191
|
+
await akmIndex({
|
|
192
|
+
stashDir,
|
|
193
|
+
implicit: true,
|
|
194
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
195
|
+
...(options.hydrateSources === false ? { hydrateSources: false } : {}),
|
|
196
|
+
});
|
|
197
|
+
return true;
|
|
207
198
|
}
|
|
208
199
|
/**
|
|
209
200
|
* Ensure the local index exists and can serve the caller.
|
|
@@ -217,8 +208,9 @@ async function runInlineReindex(stashDir, options = {}) {
|
|
|
217
208
|
* trigger and waits for it. Use this for callers like `improve` whose
|
|
218
209
|
* planning logic depends on a current `entries` table in the same process.
|
|
219
210
|
*
|
|
220
|
-
* Returns `true` only when an inline index run succeeds
|
|
221
|
-
* A rebuild attempt that fails
|
|
211
|
+
* Returns `true` only when an inline index run succeeds; `false` when no
|
|
212
|
+
* rebuild was needed. A rebuild attempt that fails propagates its error to
|
|
213
|
+
* the caller rather than silently proceeding with a stale/missing index.
|
|
222
214
|
*/
|
|
223
215
|
export async function ensureIndex(stashDir, options = {}) {
|
|
224
216
|
// §11.5: warn (once) if the configured bundle ids drifted from the persisted
|
|
@@ -257,45 +257,34 @@ export function listRelatedPathsForFile(stashRoot, filePath, limit = 5, db) {
|
|
|
257
257
|
}
|
|
258
258
|
// Confirm the target file has a graph row; without it there is nothing to
|
|
259
259
|
// relate. (Identity is file_path within the stash — one row per path.)
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
if (row === undefined)
|
|
265
|
-
return [];
|
|
266
|
-
}
|
|
267
|
-
catch {
|
|
260
|
+
const row = db
|
|
261
|
+
.prepare("SELECT 1 AS present FROM graph_files WHERE stash_root = ? AND file_path = ? LIMIT 1")
|
|
262
|
+
.get(stashRoot, filePath);
|
|
263
|
+
if (row === undefined)
|
|
268
264
|
return [];
|
|
269
|
-
}
|
|
270
265
|
const effectiveLimit = Math.max(1, limit);
|
|
271
266
|
// Shared-entity count per candidate file_path. The target's entities are the
|
|
272
267
|
// rows for `filePath`; candidates are any OTHER file_path in the stash that
|
|
273
268
|
// shares a normalized entity.
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
LIMIT ?`)
|
|
294
|
-
.all(filePath, stashRoot, effectiveLimit);
|
|
295
|
-
}
|
|
296
|
-
catch {
|
|
297
|
-
return [];
|
|
298
|
-
}
|
|
269
|
+
const candidateRows = db
|
|
270
|
+
.prepare(`SELECT gf.file_path AS file_path,
|
|
271
|
+
gf.file_type AS file_type,
|
|
272
|
+
COUNT(*) AS shared
|
|
273
|
+
FROM graph_file_entities target
|
|
274
|
+
JOIN graph_file_entities e
|
|
275
|
+
ON e.stash_root = target.stash_root
|
|
276
|
+
AND e.entity_norm = target.entity_norm
|
|
277
|
+
AND e.file_path != target.file_path
|
|
278
|
+
JOIN graph_files gf
|
|
279
|
+
ON gf.stash_root = e.stash_root
|
|
280
|
+
AND gf.file_path = e.file_path
|
|
281
|
+
AND gf.body_hash = e.body_hash
|
|
282
|
+
WHERE target.file_path = ?
|
|
283
|
+
AND target.stash_root = ?
|
|
284
|
+
GROUP BY gf.file_path
|
|
285
|
+
ORDER BY shared DESC, gf.file_path ASC
|
|
286
|
+
LIMIT ?`)
|
|
287
|
+
.all(filePath, stashRoot, effectiveLimit);
|
|
299
288
|
if (candidateRows.length === 0)
|
|
300
289
|
return [];
|
|
301
290
|
const candidatePaths = candidateRows.map((r) => r.file_path);
|
|
@@ -971,6 +971,11 @@ async function extractGraphForSingleFileRevision(db, stashRoot, filePath, opts)
|
|
|
971
971
|
if (err instanceof ConfigError)
|
|
972
972
|
throw err;
|
|
973
973
|
rethrowIfTestIsolationError(err);
|
|
974
|
+
// A genuine extraction/write failure, distinct from the deliberate
|
|
975
|
+
// "nothing to do" skips above (missing file, empty body, no model). Warn
|
|
976
|
+
// so it is visible instead of looking identical to a no-op skip; the
|
|
977
|
+
// entry stays queued and is retried on the next pass.
|
|
978
|
+
warn(`graph extraction: failed to extract graph for ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
974
979
|
return { written: false };
|
|
975
980
|
}
|
|
976
981
|
finally {
|
|
@@ -1057,12 +1062,17 @@ export function collectEligibleFiles(stashRoot, includeTypes = [...DEFAULT_GRAPH
|
|
|
1057
1062
|
const dir = path.join(stashRoot, stashDir);
|
|
1058
1063
|
if (!fs.existsSync(dir))
|
|
1059
1064
|
continue;
|
|
1060
|
-
|
|
1065
|
+
const walked = walkMarkdownFiles(dir);
|
|
1066
|
+
if (!walked.complete) {
|
|
1067
|
+
warn(`graph extraction: directory scan under ${dir} is incomplete — some files may be missing`);
|
|
1068
|
+
}
|
|
1069
|
+
for (const filePath of walked.files) {
|
|
1061
1070
|
let raw;
|
|
1062
1071
|
try {
|
|
1063
1072
|
raw = fs.readFileSync(filePath, "utf8");
|
|
1064
1073
|
}
|
|
1065
|
-
catch {
|
|
1074
|
+
catch (err) {
|
|
1075
|
+
warn(`graph extraction: failed to read candidate file ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
1066
1076
|
continue;
|
|
1067
1077
|
}
|
|
1068
1078
|
const parsed = parseFrontmatter(raw);
|
package/dist/indexer/indexer.js
CHANGED
|
@@ -30,7 +30,7 @@ import { deleteStoredGraph } from "./db/graph-db.js";
|
|
|
30
30
|
import { deriveEntryProvenance, deriveInstallations } from "./installations.js";
|
|
31
31
|
import { indexedPathMatchesOwner, resolveAdapterConceptOwner, } from "./lookup/adapter-concept-owner.js";
|
|
32
32
|
import { generateEmbeddingsForDb } from "./materialize-embeddings.js";
|
|
33
|
-
import { canUseIncrementalSkip, computeDirFingerprint,
|
|
33
|
+
import { canUseIncrementalSkip, computeDirFingerprint, getCachedDirState, getDirIndexState, inferZeroRowReason, } from "./passes/dir-staleness.js";
|
|
34
34
|
import { isEnrichmentComplete, isWorkflowSkipWarning } from "./passes/metadata.js";
|
|
35
35
|
import { drainDirDocuments } from "./scan/drain-dir.js";
|
|
36
36
|
import { buildSearchText } from "./search/search-fields.js";
|
|
@@ -208,7 +208,7 @@ async function runEmbeddingPhase(ctx) {
|
|
|
208
208
|
* verify event.
|
|
209
209
|
*/
|
|
210
210
|
async function runFinalizePhase(ctx, deferredUpdateTransaction) {
|
|
211
|
-
const { db, config, sources, sourceDirs,
|
|
211
|
+
const { db, config, sources, sourceDirs, stashDir, signal, onProgress } = ctx;
|
|
212
212
|
ctx.timing.tFinalizeStart = Date.now();
|
|
213
213
|
// `upsertEntry` and every canonical delete own their FTS projection. This is
|
|
214
214
|
// an observation point, not a second materialization pass.
|
|
@@ -360,6 +360,19 @@ export function _setIndexTransactionHookForTests(hook) {
|
|
|
360
360
|
function indexTransactionHook(point) {
|
|
361
361
|
indexTransactionHookForTests?.(point);
|
|
362
362
|
}
|
|
363
|
+
let drainObserverForTests;
|
|
364
|
+
/**
|
|
365
|
+
* TEST-ONLY. Observe every directory that actually reaches
|
|
366
|
+
* `drainDirDocuments` — the per-file read/sha256-hash/frontmatter-parse step
|
|
367
|
+
* (#900) — with the directory path and its walked file count. `undefined`
|
|
368
|
+
* restores. A directory the pre-drain gate (`getCachedDirState`) skips never
|
|
369
|
+
* fires this observer, so it is the
|
|
370
|
+
* seam #900's own tests use to assert an unchanged directory's files are
|
|
371
|
+
* never read on a no-op incremental run.
|
|
372
|
+
*/
|
|
373
|
+
export function _setDrainObserverForTests(observer) {
|
|
374
|
+
drainObserverForTests = observer;
|
|
375
|
+
}
|
|
363
376
|
/**
|
|
364
377
|
* Detect an adapter for every resolvable source that does not declare one, and
|
|
365
378
|
* persist each detection into `config.json`.
|
|
@@ -883,17 +896,19 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
883
896
|
// Incremental freshness gate shared by both branches: consult the persisted
|
|
884
897
|
// dir state and record either a skip (unchanged + eligible for incremental
|
|
885
898
|
// skip) or a scan record carrying the candidate stash.
|
|
886
|
-
const recordFreshnessDecision = (dirPath, currentStashDir, stateFiles, stash, hashByFile, conceptIdByFile, indexVariant, forceScan, pruneMissing) => {
|
|
887
|
-
const previousState = getDirIndexState(db, dirPath, stateFiles, builtAtMs, indexVariant);
|
|
899
|
+
const recordFreshnessDecision = (dirPath, currentStashDir, stateFiles, fingerprint, stash, hashByFile, conceptIdByFile, indexVariant, forceScan, pruneMissing) => {
|
|
900
|
+
const previousState = getDirIndexState(db, dirPath, stateFiles, builtAtMs, indexVariant, fingerprint);
|
|
888
901
|
if (isIncremental && !forceScan && !previousState.stale && canUseIncrementalSkip(previousState, priorDirsChanged)) {
|
|
889
902
|
skippedDirs++;
|
|
890
903
|
dirRecords.push({
|
|
891
904
|
dirPath,
|
|
892
905
|
currentStashDir,
|
|
893
906
|
files: stateFiles,
|
|
907
|
+
fingerprint,
|
|
894
908
|
stash: null,
|
|
895
909
|
skip: true,
|
|
896
910
|
reason: previousState.reason,
|
|
911
|
+
persistedRowCount: previousState.persistedRowCount,
|
|
897
912
|
indexVariant,
|
|
898
913
|
});
|
|
899
914
|
reportDirDecision("skip", dirPath, currentStashDir, previousState.reason, previousState.persistedRowCount);
|
|
@@ -906,6 +921,7 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
906
921
|
dirPath,
|
|
907
922
|
currentStashDir,
|
|
908
923
|
files: stateFiles,
|
|
924
|
+
fingerprint,
|
|
909
925
|
stash,
|
|
910
926
|
skip: false,
|
|
911
927
|
reason,
|
|
@@ -959,10 +975,13 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
959
975
|
reportDirDecision("skip", dirPath, currentStashDir, reason);
|
|
960
976
|
continue;
|
|
961
977
|
}
|
|
962
|
-
|
|
978
|
+
// #900: decide from stat data alone whether the directory can be skipped,
|
|
979
|
+
// before drainDirDocuments reads, hashes, and parses every file.
|
|
980
|
+
const fingerprint = computeDirFingerprint(dirPath, indexableFiles, indexVariant);
|
|
981
|
+
const cachedState = isIncremental &&
|
|
963
982
|
!forceScan &&
|
|
964
|
-
|
|
965
|
-
if (
|
|
983
|
+
getCachedDirState(db, dirPath, indexableFiles, builtAtMs, priorDirsChanged, indexVariant, fingerprint);
|
|
984
|
+
if (cachedState) {
|
|
966
985
|
skippedDirs++;
|
|
967
986
|
dirRecords.push({
|
|
968
987
|
dirPath,
|
|
@@ -970,15 +989,16 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
970
989
|
files: indexableFiles,
|
|
971
990
|
stash: null,
|
|
972
991
|
skip: true,
|
|
973
|
-
reason:
|
|
992
|
+
reason: cachedState.reason,
|
|
974
993
|
indexVariant,
|
|
975
994
|
});
|
|
976
|
-
reportDirDecision("skip", dirPath, currentStashDir,
|
|
995
|
+
reportDirDecision("skip", dirPath, currentStashDir, cachedState.reason, cachedState.persistedRowCount);
|
|
977
996
|
continue;
|
|
978
997
|
}
|
|
979
998
|
// F4a M-core-2 (the flip): drain the dir's `IndexDocument` stream via the
|
|
980
999
|
// component's dispatched `adapter.recognize` (broken workflows dropped-with-
|
|
981
1000
|
// warning at the drain layer) and reconstruct the durable `IndexDocument`s.
|
|
1001
|
+
drainObserverForTests?.(dirPath, ctxs.length);
|
|
982
1002
|
const drained = drainDirDocuments(adapter, component, ctxs);
|
|
983
1003
|
if (drained.warnings.length)
|
|
984
1004
|
warnings.push(...drained.warnings);
|
|
@@ -992,7 +1012,7 @@ async function scanSourceDirs(db, allSourceEntries, isIncremental, builtAtMs, ha
|
|
|
992
1012
|
if (generated.entries.length > 0) {
|
|
993
1013
|
generatedCount += generated.entries.length;
|
|
994
1014
|
}
|
|
995
|
-
recordFreshnessDecision(dirPath, currentStashDir, staleFiles, stash, drained.hashByFile, drained.conceptIdByFile, indexVariant, forceScan, walkComplete);
|
|
1015
|
+
recordFreshnessDecision(dirPath, currentStashDir, staleFiles, fingerprint, stash, drained.hashByFile, drained.conceptIdByFile, indexVariant, forceScan, walkComplete);
|
|
996
1016
|
}
|
|
997
1017
|
}
|
|
998
1018
|
return {
|
|
@@ -1089,7 +1109,7 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1089
1109
|
// tests/integration/indexer/reindex-generation-atomicity.test.ts.
|
|
1090
1110
|
indexTransactionHook("full-delete-applied");
|
|
1091
1111
|
}
|
|
1092
|
-
for (const { dirPath, currentStashDir, files, stash, skip, reason, hashByFile, conceptIdByFile, indexVariant, remove, pruneMissing, } of dirRecords) {
|
|
1112
|
+
for (const { dirPath, currentStashDir, files, fingerprint, stash, skip, reason, persistedRowCount, hashByFile, conceptIdByFile, indexVariant, remove, pruneMissing, } of dirRecords) {
|
|
1093
1113
|
const bundle = bundleByRoot.get(path.resolve(currentStashDir));
|
|
1094
1114
|
if (!bundle)
|
|
1095
1115
|
throw new Error(`Missing bundle provenance for indexed source ${currentStashDir}`);
|
|
@@ -1102,14 +1122,11 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1102
1122
|
continue;
|
|
1103
1123
|
}
|
|
1104
1124
|
if (skip) {
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
fileMtimeMaxMs: fingerprint.fileMtimeMaxMs,
|
|
1111
|
-
reason: reason.kind,
|
|
1112
|
-
});
|
|
1125
|
+
// "unchanged" is the post-drain verdict: re-persist so the row carries
|
|
1126
|
+
// row_count and the gate skips this directory before draining next
|
|
1127
|
+
// time. "unchanged-precheck" already matched the stored row.
|
|
1128
|
+
if (reason?.kind === "unchanged" && fingerprint) {
|
|
1129
|
+
upsertIndexDirState(db, { dirPath, ...fingerprint, reason: reason.kind, rowCount: persistedRowCount });
|
|
1113
1130
|
}
|
|
1114
1131
|
continue;
|
|
1115
1132
|
}
|
|
@@ -1165,7 +1182,7 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1165
1182
|
if (pruneMissing !== false) {
|
|
1166
1183
|
addEntryIds(deletedUsageEntryIds, deleteEntriesByDirExceptRefs(db, dirPath, bundle.bundleId, keptItemRefs, { cleanupUsageEvents: false }));
|
|
1167
1184
|
}
|
|
1168
|
-
const
|
|
1185
|
+
const persistedFingerprint = fingerprint ?? computeDirFingerprint(dirPath, files, indexVariant);
|
|
1169
1186
|
const persistedReason = persistedRows === 0
|
|
1170
1187
|
? inferZeroRowReason(stash, reason, warnings, dirPath, dedupedRows)
|
|
1171
1188
|
: reason?.kind === "full-rebuild"
|
|
@@ -1173,9 +1190,13 @@ function persistDirRecords(db, dirRecords, doFullDelete, warnings, sourceRoots,
|
|
|
1173
1190
|
: (reason?.kind ?? "updated");
|
|
1174
1191
|
upsertIndexDirState(db, {
|
|
1175
1192
|
dirPath,
|
|
1176
|
-
|
|
1177
|
-
fileMtimeMaxMs: fingerprint.fileMtimeMaxMs,
|
|
1193
|
+
...persistedFingerprint,
|
|
1178
1194
|
reason: persistedReason,
|
|
1195
|
+
// A directory that lost rows to per-source dedup depends on the
|
|
1196
|
+
// directories persisted before it, not only on its own files, so it
|
|
1197
|
+
// must keep draining every run (as it did before the gate) until a
|
|
1198
|
+
// drain persists it without dedup. NULL keeps the gate closed.
|
|
1199
|
+
rowCount: dedupedRows === 0 ? persistedRows : undefined,
|
|
1179
1200
|
});
|
|
1180
1201
|
if (persistedRows === 0) {
|
|
1181
1202
|
// Warn only when the dir had files that *could* produce entries (.md or
|