akm-cli 0.9.16-alpha.2 → 0.9.17-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +495 -1
- package/dist/assets/prompts/consolidate-system.md +4 -11
- package/dist/assets/prompts/graph-extract-user-prompt.md +5 -5
- package/dist/commands/health/accept-rate.js +6 -0
- package/dist/commands/health/checks.js +54 -0
- package/dist/commands/health/improve-metrics.js +1 -5
- package/dist/commands/health/report-view-model.js +0 -1
- package/dist/commands/health.js +10 -0
- package/dist/commands/improve/consolidate/chunking.js +19 -35
- package/dist/commands/improve/consolidate/merge.js +6 -9
- package/dist/commands/improve/consolidate.js +104 -91
- package/dist/commands/improve/distill/promote-memory.js +40 -2
- package/dist/commands/improve/distill/quality-gate.js +186 -23
- package/dist/commands/improve/distill.js +42 -8
- package/dist/commands/improve/eligibility.js +13 -3
- package/dist/commands/improve/improve-cli.js +32 -9
- package/dist/commands/improve/improve-strategies.js +23 -1
- package/dist/commands/improve/improve.js +121 -84
- package/dist/commands/improve/loop-stages.js +241 -108
- package/dist/commands/improve/preparation.js +50 -17
- package/dist/commands/improve/reflect.js +16 -5
- package/dist/commands/improve/shared.js +0 -10
- package/dist/commands/proposal/drain.js +79 -10
- package/dist/commands/proposal/proposal-types.js +21 -0
- package/dist/commands/proposal/repository.js +108 -29
- package/dist/core/asset/frontmatter.js +106 -1
- package/dist/core/config/schema/improve-processes.js +29 -2
- package/dist/core/improve-result.js +9 -0
- package/dist/core/paths.js +7 -0
- package/dist/indexer/ensure-index.js +52 -7
- package/dist/indexer/graph/graph-extraction.js +82 -8
- package/dist/indexer/passes/memory-inference.js +16 -1
- package/dist/llm/client.js +16 -2
- package/dist/llm/graph-extract.js +162 -18
- package/dist/output/html-render.js +2 -1
- package/dist/output/stdout.js +24 -0
- package/dist/output/text.js +4 -3
- package/dist/scripts/akm-migrate-node.js +20 -4
- package/dist/scripts/akm-migrate.js +20 -4
- package/dist/storage/repositories/index-entries-repository.js +43 -0
- package/dist/storage/repositories/proposals-repository.js +4 -1
- package/dist/storage/state-db-integrity.js +123 -0
- package/dist/workflows/program/schema.js +1 -0
- package/docs/reference/cli.md +4 -3
- package/docs/reference/data-and-telemetry.md +1 -0
- package/package.json +1 -1
- package/schemas/akm-config.json +44 -0
- package/schemas/akm-workflow.json +1 -0
- package/dist/commands/improve/eval-cases.js +0 -52
|
@@ -43,7 +43,7 @@ import { ensureAkmMarkdownType } from "../../core/asset/akm-markdown.js";
|
|
|
43
43
|
import { assetPathForName, placementTypes, stashDirFor } from "../../core/asset/asset-placement.js";
|
|
44
44
|
import { isBundleSlug, parseBundleRef } from "../../core/asset/asset-ref.js";
|
|
45
45
|
import { assembleAsset, serializeFrontmatter } from "../../core/asset/asset-serialize.js";
|
|
46
|
-
import { parseFrontmatter } from "../../core/asset/frontmatter.js";
|
|
46
|
+
import { carryForwardBookkeepingFrontmatter, computeNormalizedContentHash, parseFrontmatter, } from "../../core/asset/frontmatter.js";
|
|
47
47
|
import { conceptIdFromTypeName, parseRefInput } from "../../core/asset/resolve-ref.js";
|
|
48
48
|
import { isWithin } from "../../core/common.js";
|
|
49
49
|
import { loadConfig } from "../../core/config/config.js";
|
|
@@ -66,7 +66,7 @@ import { openSqliteReadSnapshot } from "../../storage/sqlite-read-snapshot.js";
|
|
|
66
66
|
import { pkgVersion } from "../../version.js";
|
|
67
67
|
import { runBaseChecks } from "../lint/base-linter.js";
|
|
68
68
|
import { formatNewAssetDiff, formatUnifiedDiff } from "./diff-format.js";
|
|
69
|
-
import { isAutomatedProposalSource, isValidProposalSource, PROPOSAL_SOURCES, } from "./proposal-types.js";
|
|
69
|
+
import { isAutomatedProposalSource, isStaleTargetRejection, isValidProposalSource, PROPOSAL_SOURCES, } from "./proposal-types.js";
|
|
70
70
|
import { canonicalOnlyProposalValidators, hasCanonicalProposalValidator, runProposalValidators, } from "./validators/proposal-validators.js";
|
|
71
71
|
import { repairProposalContent, validateProposal } from "./validators/proposals.js";
|
|
72
72
|
const PROMOTION_LINT_ISSUE_TYPES = new Set(["unquoted-colon", "missing-ref", "stale-path"]);
|
|
@@ -241,6 +241,40 @@ export function resolveProposalQueueTarget(stashDir, config = loadConfig()) {
|
|
|
241
241
|
}
|
|
242
242
|
return { source: bundleId, root };
|
|
243
243
|
}
|
|
244
|
+
/**
|
|
245
|
+
* Resolve the durable target ref, target root/rel-path, and current
|
|
246
|
+
* before-hash for a parsed proposal ref (WI-6.2). This is the exact
|
|
247
|
+
* computation `createProposal` uses to derive its mint-time `beforeHash`
|
|
248
|
+
* fingerprint term; {@link checkProposalGuard} (R9) calls it too, so a
|
|
249
|
+
* pre-generation guard check and `createProposal`'s post-generation check
|
|
250
|
+
* always agree on what "these inputs" means for the same ref/target.
|
|
251
|
+
*/
|
|
252
|
+
function resolveProposalTargetInfo(stashDir, parsedRef, explicitTarget) {
|
|
253
|
+
const proposalTarget = resolveCreateProposalTarget(stashDir, explicitTarget, parsedRef.origin);
|
|
254
|
+
const normalizedRef = proposalDurableRef(parsedRef, proposalTarget);
|
|
255
|
+
const targetRoot = path.resolve(proposalTarget.root);
|
|
256
|
+
let targetRelPath;
|
|
257
|
+
let mintBeforeContent;
|
|
258
|
+
try {
|
|
259
|
+
const typeRoot = path.join(targetRoot, stashDirFor(parsedRef.type));
|
|
260
|
+
const targetAbs = assetPathForName(parsedRef.type, typeRoot, parsedRef.name);
|
|
261
|
+
targetRelPath = path.relative(targetRoot, targetAbs);
|
|
262
|
+
if (fs.existsSync(targetAbs))
|
|
263
|
+
mintBeforeContent = fs.readFileSync(targetAbs, "utf8");
|
|
264
|
+
}
|
|
265
|
+
catch {
|
|
266
|
+
// Resolution failure degrades to a best-effort create — never blocks the mint.
|
|
267
|
+
targetRelPath = path.join(stashDirFor(parsedRef.type), parsedRef.name);
|
|
268
|
+
}
|
|
269
|
+
return {
|
|
270
|
+
proposalTarget,
|
|
271
|
+
normalizedRef,
|
|
272
|
+
targetRoot,
|
|
273
|
+
targetRelPath,
|
|
274
|
+
beforeHash: mintBeforeContent !== undefined ? contentHash(mintBeforeContent) : undefined,
|
|
275
|
+
beforeHashNormalized: mintBeforeContent !== undefined ? computeNormalizedContentHash(mintBeforeContent) : undefined,
|
|
276
|
+
};
|
|
277
|
+
}
|
|
244
278
|
// ── Public API ──────────────────────────────────────────────────────────────
|
|
245
279
|
/**
|
|
246
280
|
* Create a new pending proposal. The id is a stable random UUID, so two
|
|
@@ -310,27 +344,12 @@ export function createProposal(stashDir, input, ctx) {
|
|
|
310
344
|
return rejectProposal("missing_description", `Proposal for "${input.ref}" (source=consolidate) has empty or missing frontmatter description.`);
|
|
311
345
|
}
|
|
312
346
|
}
|
|
313
|
-
const proposalTarget = resolveCreateProposalTarget(stashDir, input.target, parsedRef.origin);
|
|
314
|
-
const normalizedRef = proposalDurableRef(parsedRef, proposalTarget);
|
|
315
|
-
const targetRoot = path.resolve(proposalTarget.root);
|
|
316
347
|
// WI-6.2: derive the FileChange[] envelope + mint-time beforeHash. The
|
|
317
348
|
// target is resolved against the proposal's OWN stash (a local snapshot —
|
|
318
349
|
// accept re-resolves the write target from config at apply time), and only
|
|
319
350
|
// the before-state's HASH is kept: the change's `before` body is a
|
|
320
351
|
// transaction-time capture that does not exist at mint time.
|
|
321
|
-
|
|
322
|
-
let mintBeforeContent;
|
|
323
|
-
try {
|
|
324
|
-
const typeRoot = path.join(targetRoot, stashDirFor(parsedRef.type));
|
|
325
|
-
const targetAbs = assetPathForName(parsedRef.type, typeRoot, parsedRef.name);
|
|
326
|
-
targetRelPath = path.relative(targetRoot, targetAbs);
|
|
327
|
-
if (fs.existsSync(targetAbs))
|
|
328
|
-
mintBeforeContent = fs.readFileSync(targetAbs, "utf8");
|
|
329
|
-
}
|
|
330
|
-
catch {
|
|
331
|
-
// Resolution failure degrades to a best-effort create — never blocks the mint.
|
|
332
|
-
targetRelPath = path.join(stashDirFor(parsedRef.type), parsedRef.name);
|
|
333
|
-
}
|
|
352
|
+
const { proposalTarget, normalizedRef, targetRoot, targetRelPath, beforeHash: mintedBeforeHash, beforeHashNormalized: mintedBeforeHashNormalized, } = resolveProposalTargetInfo(stashDir, parsedRef, input.target);
|
|
334
353
|
const proposalContent = targetRelPath.toLowerCase().endsWith(".md")
|
|
335
354
|
? ensureAkmMarkdownType(input.payload.content, parsedRef.type)
|
|
336
355
|
: input.payload.content;
|
|
@@ -338,10 +357,9 @@ export function createProposal(stashDir, input, ctx) {
|
|
|
338
357
|
{
|
|
339
358
|
path: targetRelPath,
|
|
340
359
|
after: proposalContent,
|
|
341
|
-
op:
|
|
360
|
+
op: mintedBeforeHash !== undefined ? "update" : "create",
|
|
342
361
|
},
|
|
343
362
|
];
|
|
344
|
-
const mintedBeforeHash = mintBeforeContent !== undefined ? contentHash(mintBeforeContent) : undefined;
|
|
345
363
|
if (hasCanonicalProposalValidator(parsedRef.type)) {
|
|
346
364
|
// Mint-time gate: structural shape only (generic + canonical-per-type),
|
|
347
365
|
// NOT the full quality-validator list — see canonicalOnlyProposalValidators'
|
|
@@ -374,7 +392,7 @@ export function createProposal(stashDir, input, ctx) {
|
|
|
374
392
|
return withProposalsDb(stashDir, ctx, (db) => {
|
|
375
393
|
return withImmediateTransaction(db, () => {
|
|
376
394
|
if (!input.force) {
|
|
377
|
-
const skip = checkFingerprintAndBackoff(db, stashDir, normalizedRef, input, fingerprint, ctx);
|
|
395
|
+
const skip = checkFingerprintAndBackoff(db, stashDir, normalizedRef, input.source, fingerprint, ctx);
|
|
378
396
|
if (skip)
|
|
379
397
|
return skip;
|
|
380
398
|
}
|
|
@@ -403,6 +421,7 @@ export function createProposal(stashDir, input, ctx) {
|
|
|
403
421
|
changes: mintedChanges,
|
|
404
422
|
proposedTarget: { source: proposalTarget.source, root: targetRoot },
|
|
405
423
|
...(mintedBeforeHash !== undefined ? { beforeHash: mintedBeforeHash } : {}),
|
|
424
|
+
...(mintedBeforeHashNormalized !== undefined ? { beforeHashNormalized: mintedBeforeHashNormalized } : {}),
|
|
406
425
|
...(sanitizedConfidence !== undefined ? { confidence: sanitizedConfidence } : {}),
|
|
407
426
|
// Attribution tagging: persist the eligibility lane so it survives to
|
|
408
427
|
// accept/reject/revert time. See EligibilitySource.
|
|
@@ -455,9 +474,9 @@ function recordProposalFingerprint(db, stashDir, fingerprint, ref, input, propos
|
|
|
455
474
|
* Evaluate the fingerprint + rejection-backoff guards. Returns the skip
|
|
456
475
|
* record when a guard fires, or undefined when the create may proceed.
|
|
457
476
|
*/
|
|
458
|
-
function checkFingerprintAndBackoff(db, stashDir, normalizedRef,
|
|
477
|
+
function checkFingerprintAndBackoff(db, stashDir, normalizedRef, source, fingerprint, ctx) {
|
|
459
478
|
const nowMs = (ctx?.now ?? Date.now)();
|
|
460
|
-
const backoffMs = cooldownMsForSource(
|
|
479
|
+
const backoffMs = cooldownMsForSource(source);
|
|
461
480
|
// §23.6: an already-processed fingerprint skips another model call's output
|
|
462
481
|
// unless explicitly forced. The row survives the proposal's lifecycle —
|
|
463
482
|
// identical inputs stay deduplicated until the target (before-hash), the
|
|
@@ -474,9 +493,13 @@ function checkFingerprintAndBackoff(db, stashDir, normalizedRef, input, fingerpr
|
|
|
474
493
|
};
|
|
475
494
|
}
|
|
476
495
|
// Rejection backoff (RETAINED cooldown semantics): a recent rejection for
|
|
477
|
-
// this ref+source suppresses new proposals until the window expires.
|
|
496
|
+
// this ref+source suppresses new proposals until the window expires. A
|
|
497
|
+
// stale-target auto-reject (STALE, R20) is excluded — it is not a
|
|
498
|
+
// judgement on the content, and counting it here would suppress a
|
|
499
|
+
// legitimate re-propose against the ref's now-current content.
|
|
478
500
|
const rejected = listStateProposals(db, { stashDir, ref: normalizedRef, status: "rejected" })
|
|
479
|
-
.filter((p) => p.source ===
|
|
501
|
+
.filter((p) => p.source === source)
|
|
502
|
+
.filter((p) => !isStaleTargetRejection(p))
|
|
480
503
|
.sort((a, b) => new Date(b.updatedAt ?? 0).getTime() - new Date(a.updatedAt ?? 0).getTime());
|
|
481
504
|
const mostRecent = rejected[0];
|
|
482
505
|
if (mostRecent !== undefined) {
|
|
@@ -487,7 +510,7 @@ function checkFingerprintAndBackoff(db, stashDir, normalizedRef, input, fingerpr
|
|
|
487
510
|
return {
|
|
488
511
|
skipped: true,
|
|
489
512
|
reason: "rejection_backoff",
|
|
490
|
-
message: `Proposal for ${normalizedRef} from source "${
|
|
513
|
+
message: `Proposal for ${normalizedRef} from source "${source}" is in rejection backoff ` +
|
|
491
514
|
`(${backoffDays}d window, ~${remainingDays}d remaining). Pass force:true to bypass.`,
|
|
492
515
|
existingProposalId: mostRecent.id,
|
|
493
516
|
};
|
|
@@ -495,6 +518,36 @@ function checkFingerprintAndBackoff(db, stashDir, normalizedRef, input, fingerpr
|
|
|
495
518
|
}
|
|
496
519
|
return undefined;
|
|
497
520
|
}
|
|
521
|
+
/**
|
|
522
|
+
* R9 (tier2-0917): pure pre-generation check of the fingerprint-match /
|
|
523
|
+
* rejection-backoff guard — the same computation `createProposal` runs AFTER
|
|
524
|
+
* generation (`checkFingerprintAndBackoff`), exposed so a caller can skip an
|
|
525
|
+
* expensive LLM call BEFORE making it. Shares {@link resolveProposalTargetInfo}
|
|
526
|
+
* and {@link checkFingerprintAndBackoff} verbatim with `createProposal`, so
|
|
527
|
+
* the two can never disagree about what "these inputs" means for the same
|
|
528
|
+
* ref/source/target/model. `createProposal`'s post-generation check remains
|
|
529
|
+
* the authoritative gate: this is a best-effort optimisation that fails open
|
|
530
|
+
* (returns `undefined`, i.e. "not skipped") on any resolution error — an
|
|
531
|
+
* unresolvable target here must never block the real dispatch.
|
|
532
|
+
*/
|
|
533
|
+
export function checkProposalGuard(input, ctx) {
|
|
534
|
+
try {
|
|
535
|
+
const parsedRef = parseRefInput(input.ref);
|
|
536
|
+
if (!stashDirFor(parsedRef.type))
|
|
537
|
+
return undefined;
|
|
538
|
+
const { normalizedRef, beforeHash } = resolveProposalTargetInfo(input.stash, parsedRef, input.target);
|
|
539
|
+
const fingerprint = computeProposalFingerprint({
|
|
540
|
+
ref: normalizedRef,
|
|
541
|
+
source: input.source,
|
|
542
|
+
...(beforeHash !== undefined ? { beforeHash } : {}),
|
|
543
|
+
...(input.modelId !== undefined ? { modelId: input.modelId } : {}),
|
|
544
|
+
});
|
|
545
|
+
return withProposalsDb(input.stash, ctx, (db) => checkFingerprintAndBackoff(db, input.stash, normalizedRef, input.source, fingerprint, ctx));
|
|
546
|
+
}
|
|
547
|
+
catch {
|
|
548
|
+
return undefined;
|
|
549
|
+
}
|
|
550
|
+
}
|
|
498
551
|
/**
|
|
499
552
|
* List proposals for one stash. By default returns only the live (pending)
|
|
500
553
|
* queue; pass `{ includeArchive: true }` to include accepted / rejected /
|
|
@@ -1571,9 +1624,24 @@ export function preflightProposalPromotion(config, proposal, options = {}, ctx)
|
|
|
1571
1624
|
if (!assetPath)
|
|
1572
1625
|
throw new UsageError(`Cannot resolve proposal target ${preparedProposal.ref}.`, "INVALID_PROPOSAL");
|
|
1573
1626
|
assertAkmAssetWrite(target.source);
|
|
1574
|
-
|
|
1627
|
+
let stampedContent = assetPath.toLowerCase().endsWith(".md")
|
|
1575
1628
|
? stampProposalProvenance(repairedContent, preparedProposal, options.gateDecision, ctx, nowIso(ctx))
|
|
1576
1629
|
: repairedContent;
|
|
1630
|
+
if (assetPath.toLowerCase().endsWith(".md") && fs.existsSync(assetPath)) {
|
|
1631
|
+
// STALE (R20): carry the live target's akm bookkeeping frontmatter
|
|
1632
|
+
// forward when the proposal's own frontmatter doesn't set it, so
|
|
1633
|
+
// promoting never drops e.g. `inferenceProcessed` and makes memory
|
|
1634
|
+
// inference reprocess the memory. Best-effort — an unreadable live file
|
|
1635
|
+
// here doesn't block promotion; the freshness guard below is the real
|
|
1636
|
+
// gate on staleness.
|
|
1637
|
+
try {
|
|
1638
|
+
const liveRaw = fs.readFileSync(assetPath, "utf8");
|
|
1639
|
+
stampedContent = carryForwardBookkeepingFrontmatter(stampedContent, liveRaw);
|
|
1640
|
+
}
|
|
1641
|
+
catch {
|
|
1642
|
+
// Best-effort — see comment above.
|
|
1643
|
+
}
|
|
1644
|
+
}
|
|
1577
1645
|
const lintBlockers = promotionLintBlockers(stampedContent, assetPath, target.source.path, ref.type, config);
|
|
1578
1646
|
if (lintBlockers.length > 0) {
|
|
1579
1647
|
const summary = lintBlockers.map((finding) => `[${finding.issue}] ${finding.detail}`).join("; ");
|
|
@@ -1707,8 +1775,19 @@ async function promoteProposalWithLease(stashDir, config, id, options, ctx) {
|
|
|
1707
1775
|
throw new Error(`Proposal backup read failed for ${assetPath}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1708
1776
|
}
|
|
1709
1777
|
}
|
|
1710
|
-
if (proposal.beforeHash !== undefined
|
|
1711
|
-
|
|
1778
|
+
if (proposal.beforeHash !== undefined) {
|
|
1779
|
+
// STALE (R20): a proposal minted with a normalized before-hash is fresh
|
|
1780
|
+
// when the CURRENT target's bookkeeping-stripped content still matches —
|
|
1781
|
+
// insensitive to a same-run bookkeeping rewrite (salience scoring,
|
|
1782
|
+
// inference dedup marking) of the target after mint. A legacy proposal
|
|
1783
|
+
// without one keeps the exact raw-hash check it always had.
|
|
1784
|
+
const fresh = proposal.beforeHashNormalized !== undefined
|
|
1785
|
+
? backup !== undefined &&
|
|
1786
|
+
computeNormalizedContentHash(backup.toString("utf8")) === proposal.beforeHashNormalized
|
|
1787
|
+
: backup !== undefined && proposalHash(backup) === proposal.beforeHash;
|
|
1788
|
+
if (!fresh) {
|
|
1789
|
+
throw new UsageError(`Proposal target changed after proposal ${id} was created; refusing to overwrite newer content.`, "INVALID_FLAG_VALUE");
|
|
1790
|
+
}
|
|
1712
1791
|
}
|
|
1713
1792
|
if (proposal.beforeHash === undefined &&
|
|
1714
1793
|
backup !== undefined &&
|
|
@@ -8,11 +8,12 @@
|
|
|
8
8
|
* (block scalars, multi-line strings, nested objects, flow sequences, escape
|
|
9
9
|
* sequences) is handled correctly without a brittle hand-rolled state machine.
|
|
10
10
|
*/
|
|
11
|
+
import { createHash } from "node:crypto";
|
|
11
12
|
import fs from "node:fs";
|
|
12
13
|
import { parse as yamlParse, stringify as yamlStringify } from "yaml";
|
|
13
14
|
import { existingFileMode, writeFileAtomic } from "../common.js";
|
|
14
15
|
import { recordWrittenPath } from "../write-provenance.js";
|
|
15
|
-
import { assembleAsset, serializeFrontmatter } from "./asset-serialize.js";
|
|
16
|
+
import { assembleAsset, assembleAssetFromString, serializeFrontmatter } from "./asset-serialize.js";
|
|
16
17
|
/**
|
|
17
18
|
* Parse YAML frontmatter from a Markdown (or similar) string.
|
|
18
19
|
*
|
|
@@ -451,6 +452,110 @@ export function parseYamlScalar(value) {
|
|
|
451
452
|
}
|
|
452
453
|
return value;
|
|
453
454
|
}
|
|
455
|
+
// ── Bookkeeping-insensitive freshness (STALE, R20) ────────────────────────────
|
|
456
|
+
/**
|
|
457
|
+
* Frontmatter keys akm's own improve/index passes write into an EXISTING
|
|
458
|
+
* asset as pipeline bookkeeping — never a change to the asset's authored
|
|
459
|
+
* meaning. `salience`/`salienceInputs` come from {@link writeSalienceToFrontmatter}
|
|
460
|
+
* (distill); `inferenceProcessed` comes from memory inference
|
|
461
|
+
* (`indexer/passes/memory-inference.ts`). A pending proposal's mint-time
|
|
462
|
+
* before-hash is sensitive to these rewrites even though nothing a reviewer
|
|
463
|
+
* or the proposal's own diff cares about actually changed, which stales out
|
|
464
|
+
* proposals that a same-run bookkeeping pass touches after mint. See
|
|
465
|
+
* {@link computeNormalizedContentHash} and {@link carryForwardBookkeepingFrontmatter}.
|
|
466
|
+
*
|
|
467
|
+
* Deliberately excludes `beliefState`/`contradictedBy`/`supersededBy`: those
|
|
468
|
+
* are editorial state (contradiction/supersession demotions), not derived
|
|
469
|
+
* audit metadata — a promote should still refuse when one of those changed
|
|
470
|
+
* underneath a pending proposal.
|
|
471
|
+
*/
|
|
472
|
+
export const BOOKKEEPING_FRONTMATTER_KEYS = ["salience", "salienceInputs", "inferenceProcessed"];
|
|
473
|
+
function sha256Hex(content) {
|
|
474
|
+
return createHash("sha256").update(content, "utf8").digest("hex");
|
|
475
|
+
}
|
|
476
|
+
/**
|
|
477
|
+
* Hash of `raw` with {@link BOOKKEEPING_FRONTMATTER_KEYS} removed from its
|
|
478
|
+
* frontmatter and the remaining frontmatter re-serialized with sorted keys,
|
|
479
|
+
* so pipeline-only bookkeeping rewrites (and incidental key-order churn)
|
|
480
|
+
* cannot change the result. The body's boundary is normalized the same way
|
|
481
|
+
* {@link assembleAssetFromString} normalizes it (leading newlines stripped,
|
|
482
|
+
* exactly one trailing newline) before hashing, so the body-boundary shift
|
|
483
|
+
* that `writeSalienceToFrontmatter` and the `assembleAsset` bookkeeping
|
|
484
|
+
* rewrite both introduce cannot change the result either; the body's
|
|
485
|
+
* interior is still hashed verbatim.
|
|
486
|
+
*
|
|
487
|
+
* An empty frontmatter block (`---\n---\n…`) is treated as `{}` rather than
|
|
488
|
+
* falling back to the raw hash, so adding bookkeeping keys to a memory with
|
|
489
|
+
* no prior frontmatter fields is still insensitive.
|
|
490
|
+
*
|
|
491
|
+
* Falls back to hashing `raw` verbatim when there is no frontmatter block at
|
|
492
|
+
* all or it fails to parse — normalizing unparsable frontmatter isn't safe,
|
|
493
|
+
* and the caller's existing raw-hash check already covers that case.
|
|
494
|
+
*/
|
|
495
|
+
export function computeNormalizedContentHash(raw) {
|
|
496
|
+
const block = parseFrontmatterBlock(raw);
|
|
497
|
+
if (!block)
|
|
498
|
+
return sha256Hex(raw);
|
|
499
|
+
let data;
|
|
500
|
+
try {
|
|
501
|
+
data = block.frontmatter.trim() ? yamlParse(block.frontmatter) : {};
|
|
502
|
+
}
|
|
503
|
+
catch {
|
|
504
|
+
return sha256Hex(raw);
|
|
505
|
+
}
|
|
506
|
+
if (data === null || typeof data !== "object" || Array.isArray(data))
|
|
507
|
+
return sha256Hex(raw);
|
|
508
|
+
const normalized = { ...data };
|
|
509
|
+
for (const key of BOOKKEEPING_FRONTMATTER_KEYS)
|
|
510
|
+
delete normalized[key];
|
|
511
|
+
const canonicalFrontmatter = yamlStringify(normalized, { sortMapEntries: true }).trimEnd();
|
|
512
|
+
return sha256Hex(assembleAssetFromString(canonicalFrontmatter, block.content));
|
|
513
|
+
}
|
|
514
|
+
/**
|
|
515
|
+
* Fold the live target's {@link BOOKKEEPING_FRONTMATTER_KEYS} into
|
|
516
|
+
* `proposedRaw` wherever the proposal's own frontmatter does not already set
|
|
517
|
+
* them, so promoting a proposal minted before a bookkeeping rewrite never
|
|
518
|
+
* drops that bookkeeping (STALE, R20) — e.g. dropping `inferenceProcessed`
|
|
519
|
+
* would make memory inference reprocess the memory.
|
|
520
|
+
*
|
|
521
|
+
* Returns `proposedRaw` unchanged when either side has no parseable
|
|
522
|
+
* frontmatter block or nothing needs to carry forward.
|
|
523
|
+
*/
|
|
524
|
+
export function carryForwardBookkeepingFrontmatter(proposedRaw, liveRaw) {
|
|
525
|
+
const proposedBlock = parseFrontmatterBlock(proposedRaw);
|
|
526
|
+
const liveBlock = parseFrontmatterBlock(liveRaw);
|
|
527
|
+
if (!proposedBlock || !liveBlock)
|
|
528
|
+
return proposedRaw;
|
|
529
|
+
let proposedData;
|
|
530
|
+
let liveData;
|
|
531
|
+
try {
|
|
532
|
+
const parsedProposed = proposedBlock.frontmatter.trim() ? yamlParse(proposedBlock.frontmatter) : {};
|
|
533
|
+
const parsedLive = liveBlock.frontmatter.trim() ? yamlParse(liveBlock.frontmatter) : {};
|
|
534
|
+
if (parsedProposed === null || typeof parsedProposed !== "object" || Array.isArray(parsedProposed)) {
|
|
535
|
+
return proposedRaw;
|
|
536
|
+
}
|
|
537
|
+
if (parsedLive === null || typeof parsedLive !== "object" || Array.isArray(parsedLive))
|
|
538
|
+
return proposedRaw;
|
|
539
|
+
proposedData = parsedProposed;
|
|
540
|
+
liveData = parsedLive;
|
|
541
|
+
}
|
|
542
|
+
catch {
|
|
543
|
+
return proposedRaw;
|
|
544
|
+
}
|
|
545
|
+
const merged = { ...proposedData };
|
|
546
|
+
let changed = false;
|
|
547
|
+
for (const key of BOOKKEEPING_FRONTMATTER_KEYS) {
|
|
548
|
+
if (!(key in merged) && key in liveData) {
|
|
549
|
+
merged[key] = liveData[key];
|
|
550
|
+
changed = true;
|
|
551
|
+
}
|
|
552
|
+
}
|
|
553
|
+
if (!changed)
|
|
554
|
+
return proposedRaw;
|
|
555
|
+
const newFrontmatter = yamlStringify(merged).trimEnd();
|
|
556
|
+
const separator = proposedBlock.content.startsWith("\n") ? "" : "\n";
|
|
557
|
+
return `---\n${newFrontmatter}\n---\n${separator}${proposedBlock.content}`;
|
|
558
|
+
}
|
|
454
559
|
// ── Minimum score delta to trigger a frontmatter salience rewrite ─────────────
|
|
455
560
|
const SALIENCE_WRITE_DELTA_THRESHOLD = 0.05;
|
|
456
561
|
/**
|
|
@@ -42,6 +42,15 @@ const IMPROVE_PROCESS_BASE_FIELDS = {
|
|
|
42
42
|
};
|
|
43
43
|
/** Reflect/distill/consolidate: process-level asset-type filter (see improve-strategies.ts shouldSkipRef/DEFAULT_ALLOWED_TYPES). */
|
|
44
44
|
const allowedTypesField = z.array(z.string().min(1)).optional();
|
|
45
|
+
/**
|
|
46
|
+
* Reflect only (R12): conceptId prefixes to exclude, matched after stripping
|
|
47
|
+
* an optional `bundle//` from both the ref and each prefix (see
|
|
48
|
+
* improve-strategies.ts shouldSkipRef). `allowedTypes` is type-only and can't
|
|
49
|
+
* exclude a subset of one type, e.g. raw wiki-ingest snapshots indexed as
|
|
50
|
+
* `knowledge/wikis/articles/raw/*`. distill/consolidate are memory-only and
|
|
51
|
+
* never read this field.
|
|
52
|
+
*/
|
|
53
|
+
const excludeRefPrefixesField = z.array(z.string().min(1)).optional();
|
|
45
54
|
/**
|
|
46
55
|
* Consolidate process: hard cap on memories processed per pass.
|
|
47
56
|
* Reflect/distill: max refs processed (same as profile-level `limit`).
|
|
@@ -160,6 +169,7 @@ const antiCollapseField = z
|
|
|
160
169
|
.optional();
|
|
161
170
|
const REFLECT_PROCESS_FIELDS = {
|
|
162
171
|
allowedTypes: allowedTypesField,
|
|
172
|
+
excludeRefPrefixes: excludeRefPrefixesField,
|
|
163
173
|
limit: processLimitField,
|
|
164
174
|
qualityGate: qualityGateField,
|
|
165
175
|
lowValueFilter: lowValueFilterField,
|
|
@@ -215,6 +225,11 @@ const GRAPH_EXTRACTION_PROCESS_FIELDS = {
|
|
|
215
225
|
// instead of only files touched by actionable refs in the current run.
|
|
216
226
|
// Used by the `graph-refresh` built-in profile / a scheduled weekly task.
|
|
217
227
|
fullScan: z.boolean().optional(),
|
|
228
|
+
// R12b + R20: cap on chunks processed per asset. A body chunked beyond this
|
|
229
|
+
// is truncated to the first N chunks instead of paying for unbounded
|
|
230
|
+
// per-asset LLM calls; the coverage loss is recorded as truncatedChunks.
|
|
231
|
+
// Absent = default 8 (src/llm/graph-extract.ts DEFAULT_MAX_CHUNKS_PER_ASSET).
|
|
232
|
+
maxChunksPerAsset: positiveInt.optional(),
|
|
218
233
|
};
|
|
219
234
|
const EXTRACT_PROCESS_FIELDS = {
|
|
220
235
|
defaultSince: z.string().min(1).optional(),
|
|
@@ -293,6 +308,16 @@ function checkRetiredProcessKeys(value, ctx) {
|
|
|
293
308
|
}
|
|
294
309
|
}
|
|
295
310
|
}
|
|
311
|
+
/** distill/consolidate are memory-only and never read `excludeRefPrefixes` (reflect only, R12). */
|
|
312
|
+
function rejectExcludeRefPrefixesOutsideReflect(value, ctx) {
|
|
313
|
+
if ("excludeRefPrefixes" in value) {
|
|
314
|
+
ctx.addIssue({
|
|
315
|
+
code: z.ZodIssueCode.custom,
|
|
316
|
+
path: ["excludeRefPrefixes"],
|
|
317
|
+
message: "excludeRefPrefixes is only valid on the reflect process",
|
|
318
|
+
});
|
|
319
|
+
}
|
|
320
|
+
}
|
|
296
321
|
export const ImproveProcessConfigSchema = z
|
|
297
322
|
.object({
|
|
298
323
|
...IMPROVE_PROCESS_BASE_FIELDS,
|
|
@@ -316,12 +341,14 @@ export const ReflectProcessConfigSchema = z
|
|
|
316
341
|
export const DistillProcessConfigSchema = z
|
|
317
342
|
.object({ ...IMPROVE_PROCESS_BASE_FIELDS, ...DISTILL_PROCESS_FIELDS })
|
|
318
343
|
.passthrough()
|
|
319
|
-
.superRefine(checkRetiredProcessKeys)
|
|
344
|
+
.superRefine(checkRetiredProcessKeys)
|
|
345
|
+
.superRefine(rejectExcludeRefPrefixesOutsideReflect);
|
|
320
346
|
/** `processes.consolidate` — narrow per-process schema (WI-9.6). */
|
|
321
347
|
export const ConsolidateProcessConfigSchema = z
|
|
322
348
|
.object({ ...IMPROVE_PROCESS_BASE_FIELDS, ...CONSOLIDATE_PROCESS_FIELDS })
|
|
323
349
|
.passthrough()
|
|
324
|
-
.superRefine(checkRetiredProcessKeys)
|
|
350
|
+
.superRefine(checkRetiredProcessKeys)
|
|
351
|
+
.superRefine(rejectExcludeRefPrefixesOutsideReflect);
|
|
325
352
|
/** `processes.memoryInference` — narrow per-process schema (WI-9.6). */
|
|
326
353
|
export const MemoryInferenceProcessConfigSchema = z
|
|
327
354
|
.object({ ...IMPROVE_PROCESS_BASE_FIELDS, ...MEMORY_INFERENCE_PROCESS_FIELDS })
|
|
@@ -19,6 +19,7 @@ const COMMON_FIELDS = [
|
|
|
19
19
|
"plan",
|
|
20
20
|
"actions",
|
|
21
21
|
"skippedProcesses",
|
|
22
|
+
"engineProbe",
|
|
22
23
|
"distillSkipped",
|
|
23
24
|
"validationFailures",
|
|
24
25
|
"schemaRepairs",
|
|
@@ -27,6 +28,10 @@ const COMMON_FIELDS = [
|
|
|
27
28
|
"lintSummary",
|
|
28
29
|
"memoryIndexHealth",
|
|
29
30
|
"coverageGaps",
|
|
31
|
+
// R10: no longer written (the write-only eval-cases path was removed),
|
|
32
|
+
// but kept in the allow-list so `decodeImproveResult` still reads
|
|
33
|
+
// improve-result envelopes a prior release wrote with this field —
|
|
34
|
+
// AGENTS.md "Reading persisted data".
|
|
30
35
|
"evalCasesWritten",
|
|
31
36
|
"deadUrls",
|
|
32
37
|
"deadUrlCoverage",
|
|
@@ -35,6 +40,7 @@ const COMMON_FIELDS = [
|
|
|
35
40
|
"graphExtraction",
|
|
36
41
|
"memoryInferenceDurationMs",
|
|
37
42
|
"graphExtractionDurationMs",
|
|
43
|
+
"ensureIndexDurationMs",
|
|
38
44
|
"orphansPurged",
|
|
39
45
|
"proposalsExpired",
|
|
40
46
|
"reflectCooldownActions",
|
|
@@ -442,6 +448,7 @@ function validateCommon(value) {
|
|
|
442
448
|
for (const field of [
|
|
443
449
|
"actions",
|
|
444
450
|
"skippedProcesses",
|
|
451
|
+
"engineProbe",
|
|
445
452
|
"validationFailures",
|
|
446
453
|
"schemaRepairs",
|
|
447
454
|
"extract",
|
|
@@ -457,10 +464,12 @@ function validateCommon(value) {
|
|
|
457
464
|
}
|
|
458
465
|
for (const field of [
|
|
459
466
|
"cyclesRun",
|
|
467
|
+
// R10: retired write, kept readable — see the COMMON_FIELDS comment above.
|
|
460
468
|
"evalCasesWritten",
|
|
461
469
|
"reflectsWithErrorContext",
|
|
462
470
|
"memoryInferenceDurationMs",
|
|
463
471
|
"graphExtractionDurationMs",
|
|
472
|
+
"ensureIndexDurationMs",
|
|
464
473
|
"orphansPurged",
|
|
465
474
|
"proposalsExpired",
|
|
466
475
|
"reflectCooldownActions",
|
package/dist/core/paths.js
CHANGED
|
@@ -352,6 +352,13 @@ export function getDistillRejectedDir(stashDir) {
|
|
|
352
352
|
* `$STATE/improve/eval-cases/<stash>/` — regression eval cases captured from
|
|
353
353
|
* rejected distill/proposal output. Moved out of `$STASH/.akm/eval-cases/`
|
|
354
354
|
* (itlackey/akm#890).
|
|
355
|
+
*
|
|
356
|
+
* R10: the writer (`src/commands/improve/eval-cases.ts`) was removed —
|
|
357
|
+
* nothing ever read the files it wrote, and a rejected proposal row now
|
|
358
|
+
* carries the same information (see `quality-gate.ts`'s
|
|
359
|
+
* `writeQualityRejection`). This resolver stays: `scripts/akm-migrate/migrate/
|
|
360
|
+
* writer-relocation.ts` still uses it to relocate pre-existing eval-case
|
|
361
|
+
* files an older release wrote to the legacy `$STASH/.akm/eval-cases/` path.
|
|
355
362
|
*/
|
|
356
363
|
export function getEvalCasesDir(stashDir) {
|
|
357
364
|
return stashScopedDir(path.join(getStateDir(), "improve", "eval-cases"), stashDir);
|
|
@@ -21,11 +21,13 @@
|
|
|
21
21
|
*/
|
|
22
22
|
import fs from "node:fs";
|
|
23
23
|
import path from "node:path";
|
|
24
|
+
import { hashContent } from "../core/adapter/adapters/shared.js";
|
|
24
25
|
import { placementSpecList } from "../core/asset/asset-placement.js";
|
|
25
26
|
import { classifyPathAccess } from "../core/path-access.js";
|
|
26
27
|
import { getDbPath } from "../core/paths.js";
|
|
28
|
+
import { warnVerbose } from "../core/warn.js";
|
|
27
29
|
import { assertIndexPathReadable, closeDatabase, openExistingDatabase } from "../storage/repositories/index-connection.js";
|
|
28
|
-
import { getEntryCount, getIndexedFilePaths } from "../storage/repositories/index-entries-repository.js";
|
|
30
|
+
import { getEntryCount, getIndexedFileHashes, getIndexedFilePaths, } from "../storage/repositories/index-entries-repository.js";
|
|
29
31
|
import { isCanonicalIndexGeneration } from "../storage/repositories/index-entry-schema.js";
|
|
30
32
|
import { getMeta } from "../storage/repositories/index-meta-repository.js";
|
|
31
33
|
import { warnOnBundleRenameDrift } from "./bundle-identity-guard.js";
|
|
@@ -77,11 +79,20 @@ function getIndexableFiles(root, spec) {
|
|
|
77
79
|
* millisecond-truncated), so the mtime test alone silently misses
|
|
78
80
|
* additions made within ~a millisecond of the previous build.
|
|
79
81
|
*
|
|
82
|
+
* A file whose mtime is newer than `builtAt` is NOT automatically stale
|
|
83
|
+
* (R6): `indexWrittenAssets` upserts a fresh `content_hash` without bumping
|
|
84
|
+
* `builtAt`, so a file `ensureIndex` itself just incrementally re-indexed
|
|
85
|
+
* (for example a proposal triage just promoted into `knowledge/`) would
|
|
86
|
+
* otherwise keep tripping this check on every subsequent call, forcing the
|
|
87
|
+
* full rescan the write-path fast path exists to avoid. Only files newer
|
|
88
|
+
* than `builtAt` are hashed here, so this stays cheap — the common case is
|
|
89
|
+
* zero or a handful of such files.
|
|
90
|
+
*
|
|
80
91
|
* `getIndexableFiles` applies each asset type's own relevance filter, so
|
|
81
92
|
* non-indexed companion files (e.g. `package.json` next to a knowledge doc) are
|
|
82
93
|
* never considered and do not produce false "new file" positives.
|
|
83
94
|
*/
|
|
84
|
-
function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths) {
|
|
95
|
+
function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths, indexedHashes) {
|
|
85
96
|
const builtAtMs = builtAt ? new Date(builtAt).getTime() : Number.NaN;
|
|
86
97
|
const builtAtUsable = Number.isFinite(builtAtMs);
|
|
87
98
|
for (const spec of placementSpecList()) {
|
|
@@ -92,13 +103,31 @@ function hasNewerIndexableFiles(stashDir, builtAt, indexedPaths) {
|
|
|
92
103
|
return true;
|
|
93
104
|
if (!builtAtUsable)
|
|
94
105
|
return true;
|
|
106
|
+
let mtimeMs;
|
|
95
107
|
try {
|
|
96
|
-
|
|
97
|
-
|
|
108
|
+
mtimeMs = fs.statSync(file).mtimeMs;
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return true;
|
|
112
|
+
}
|
|
113
|
+
if (mtimeMs <= builtAtMs)
|
|
114
|
+
continue;
|
|
115
|
+
// Newer than the last full build — only stale if its current content
|
|
116
|
+
// actually differs from what is indexed. No stored hash means the row
|
|
117
|
+
// predates content-hash tracking (or a hash-less enrichment pass), so
|
|
118
|
+
// fall back to the conservative mtime-stale answer.
|
|
119
|
+
const indexedHash = indexedHashes.get(file);
|
|
120
|
+
if (indexedHash === undefined)
|
|
121
|
+
return true;
|
|
122
|
+
let currentHash;
|
|
123
|
+
try {
|
|
124
|
+
currentHash = hashContent(fs.readFileSync(file, "utf8"));
|
|
98
125
|
}
|
|
99
126
|
catch {
|
|
100
127
|
return true;
|
|
101
128
|
}
|
|
129
|
+
if (currentHash !== indexedHash)
|
|
130
|
+
return true;
|
|
102
131
|
}
|
|
103
132
|
}
|
|
104
133
|
return false;
|
|
@@ -125,7 +154,7 @@ export function isIndexStale(stashDir) {
|
|
|
125
154
|
if (entryCount === 0)
|
|
126
155
|
return true;
|
|
127
156
|
const builtAt = getMeta(db, "builtAt");
|
|
128
|
-
if (hasNewerIndexableFiles(stashDir, builtAt, getIndexedFilePaths(db)))
|
|
157
|
+
if (hasNewerIndexableFiles(stashDir, builtAt, getIndexedFilePaths(db), getIndexedFileHashes(db)))
|
|
129
158
|
return true;
|
|
130
159
|
const storedStashDir = getMeta(db, "stashDir");
|
|
131
160
|
if (storedStashDir !== stashDir) {
|
|
@@ -193,12 +222,24 @@ function indexCanServeStash(stashDir) {
|
|
|
193
222
|
}
|
|
194
223
|
async function runInlineReindex(stashDir, options = {}) {
|
|
195
224
|
const { akmIndex } = await import("./indexer.js");
|
|
196
|
-
|
|
225
|
+
const startedMs = Date.now();
|
|
226
|
+
const response = await akmIndex({
|
|
197
227
|
stashDir,
|
|
198
228
|
implicit: true,
|
|
199
229
|
...(options.signal ? { signal: options.signal } : {}),
|
|
200
230
|
...(options.hydrateSources === false ? { hydrateSources: false } : {}),
|
|
201
231
|
});
|
|
232
|
+
// R6: the implicit reindex's cost was previously discarded entirely
|
|
233
|
+
// (`await akmIndex(...)` and nothing else), making a 27-minute blocking
|
|
234
|
+
// rebuild invisible to both the operator and the improve result. Fall back
|
|
235
|
+
// to a wall-clock measurement when the response carries no `timing` block.
|
|
236
|
+
const durationMs = response.timing?.totalMs ?? Date.now() - startedMs;
|
|
237
|
+
const timing = response.timing;
|
|
238
|
+
warnVerbose(`[ensure-index] implicit reindex completed in ${durationMs}ms` +
|
|
239
|
+
(timing
|
|
240
|
+
? ` (walk=${timing.walkMs}ms llm=${timing.llmMs}ms embed=${timing.embedMs}ms finalize=${timing.finalizeMs}ms)`
|
|
241
|
+
: ""));
|
|
242
|
+
options.onReindexTiming?.({ durationMs, timing });
|
|
202
243
|
return true;
|
|
203
244
|
}
|
|
204
245
|
/**
|
|
@@ -226,7 +267,10 @@ export async function ensureIndex(stashDir, options = {}) {
|
|
|
226
267
|
// materialization point — hydrate cache-backed sources as usual.
|
|
227
268
|
if (!isIndexStale(stashDir))
|
|
228
269
|
return false;
|
|
229
|
-
return runInlineReindex(stashDir, {
|
|
270
|
+
return runInlineReindex(stashDir, {
|
|
271
|
+
...(options.signal ? { signal: options.signal } : {}),
|
|
272
|
+
...(options.onReindexTiming ? { onReindexTiming: options.onReindexTiming } : {}),
|
|
273
|
+
});
|
|
230
274
|
}
|
|
231
275
|
// Background = the READ path (`show` auto-index): query time must never clone/
|
|
232
276
|
// pull/fetch (spec §14.3 / D11). Build from already-materialized content only;
|
|
@@ -236,5 +280,6 @@ export async function ensureIndex(stashDir, options = {}) {
|
|
|
236
280
|
return runInlineReindex(stashDir, {
|
|
237
281
|
...(options.signal ? { signal: options.signal } : {}),
|
|
238
282
|
hydrateSources: false,
|
|
283
|
+
...(options.onReindexTiming ? { onReindexTiming: options.onReindexTiming } : {}),
|
|
239
284
|
});
|
|
240
285
|
}
|