akm-cli 0.9.0-beta.9 → 0.9.0-rc.0

package/dist/commands/improve/extract.js

@@ -23,23 +23,35 @@
  *     descriptionQualityValidator passes — same pattern as the
  *     consolidate-writer fix.
  */
+import fs from "node:fs";
+import path from "node:path";
 import { assembleAsset } from "../../core/asset/asset-serialize.js";
 import { resolveStashDir, timestampForFilename } from "../../core/common.js";
-import { getDefaultLlmConfig, loadConfig } from "../../core/config/config.js";
+import { getDefaultLlmConfig, getImproveProcessConfig, loadConfig } from "../../core/config/config.js";
 import { ConfigError, UsageError } from "../../core/errors.js";
 import { appendEvent } from "../../core/events.js";
-import { getExtractedSessionsMap, openStateDatabase, shouldSkipAlreadyExtractedSession, upsertExtractedSession, } from "../../core/state-db.js";
+import { probeLock, releaseLock, tryAcquireLockSync } from "../../core/file-lock.js";
+import { resolveStashStandards } from "../../core/standards/resolve-stash-standards.js";
+import { getStateDbPath, openStateDatabase, withStateDb } from "../../core/state-db.js";
 import { repairTruncatedDescription } from "../../core/text-truncation.js";
 import { warn } from "../../core/warn.js";
+import { indexWrittenAssets } from "../../indexer/index-written-assets.js";
 import { resolveImproveProcessRunnerFromProfile, runnerIsLlm } from "../../integrations/agent/runner.js";
 import { normalizeHarnessId } from "../../integrations/harnesses/index.js";
 import { getAvailableHarnesses } from "../../integrations/session-logs/index.js";
 import { preFilterSession } from "../../integrations/session-logs/pre-filter.js";
 import { chatCompletion } from "../../llm/client.js";
-import { isLlmFeatureEnabled, tryLlmFeature } from "../../llm/feature-gate.js";
-import { createProposal, isProposalSkipped } from "../proposal/validators/proposals.js";
+import { embed } from "../../llm/embedder.js";
+import { tryLlmFeature } from "../../llm/feature-gate.js";
+import { sha256Hex } from "../../runtime.js";
+import { getExtractedSessionsMap, getLastExtractRunAt, shouldSkipAlreadyExtractedSession, upsertExtractedSession, } from "../../storage/repositories/extract-sessions-repository.js";
+import { createProposal, isProposalSkipped } from "../proposal/repository.js";
 import { buildExtractPrompt, EXTRACT_JSON_SCHEMA, parseExtractPayload } from "./extract-prompt.js";
+import { buildHotProbationFrontmatter } from "./hot-probation.js";
+import { resolveProcessEnabled } from "./improve-profiles.js";
+import { applySchemaSimilarityPenalty, loadDerivedLayerEmbeddings, } from "./schema-similarity-gate.js";
 import { buildSessionSummaryPrompt, parseSessionSummary, SESSION_SUMMARY_JSON_SCHEMA, sessionMeetsDurationGate, writeSessionAsset, } from "./session-asset.js";
+import { resolveTriageConfig, scoreSessionTriage } from "./triage.js";
 /** Default minimum session duration (minutes) for session indexing (#561). */
 const DEFAULT_MIN_SESSION_DURATION_MINUTES = 5;
 /**
@@ -50,6 +62,82 @@ const DEFAULT_MIN_SESSION_DURATION_MINUTES = 5;
  * (0 chars, journal files) are safe to skip.
  */
 const DEFAULT_MIN_CONTENT_CHARS = 10;
+/**
+ * Default cap on NEW sessions the extract pass will LLM-process in a single run
+ * (`processes.extract.maxSessionsPerRun` overrides; `0` disables). Bounds per-run
+ * wall time + token spend so a backlog of accumulated sessions can't run a single
+ * pass past its scheduled-task timeout. Overflow sessions stay unseen and are
+ * processed by subsequent runs, so coverage is preserved — just spread out.
+ */
+const DEFAULT_MAX_SESSIONS_PER_RUN = 25;
+/**
+ * Floor for the default discovery window (48h). When no explicit `--since` /
+ * `defaultSince` is configured, discovery looks back to the LAST recorded
+ * extract run for the harness (so an intermittently-online host that was off for
+ * days still rediscovers sessions that ended during the gap), but never LESS
+ * than this — looking back less than the prior window could drop a session that
+ * a previous run deferred via `maxSessionsPerRun`. Widening is free of redundant
+ * LLM cost: the content-hash ledger skips unchanged sessions with zero LLM calls.
+ */
+const DEFAULT_SINCE_FLOOR_MS = 48 * 60 * 60 * 1000;
+/**
+ * Staleness window for the per-session extract lock. A single session's
+ * processing is bounded by the per-session LLM timeout (default 60s) plus the
+ * session-summary call, so a lock older than this must belong to a crashed
+ * holder and is safe to reclaim.
+ */
+const EXTRACT_SESSION_LOCK_STALE_MS = 5 * 60 * 1000;
+/**
+ * Resolve the discovery `sinceMs` cutoff when no explicit `since`/`defaultSince`
+ * is set: the later of (last recorded extract run for this harness) and
+ * (now − 48h). See {@link DEFAULT_SINCE_FLOOR_MS}. Best-effort — any state.db
+ * error falls back to the 48h floor.
+ */
+function resolveDefaultSinceMs(harnessName, now, opts) {
+    const floor = now - DEFAULT_SINCE_FLOOR_MS;
+    if (opts.skipTracking)
+        return floor;
+    try {
+        return withStateDb((db) => {
+            const lastRun = getLastExtractRunAt(db, harnessName);
+            return lastRun != null ? Math.min(lastRun, floor) : floor;
+        }, { path: opts.stateDbPath, borrowed: opts.stateDb });
+    }
+    catch {
+        return floor;
+    }
+}
+/** Filesystem-safe per-session lock path, co-located with the state.db. */
+function getExtractSessionLockPath(harness, sessionId, stateDbPath) {
+    const safe = `${harness}-${sessionId}`.replace(/[^A-Za-z0-9._-]/g, "_");
+    return path.join(path.dirname(stateDbPath), "extract-locks", `extract-${safe}.lock`);
+}
+/**
+ * Try to claim the per-session extract lock so a concurrent extract (e.g. a
+ * session-end hook firing `--session-id` while the hourly improve pass runs
+ * discovery) cannot double-process the SAME session — duplicate LLM spend and
+ * near-duplicate proposals. Reclaims a stale lock (dead holder PID or age past
+ * {@link EXTRACT_SESSION_LOCK_STALE_MS}). Returns false when another LIVE run
+ * holds it — the caller then skips the session without any LLM call. Best-effort:
+ * any filesystem error resolves to `true` (proceed) so locking never blocks
+ * extraction outright.
+ */
+function acquireExtractSessionLock(lockPath) {
+    try {
+        fs.mkdirSync(path.dirname(lockPath), { recursive: true });
+        if (tryAcquireLockSync(lockPath, String(process.pid)))
+            return true;
+        const probe = probeLock(lockPath, { staleAfterMs: EXTRACT_SESSION_LOCK_STALE_MS });
+        if (probe.state === "held")
+            return false;
+        // absent (released between attempt + probe) or stale → reclaim and retry once.
+        releaseLock(lockPath);
+        return tryAcquireLockSync(lockPath, String(process.pid));
+    }
+    catch {
+        return true;
+    }
+}
 // ── Helpers ──────────────────────────────────────────────────────────────────
 /**
  * Parse a since-string into an absolute ms-epoch cutoff. Accepts:
@@ -112,9 +200,43 @@ function buildCandidateProposal(candidate, sourceRef) {
     if (candidate.type === "lesson" && candidate.when_to_use) {
         fm.when_to_use = candidate.when_to_use;
     }
+    // #615 WS-0: preserve ordered-action + outcome data in frontmatter so the data
+    // survives even if source transcripts are not re-extractable later. The
+    // procedural-compilation feature (detection/compilation) is deferred to 0.10+.
+    if (candidate.orderedActions && candidate.orderedActions.length > 0) {
+        fm.orderedActions = candidate.orderedActions;
+        if (candidate.outcomeData) {
+            fm.outcomeData = candidate.outcomeData;
+        }
+    }
     const content = assembleAsset(fm, candidate.body);
     return { ref, content, description };
 }
+/**
+ * Canonicalize a session's content into a single deterministic string for
+ * hashing (#602). Each event is rendered `<role>\n<text>` and events are joined
+ * with a NUL-delimited separator (`\n\0\n`) so event boundaries cannot be forged
+ * by text that itself contains newlines.
+ *
+ * The input is the RAW `data.events` stream — NOT the pre-filtered / truncated
+ * set — so the hash is stable across `maxTotalChars` (and any other pre-filter)
+ * config changes: changing config must NEVER change the hash (idempotency AC).
+ * `inlineRefs` and ref metadata (title, startedAt/endedAt timestamps) are
+ * deliberately EXCLUDED so clock/title churn (and an agent adding an inline
+ * `akm remember` mid-session) does not change the hash.
+ */
+function canonicalizeSessionContent(data) {
+    return data.events.map((e) => `${e.role ?? "unknown"}\n${e.text}`).join("\n\0\n");
+}
+/**
+ * sha256 (hex) of the normalized session content (#602). This is the byte-exact,
+ * clock-independent skip authority that replaced the old `session_ended_at`
+ * timestamp comparison. See {@link canonicalizeSessionContent} for exactly what
+ * is (and is not) hashed.
+ */
+export function hashSessionContent(data) {
+    return sha256Hex(canonicalizeSessionContent(data));
+}
 /**
  * Process one session through the full pipeline: read → pre-filter → LLM →
  * parse → createProposal-per-candidate. Returns the per-session result.
@@ -123,7 +245,21 @@ function buildCandidateProposal(candidate, sourceRef) {
  * proposal validation failure) the session result records a warning and
  * keeps going — one session's bad luck never aborts a multi-session run.
  */
-async function processSession(harness, sessionRef, stashDir, config, llmConfig, chat, ctx, sourceRun, dryRun, timeoutMs, maxTotalChars, minContentChars, sessionIndexing) {
+async function processSession(harness, sessionRef, stashDir, config, llmConfig, chat, ctx, sourceRun, dryRun, timeoutMs, maxTotalChars, minContentChars, 
+// #626 — pre-LLM heuristic triage gate. Default-off (enabled:false) takes the
+// exact pre-change path (no scorer call, no new skipReason).
+// #641 — proceduralAwareFloor is opt-in, DEFAULT OFF.
+triage, sessionIndexing, schemaSimilarityCtx, 
+// #602 — already-extracted skip moved INSIDE processSession: the content hash
+// can only be computed after readSession, so the skip decision lives here. The
+// prior row + bypass flags are threaded in from the caller. Skipping here still
+// costs ZERO LLM calls (the expensive resource #602 protects); only the cheap
+// file read is incurred.
+prior, force, 
+// Stash authoring standards (convention/meta fact bodies) for non-wiki
+// output. Resolved ONCE per run by the caller and threaded in so facts are
+// not re-read per session. Empty string when none exist.
+standardsContext) {
     const warnings = [];
     let data;
     try {
@@ -141,6 +277,27 @@ async function processSession(harness, sessionRef, stashDir, config, llmConfig,
             skipReason: "read_failed",
         };
     }
+    // #602 — content-hash skip. Computed on the RAW event stream immediately after
+    // a successful read, BEFORE the pre-filter / minContentChars / triage gates, so
+    // an unchanged session never reaches the LLM. Hash-based ⇒ clock-independent
+    // (immune to the Jun 11-12 timestamp double-extract/over-throttle bug). The skip
+    // applies UNIFORMLY — including explicit `--session-id` targeting (so a
+    // session-end hook firing `extract --session-id <id>` is idempotent). ONLY
+    // `--force` overrides it to re-extract a previously-extracted session.
+    const contentHash = hashSessionContent(data);
+    if (!force && shouldSkipAlreadyExtractedSession(prior, contentHash)) {
+        return {
+            sessionId: sessionRef.sessionId,
+            harness: harness.name,
+            candidateCount: 0,
+            proposalIds: [],
+            preFilter: { inputCount: 0, outputCount: 0, truncatedCount: 0 },
+            warnings: [`already extracted (content unchanged) at ${prior?.processed_at}; pass --force to re-process`],
+            skipped: true,
+            skipReason: "already_extracted",
+            contentHash,
+        };
+    }
     const filtered = preFilterSession(data, {
         ...(typeof maxTotalChars === "number" ? { maxTotalChars } : {}),
     });
@@ -167,9 +324,42 @@ async function processSession(harness, sessionRef, stashDir, config, llmConfig,
             warnings: [],
             skipped: true,
             skipReason: "too_short",
+            contentHash,
         };
     }
-    const prompt = buildExtractPrompt({ data, events: filtered.events, inlineRefs: data.inlineRefs });
+    // #626 — pre-LLM heuristic triage gate. Runs AFTER minContentChars + the
+    // already-extracted skip check (both in the caller / above), BEFORE the
+    // extraction prompt and the session-asset write. When the session scores below
+    // the configured threshold we triage it out: no chat() call, no session asset,
+    // no proposals. Pure-heuristic — zero added LLM cost. Default-off → skipped.
+    if (triage.enabled) {
+        const t = scoreSessionTriage(data, triage.minScore, {
+            proceduralAwareFloor: triage.proceduralAwareFloor,
+        });
+        if (!t.pass) {
+            return {
+                sessionId: sessionRef.sessionId,
+                harness: harness.name,
+                candidateCount: 0,
+                proposalIds: [],
+                preFilter: {
+                    inputCount: filtered.stats.inputCount,
+                    outputCount: filtered.stats.outputCount,
+                    truncatedCount: filtered.stats.truncatedCount,
+                },
+                warnings: [],
+                skipped: true,
+                skipReason: "triaged_out",
+                contentHash,
+            };
+        }
+    }
+    const prompt = buildExtractPrompt({
+        data,
+        events: filtered.events,
+        inlineRefs: data.inlineRefs,
+        ...(standardsContext.trim() ? { standardsContext } : {}),
+    });
     // #561 — ADDITIVE session indexing. Generate + write the session asset
     // (`sessions/<harness>/<id>.md`). FAIL-OPEN: any failure only records a
     // warning; it NEVER changes the proposal/skip outcome of extract. Returns the
@@ -183,6 +373,10 @@ async function processSession(harness, sessionRef, stashDir, config, llmConfig,
         try {
             const result = await writeSessionAsset(data, stashDir, sessionIndexing.generate);
             if (result.written) {
+                // Write-path indexing (itself fail-open): standalone `akm extract`
+                // (session-end hook) has no post-loop reindex to pick this file up.
+                if (result.filePath)
+                    await indexWrittenAssets(stashDir, [result.filePath]);
                 return {
                     ...(result.ref ? { sessionAssetRef: result.ref } : {}),
                     ...(result.logPath ? { sessionLogPath: result.logPath } : {}),
@@ -217,6 +411,7 @@ async function processSession(harness, sessionRef, stashDir, config, llmConfig,
             warnings: ["session_extraction feature returned empty (disabled / timeout / error)"],
             skipped: true,
             skipReason: "llm_unavailable",
+            contentHash,
         };
     }
     const payload = parseExtractPayload(llmRaw);
@@ -248,15 +443,38 @@ async function processSession(harness, sessionRef, stashDir, config, llmConfig,
                 truncatedCount: filtered.stats.truncatedCount,
             },
             warnings,
+            contentHash,
             ...sessionAsset,
         };
     }
+    // WS-3b step 0c: hot-probation intake buffer (#604).
+    // When enabled, system-generated extractions enter captureMode: hot-probation
+    // so they spend ONE consolidation cycle in probation before the deterministic
+    // dedup+quality pass promotes them. Default OFF.
+    // Reads the `default` profile deliberately: this per-session hot-probation
+    // flag lives inside the 18-arg `processSession`, which has no handle to the
+    // active profile, and threading a 19th positional arg for a DEFAULT-OFF flag
+    // isn't worth the param bloat. The extract STAGE toggle (in `akmExtract`,
+    // below) already honors `--profile`.
+    const hotProbationEnabled = getImproveProcessConfig(config, "extract")?.hotProbation?.enabled === true;
     for (const candidate of payload.candidates) {
         if (dryRun) {
             proposalIds.push(`dry-run:${candidate.type}:${candidate.name}`);
             continue;
         }
         try {
+            // WS-3b Step-0b: schema-similarity intake gate. When enabled and the
+            // candidate is a lesson/knowledge whose body embedding is within ε of an
+            // existing derived-layer node, down-prioritize by multiplying confidence by
+            // the penalty. PARITY: schemaSimilarityCtx is null when the flag is off →
+            // applySchemaSimilarityPenalty returns the original confidence untouched and
+            // never embeds. (Logic lives in schema-similarity-gate.ts so it is unit-testable.)
+            const gateResult = await applySchemaSimilarityPenalty(candidate, schemaSimilarityCtx, (text) => schemaSimilarityCtx?.embedFn
+                ? schemaSimilarityCtx.embedFn(text)
+                : embed(text, schemaSimilarityCtx?.embeddingConfig));
+            const effectiveConfidence = gateResult.effectiveConfidence;
+            if (gateResult.warning)
+                warn(gateResult.warning);
             const { ref, content, description } = buildCandidateProposal(candidate, sessionRef);
             const result = createProposal(stashDir, {
                 ref,
@@ -267,9 +485,22 @@ async function processSession(harness, sessionRef, stashDir, config, llmConfig,
                     frontmatter: {
                         description,
                         ...(candidate.when_to_use ? { when_to_use: candidate.when_to_use } : {}),
-                        ...(typeof candidate.confidence === "number" ? { confidence: candidate.confidence } : {}),
+                        ...(effectiveConfidence !== undefined ? { confidence: effectiveConfidence } : {}),
                         sources: [`session:${sessionRef.harness}:${sessionRef.sessionId}`],
                         evidence: candidate.evidence,
+                        // #615 WS-0: mirror ordered-action + outcome data in the proposal
+                        // frontmatter record so downstream tooling can read it without
+                        // re-parsing the content body. Omitted when not present.
+                        ...(candidate.orderedActions && candidate.orderedActions.length > 0
+                            ? { orderedActions: candidate.orderedActions }
+                            : {}),
+                        ...(candidate.outcomeData ? { outcomeData: candidate.outcomeData } : {}),
+                        // WS-3b step 0c: tag system-generated extractions as hot-probation
+                        // when the feature is enabled. The consolidation pass will exclude
+                        // them from the LLM merge pool until the intake dedup+quality pass
+                        // runs against them. User-explicit `akm remember` (captureMode: hot)
+                        // is unaffected — this only applies to extract-generated proposals.
+                        ...(hotProbationEnabled ? buildHotProbationFrontmatter() : {}),
                     },
                 },
             }, ctx);
@@ -310,6 +541,7 @@ async function processSession(harness, sessionRef, stashDir, config, llmConfig,
             truncatedCount: filtered.stats.truncatedCount,
         },
         warnings,
+        contentHash,
         ...sessionAsset,
     };
 }
@@ -323,13 +555,22 @@ export async function akmExtract(options) {
     const stashDir = options.stashDir ?? resolveStashDir();
     const dryRun = options.dryRun ?? false;
     const sourceRun = options.sourceRun ?? `extract-${timestampForFilename()}`;
-    // Read the per-process extract config from the active improve profile. Matches
-    // the pattern reflect/distill/consolidate use: `profiles.improve.<active>.processes.extract`.
-    // Only the `default` improve profile is consulted here — extract isn't invoked
-    // with a profile flag yet (parity item for a future change).
-    const extractProcess = config.profiles?.improve?.default?.processes?.extract;
+    // Read the per-process extract config + enabled-state from the ACTIVE improve
+    // profile when `akmImprove` threaded it; otherwise fall back to the `default`
+    // profile (the explicit `akm extract` path, which has no active profile). This
+    // is what stops a non-default profile's `extract.enabled` from being silently
+    // overridden by the default profile and vice-versa.
+    const activeProfile = options.improveProfile;
+    const extractProcess = activeProfile ? activeProfile.processes?.extract : getImproveProcessConfig(config, "extract");
+    // The `extract.enabled` process toggle gates extract as a STAGE of `akm improve`
+    // (the activeProfile path) — consistent with #593/#594 where the active profile,
+    // not `default`, is the source of truth. An EXPLICIT `akm extract` invocation
+    // (no activeProfile) is a direct user/cron action and always runs; gating it on
+    // the default improve profile's stage toggle was a footgun — dropping extract
+    // from the daily improve profile would silently disable the standalone command.
+    const extractEnabled = activeProfile ? resolveProcessEnabled("extract", activeProfile) : true;
     // Feature-gate early so we get a clean "skipped because disabled" envelope.
-    if (!isLlmFeatureEnabled(config, "session_extraction")) {
+    if (!extractEnabled) {
         return {
             schemaVersion: 1,
             ok: true,
@@ -375,8 +616,17 @@ export async function akmExtract(options) {
     // #595/#596 — minimum raw session size; sessions below it skip the LLM call
     // entirely. Set `processes.extract.minContentChars: 0` to disable the gate.
     const minContentChars = typeof extractProcess?.minContentChars === "number" ? extractProcess.minContentChars : DEFAULT_MIN_CONTENT_CHARS;
+    // Cap on NEW sessions LLM-processed per run; 0 disables. Absent = default.
+    // Bounds per-run wall time / LLM cost so a backlog can't push a run past its
+    // task timeout — the overflow stays unseen and is picked up by later runs.
+    const maxSessionsPerRun = typeof extractProcess?.maxSessionsPerRun === "number"
+        ? extractProcess.maxSessionsPerRun
+        : DEFAULT_MAX_SESSIONS_PER_RUN;
     // Default discovery window — process config can override the built-in 24h.
     const effectiveSince = options.since ?? extractProcess?.defaultSince;
+    // #626 — resolve the triage gate config once per run. Default-off → the
+    // per-session path never calls the scorer and emits no telemetry.
+    const triage = resolveTriageConfig(extractProcess);
     // #561 — resolve session-indexing config. Default ON: we only reach this code
     // when `session_extraction` is enabled AND an LLM is configured (both checked
     // above), so defaulting on costs nothing offline (the summary call fails open)
@@ -465,7 +715,16 @@ export async function akmExtract(options) {
         candidates = [target];
     }
     else {
-        const sinceMs = parseSinceArg(effectiveSince);
+        // No explicit `--since`/`defaultSince` → default to "since the last run"
+        // (floored at 48h) so an intermittently-online host doesn't lose sessions
+        // that ended while it was off. See {@link resolveDefaultSinceMs}.
+        const sinceMs = effectiveSince
+            ? parseSinceArg(effectiveSince)
+            : resolveDefaultSinceMs(harness.name, startMs, {
+                ...(options.stateDb ? { stateDb: options.stateDb } : {}),
+                ...(options.stateDbPath ? { stateDbPath: options.stateDbPath } : {}),
+                ...(options.skipTracking ? { skipTracking: options.skipTracking } : {}),
+            });
         candidates = harness.listSessions({
             sinceMs,
             ...(options.location ? { location: options.location } : {}),
@@ -474,6 +733,10 @@ export async function akmExtract(options) {
     const sessions = [];
     let processedCount = 0;
     let skippedCount = 0;
+    // #626 — per-run triage aggregation counters (counts-only telemetry, AC4).
+    let triageEvaluated = 0;
+    let triagePassed = 0;
+    let triagedOut = 0;
     const allProposalIds = [];
     const topLevelWarnings = [];
     const chat = options.chat ?? chatCompletion;
@@ -497,39 +760,97 @@ export async function akmExtract(options) {
             stateDb = undefined;
         }
     }
+    // WS-3b Step-0b: schema-similarity intake gate.
+    // DEFAULT ON since R3 (docs/design/improve-self-learning-analysis.md G5):
+    // extract is the highest-volume acquisition path with no LLM judge, so the
+    // cheap embedding-dedup check (one embed per lesson/knowledge candidate,
+    // fail-open) is the intake quality gate. Opt out via
+    // processes.extract.schemaSimilarity.enabled: false. The gate is inert in
+    // practice when no derived-layer embeddings exist (empty ctx → no penalty).
+    const schemaSimilarityCfg = extractProcess?.schemaSimilarity;
+    let schemaSimilarityCtx = null;
+    if (schemaSimilarityCfg?.enabled !== false) {
+        const derivedEmbeddings = options.schemaSimilarityEmbeddings ?? loadDerivedLayerEmbeddings();
+        schemaSimilarityCtx = {
+            config: { ...schemaSimilarityCfg, enabled: true },
+            derivedEmbeddings,
+            embeddingConfig: config.embedding,
+            embedFn: options.schemaSimilarityEmbedFn,
+        };
+    }
+    // Stash authoring standards (convention/meta fact bodies) for non-wiki
+    // extract output. Resolved ONCE per run and threaded into each session's
+    // prompt so facts are not re-read per session.
+    const extractStandardsContext = resolveStashStandards(stashDir);
     for (const summary of candidates) {
-        // Skip-tracking: if this session was already processed AND no new events
-        // have arrived since (live endedAt <= recorded endedAt), don't burn an LLM
-        // call. --force or single-session mode (explicit sessionId) bypasses.
+        // #602 — the already-extracted skip moved INTO processSession (the content
+        // hash needs the session body, only available after readSession). The prior
+        // row + bypass flags are threaded through; an unchanged session returns
+        // skipReason 'already_extracted' WITHOUT any LLM call.
         const prior = seenMap.get(summary.sessionId);
-        if (!options.force && !options.sessionId && shouldSkipAlreadyExtractedSession(prior, summary.endedAt)) {
-            sessions.push({
-                sessionId: summary.sessionId,
-                harness: harness.name,
-                candidateCount: 0,
-                proposalIds: [],
-                preFilter: { inputCount: 0, outputCount: 0, truncatedCount: 0 },
-                warnings: [
-                    `already extracted at ${prior?.processed_at}; pass --force to re-process or wait until the session has new content`,
-                ],
-                skipped: true,
-                skipReason: "already_extracted",
-            });
-            skippedCount += 1;
-            continue;
+        // Per-run cap on LLM-processed sessions (skip-tracked seen sessions above
+        // don't count). Single-session / --force modes bypass the cap (explicit
+        // intent). Overflow sessions are left unseen for the next run.
+        if (!options.sessionId && !options.force && maxSessionsPerRun > 0 && processedCount >= maxSessionsPerRun) {
+            topLevelWarnings.push(`Reached maxSessionsPerRun=${maxSessionsPerRun}; ${candidates.length - processedCount - skippedCount} session(s) deferred to a later run.`);
+            break;
+        }
+        // Q5 — per-session lock so two concurrent extracts (e.g. a session-end hook
+        // firing `--session-id` while the hourly improve discovery pass runs) can't
+        // both LLM-process the SAME session. The holder records the outcome; a
+        // second run skips without any LLM call. Engaged only for real cross-process
+        // runs (those that open their own state.db): dry-run is read-only, an
+        // injected `stateDb` handle is an in-process/test scenario with no cross-
+        // process race, and skip-tracking-off opts out entirely.
+        let sessionLockPath;
+        if (trackingEnabled && !dryRun && !options.stateDb) {
+            sessionLockPath = getExtractSessionLockPath(harness.name, summary.sessionId, options.stateDbPath ?? getStateDbPath());
+            if (!acquireExtractSessionLock(sessionLockPath)) {
+                sessions.push({
+                    sessionId: summary.sessionId,
+                    harness: harness.name,
+                    candidateCount: 0,
+                    proposalIds: [],
+                    preFilter: { inputCount: 0, outputCount: 0, truncatedCount: 0 },
+                    warnings: ["concurrent extract holds this session's lock — skipped (handled by the other run)"],
+                    skipped: true,
+                    skipReason: "locked_concurrent",
+                });
+                skippedCount += 1;
+                continue;
+            }
         }
         try {
-            const result = await processSession(harness, summary, stashDir, config, llmConfig, chat, options.ctx, sourceRun, dryRun, timeoutMs, maxTotalChars, minContentChars, sessionIndexing);
+            const result = await processSession(harness, summary, stashDir, config, llmConfig, chat, options.ctx, sourceRun, dryRun, timeoutMs, maxTotalChars, minContentChars, triage, sessionIndexing, schemaSimilarityCtx, prior, options.force === true, extractStandardsContext);
             sessions.push(result);
+            // #626 — triage aggregation. A session reached the triage gate only when it
+            // was NOT already preempted by an earlier skip (read_failed / too_short /
+            // already_extracted handled above the processSession call). When triage is
+            // enabled, processSession either triages-out (skipReason 'triaged_out') or
+            // proceeds past the gate — both count as "evaluated".
+            if (triage.enabled) {
+                const preemptedBeforeTriage = result.skipReason === "read_failed" ||
+                    result.skipReason === "too_short" ||
+                    result.skipReason === "already_extracted";
+                if (!preemptedBeforeTriage) {
+                    triageEvaluated += 1;
+                    if (result.skipReason === "triaged_out")
+                        triagedOut += 1;
+                    else
+                        triagePassed += 1;
+                }
+            }
             if (result.skipped)
                 skippedCount += 1;
             else
                 processedCount += 1;
             allProposalIds.push(...result.proposalIds);
-            // Persist outcome so the next run skips this session unless new events
-            // arrive. We only track non-dry-run paths — dry-run is for inspection
-            // and should never poison the seen-table.
-            if (trackingEnabled && stateDb && !dryRun) {
+            // Persist outcome so the next run skips this session unless its content
+            // changes. We only track non-dry-run paths — dry-run is for inspection
+            // and should never poison the seen-table. #602: an `already_extracted`
+            // skip is a no-op (the row already carries the matching hash), so don't
+            // re-write it — that keeps `processed_at` stable across unchanged runs.
+            if (trackingEnabled && stateDb && !dryRun && result.skipReason !== "already_extracted") {
                 try {
                     const outcome = result.skipped
                         ? result.skipReason === "read_failed" || result.skipReason === "exception"
@@ -548,6 +869,16 @@ export async function akmExtract(options) {
                         proposalCount: result.proposalIds.length,
                         rationale: result.rationaleIfEmpty ?? null,
                         sourceRun,
+                        // #602 — persist the freshly computed content hash so the NEXT run
+                        // can compare byte-for-byte. read_failed (before hash) → null, which
+                        // keeps the row eligible for retry (matches failed-row semantics).
+                        // R4 — llm_unavailable (LLM was down) and triaged_out (deferred by the
+                        // triage gate) are transient outcomes: persist null so the null-hash
+                        // retry re-processes them on a later run instead of pinning them as
+                        // "seen" forever against the current byte content.
+                        contentHash: result.skipReason === "llm_unavailable" || result.skipReason === "triaged_out"
+                            ? null
+                            : (result.contentHash ?? null),
                         metadata: {
                             preFilterInputCount: result.preFilter.inputCount,
                             preFilterOutputCount: result.preFilter.outputCount,
@@ -584,6 +915,10 @@ export async function akmExtract(options) {
             });
             skippedCount += 1;
         }
+        finally {
+            if (sessionLockPath)
+                releaseLock(sessionLockPath);
+        }
     }
     // Close the state.db connection we opened. Callers that injected stateDb
     // via the test seam own its lifecycle.
@@ -595,6 +930,20 @@ export async function akmExtract(options) {
             // best-effort close
         }
     }
+    // #626 — counts-only triage telemetry (AC4). Exactly ONE aggregated event per
+    // run, emitted only when the gate was enabled and actually evaluated at least
+    // one session. No per-session events (avoids the log-spam the issue warns of).
+    if (triage.enabled && triageEvaluated > 0) {
+        appendEvent({
+            eventType: "extract_triaged",
+            metadata: {
+                evaluated: triageEvaluated,
+                passed: triagePassed,
+                triagedOut,
+                sourceRun,
+            },
+        }, options.ctx);
+    }
     return {
         schemaVersion: 1,
         ok: true,
@@ -616,20 +965,34 @@ export async function akmExtract(options) {
  * logic in {@link akmExtract} so the `#554 minNewSessions` gate in `improve`
  * can decide whether the extract pass is worth running before any work begins.
  *
- * A session is a "new candidate" when it is in the `since` window AND it would
- * not be skipped by {@link shouldSkipAlreadyExtractedSession} (i.e. it has never
- * been extracted, or new events have arrived since it was last extracted).
+ * #602 — this gate is intentionally CHEAP: it does NOT read session bodies, so
+ * it cannot compute the content hash that {@link shouldSkipAlreadyExtractedSession}
+ * now uses. It therefore uses a CONSERVATIVE row-presence approximation: a
+ * session counts as "new" when there is NO prior row OR the prior row's
+ * `content_hash` is null (never-seen or backfill-eligible). A prior row WITH a
+ * non-null content_hash counts as NOT new — it MIGHT have changed, but the
+ * precise per-session hash check happens downstream in processSession, so an
+ * over-/under-count here only affects whether the pass RUNS, never whether a
+ * changed session is actually re-processed.
  */
 export function countNewExtractCandidates(config, options = {}) {
-    const extractProcess = config.profiles?.improve?.default?.processes?.extract;
+    const extractProcess = getImproveProcessConfig(config, "extract", options.improveProfile);
     const effectiveSince = options.since ?? extractProcess?.defaultSince;
-    const sinceMs = parseSinceArg(effectiveSince);
+    // Mirror akmExtract: when no explicit window is set, default per-harness to
+    // "since the last run" (floored at 48h) instead of a fixed 24h. Keeps this
+    // gate's discovery window identical to what akmExtract will actually scan.
+    const explicitSinceMs = effectiveSince ? parseSinceArg(effectiveSince) : undefined;
     const harnesses = (options.harnesses ?? getAvailableHarnesses()).filter((h) => h.isAvailable());
     let stateDb = options.stateDb;
     let openedStateDb = false;
     let total = 0;
     try {
         for (const harness of harnesses) {
+            const sinceMs = explicitSinceMs ??
+                resolveDefaultSinceMs(harness.name, Date.now(), {
+                    ...(options.stateDb ? { stateDb: options.stateDb } : {}),
+                    ...(options.stateDbPath ? { stateDbPath: options.stateDbPath } : {}),
+                });
             const candidates = harness.listSessions({ sinceMs });
             if (candidates.length === 0)
                 continue;
@@ -652,7 +1015,10 @@ export function countNewExtractCandidates(config, options = {}) {
             }
             for (const summary of candidates) {
                 const prior = seenMap.get(summary.sessionId);
-                if (shouldSkipAlreadyExtractedSession(prior, summary.endedAt))
+                // #602 row-presence approximation (see fn doc): a prior row WITH a
+                // non-null content_hash is treated as not-new here; everything else
+                // (never-seen, or null-hash backfill-eligible) counts as new.
+                if (prior && prior.content_hash != null)
                     continue;
                 total += 1;
             }