@evo-dev/core 0.0.1-alpha → 0.0.1-alpha.10

package/src/evolution/processor/process.ts

@@ -0,0 +1,528 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, open, readFile, rm, stat } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { resolveEvoDevPaths } from "../../config/paths.ts";
+import { isFileExistsError, normalizeTimestamp } from "../../utils/index.ts";
+import { analyzeEvolutionRun } from "../evidence/analysis.ts";
+import { analyzeSessionEvidenceSegment } from "../evidence/session-memory/analysis.ts";
+import {
+  readSessionEvidenceSegment,
+  updateSessionEvidenceSegmentLifecycle,
+} from "../evidence/session-memory/storage.ts";
+import type {
+  EvolutionProcessInput,
+  EvolutionProcessProgress,
+  EvolutionProcessResult,
+  EvolutionTriggerStatus,
+} from "../schema.ts";
+import { sanitizeText } from "../shared.ts";
+import {
+  groupTriggersByRun,
+  listEvolutionTriggers,
+  listSegmentEvolutionTriggers,
+  updateSegmentTriggers,
+  updateTriggers,
+} from "../triggers/index.ts";
+import {
+  activateCuratedSessionKnowledgePlan,
+  activateEvolutionDistillationBatch,
+  createEvolutionDistillationBatch,
+} from "./distillation.ts";
+
+const PROCESS_LOCK_STALE_MS = 5 * 60 * 1000;
+const PROCESS_LOCK_HEARTBEAT_MS = 30 * 1000;
+const PROCESSING_TRIGGER_STALE_MS = PROCESS_LOCK_STALE_MS;
+const MAX_TRANSIENT_TRIGGER_ATTEMPTS = 3;
+
+interface EvolutionProcessLock {
+  path: string;
+  ownerId: string;
+  handle: Awaited<ReturnType<typeof open>>;
+  timer: ReturnType<typeof setInterval>;
+  heartbeat: Promise<void> | null;
+  stopped: boolean;
+  lost: boolean;
+}
+
+export async function processEvolutionTriggers(
+  input: EvolutionProcessInput,
+): Promise<EvolutionProcessResult> {
+  const now = normalizeTimestamp(input.now);
+  const limit = input.limit ?? 20;
+  const dryRun = input.dryRun === true;
+  const warnings: string[] = [];
+  const lock = dryRun ? null : await acquireEvolutionProcessLock(input.homeDir, now);
+  if (!dryRun && lock === null) {
+    warnings.push("Evolution trigger processor is already running; this pass was skipped.");
+    const result: EvolutionProcessResult = {
+      processed: 0,
+      consumed: 0,
+      skipped: 0,
+      failed: 0,
+      pending: 0,
+      triggerIds: [],
+      batchIds: [],
+      warnings,
+      dryRun,
+    };
+    await reportProgress(input, result, "completed");
+    return result;
+  }
+
+  try {
+    if (!dryRun) {
+      await reconcileRetryableEvolutionTriggers(input, now);
+    }
+    const pendingSegments =
+      dryRun || input.distillSessionKnowledge !== undefined
+        ? (
+            await listSegmentEvolutionTriggers({
+              homeDir: input.homeDir,
+              projectKey: input.projectKey,
+              runId: input.runId,
+              status: "pending",
+            })
+          ).slice(0, limit)
+        : [];
+    const remainingLimit = Math.max(0, limit - pendingSegments.length);
+    const pending =
+      remainingLimit === 0
+        ? []
+        : (
+            await listEvolutionTriggers({
+              homeDir: input.homeDir,
+              projectKey: input.projectKey,
+              runId: input.runId,
+              status: "pending",
+            })
+          ).slice(0, remainingLimit);
+    const grouped = groupTriggersByRun(pending);
+    const result: EvolutionProcessResult = {
+      processed: 0,
+      consumed: 0,
+      skipped: 0,
+      failed: 0,
+      pending: 0,
+      triggerIds: [
+        ...pendingSegments.map((trigger) => trigger.id),
+        ...pending.map((trigger) => trigger.id),
+      ],
+      batchIds: [],
+      warnings,
+      dryRun,
+    };
+    await reportProgress(input, result, "started");
+
+    for (const trigger of pendingSegments) {
+      result.processed += 1;
+      if (dryRun) {
+        result.pending += 1;
+        await reportProgress(input, result, "updated");
+        continue;
+      }
+      await updateSegmentTriggers(input.homeDir, [trigger], {
+        status: "processing",
+        updatedAt: now,
+        attempts: (current) => current.attempts + 1,
+      });
+      const attemptedTrigger = {
+        ...trigger,
+        status: "processing" as const,
+        updatedAt: now,
+        attempts: trigger.attempts + 1,
+      };
+      try {
+        const segmentIdentity = {
+          homeDir: input.homeDir,
+          projectKey: trigger.projectKey,
+          sessionKey: trigger.sessionKey,
+          segmentId: trigger.segmentId,
+        };
+        const segment = await readSessionEvidenceSegment(segmentIdentity);
+        const existingBatchId = segment.lifecycle.consumedByBatchIds.at(-1);
+        if (segment.lifecycle.status === "distilled" && existingBatchId !== undefined) {
+          result.consumed += 1;
+          result.batchIds.push(existingBatchId);
+          await updateSegmentTriggers(input.homeDir, [attemptedTrigger], {
+            status: "consumed",
+            updatedAt: now,
+            processedBatchId: existingBatchId,
+            lastError: null,
+          });
+          continue;
+        }
+
+        const analysis = analyzeSessionEvidenceSegment({
+          homeDir: input.homeDir,
+          segment,
+          now,
+        });
+        warnings.push(...analysis.warnings);
+        if (analysis.evidenceWindow === null) {
+          const skipReason = analysis.skipReason ?? "no-distillation-signal";
+          if (skipReason === "sensitive-segment" || skipReason === "no-distillation-signal") {
+            await updateSessionEvidenceSegmentLifecycle({
+              ...segmentIdentity,
+              status: "ignored",
+            });
+          }
+          result.skipped += 1;
+          await updateSegmentTriggers(input.homeDir, [attemptedTrigger], {
+            status: "skipped",
+            updatedAt: now,
+            lastError: skipReason,
+          });
+          continue;
+        }
+        if (!analysis.evidenceWindow.triggerPolicy.distillRecommended) {
+          await updateSessionEvidenceSegmentLifecycle({
+            ...segmentIdentity,
+            status: "ignored",
+          });
+          result.skipped += 1;
+          await updateSegmentTriggers(input.homeDir, [attemptedTrigger], {
+            status: "skipped",
+            updatedAt: now,
+            lastError: "no-distillation-signal",
+          });
+          continue;
+        }
+
+        const distiller = input.distillSessionKnowledge;
+        if (distiller === undefined) {
+          throw new Error("Semantic session knowledge distiller is unavailable.");
+        }
+        const plan = await distiller({
+          homeDir: input.homeDir,
+          segment,
+          evidenceWindow: analysis.evidenceWindow,
+          now,
+        });
+        if (lock !== null) await assertEvolutionProcessLockOwned(lock);
+        const curated = await activateCuratedSessionKnowledgePlan({
+          homeDir: input.homeDir,
+          evidenceWindow: analysis.evidenceWindow,
+          plan,
+          warnings: analysis.warnings,
+          now,
+          overwrite: true,
+        });
+        const batch = curated.batch;
+        await updateSessionEvidenceSegmentLifecycle({
+          ...segmentIdentity,
+          status: "distilled",
+          consumedByBatchId: batch.id,
+        });
+        result.consumed += 1;
+        result.batchIds.push(batch.id);
+        await updateSegmentTriggers(input.homeDir, [attemptedTrigger], {
+          status: "consumed",
+          updatedAt: now,
+          processedBatchId: batch.id,
+          lastError: null,
+        });
+      } catch (error) {
+        const message = sanitizeText(error instanceof Error ? error.message : String(error));
+        warnings.push(message);
+        if (lock?.lost) {
+          result.pending += 1;
+          await reportProgress(input, result, "completed");
+          return result;
+        }
+        const retry =
+          isTransientEvolutionError(error) &&
+          attemptedTrigger.attempts < MAX_TRANSIENT_TRIGGER_ATTEMPTS;
+        if (retry) result.pending += 1;
+        else result.failed += 1;
+        await updateSegmentTriggers(input.homeDir, [attemptedTrigger], {
+          status: retry ? "pending" : "failed",
+          updatedAt: now,
+          lastError: message,
+        });
+      } finally {
+        await reportProgress(input, result, "updated");
+      }
+    }
+
+    for (const triggers of grouped) {
+      result.processed += triggers.length;
+      if (dryRun) {
+        result.pending += triggers.length;
+        await reportProgress(input, result, "updated");
+        continue;
+      }
+      await updateTriggers(input.homeDir, triggers, {
+        status: "processing",
+        updatedAt: now,
+        attempts: (trigger) => trigger.attempts + 1,
+      });
+      const attemptedTriggers = triggers.map((trigger) => ({
+        ...trigger,
+        status: "processing" as const,
+        updatedAt: now,
+        attempts: trigger.attempts + 1,
+      }));
+
+      try {
+        const first = attemptedTriggers[0];
+        if (first === undefined) continue;
+        const analysis = await analyzeEvolutionRun({
+          homeDir: input.homeDir,
+          projectKey: first.projectKey,
+          runId: first.runId,
+          now,
+        });
+        warnings.push(...analysis.warnings);
+        const missingTrace = attemptedTriggers.some(
+          (trigger) =>
+            trigger.eventId !== null &&
+            !analysis.evidenceWindow.events.some((event) => event.hookEventId === trigger.eventId),
+        );
+        if (missingTrace) {
+          result.pending += triggers.length;
+          await updateTriggers(input.homeDir, attemptedTriggers, {
+            status: "pending",
+            updatedAt: now,
+            lastError: "waiting-for-trace",
+          });
+          continue;
+        }
+        if (!analysis.evidenceWindow.triggerPolicy.distillRecommended) {
+          result.skipped += triggers.length;
+          await updateTriggers(input.homeDir, attemptedTriggers, {
+            status: "skipped",
+            updatedAt: now,
+            lastError: "no-distillation-signal",
+          });
+          continue;
+        }
+
+        const batch = createEvolutionDistillationBatch({
+          evidenceWindow: analysis.evidenceWindow,
+          warnings: analysis.warnings,
+          now,
+        });
+        if (lock !== null) await assertEvolutionProcessLockOwned(lock);
+        await activateEvolutionDistillationBatch({
+          homeDir: input.homeDir,
+          batch,
+          overwrite: true,
+        });
+        result.consumed += triggers.length;
+        result.batchIds.push(batch.id);
+        await updateTriggers(input.homeDir, attemptedTriggers, {
+          status: "consumed",
+          updatedAt: now,
+          processedBatchId: batch.id,
+          lastError: null,
+        });
+      } catch (error) {
+        const message = sanitizeText(error instanceof Error ? error.message : String(error));
+        warnings.push(message);
+        if (lock?.lost) {
+          result.pending += triggers.length;
+          await reportProgress(input, result, "completed");
+          return result;
+        }
+        const retry =
+          isTransientEvolutionError(error) &&
+          attemptedTriggers.every((trigger) => trigger.attempts < MAX_TRANSIENT_TRIGGER_ATTEMPTS);
+        if (retry) result.pending += triggers.length;
+        else result.failed += triggers.length;
+        await updateTriggers(input.homeDir, attemptedTriggers, {
+          status: retry ? "pending" : "failed",
+          updatedAt: now,
+          lastError: message,
+        });
+      } finally {
+        await reportProgress(input, result, "updated");
+      }
+    }
+
+    await reportProgress(input, result, "completed");
+    return result;
+  } finally {
+    if (lock !== null) await releaseEvolutionProcessLock(lock);
+  }
+}
+
+async function reportProgress(
+  input: EvolutionProcessInput,
+  result: EvolutionProcessResult,
+  phase: EvolutionProcessProgress["phase"],
+): Promise<void> {
+  if (input.onProgress === undefined) return;
+  try {
+    await input.onProgress({
+      phase,
+      total: result.triggerIds.length,
+      processed: result.processed,
+      consumed: result.consumed,
+      skipped: result.skipped,
+      failed: result.failed,
+      pending: result.pending,
+    });
+  } catch {
+    // Progress persistence/observers must not change evolution processing outcomes.
+  }
+}
+
+async function acquireEvolutionProcessLock(
+  homeDir: string,
+  now: string,
+): Promise<EvolutionProcessLock | null> {
+  const paths = resolveEvoDevPaths(homeDir);
+  const lockPath = join(paths.stateDir, "evolution", ".process.lock");
+  const ownerId = randomUUID();
+  await mkdir(dirname(lockPath), { recursive: true });
+  try {
+    const handle = await open(lockPath, "wx");
+    try {
+      await handle.writeFile(
+        `${JSON.stringify(
+          {
+            schemaVersion: 1,
+            kind: "evolution-process-lock",
+            ownerId,
+            createdAt: now,
+            heartbeatAt: now,
+            pid: process.pid,
+          },
+          null,
+          2,
+        )}\n`,
+        "utf8",
+      );
+    } catch (error) {
+      await handle.close().catch(() => undefined);
+      await rm(lockPath, { force: true }).catch(() => undefined);
+      throw error;
+    }
+    const lock: EvolutionProcessLock = {
+      path: lockPath,
+      ownerId,
+      handle,
+      timer: undefined as unknown as ReturnType<typeof setInterval>,
+      heartbeat: null,
+      stopped: false,
+      lost: false,
+    };
+    lock.timer = setInterval(() => {
+      if (lock.stopped || lock.heartbeat !== null) return;
+      lock.heartbeat = heartbeatEvolutionProcessLock(lock).finally(() => {
+        lock.heartbeat = null;
+      });
+    }, PROCESS_LOCK_HEARTBEAT_MS);
+    lock.timer.unref?.();
+    return lock;
+  } catch (error) {
+    if (!isFileExistsError(error)) throw error;
+    const current = await stat(lockPath).catch(() => null);
+    if (current !== null && Date.now() - current.mtimeMs > PROCESS_LOCK_STALE_MS) {
+      await rm(lockPath, { force: true });
+      return acquireEvolutionProcessLock(homeDir, now);
+    }
+    return null;
+  }
+}
+
+async function heartbeatEvolutionProcessLock(lock: EvolutionProcessLock): Promise<void> {
+  if (!(await evolutionProcessLockIsOwned(lock))) {
+    lock.lost = true;
+    return;
+  }
+  try {
+    const now = new Date();
+    await lock.handle.utimes(now, now);
+  } catch {
+    lock.lost = true;
+  }
+}
+
+async function assertEvolutionProcessLockOwned(lock: EvolutionProcessLock): Promise<void> {
+  if (lock.lost || !(await evolutionProcessLockIsOwned(lock))) {
+    lock.lost = true;
+    throw new Error("Evolution process lock ownership was lost during processing.");
+  }
+}
+
+async function evolutionProcessLockIsOwned(lock: EvolutionProcessLock): Promise<boolean> {
+  try {
+    const value = JSON.parse(await readFile(lock.path, "utf8")) as { ownerId?: unknown };
+    return value.ownerId === lock.ownerId;
+  } catch {
+    return false;
+  }
+}
+
+async function releaseEvolutionProcessLock(lock: EvolutionProcessLock): Promise<void> {
+  lock.stopped = true;
+  clearInterval(lock.timer);
+  await lock.heartbeat?.catch(() => undefined);
+  await lock.handle.close().catch(() => undefined);
+  if (await evolutionProcessLockIsOwned(lock)) {
+    await rm(lock.path, { force: true });
+  }
+}
+
+async function reconcileRetryableEvolutionTriggers(
+  input: EvolutionProcessInput,
+  now: string,
+): Promise<void> {
+  const [segmentTriggers, legacyTriggers] = await Promise.all([
+    listSegmentEvolutionTriggers({
+      homeDir: input.homeDir,
+      projectKey: input.projectKey,
+      runId: input.runId,
+    }),
+    listEvolutionTriggers({
+      homeDir: input.homeDir,
+      projectKey: input.projectKey,
+      runId: input.runId,
+    }),
+  ]);
+  const nowMs = Date.parse(now);
+  const retryableSegments = segmentTriggers.filter((trigger) =>
+    shouldRequeueTrigger(trigger, nowMs),
+  );
+  const retryableLegacy = legacyTriggers.filter((trigger) => shouldRequeueTrigger(trigger, nowMs));
+  await Promise.all([
+    updateSegmentTriggers(input.homeDir, retryableSegments, {
+      status: "pending",
+      updatedAt: now,
+      lastError: "Retrying a transient failure or stale processing lease.",
+    }),
+    updateTriggers(input.homeDir, retryableLegacy, {
+      status: "pending",
+      updatedAt: now,
+      lastError: "Retrying a transient failure or stale processing lease.",
+    }),
+  ]);
+}
+
+function shouldRequeueTrigger(
+  trigger: {
+    status: EvolutionTriggerStatus;
+    attempts: number;
+    updatedAt: string;
+    lastError: string | null;
+  },
+  nowMs: number,
+): boolean {
+  if (trigger.attempts >= MAX_TRANSIENT_TRIGGER_ATTEMPTS) return false;
+  if (trigger.status === "failed") return isTransientEvolutionError(trigger.lastError ?? "");
+  if (trigger.status !== "processing") return false;
+  const updatedAt = Date.parse(trigger.updatedAt);
+  return Number.isFinite(nowMs) && Number.isFinite(updatedAt)
+    ? nowMs - updatedAt >= PROCESSING_TRIGGER_STALE_MS
+    : false;
+}
+
+function isTransientEvolutionError(error: unknown): boolean {
+  const message = (error instanceof Error ? `${error.name}: ${error.message}` : String(error))
+    .toLowerCase()
+    .trim();
+  return /(?:timed?\s*out|timeout|rate[\s-]*limit|too many requests|\b429\b|network|econn(?:reset|refused|aborted)|eai_again|etimedout|socket|temporar|service unavailable|\b50[0234]\b|fetch failed)/u.test(
+    message,
+  );
+}

package/src/{learning → evolution/review}/index.ts

@@ -1,5 +1,9 @@
 import { mkdir, readFile, stat, writeFile } from "node:fs/promises";
 import { dirname, join } from "node:path";
+import {
+  detectSessionMemorySensitivity,
+  redactSessionMemoryCredentialText,
+} from "../evidence/session-memory/sensitivity.ts";
 
 export type LearningCandidateStatus = "candidate" | "rejected" | "deferred";
 export type LearningCandidateKind =
@@ -31,7 +35,6 @@ export interface LearningCandidate {
   };
   provenance: {
     taskId: string | null;
-    taskContractRef: string | null;
     workflowRunId: string | null;
     evidenceRefs: string[];
     sourceType:
@@ -167,8 +170,6 @@ const FORBIDDEN_RAW_KEYS = new Set([
   "transcriptbody",
   "transcripttext",
 ]);
-const SENSITIVE_TEXT_PATTERN =
-  /https?:\/\/\S+|\b(secret|token|password|passwd|private|internal|api[_-]?key|apikey|credential|credentials|raw log|raw logs|raw output|raw source|raw prompt|shell history|command history)\b/i;
 const PROTECTED_PATH_PATTERN =
   /(^|[~/\\])(?:USER|KNOWLEDGE|LEARNING|OBSERVABILITY|PACKS|RELEASES|logs?|memory|\.env[^/\\]*)(?:$|[/\\])|PROJECTS[/\\][^/\\]+[/\\]LEARNING(?:$|[/\\])|\.evodev[/\\](?:USER|KNOWLEDGE|LEARNING|OBSERVABILITY|PACKS|RELEASES)(?:$|[/\\])/i;
 
@@ -198,7 +199,6 @@ export function createLearningCandidate(input: {
     },
     provenance: {
       taskId: sanitizeNullableId(input.provenance.taskId),
-      taskContractRef: sanitizeNullableText(input.provenance.taskContractRef),
       workflowRunId: sanitizeNullableId(input.provenance.workflowRunId),
       evidenceRefs: input.provenance.evidenceRefs.map(sanitizeText),
       sourceType: input.provenance.sourceType,
@@ -626,13 +626,9 @@ async function parseJsonOrJsonlFile<T>(
 }
 
 function candidatePathFields(candidate: LearningCandidate): Array<[string, string]> {
-  return [
-    ["provenance.taskContractRef", candidate.provenance.taskContractRef],
-    ...candidate.provenance.evidenceRefs.map((ref, index): [string, string] => [
-      `provenance.evidenceRefs[${index}]`,
-      ref,
-    ]),
-  ].filter((entry): entry is [string, string] => typeof entry[1] === "string");
+  return candidate.provenance.evidenceRefs
+    .map((ref, index): [string, string] => [`provenance.evidenceRefs[${index}]`, ref])
+    .filter((entry): entry is [string, string] => typeof entry[1] === "string");
 }
 
 function isCandidateStale(candidate: LearningCandidate, now: string | undefined): boolean {
@@ -663,8 +659,8 @@ function assertStringArrayField(field: string, value: unknown): void {
 function assertNoForbiddenContent(value: unknown): void {
   if (typeof value === "string") {
     if (value === "local-private") return;
-    if (SENSITIVE_TEXT_PATTERN.test(value)) {
-      throw new Error("Learning candidate contains sensitive content.");
+    if (detectSessionMemorySensitivity(value).classification === "credential") {
+      throw new Error("Learning candidate contains a credential.");
     }
     return;
   }
@@ -684,7 +680,7 @@ function assertNoForbiddenContent(value: unknown): void {
 }
 
 function sanitizeText(value: string): string {
-  return value.replace(SENSITIVE_TEXT_PATTERN, "[redacted]").slice(0, 500);
+  return redactSessionMemoryCredentialText(value).value.slice(0, 500);
 }
 
 function sanitizeNullableText(value: string | null): string | null {