@evo-dev/core 0.0.1-alpha.2 → 0.0.1-alpha.20

package/src/evolution/evidence/session-memory/types.ts

@@ -0,0 +1,296 @@
+export type SessionMemoryTarget = "claude" | "codex" | "unknown";
+export type SessionMemorySegmentReason =
+  | "session-memory-init"
+  | "session-memory-threshold"
+  | "intent-refinement"
+  | "user-interruption"
+  | "failure-signal"
+  | "permission-denied"
+  | "verification-after-fix"
+  | "explicit-memory-intent"
+  | "historical-import";
+export type SessionMemorySignalStrength = "normal" | "strong";
+export type SessionMemorySensitivity = "safe-metadata" | "private-content" | "credential";
+export type SessionMemorySensitivityReason =
+  | "credential-field"
+  | "credential-shape"
+  | "credential-url-parameter"
+  | "credential-url-userinfo"
+  | "legacy-sensitive-flag"
+  | "private-marker"
+  | "url";
+
+export interface SessionMemoryPolicySnapshot {
+  enabled: boolean;
+  storeRawSegments: boolean;
+  maxRawSegmentBytes: number;
+  retentionDays: number;
+  minimumMessageTokensToInit: number;
+  minimumTokensBetweenUpdate: number;
+  toolCallsBetweenUpdates: number;
+}
+
+export interface SessionMemorySignal {
+  eventId: string;
+  eventType: string;
+  reason: SessionMemorySegmentReason | "observed";
+  strength: SessionMemorySignalStrength;
+  occurredAt: string | null;
+  summary: string;
+  rawEventLine: number;
+}
+
+export interface SessionMemorySpan {
+  spanId: string;
+  status: "open" | "closed" | "interrupted";
+  openedAt: string;
+  closedAt: string | null;
+  openedByEventId: string;
+  lastEventId: string;
+  userPromptCount: number;
+  toolCallCount: number;
+  workStarted: boolean;
+  tokenEstimate: number;
+}
+
+export interface SessionMemoryStateV1 {
+  schemaVersion: 1;
+  kind: "session-memory-state";
+  projectKey: string;
+  sessionKey: string;
+  target: SessionMemoryTarget;
+  runId: string | null;
+  roleId: string | null;
+  initialized: boolean;
+  createdAt: string;
+  updatedAt: string;
+  counters: {
+    messageTokenEstimate: number;
+    tokensSinceLastUpdate: number;
+    toolCallsSinceLastUpdate: number;
+    userPromptCount: number;
+    assistantTurnCount: number;
+    toolCallCount: number;
+    failureSignalCount: number;
+    verificationSignalCount: number;
+  };
+  activeSpan: SessionMemorySpan | null;
+  pendingSignals: SessionMemorySignal[];
+  lastSegmentId: string | null;
+  lastMemoryUpdateAt: string | null;
+  policy: SessionMemoryPolicySnapshot;
+}
+
+export interface SessionMemoryCursorV1 {
+  schemaVersion: 1;
+  kind: "session-memory-cursor";
+  sessionKey: string;
+  traceRefId: string | null;
+  sourcePath: string | null;
+  lastCapturedOffset: number | null;
+  lastCapturedLine: number | null;
+  lastCapturedEventId: string | null;
+  updatedAt: string;
+}
+
+export interface HistoricalImportOriginV1 {
+  kind: "historical-import";
+  importId: string;
+  generationId: string;
+  snapshotId: string;
+  sourceKey: string;
+  firstRecordKeyHash: string;
+  lastRecordKeyHash: string;
+  recordCount: number;
+}
+
+export interface SessionEvidenceRetentionV1 {
+  policyDays: number;
+  expiresAt: string;
+  rawState: "available" | "purged";
+  rawPurgedAt: string | null;
+  originalRawSha256: string;
+}
+
+export type HistoricalImportRetentionV1 = SessionEvidenceRetentionV1;
+
+export interface HistoricalImportStoredRecordV1 {
+  schemaVersion: 1;
+  kind: "historical-import-record";
+  recordKeyHash: string;
+  recordType: "user" | "reasoning" | "assistant" | "assistant-tool-call" | "tool";
+  canonicalJson: string;
+  canonicalJsonTruncated: boolean;
+}
+
+export interface SemanticEvidencePacketV1 {
+  schemaVersion: 1;
+  kind: "semantic-evidence-packet";
+  projectKey: string;
+  segmentId: string;
+  conversation: Array<{
+    role: "user" | "assistant";
+    text: string;
+    recordKeyHash: string;
+  }>;
+  toolEvidence: Array<{
+    reason: "failure" | "verification" | "permission" | "rollback";
+    toolName: string;
+    safeInputExcerpt: string | null;
+    outputExcerpt: string | null;
+    status: string | null;
+    exitCode: number | null;
+    inputTruncated: boolean;
+    outputTruncated: boolean;
+  }>;
+  touchedPaths: string[];
+  selection: {
+    selectedRecords: number;
+    droppedRecords: number;
+    redactionCount: number;
+    truncated: boolean;
+  };
+  privacy: {
+    containsPrivateContent: true;
+    credentialRedacted: true;
+    rawSegmentIncluded: false;
+  };
+}
+
+export interface SessionEvidenceSegmentV1 {
+  schemaVersion: 1;
+  kind: "session-evidence-segment";
+  id: string;
+  projectKey: string;
+  runId: string | null;
+  roleId: string | null;
+  sessionKey: string;
+  target: SessionMemoryTarget;
+  createdAt: string;
+  reason: SessionMemorySegmentReason;
+  strength: SessionMemorySignalStrength;
+  origin?: HistoricalImportOriginV1;
+  retention?: SessionEvidenceRetentionV1;
+  source: {
+    traceRefId: string | null;
+    sourcePath: string | null;
+    fromOffset: number | null;
+    toOffset: number | null;
+    fromLine: number | null;
+    toLine: number | null;
+    fromEventId: string | null;
+    toEventId: string | null;
+  };
+  signals: SessionMemorySignal[];
+  rawExcerpt: {
+    stored: boolean;
+    encoding: "utf8";
+    content: string;
+    truncated: boolean;
+    byteLength: number;
+    sha256: string;
+  };
+  normalized: {
+    summary: string;
+    userIntent: string | null;
+    intentDelta: string | null;
+    decisions: string[];
+    failures: string[];
+    verifications: string[];
+    touchedTools: string[];
+  };
+  privacy: {
+    localOnly: true;
+    rawPromptStored: boolean;
+    rawOutputStored: boolean;
+    sourceContentStored: boolean;
+    secretsDetected: boolean;
+    sensitivity?: SessionMemorySensitivity;
+    sensitivityReasons?: SessionMemorySensitivityReason[];
+    redactionApplied: boolean;
+    externalUploadAllowed: false;
+  };
+  lifecycle: {
+    status:
+      | "captured"
+      | "pending-review"
+      | "reviewed"
+      | "distilled"
+      | "ignored"
+      | "deleted"
+      | "raw-expired";
+    reviewState: "not-required" | "unreviewed" | "accepted" | "rejected" | "deferred";
+    consumedByBatchIds: string[];
+  };
+}
+
+export interface SessionMemoryRawEventV1 {
+  schemaVersion: 1;
+  kind: "session-memory-raw-event";
+  eventId: string;
+  eventType: string;
+  occurredAt: string | null;
+  receivedAt: string;
+  summary: string;
+  rawPayloadJson: string;
+  rawPayloadByteLength: number;
+  rawPayloadTruncated: boolean;
+  rawPayloadExpired?: boolean;
+  secretsDetected: boolean;
+  sensitivity?: SessionMemorySensitivity;
+  sensitivityReasons?: SessionMemorySensitivityReason[];
+  credentialRedactionApplied?: boolean;
+}
+
+export interface SessionMemoryPaths {
+  rootDir: string;
+  projectDir: string;
+  sessionDir: string;
+  statePath: string;
+  cursorPath: string;
+  eventsPath: string;
+  segmentsDir: string;
+  segmentPath: (segmentId: string) => string;
+  indexPath: string;
+}
+
+export interface UpdateSessionMemoryInput {
+  homeDir: string;
+  target: SessionMemoryTarget;
+  event: {
+    eventId: string;
+    type: string;
+    time: {
+      occurredAt: string | null;
+      receivedAt: string;
+    };
+    scope: {
+      taskId: string | null;
+    };
+    payload: {
+      summary: string;
+      metadata: Record<string, string | number | boolean | string[]>;
+    };
+  };
+  rawPayload: Record<string, unknown>;
+  environment?: Record<string, string | undefined>;
+  receivedAt?: string;
+  policy?: Partial<SessionMemoryPolicySnapshot>;
+}
+
+export interface SessionMemoryUpdateResult {
+  state: SessionMemoryStateV1 | null;
+  cursor: SessionMemoryCursorV1 | null;
+  segment: SessionEvidenceSegmentV1 | null;
+  stateWrites: string[];
+  warnings: string[];
+  queuedSegmentTriggerId: string | null;
+}
+
+export interface SessionMemoryIndexSegment {
+  id: string;
+  reason: SessionMemorySegmentReason;
+  strength: SessionMemorySignalStrength;
+  createdAt: string;
+  reviewState: "not-required" | "unreviewed" | "accepted" | "rejected" | "deferred";
+}

package/src/evolution/evidence/session-memory/updater.ts

@@ -0,0 +1,199 @@
+import { join } from "node:path";
+import { resolveProjectWorkspaceFromCwd } from "../../../projects/index.ts";
+import {
+  resolveProjectLogKey,
+  resolveTraceSessionKey,
+  resolveTraceTeamContext,
+} from "../../../runtime-logs/index.ts";
+import { normalizeTimestamp, optionalString, sanitizeStorageId } from "../../../utils/index.ts";
+import { enqueueSegmentEvolutionTrigger } from "../../triggers/index.ts";
+import { resolveSessionMemoryPaths } from "./paths.ts";
+import { parseSessionMemoryPolicy } from "./policy.ts";
+import { createRawEvent, createSegment, createSignal, estimateTokenCount } from "./segment.ts";
+import {
+  appendBoundedSignal,
+  applyEventToState,
+  cloneState,
+  createInitialState,
+  decideSegment,
+} from "./state-machine.ts";
+import {
+  appendRawEvent,
+  readSessionCursor,
+  readSessionState,
+  resetSessionRawEvents,
+  writeJson,
+  writeSessionIndex,
+} from "./storage.ts";
+import type {
+  SessionEvidenceSegmentV1,
+  SessionMemoryUpdateResult,
+  UpdateSessionMemoryInput,
+} from "./types.ts";
+
+export async function updateSessionMemoryFromHook(
+  input: UpdateSessionMemoryInput,
+): Promise<SessionMemoryUpdateResult> {
+  const policy = parseSessionMemoryPolicy(input.policy);
+  if (!policy.enabled || !policy.storeRawSegments) {
+    return emptyResult();
+  }
+
+  const team = resolveTraceTeamContext({
+    homeDir: input.homeDir,
+    environment: input.environment,
+    payload: input.rawPayload,
+  });
+  const cwd = optionalString(input.rawPayload.cwd);
+  const workspace =
+    team === null && cwd !== null
+      ? await resolveProjectWorkspaceFromCwd({ homeDir: input.homeDir, cwd })
+      : null;
+  const projectKey = sanitizeStorageId(
+    team?.projectKey ??
+      workspace?.projectKey ??
+      resolveProjectLogKey(input.homeDir, cwd ?? input.homeDir),
+    "project",
+  );
+  const sessionKey = sanitizeStorageId(resolveTraceSessionKey(input.rawPayload), "session");
+  const paths = resolveSessionMemoryPaths({ homeDir: input.homeDir, projectKey, sessionKey });
+  const now = normalizeTimestamp(input.receivedAt ?? input.event.time.receivedAt);
+  const existingState = await readSessionState(paths.statePath);
+
+  if (existingState === null && input.event.type !== "UserPromptSubmit") {
+    return emptyResult();
+  }
+
+  const rawEvent = createRawEvent(input, now);
+  const appended = await appendRawEvent(paths.eventsPath, rawEvent);
+  const tokenEstimate = estimateTokenCount(rawEvent.rawPayloadJson, input.event.payload.summary);
+  const currentSignal = createSignal(input, rawEvent, appended.lineNumber);
+  const state =
+    existingState ??
+    createInitialState({
+      projectKey,
+      sessionKey,
+      target: input.target,
+      runId: team?.runId ?? null,
+      roleId: team?.roleId ?? null,
+      now,
+      policy,
+      eventId: input.event.eventId,
+      tokenEstimate,
+    });
+  const cursor = await readSessionCursor(paths.cursorPath, sessionKey, paths.eventsPath, now);
+  const previous = cloneState(state);
+
+  applyEventToState(state, {
+    input,
+    now,
+    tokenEstimate,
+    policy,
+    teamRunId: team?.runId ?? null,
+    teamRoleId: team?.roleId ?? null,
+  });
+
+  const decision = decideSegment({
+    previous,
+    next: state,
+    input,
+    rawEvent,
+    currentSignal,
+    tokenEstimate,
+  });
+
+  let segment: SessionEvidenceSegmentV1 | null = null;
+  let segmentPath: string | null = null;
+  let queuedSegmentTriggerId: string | null = null;
+  const stateWrites: string[] = [paths.eventsPath];
+
+  if (decision === null) {
+    state.pendingSignals = appendBoundedSignal(state.pendingSignals, currentSignal);
+  } else {
+    const signals = [
+      ...state.pendingSignals,
+      { ...currentSignal, reason: decision.reason, strength: decision.strength },
+    ];
+    segment = await createSegment({
+      paths,
+      state,
+      cursor,
+      reason: decision.reason,
+      strength: decision.strength,
+      signals,
+      currentLine: appended.lineNumber,
+      now,
+    });
+    segmentPath = paths.segmentPath(segment.id);
+    await writeJson(segmentPath, segment);
+    await writeSessionIndex(paths, state, segment, now);
+    if (shouldQueueSegmentForEvolution(segment)) {
+      const trigger = await enqueueSegmentEvolutionTrigger({
+        homeDir: input.homeDir,
+        projectKey: segment.projectKey,
+        sessionKey: segment.sessionKey,
+        runId: segment.runId,
+        roleId: segment.roleId,
+        segmentId: segment.id,
+        segmentPath,
+        strength: segment.strength,
+        reason: segment.reason,
+        summary: segment.normalized.summary,
+        now,
+      });
+      queuedSegmentTriggerId = trigger.id;
+    }
+    await resetSessionRawEvents(paths.eventsPath);
+    cursor.lastCapturedLine = 0;
+    cursor.lastCapturedEventId = input.event.eventId;
+    cursor.updatedAt = now;
+    state.lastSegmentId = segment.id;
+    state.lastMemoryUpdateAt = now;
+    state.counters.tokensSinceLastUpdate = 0;
+    state.counters.toolCallsSinceLastUpdate = 0;
+    state.pendingSignals = [];
+    stateWrites.push(segmentPath, paths.indexPath);
+  }
+
+  state.updatedAt = now;
+  await writeJson(paths.statePath, state);
+  await writeJson(paths.cursorPath, cursor);
+  stateWrites.push(paths.statePath, paths.cursorPath);
+  if (queuedSegmentTriggerId !== null) {
+    stateWrites.push(
+      join(
+        input.homeDir,
+        ".evodev",
+        "state",
+        "evolution",
+        projectKey,
+        "segments",
+        `${queuedSegmentTriggerId}.json`,
+      ),
+    );
+  }
+
+  return {
+    state,
+    cursor,
+    segment,
+    stateWrites,
+    warnings: [],
+    queuedSegmentTriggerId,
+  };
+}
+
+function shouldQueueSegmentForEvolution(segment: SessionEvidenceSegmentV1): boolean {
+  return segment.reason !== "session-memory-threshold" && segment.reason !== "session-memory-init";
+}
+
+function emptyResult(): SessionMemoryUpdateResult {
+  return {
+    state: null,
+    cursor: null,
+    segment: null,
+    stateWrites: [],
+    warnings: [],
+    queuedSegmentTriggerId: null,
+  };
+}

package/src/evolution/formatters.ts

@@ -0,0 +1,169 @@
+import type {
+  EvolutionActivationResult,
+  EvolutionAnalyzeResult,
+  EvolutionDistillationBatch,
+  EvolutionProcessResult,
+  EvolutionReviewSnapshot,
+  EvolutionWriteResult,
+} from "./schema.ts";
+import { collectRoleIds } from "./shared.ts";
+
+export function formatEvolutionAnalyzeResult(result: EvolutionAnalyzeResult): string {
+  const roles = collectRoleIds(result.evidenceWindow);
+  return [
+    "EvoDev evolution analyze",
+    "",
+    `Project: ${result.evidenceWindow.projectKey}`,
+    `Run: ${result.evidenceWindow.runId}`,
+    `Evidence window: ${result.evidenceWindow.id}`,
+    `Sources: ${result.evidenceWindow.sourceRefs.length}`,
+    `Events: ${result.evidenceWindow.events.length}`,
+    `Episodes: ${result.evidenceWindow.episodes.length}`,
+    `Trigger: ${result.evidenceWindow.triggerPolicy.strongest} (${result.evidenceWindow.triggerPolicy.reasons.join(", ") || "none"}); distillRecommended=${result.evidenceWindow.triggerPolicy.distillRecommended}`,
+    `Roles: ${roles.length === 0 ? "none" : roles.join(", ")}`,
+    "Privacy: metadata-only; rawLogs=false; rawPrompts=false; sourceDumps=false; rawCommandOutput=false",
+    ...(result.warnings.length === 0
+      ? ["Warnings: none"]
+      : ["Warnings:", ...result.warnings.map((warning) => `  - ${warning}`)]),
+  ].join("\n");
+}
+
+export function formatEvolutionDistillationBatch(batch: EvolutionDistillationBatch): string {
+  return [
+    "EvoDev evolution distillation",
+    "",
+    `Batch: ${batch.id}`,
+    `Project: ${batch.projectKey}`,
+    `Run: ${batch.runId}`,
+    `Evidence events: ${batch.evidenceWindow.events.length}`,
+    `Episodes: ${batch.evidenceWindow.episodes.length}`,
+    `Trigger: ${batch.evidenceWindow.triggerPolicy.strongest}; distillRecommended=${batch.evidenceWindow.triggerPolicy.distillRecommended}`,
+    `Knowledge records: ${batch.knowledgeRecords.length}`,
+    `Evos cases: ${batch.evosCases.length}`,
+    `Repo proposals: ${batch.repoProposals.length}`,
+    "Default batch review state: contextual until the OKF plan auto-accepts or queues review.",
+    "Runtime authority: accepted/auto-accepted only; hardBlocking=false.",
+    ...(batch.knowledgeRecords.length === 0
+      ? ["Knowledge: none"]
+      : [
+          "Knowledge:",
+          ...batch.knowledgeRecords.map(
+            (record) =>
+              `  - ${record.id} (${record.kind}, ${record.reviewState}, ${record.confidence}): ${record.title}`,
+          ),
+        ]),
+    ...(batch.evosCases.length === 0
+      ? ["Evos cases: none"]
+      : [
+          "Evos cases:",
+          ...batch.evosCases.map(
+            (evosCase) =>
+              `  - ${evosCase.id} (${evosCase.reviewState}, ${evosCase.confidence}): ${evosCase.title}`,
+          ),
+        ]),
+    ...(batch.repoProposals.length === 0
+      ? ["Repo proposals: none"]
+      : [
+          "Repo proposals:",
+          ...batch.repoProposals.map(
+            (proposal) =>
+              `  - ${proposal.id} (${proposal.kind}, ${proposal.reviewState}, ${proposal.confidence}): ${proposal.title}`,
+          ),
+        ]),
+    ...(batch.warnings.length === 0
+      ? []
+      : ["Warnings:", ...batch.warnings.map((warning) => `  - ${warning}`)]),
+  ].join("\n");
+}
+
+export function formatEvolutionWriteResult(result: EvolutionWriteResult): string {
+  return [
+    "EvoDev evolution write",
+    "",
+    `Evidence window: ${result.evidenceWindowPath}`,
+    `Distillation batch: ${result.batchPath}`,
+    `Knowledge records: ${result.knowledgePaths.length}`,
+    ...result.knowledgePaths.map((path) => `  - ${path}`),
+    `Evos cases: ${result.evosCasePaths.length}`,
+    ...result.evosCasePaths.map((path) => `  - ${path}`),
+    `Repo proposals: ${result.repoProposalPaths.length}`,
+    ...result.repoProposalPaths.map((path) => `  - ${path}`),
+    `Knowledge index: ${result.knowledgeIndexPath}`,
+    `Evos index: ${result.evosIndexPath}`,
+  ].join("\n");
+}
+
+export function formatEvolutionActivationResult(result: EvolutionActivationResult): string {
+  return [
+    "EvoDev evolution activate",
+    "",
+    `Evidence window: ${result.evidenceWindowPath}`,
+    `Distillation batch: ${result.batchPath}`,
+    `OKF concepts: ${result.okf.conceptPaths.length}`,
+    ...result.okf.conceptPaths.map((path) => `  - ${path}`),
+    `OKF overlays: ${result.okf.overlayPaths.length}`,
+    ...result.okf.overlayPaths.map((path) => `  - ${path}`),
+    `Skipped candidates: ${result.okf.skippedCandidates.length}`,
+    ...result.okf.skippedCandidates.map((id) => `  - ${id}`),
+    `Needs human: ${result.okf.needsHumanCandidates.length}`,
+    ...result.okf.needsHumanCandidates.map((id) => `  - ${id}`),
+    `Evos cases: ${result.evosCasePaths.length}`,
+    ...result.evosCasePaths.map((path) => `  - ${path}`),
+    `Repo proposals: ${result.repoProposalPaths.length}`,
+    ...result.repoProposalPaths.map((path) => `  - ${path}`),
+    `OKF derived indexes: ${result.okf.derivedIndexPaths.length}`,
+    ...result.okf.derivedIndexPaths.map((path) => `  - ${path}`),
+    `Evos index: ${result.evosIndexPath}`,
+  ].join("\n");
+}
+
+export function formatEvolutionProcessResult(result: EvolutionProcessResult): string {
+  return [
+    "EvoDev evolution process",
+    "",
+    `Mode: ${result.dryRun ? "dry-run" : "write"}`,
+    `Processed triggers: ${result.processed}`,
+    `Consumed: ${result.consumed}`,
+    `Skipped: ${result.skipped}`,
+    `Failed: ${result.failed}`,
+    `Still pending: ${result.pending}`,
+    `Batches: ${result.batchIds.length}`,
+    ...result.batchIds.map((batchId) => `  - ${batchId}`),
+    ...(result.warnings.length === 0
+      ? ["Warnings: none"]
+      : ["Warnings:", ...result.warnings.map((warning) => `  - ${warning}`)]),
+  ].join("\n");
+}
+
+export function formatEvolutionReviewSnapshot(snapshot: EvolutionReviewSnapshot): string {
+  return [
+    "EvoDev evolution review snapshot",
+    "",
+    `Project: ${snapshot.projectKey ?? "all"}`,
+    `Knowledge records: ${snapshot.knowledgeRecords.length}`,
+    ...snapshot.knowledgeRecords.map(
+      (record) =>
+        `  - ${record.id} (${record.projectKey}, ${record.kind}, ${record.reviewState}, ${record.confidence}): ${record.title}`,
+    ),
+    `Evos cases: ${snapshot.evosCases.length}`,
+    ...snapshot.evosCases.map(
+      (evosCase) =>
+        `  - ${evosCase.id} (${evosCase.projectKey}, ${evosCase.reviewState}, ${evosCase.confidence}): ${evosCase.title}`,
+    ),
+    `Repo proposals: ${snapshot.repoProposals.length}`,
+    ...snapshot.repoProposals.map(
+      (proposal) =>
+        `  - ${proposal.id} (${proposal.projectKey}, ${proposal.kind}, ${proposal.reviewState}, ${proposal.confidence}): ${proposal.title}`,
+    ),
+    `Review candidates: ${snapshot.reviewCandidates.length}`,
+    ...snapshot.reviewCandidates.map(
+      (candidate) =>
+        `  - ${candidate.id} (${candidate.projectKey}, ${candidate.candidateKind}, ${candidate.reviewState}): ${candidate.title}`,
+    ),
+    `Triggers: ${snapshot.triggers.length}`,
+    ...snapshot.triggers.map(
+      (trigger) =>
+        `  - ${trigger.id} (${trigger.projectKey}, ${trigger.runId}, ${trigger.eventType}, ${trigger.status}, ${trigger.triggerStrength}): ${trigger.summary}`,
+    ),
+  ].join("\n");
+}