@evo-dev/core 0.0.1-alpha → 0.0.1-alpha.10

package/src/evolution/evidence/session-memory/storage.ts

@@ -0,0 +1,379 @@
+import { appendFile, mkdir, readFile, readdir, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { resolveEvoDevPaths } from "../../../config/paths.ts";
+import { isNotFoundError } from "../../../utils/index.ts";
+import { resolveSessionMemoryPaths } from "./paths.ts";
+import { detectSessionMemorySensitivity, mergeSessionMemorySensitivity } from "./sensitivity.ts";
+import type {
+  SessionEvidenceSegmentV1,
+  SessionMemoryCursorV1,
+  SessionMemoryIndexSegment,
+  SessionMemoryPaths,
+  SessionMemoryRawEventV1,
+  SessionMemorySensitivity,
+  SessionMemorySensitivityReason,
+  SessionMemoryStateV1,
+} from "./types.ts";
+
+export class SessionMemoryEvidenceError extends Error {
+  readonly code: "not-found" | "invalid";
+
+  constructor(code: "not-found" | "invalid", message: string) {
+    super(message);
+    this.name = "SessionMemoryEvidenceError";
+    this.code = code;
+  }
+}
+
+export async function listSessionEvidenceSegments(input: {
+  homeDir: string;
+  projectKey?: string;
+}): Promise<SessionEvidenceSegmentV1[]> {
+  const rootDir = join(resolveEvoDevPaths(input.homeDir).stateDir, "session-memory");
+  const projectDirs =
+    input.projectKey === undefined ? await listDirectoryNames(rootDir) : [input.projectKey];
+  const segments: SessionEvidenceSegmentV1[] = [];
+  for (const projectKey of projectDirs) {
+    const paths = resolveSessionMemoryPaths({
+      homeDir: input.homeDir,
+      projectKey,
+      sessionKey: "list",
+    });
+    for (const sessionKey of await listDirectoryNames(paths.projectDir)) {
+      const sessionPaths = resolveSessionMemoryPaths({
+        homeDir: input.homeDir,
+        projectKey,
+        sessionKey,
+      });
+      for (const entry of await listJsonFiles(sessionPaths.segmentsDir)) {
+        try {
+          segments.push(parseSessionEvidenceSegment(JSON.parse(await readFile(entry, "utf8"))));
+        } catch {
+          // Invalid local evidence is omitted from listings and remains available for diagnostics.
+        }
+      }
+    }
+  }
+  return segments.sort((left, right) => right.createdAt.localeCompare(left.createdAt));
+}
+
+export async function listSessionMemoryStates(input: {
+  homeDir: string;
+  projectKey?: string;
+}): Promise<SessionMemoryStateV1[]> {
+  const rootDir = join(resolveEvoDevPaths(input.homeDir).stateDir, "session-memory");
+  const projectDirs =
+    input.projectKey === undefined ? await listDirectoryNames(rootDir) : [input.projectKey];
+  const states: SessionMemoryStateV1[] = [];
+  for (const projectKey of projectDirs) {
+    const projectDir = join(rootDir, projectKey);
+    for (const sessionKey of await listDirectoryNames(projectDir)) {
+      try {
+        const state = parseSessionMemoryState(
+          JSON.parse(await readFile(join(projectDir, sessionKey, "state.json"), "utf8")) as unknown,
+        );
+        if (state.projectKey === projectKey && state.sessionKey === sessionKey) states.push(state);
+      } catch {
+        // Invalid local state is omitted from listings and remains available for diagnostics.
+      }
+    }
+  }
+  return states.sort(
+    (left, right) =>
+      right.updatedAt.localeCompare(left.updatedAt) ||
+      left.sessionKey.localeCompare(right.sessionKey),
+  );
+}
+
+export async function readSessionEvidenceSegment(input: {
+  homeDir: string;
+  projectKey: string;
+  sessionKey: string;
+  segmentId: string;
+}): Promise<SessionEvidenceSegmentV1> {
+  const paths = resolveSessionMemoryPaths(input);
+  try {
+    const segment = parseSessionEvidenceSegment(
+      JSON.parse(await readFile(paths.segmentPath(input.segmentId), "utf8")) as unknown,
+    );
+    if (
+      segment.id !== input.segmentId ||
+      segment.projectKey !== input.projectKey ||
+      segment.sessionKey !== input.sessionKey
+    ) {
+      throw new SessionMemoryEvidenceError(
+        "invalid",
+        "Session evidence identity does not match its storage path.",
+      );
+    }
+    return segment;
+  } catch (error) {
+    if (error instanceof SessionMemoryEvidenceError) throw error;
+    if (isNotFoundError(error)) {
+      throw new SessionMemoryEvidenceError("not-found", "Session evidence segment was not found.");
+    }
+    throw new SessionMemoryEvidenceError("invalid", "Session evidence segment is invalid.");
+  }
+}
+
+export async function updateSessionEvidenceSegmentLifecycle(input: {
+  homeDir: string;
+  projectKey: string;
+  sessionKey: string;
+  segmentId: string;
+  status: "distilled" | "ignored";
+  consumedByBatchId?: string;
+}): Promise<SessionEvidenceSegmentV1> {
+  const segment = await readSessionEvidenceSegment(input);
+  const consumedByBatchIds =
+    input.consumedByBatchId === undefined
+      ? segment.lifecycle.consumedByBatchIds
+      : [...new Set([...segment.lifecycle.consumedByBatchIds, input.consumedByBatchId])];
+  const updated: SessionEvidenceSegmentV1 = {
+    ...segment,
+    lifecycle: {
+      ...segment.lifecycle,
+      status: input.status,
+      consumedByBatchIds,
+    },
+  };
+  const paths = resolveSessionMemoryPaths(input);
+  await writeJson(paths.segmentPath(input.segmentId), updated);
+  return updated;
+}
+
+export async function readSessionState(path: string): Promise<SessionMemoryStateV1 | null> {
+  try {
+    return JSON.parse(await readFile(path, "utf8")) as SessionMemoryStateV1;
+  } catch (error) {
+    if (isNotFoundError(error)) return null;
+    throw error;
+  }
+}
+
+export async function readSessionCursor(
+  path: string,
+  sessionKey: string,
+  sourcePath: string,
+  now: string,
+): Promise<SessionMemoryCursorV1> {
+  try {
+    return JSON.parse(await readFile(path, "utf8")) as SessionMemoryCursorV1;
+  } catch (error) {
+    if (!isNotFoundError(error)) throw error;
+    return {
+      schemaVersion: 1,
+      kind: "session-memory-cursor",
+      sessionKey,
+      traceRefId: null,
+      sourcePath,
+      lastCapturedOffset: null,
+      lastCapturedLine: null,
+      lastCapturedEventId: null,
+      updatedAt: now,
+    };
+  }
+}
+
+export async function appendRawEvent(
+  path: string,
+  event: SessionMemoryRawEventV1,
+): Promise<{ lineNumber: number }> {
+  const existingLineCount = await countJsonlLines(path);
+  await mkdir(dirname(path), { recursive: true });
+  await appendFile(path, `${JSON.stringify(event)}\n`, "utf8");
+  return { lineNumber: existingLineCount + 1 };
+}
+
+export async function readLineRange(
+  path: string,
+  fromLine: number,
+  toLine: number,
+): Promise<{
+  content: string;
+  truncated: boolean;
+  secretsDetected: boolean;
+  sensitivity: SessionMemorySensitivity;
+  sensitivityReasons: SessionMemorySensitivityReason[];
+  redactionApplied: boolean;
+}> {
+  const text = await readFile(path, "utf8");
+  const lines = text.split("\n").filter((line) => line.trim() !== "");
+  const selected = lines.slice(Math.max(0, fromLine - 1), toLine);
+  const detections = [];
+  let redactionApplied = false;
+  for (const line of selected) {
+    try {
+      const parsed = JSON.parse(line) as Partial<SessionMemoryRawEventV1>;
+      redactionApplied ||= parsed.credentialRedactionApplied === true;
+      if (parsed.sensitivity !== undefined) {
+        detections.push({
+          classification: parsed.sensitivity,
+          reasons: [...(parsed.sensitivityReasons ?? [])],
+        });
+      } else if (typeof parsed.rawPayloadJson === "string") {
+        try {
+          detections.push(
+            detectSessionMemorySensitivity(JSON.parse(parsed.rawPayloadJson) as unknown),
+          );
+        } catch {
+          detections.push(detectSessionMemorySensitivity(parsed.rawPayloadJson));
+        }
+      } else if (parsed.secretsDetected === true) {
+        detections.push({
+          classification: "credential" as const,
+          reasons: ["legacy-sensitive-flag" as const],
+        });
+      }
+    } catch {
+      detections.push(detectSessionMemorySensitivity(line));
+    }
+  }
+  const sensitivity = mergeSessionMemorySensitivity(detections);
+  return {
+    content: selected.join("\n"),
+    truncated: false,
+    secretsDetected: sensitivity.classification === "credential",
+    sensitivity: sensitivity.classification,
+    sensitivityReasons: sensitivity.reasons,
+    redactionApplied,
+  };
+}
+
+export async function writeSessionIndex(
+  paths: SessionMemoryPaths,
+  state: SessionMemoryStateV1,
+  segment: SessionEvidenceSegmentV1,
+  now: string,
+): Promise<void> {
+  const existing = await readSessionIndex(paths.indexPath);
+  const segments = [
+    ...existing.segments.filter((item) => item.id !== segment.id),
+    {
+      id: segment.id,
+      reason: segment.reason,
+      strength: segment.strength,
+      createdAt: segment.createdAt,
+      reviewState: segment.lifecycle.reviewState,
+    },
+  ];
+  await writeJson(paths.indexPath, {
+    schemaVersion: 1,
+    kind: "session-memory-index",
+    projectKey: state.projectKey,
+    sessionKey: state.sessionKey,
+    updatedAt: now,
+    lastSegmentId: segment.id,
+    segments,
+  });
+}
+
+export async function writeJson(path: string, value: unknown): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+}
+
+async function countJsonlLines(path: string): Promise<number> {
+  try {
+    const text = await readFile(path, "utf8");
+    return text.split("\n").filter((line) => line.trim() !== "").length;
+  } catch (error) {
+    if (isNotFoundError(error)) return 0;
+    throw error;
+  }
+}
+
+async function readSessionIndex(path: string): Promise<{ segments: SessionMemoryIndexSegment[] }> {
+  try {
+    const parsed = JSON.parse(await readFile(path, "utf8")) as {
+      segments?: unknown;
+    };
+    if (!Array.isArray(parsed.segments)) return { segments: [] };
+    return {
+      segments: parsed.segments.filter(isSessionIndexSegment),
+    };
+  } catch (error) {
+    if (isNotFoundError(error)) return { segments: [] };
+    throw error;
+  }
+}
+
+function isSessionIndexSegment(value: unknown): value is SessionMemoryIndexSegment {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
+  const item = value as Record<string, unknown>;
+  return (
+    typeof item.id === "string" &&
+    typeof item.reason === "string" &&
+    typeof item.strength === "string" &&
+    typeof item.createdAt === "string" &&
+    typeof item.reviewState === "string"
+  );
+}
+
+function parseSessionEvidenceSegment(value: unknown): SessionEvidenceSegmentV1 {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new SessionMemoryEvidenceError("invalid", "Session evidence must be an object.");
+  }
+  const segment = value as Partial<SessionEvidenceSegmentV1>;
+  if (
+    segment.schemaVersion !== 1 ||
+    segment.kind !== "session-evidence-segment" ||
+    typeof segment.id !== "string" ||
+    typeof segment.projectKey !== "string" ||
+    typeof segment.sessionKey !== "string" ||
+    typeof segment.createdAt !== "string" ||
+    typeof segment.normalized !== "object" ||
+    segment.normalized === null ||
+    typeof segment.rawExcerpt !== "object" ||
+    segment.rawExcerpt === null
+  ) {
+    throw new SessionMemoryEvidenceError("invalid", "Session evidence fields are invalid.");
+  }
+  return segment as SessionEvidenceSegmentV1;
+}
+
+function parseSessionMemoryState(value: unknown): SessionMemoryStateV1 {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) {
+    throw new SessionMemoryEvidenceError("invalid", "Session Memory state must be an object.");
+  }
+  const state = value as Partial<SessionMemoryStateV1>;
+  if (
+    state.schemaVersion !== 1 ||
+    state.kind !== "session-memory-state" ||
+    typeof state.projectKey !== "string" ||
+    typeof state.sessionKey !== "string" ||
+    !(state.runId === null || typeof state.runId === "string") ||
+    !(state.roleId === null || typeof state.roleId === "string") ||
+    typeof state.createdAt !== "string" ||
+    typeof state.updatedAt !== "string" ||
+    typeof state.counters !== "object" ||
+    state.counters === null
+  ) {
+    throw new SessionMemoryEvidenceError("invalid", "Session Memory state fields are invalid.");
+  }
+  return state as SessionMemoryStateV1;
+}
+
+async function listDirectoryNames(path: string): Promise<string[]> {
+  try {
+    return (await readdir(path, { withFileTypes: true }))
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name)
+      .sort();
+  } catch (error) {
+    if (isNotFoundError(error)) return [];
+    throw error;
+  }
+}
+
+async function listJsonFiles(path: string): Promise<string[]> {
+  try {
+    return (await readdir(path, { withFileTypes: true }))
+      .filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
+      .map((entry) => join(path, entry.name))
+      .sort();
+  } catch (error) {
+    if (isNotFoundError(error)) return [];
+    throw error;
+  }
+}

package/src/evolution/evidence/session-memory/types.ts

@@ -0,0 +1,221 @@
+export type SessionMemoryTarget = "claude" | "codex" | "unknown";
+export type SessionMemorySegmentReason =
+  | "session-memory-init"
+  | "session-memory-threshold"
+  | "intent-refinement"
+  | "user-interruption"
+  | "failure-signal"
+  | "permission-denied"
+  | "verification-after-fix"
+  | "explicit-memory-intent";
+export type SessionMemorySignalStrength = "normal" | "strong";
+export type SessionMemorySensitivity = "safe-metadata" | "private-content" | "credential";
+export type SessionMemorySensitivityReason =
+  | "credential-field"
+  | "credential-shape"
+  | "credential-url-parameter"
+  | "credential-url-userinfo"
+  | "legacy-sensitive-flag"
+  | "private-marker"
+  | "url";
+
+export interface SessionMemoryPolicySnapshot {
+  enabled: boolean;
+  storeRawSegments: boolean;
+  maxRawSegmentBytes: number;
+  retentionDays: number;
+  minimumMessageTokensToInit: number;
+  minimumTokensBetweenUpdate: number;
+  toolCallsBetweenUpdates: number;
+}
+
+export interface SessionMemorySignal {
+  eventId: string;
+  eventType: string;
+  reason: SessionMemorySegmentReason | "observed";
+  strength: SessionMemorySignalStrength;
+  occurredAt: string | null;
+  summary: string;
+  rawEventLine: number;
+}
+
+export interface SessionMemorySpan {
+  spanId: string;
+  status: "open" | "closed" | "interrupted";
+  openedAt: string;
+  closedAt: string | null;
+  openedByEventId: string;
+  lastEventId: string;
+  userPromptCount: number;
+  toolCallCount: number;
+  workStarted: boolean;
+  tokenEstimate: number;
+}
+
+export interface SessionMemoryStateV1 {
+  schemaVersion: 1;
+  kind: "session-memory-state";
+  projectKey: string;
+  sessionKey: string;
+  target: SessionMemoryTarget;
+  runId: string | null;
+  roleId: string | null;
+  initialized: boolean;
+  createdAt: string;
+  updatedAt: string;
+  counters: {
+    messageTokenEstimate: number;
+    tokensSinceLastUpdate: number;
+    toolCallsSinceLastUpdate: number;
+    userPromptCount: number;
+    assistantTurnCount: number;
+    toolCallCount: number;
+    failureSignalCount: number;
+    verificationSignalCount: number;
+  };
+  activeSpan: SessionMemorySpan | null;
+  pendingSignals: SessionMemorySignal[];
+  lastSegmentId: string | null;
+  lastMemoryUpdateAt: string | null;
+  policy: SessionMemoryPolicySnapshot;
+}
+
+export interface SessionMemoryCursorV1 {
+  schemaVersion: 1;
+  kind: "session-memory-cursor";
+  sessionKey: string;
+  traceRefId: string | null;
+  sourcePath: string | null;
+  lastCapturedOffset: number | null;
+  lastCapturedLine: number | null;
+  lastCapturedEventId: string | null;
+  updatedAt: string;
+}
+
+export interface SessionEvidenceSegmentV1 {
+  schemaVersion: 1;
+  kind: "session-evidence-segment";
+  id: string;
+  projectKey: string;
+  runId: string | null;
+  roleId: string | null;
+  sessionKey: string;
+  target: SessionMemoryTarget;
+  createdAt: string;
+  reason: SessionMemorySegmentReason;
+  strength: SessionMemorySignalStrength;
+  source: {
+    traceRefId: string | null;
+    sourcePath: string | null;
+    fromOffset: number | null;
+    toOffset: number | null;
+    fromLine: number | null;
+    toLine: number | null;
+    fromEventId: string | null;
+    toEventId: string | null;
+  };
+  signals: SessionMemorySignal[];
+  rawExcerpt: {
+    stored: true;
+    encoding: "utf8";
+    content: string;
+    truncated: boolean;
+    byteLength: number;
+    sha256: string;
+  };
+  normalized: {
+    summary: string;
+    userIntent: string | null;
+    intentDelta: string | null;
+    decisions: string[];
+    failures: string[];
+    verifications: string[];
+    touchedTools: string[];
+  };
+  privacy: {
+    localOnly: true;
+    rawPromptStored: boolean;
+    rawOutputStored: boolean;
+    sourceContentStored: boolean;
+    secretsDetected: boolean;
+    sensitivity?: SessionMemorySensitivity;
+    sensitivityReasons?: SessionMemorySensitivityReason[];
+    redactionApplied: boolean;
+    externalUploadAllowed: false;
+  };
+  lifecycle: {
+    status: "captured" | "pending-review" | "reviewed" | "distilled" | "ignored" | "deleted";
+    reviewState: "not-required" | "unreviewed" | "accepted" | "rejected" | "deferred";
+    consumedByBatchIds: string[];
+  };
+}
+
+export interface SessionMemoryRawEventV1 {
+  schemaVersion: 1;
+  kind: "session-memory-raw-event";
+  eventId: string;
+  eventType: string;
+  occurredAt: string | null;
+  receivedAt: string;
+  summary: string;
+  rawPayloadJson: string;
+  rawPayloadByteLength: number;
+  rawPayloadTruncated: boolean;
+  secretsDetected: boolean;
+  sensitivity?: SessionMemorySensitivity;
+  sensitivityReasons?: SessionMemorySensitivityReason[];
+  credentialRedactionApplied?: boolean;
+}
+
+export interface SessionMemoryPaths {
+  rootDir: string;
+  projectDir: string;
+  sessionDir: string;
+  statePath: string;
+  cursorPath: string;
+  eventsPath: string;
+  segmentsDir: string;
+  segmentPath: (segmentId: string) => string;
+  indexPath: string;
+}
+
+export interface UpdateSessionMemoryInput {
+  homeDir: string;
+  target: SessionMemoryTarget;
+  event: {
+    eventId: string;
+    type: string;
+    time: {
+      occurredAt: string | null;
+      receivedAt: string;
+    };
+    scope: {
+      taskId: string | null;
+    };
+    payload: {
+      summary: string;
+      metadata: Record<string, string | number | boolean | string[]>;
+    };
+  };
+  rawPayload: Record<string, unknown>;
+  environment?: Record<string, string | undefined>;
+  receivedAt?: string;
+  policy?: Partial<SessionMemoryPolicySnapshot>;
+}
+
+export interface SessionMemoryUpdateResult {
+  state: SessionMemoryStateV1 | null;
+  cursor: SessionMemoryCursorV1 | null;
+  segment: SessionEvidenceSegmentV1 | null;
+  stateWrites: string[];
+  warnings: string[];
+  queuedSegmentTriggerId: string | null;
+}
+
+export interface SessionMemoryIndexSegment {
+  id: string;
+  reason: SessionMemorySegmentReason;
+  strength: SessionMemorySignalStrength;
+  createdAt: string;
+  reviewState: "not-required" | "unreviewed" | "accepted" | "rejected" | "deferred";
+}