@evo-dev/core 0.0.1-alpha.3 → 0.0.1-alpha.4

package/src/evolution/evidence/session-memory/segment.ts

@@ -0,0 +1,192 @@
+import {
+  createStableId,
+  sanitizeStorageId,
+  sanitizeSummary,
+  sha256Hex,
+  truncateUtf8Tail,
+} from "../../../utils/index.ts";
+import {
+  DEFAULT_MAX_RAW_EVENT_BYTES,
+  SENSITIVE_TEXT_PATTERN,
+  VERIFICATION_PATTERN,
+} from "./constants.ts";
+import { readLineRange } from "./storage.ts";
+import type {
+  SessionEvidenceSegmentV1,
+  SessionMemoryCursorV1,
+  SessionMemoryPaths,
+  SessionMemoryRawEventV1,
+  SessionMemorySegmentReason,
+  SessionMemorySignal,
+  SessionMemorySignalStrength,
+  SessionMemoryStateV1,
+  UpdateSessionMemoryInput,
+} from "./types.ts";
+
+export function createRawEvent(
+  input: UpdateSessionMemoryInput,
+  now: string,
+): SessionMemoryRawEventV1 {
+  const rawJson = stringifyPayload(input.rawPayload);
+  const truncated = truncateUtf8Tail(rawJson, DEFAULT_MAX_RAW_EVENT_BYTES);
+  return {
+    schemaVersion: 1,
+    kind: "session-memory-raw-event",
+    eventId: input.event.eventId,
+    eventType: input.event.type,
+    occurredAt: input.event.time.occurredAt,
+    receivedAt: now,
+    summary: sanitizeSummary(input.event.payload.summary),
+    rawPayloadJson: truncated.content,
+    rawPayloadByteLength: Buffer.byteLength(rawJson, "utf8"),
+    rawPayloadTruncated: truncated.truncated,
+    secretsDetected: SENSITIVE_TEXT_PATTERN.test(rawJson),
+  };
+}
+
+export function createSignal(
+  input: UpdateSessionMemoryInput,
+  rawEvent: SessionMemoryRawEventV1,
+  rawEventLine: number,
+): SessionMemorySignal {
+  return {
+    eventId: input.event.eventId,
+    eventType: input.event.type,
+    reason: "observed",
+    strength: "normal",
+    occurredAt: input.event.time.occurredAt,
+    summary: rawEvent.summary,
+    rawEventLine,
+  };
+}
+
+export async function createSegment(input: {
+  paths: SessionMemoryPaths;
+  state: SessionMemoryStateV1;
+  cursor: SessionMemoryCursorV1;
+  reason: SessionMemorySegmentReason;
+  strength: SessionMemorySignalStrength;
+  signals: SessionMemorySignal[];
+  currentLine: number;
+  now: string;
+}): Promise<SessionEvidenceSegmentV1> {
+  const fromLine = (input.cursor.lastCapturedLine ?? 0) + 1;
+  const toLine = input.currentLine;
+  const raw = await readLineRange(input.paths.eventsPath, fromLine, toLine);
+  const truncated = truncateUtf8Tail(raw.content, input.state.policy.maxRawSegmentBytes);
+  const signalSummaries = input.signals.map((signal) => signal.summary).filter(Boolean);
+  const failures = input.signals
+    .filter((signal) => signal.reason === "failure-signal")
+    .map((signal) => signal.summary);
+  const verifications = input.signals
+    .filter(
+      (signal) =>
+        signal.reason === "verification-after-fix" || VERIFICATION_PATTERN.test(signal.summary),
+    )
+    .map((signal) => signal.summary);
+  const segmentId = createStableId("segment", [
+    input.state.projectKey,
+    input.state.sessionKey,
+    input.reason,
+    String(fromLine),
+    String(toLine),
+    input.signals.at(-1)?.eventId ?? input.now,
+  ]);
+  return {
+    schemaVersion: 1,
+    kind: "session-evidence-segment",
+    id: segmentId,
+    projectKey: input.state.projectKey,
+    runId: input.state.runId,
+    roleId: input.state.roleId,
+    sessionKey: input.state.sessionKey,
+    target: input.state.target,
+    createdAt: input.now,
+    reason: input.reason,
+    strength: input.strength,
+    source: {
+      traceRefId: input.cursor.traceRefId,
+      sourcePath: input.paths.eventsPath,
+      fromOffset: null,
+      toOffset: null,
+      fromLine,
+      toLine,
+      fromEventId: input.signals[0]?.eventId ?? null,
+      toEventId: input.signals.at(-1)?.eventId ?? null,
+    },
+    signals: input.signals,
+    rawExcerpt: {
+      stored: true,
+      encoding: "utf8",
+      content: truncated.content,
+      truncated: raw.truncated || truncated.truncated,
+      byteLength: Buffer.byteLength(truncated.content, "utf8"),
+      sha256: sha256Hex(truncated.content),
+    },
+    normalized: {
+      summary: createSegmentSummary(input.reason, input.strength, signalSummaries),
+      userIntent: firstUserPromptSummary(input.signals),
+      intentDelta: intentDeltaSummary(input.signals),
+      decisions: [],
+      failures,
+      verifications,
+      touchedTools: collectTouchedTools(raw.content),
+    },
+    privacy: {
+      localOnly: true,
+      rawPromptStored: input.signals.some((signal) => signal.eventType === "UserPromptSubmit"),
+      rawOutputStored: input.signals.some((signal) =>
+        ["PostToolUse", "PostToolUseFailure", "PostToolBatch"].includes(signal.eventType),
+      ),
+      sourceContentStored: true,
+      secretsDetected: raw.secretsDetected,
+      redactionApplied: false,
+      externalUploadAllowed: false,
+    },
+    lifecycle: {
+      status: "pending-review",
+      reviewState: "unreviewed",
+      consumedByBatchIds: [],
+    },
+  };
+}
+
+export function estimateTokenCount(rawPayloadJson: string, summary: string): number {
+  return Math.max(1, Math.ceil((rawPayloadJson.length + summary.length) / 4));
+}
+
+function createSegmentSummary(
+  reason: SessionMemorySegmentReason,
+  strength: SessionMemorySignalStrength,
+  summaries: string[],
+): string {
+  const suffix = summaries.length === 0 ? "No event summary." : summaries.slice(-3).join(" / ");
+  return sanitizeSummary(`Session segment ${reason} (${strength}): ${suffix}`);
+}
+
+function firstUserPromptSummary(signals: SessionMemorySignal[]): string | null {
+  return signals.find((signal) => signal.eventType === "UserPromptSubmit")?.summary ?? null;
+}
+
+function intentDeltaSummary(signals: SessionMemorySignal[]): string | null {
+  const prompts = signals.filter((signal) => signal.eventType === "UserPromptSubmit");
+  if (prompts.length < 2) return null;
+  return prompts.at(-1)?.summary ?? null;
+}
+
+function collectTouchedTools(rawContent: string): string[] {
+  const tools = new Set<string>();
+  for (const match of rawContent.matchAll(/"tool(?:Name|_name)"\s*:\s*"([^"]+)"/g)) {
+    const tool = match[1]?.trim();
+    if (tool !== undefined && tool !== "") tools.add(sanitizeStorageId(tool, "tool"));
+  }
+  return [...tools].slice(0, 20);
+}
+
+function stringifyPayload(payload: Record<string, unknown>): string {
+  try {
+    return JSON.stringify(payload);
+  } catch {
+    return JSON.stringify({ unserializable: true });
+  }
+}

package/src/evolution/evidence/session-memory/state-machine.ts

@@ -0,0 +1,249 @@
+import { createStableId, optionalString } from "../../../utils/index.ts";
+import { EXPLICIT_MEMORY_PATTERN, FAILURE_PATTERN, VERIFICATION_PATTERN } from "./constants.ts";
+import type {
+  SessionMemoryPolicySnapshot,
+  SessionMemoryRawEventV1,
+  SessionMemorySegmentReason,
+  SessionMemorySignal,
+  SessionMemorySignalStrength,
+  SessionMemoryStateV1,
+  SessionMemoryTarget,
+  UpdateSessionMemoryInput,
+} from "./types.ts";
+
+export function createInitialState(input: {
+  projectKey: string;
+  sessionKey: string;
+  target: SessionMemoryTarget;
+  runId: string | null;
+  roleId: string | null;
+  now: string;
+  policy: SessionMemoryPolicySnapshot;
+  eventId: string;
+  tokenEstimate: number;
+}): SessionMemoryStateV1 {
+  return {
+    schemaVersion: 1,
+    kind: "session-memory-state",
+    projectKey: input.projectKey,
+    sessionKey: input.sessionKey,
+    target: input.target,
+    runId: input.runId,
+    roleId: input.roleId,
+    initialized: true,
+    createdAt: input.now,
+    updatedAt: input.now,
+    counters: {
+      messageTokenEstimate: 0,
+      tokensSinceLastUpdate: 0,
+      toolCallsSinceLastUpdate: 0,
+      userPromptCount: 0,
+      assistantTurnCount: 0,
+      toolCallCount: 0,
+      failureSignalCount: 0,
+      verificationSignalCount: 0,
+    },
+    activeSpan: {
+      spanId: createStableId("span", [input.projectKey, input.sessionKey, input.eventId]),
+      status: "open",
+      openedAt: input.now,
+      closedAt: null,
+      openedByEventId: input.eventId,
+      lastEventId: input.eventId,
+      userPromptCount: 0,
+      toolCallCount: 0,
+      workStarted: false,
+      tokenEstimate: input.tokenEstimate,
+    },
+    pendingSignals: [],
+    lastSegmentId: null,
+    lastMemoryUpdateAt: null,
+    policy: input.policy,
+  };
+}
+
+export function applyEventToState(
+  state: SessionMemoryStateV1,
+  input: {
+    input: UpdateSessionMemoryInput;
+    now: string;
+    tokenEstimate: number;
+    policy: SessionMemoryPolicySnapshot;
+    teamRunId: string | null;
+    teamRoleId: string | null;
+  },
+): void {
+  state.policy = input.policy;
+  state.runId = input.teamRunId ?? state.runId;
+  state.roleId = input.teamRoleId ?? state.roleId;
+  state.target = input.input.target;
+  state.counters.messageTokenEstimate += input.tokenEstimate;
+  state.counters.tokensSinceLastUpdate += input.tokenEstimate;
+  if (input.input.event.type === "UserPromptSubmit") {
+    state.counters.userPromptCount += 1;
+  }
+  if (isToolCallCompletion(input.input.event.type)) {
+    state.counters.toolCallCount += 1;
+    state.counters.toolCallsSinceLastUpdate += 1;
+  }
+  if (isAssistantTurn(input.input.event.type)) {
+    state.counters.assistantTurnCount += 1;
+  }
+  if (isFailureSignal(input.input.event.type, input.input.event.payload.summary)) {
+    state.counters.failureSignalCount += 1;
+  }
+  if (isVerificationSignal(input.input.event.payload.summary)) {
+    state.counters.verificationSignalCount += 1;
+  }
+
+  if (state.activeSpan !== null) {
+    state.activeSpan.lastEventId = input.input.event.eventId;
+    state.activeSpan.tokenEstimate += input.tokenEstimate;
+    if (input.input.event.type === "UserPromptSubmit") {
+      state.activeSpan.userPromptCount += 1;
+    }
+    if (isWorkStartedEvent(input.input.event.type)) {
+      state.activeSpan.workStarted = true;
+    }
+    if (isToolCallCompletion(input.input.event.type)) {
+      state.activeSpan.toolCallCount += 1;
+    }
+    if (isSpanCloseEvent(input.input.event.type)) {
+      state.activeSpan.status = "closed";
+      state.activeSpan.closedAt = input.now;
+    }
+  }
+}
+
+export function decideSegment(input: {
+  previous: SessionMemoryStateV1;
+  next: SessionMemoryStateV1;
+  input: UpdateSessionMemoryInput;
+  rawEvent: SessionMemoryRawEventV1;
+  currentSignal: SessionMemorySignal;
+  tokenEstimate: number;
+}): { reason: SessionMemorySegmentReason; strength: SessionMemorySignalStrength } | null {
+  const eventType = input.input.event.type;
+  const promptText = extractPromptText(input.input.rawPayload);
+  if (eventType === "UserPromptSubmit") {
+    if (input.previous.counters.userPromptCount === 0) return null;
+    if (EXPLICIT_MEMORY_PATTERN.test(promptText)) {
+      return { reason: "explicit-memory-intent", strength: "strong" };
+    }
+    if (input.previous.activeSpan?.workStarted === true) {
+      reopenInterruptedSpan(
+        input.next,
+        input.input.event.eventId,
+        input.input.receivedAt ?? input.rawEvent.receivedAt,
+      );
+      return { reason: "user-interruption", strength: "strong" };
+    }
+    return { reason: "intent-refinement", strength: "strong" };
+  }
+
+  if (eventType === "PermissionDenied") return { reason: "permission-denied", strength: "strong" };
+  if (isFailureSignal(eventType, input.input.event.payload.summary)) {
+    return { reason: "failure-signal", strength: "strong" };
+  }
+  if (
+    input.previous.counters.failureSignalCount > 0 &&
+    isVerificationSignal(input.input.event.payload.summary)
+  ) {
+    return { reason: "verification-after-fix", strength: "strong" };
+  }
+
+  if (
+    input.next.lastMemoryUpdateAt === null &&
+    input.next.counters.messageTokenEstimate >= input.next.policy.minimumMessageTokensToInit
+  ) {
+    return { reason: "session-memory-threshold", strength: "normal" };
+  }
+  if (
+    input.next.lastMemoryUpdateAt !== null &&
+    (input.next.counters.tokensSinceLastUpdate >= input.next.policy.minimumTokensBetweenUpdate ||
+      input.next.counters.toolCallsSinceLastUpdate >= input.next.policy.toolCallsBetweenUpdates)
+  ) {
+    return { reason: "session-memory-threshold", strength: "normal" };
+  }
+  return null;
+}
+
+export function cloneState(state: SessionMemoryStateV1): SessionMemoryStateV1 {
+  return JSON.parse(JSON.stringify(state)) as SessionMemoryStateV1;
+}
+
+export function appendBoundedSignal(
+  signals: SessionMemorySignal[],
+  signal: SessionMemorySignal,
+): SessionMemorySignal[] {
+  return [...signals, signal].slice(-50);
+}
+
+function reopenInterruptedSpan(state: SessionMemoryStateV1, eventId: string, now: string): void {
+  if (state.activeSpan !== null) {
+    state.activeSpan.status = "interrupted";
+    state.activeSpan.closedAt = now;
+  }
+  state.activeSpan = {
+    spanId: createStableId("span", [state.projectKey, state.sessionKey, eventId, now]),
+    status: "open",
+    openedAt: now,
+    closedAt: null,
+    openedByEventId: eventId,
+    lastEventId: eventId,
+    userPromptCount: 1,
+    toolCallCount: 0,
+    workStarted: false,
+    tokenEstimate: 0,
+  };
+}
+
+function isWorkStartedEvent(eventType: string): boolean {
+  return [
+    "PreToolUse",
+    "PermissionRequest",
+    "PostToolUse",
+    "PostToolUseFailure",
+    "PostToolBatch",
+    "TaskCreated",
+    "SubagentStart",
+  ].includes(eventType);
+}
+
+function isToolCallCompletion(eventType: string): boolean {
+  return eventType === "PostToolUse" || eventType === "PostToolUseFailure";
+}
+
+function isAssistantTurn(eventType: string): boolean {
+  return ["PostToolUse", "PostToolUseFailure", "PostToolBatch", "Stop", "StopFailure"].includes(
+    eventType,
+  );
+}
+
+function isSpanCloseEvent(eventType: string): boolean {
+  return ["TaskCompleted", "Stop", "SessionEnd", "SubagentStop"].includes(eventType);
+}
+
+function isFailureSignal(eventType: string, summary: string): boolean {
+  return (
+    eventType === "PostToolUseFailure" ||
+    eventType === "StopFailure" ||
+    FAILURE_PATTERN.test(summary)
+  );
+}
+
+function isVerificationSignal(summary: string): boolean {
+  return VERIFICATION_PATTERN.test(summary);
+}
+
+function extractPromptText(payload: Record<string, unknown>): string {
+  return (
+    optionalString(payload.prompt) ??
+    optionalString(payload.user_prompt) ??
+    optionalString(payload.userPrompt) ??
+    optionalString(payload.query) ??
+    optionalString(payload.message) ??
+    optionalString(payload.text) ??
+    ""
+  );
+}

package/src/evolution/evidence/session-memory/storage.ts

@@ -0,0 +1,145 @@
+import { appendFile, mkdir, readFile, writeFile } from "node:fs/promises";
+import { dirname } from "node:path";
+import { isNotFoundError } from "../../../utils/index.ts";
+import { SENSITIVE_TEXT_PATTERN } from "./constants.ts";
+import type {
+  SessionEvidenceSegmentV1,
+  SessionMemoryCursorV1,
+  SessionMemoryIndexSegment,
+  SessionMemoryPaths,
+  SessionMemoryRawEventV1,
+  SessionMemoryStateV1,
+} from "./types.ts";
+
+export async function readSessionState(path: string): Promise<SessionMemoryStateV1 | null> {
+  try {
+    return JSON.parse(await readFile(path, "utf8")) as SessionMemoryStateV1;
+  } catch (error) {
+    if (isNotFoundError(error)) return null;
+    throw error;
+  }
+}
+
+export async function readSessionCursor(
+  path: string,
+  sessionKey: string,
+  sourcePath: string,
+  now: string,
+): Promise<SessionMemoryCursorV1> {
+  try {
+    return JSON.parse(await readFile(path, "utf8")) as SessionMemoryCursorV1;
+  } catch (error) {
+    if (!isNotFoundError(error)) throw error;
+    return {
+      schemaVersion: 1,
+      kind: "session-memory-cursor",
+      sessionKey,
+      traceRefId: null,
+      sourcePath,
+      lastCapturedOffset: null,
+      lastCapturedLine: null,
+      lastCapturedEventId: null,
+      updatedAt: now,
+    };
+  }
+}
+
+export async function appendRawEvent(
+  path: string,
+  event: SessionMemoryRawEventV1,
+): Promise<{ lineNumber: number }> {
+  const existingLineCount = await countJsonlLines(path);
+  await mkdir(dirname(path), { recursive: true });
+  await appendFile(path, `${JSON.stringify(event)}\n`, "utf8");
+  return { lineNumber: existingLineCount + 1 };
+}
+
+export async function readLineRange(
+  path: string,
+  fromLine: number,
+  toLine: number,
+): Promise<{ content: string; truncated: boolean; secretsDetected: boolean }> {
+  const text = await readFile(path, "utf8");
+  const lines = text.split("\n").filter((line) => line.trim() !== "");
+  const selected = lines.slice(Math.max(0, fromLine - 1), toLine);
+  let secretsDetected = false;
+  for (const line of selected) {
+    try {
+      const parsed = JSON.parse(line) as Partial<SessionMemoryRawEventV1>;
+      secretsDetected = secretsDetected || parsed.secretsDetected === true;
+    } catch {
+      secretsDetected = secretsDetected || SENSITIVE_TEXT_PATTERN.test(line);
+    }
+  }
+  return { content: selected.join("\n"), truncated: false, secretsDetected };
+}
+
+export async function writeSessionIndex(
+  paths: SessionMemoryPaths,
+  state: SessionMemoryStateV1,
+  segment: SessionEvidenceSegmentV1,
+  now: string,
+): Promise<void> {
+  const existing = await readSessionIndex(paths.indexPath);
+  const segments = [
+    ...existing.segments.filter((item) => item.id !== segment.id),
+    {
+      id: segment.id,
+      reason: segment.reason,
+      strength: segment.strength,
+      createdAt: segment.createdAt,
+      reviewState: segment.lifecycle.reviewState,
+    },
+  ];
+  await writeJson(paths.indexPath, {
+    schemaVersion: 1,
+    kind: "session-memory-index",
+    projectKey: state.projectKey,
+    sessionKey: state.sessionKey,
+    updatedAt: now,
+    lastSegmentId: segment.id,
+    segments,
+  });
+}
+
+export async function writeJson(path: string, value: unknown): Promise<void> {
+  await mkdir(dirname(path), { recursive: true });
+  await writeFile(path, `${JSON.stringify(value, null, 2)}\n`, "utf8");
+}
+
+async function countJsonlLines(path: string): Promise<number> {
+  try {
+    const text = await readFile(path, "utf8");
+    return text.split("\n").filter((line) => line.trim() !== "").length;
+  } catch (error) {
+    if (isNotFoundError(error)) return 0;
+    throw error;
+  }
+}
+
+async function readSessionIndex(path: string): Promise<{ segments: SessionMemoryIndexSegment[] }> {
+  try {
+    const parsed = JSON.parse(await readFile(path, "utf8")) as {
+      segments?: unknown;
+    };
+    if (!Array.isArray(parsed.segments)) return { segments: [] };
+    return {
+      segments: parsed.segments.filter(isSessionIndexSegment),
+    };
+  } catch (error) {
+    if (isNotFoundError(error)) return { segments: [] };
+    throw error;
+  }
+}
+
+function isSessionIndexSegment(value: unknown): value is SessionMemoryIndexSegment {
+  if (typeof value !== "object" || value === null || Array.isArray(value)) return false;
+  const item = value as Record<string, unknown>;
+  return (
+    typeof item.id === "string" &&
+    typeof item.reason === "string" &&
+    typeof item.strength === "string" &&
+    typeof item.createdAt === "string" &&
+    typeof item.reviewState === "string"
+  );
+}