@evo-dev/core 0.0.1-alpha.2 → 0.0.1-alpha.20

package/src/evolution/evidence/session-memory/segment.ts

@@ -0,0 +1,216 @@
+import {
+  createStableId,
+  sanitizeStorageId,
+  sanitizeSummary,
+  sha256Hex,
+  truncateUtf8Tail,
+} from "../../../utils/index.ts";
+import { DEFAULT_MAX_RAW_EVENT_BYTES, VERIFICATION_PATTERN } from "./constants.ts";
+import {
+  detectSessionMemorySensitivity,
+  redactSessionMemoryCredentialText,
+  redactSessionMemoryCredentials,
+} from "./sensitivity.ts";
+import { readLineRange } from "./storage.ts";
+import type {
+  SessionEvidenceSegmentV1,
+  SessionMemoryCursorV1,
+  SessionMemoryPaths,
+  SessionMemoryRawEventV1,
+  SessionMemorySegmentReason,
+  SessionMemorySignal,
+  SessionMemorySignalStrength,
+  SessionMemoryStateV1,
+  UpdateSessionMemoryInput,
+} from "./types.ts";
+
+export function createRawEvent(
+  input: UpdateSessionMemoryInput,
+  now: string,
+): SessionMemoryRawEventV1 {
+  const originalRawJson = stringifyPayload(input.rawPayload);
+  const redactedPayload = redactSessionMemoryCredentials(input.rawPayload);
+  const redactedSummary = redactSessionMemoryCredentialText(input.event.payload.summary);
+  const rawJson = stringifyPayload(redactedPayload.value);
+  const truncated = truncateUtf8Tail(rawJson, DEFAULT_MAX_RAW_EVENT_BYTES);
+  const sensitivity = detectSessionMemorySensitivity(redactedPayload.value);
+  return {
+    schemaVersion: 1,
+    kind: "session-memory-raw-event",
+    eventId: input.event.eventId,
+    eventType: input.event.type,
+    occurredAt: input.event.time.occurredAt,
+    receivedAt: now,
+    summary: sanitizeSummary(redactedSummary.value),
+    rawPayloadJson: truncated.content,
+    rawPayloadByteLength: Buffer.byteLength(originalRawJson, "utf8"),
+    rawPayloadTruncated: truncated.truncated,
+    secretsDetected: sensitivity.classification === "credential",
+    sensitivity: sensitivity.classification,
+    sensitivityReasons: sensitivity.reasons,
+    credentialRedactionApplied: redactedPayload.redacted || redactedSummary.redacted,
+  };
+}
+
+export function createSignal(
+  input: UpdateSessionMemoryInput,
+  rawEvent: SessionMemoryRawEventV1,
+  rawEventLine: number,
+): SessionMemorySignal {
+  return {
+    eventId: input.event.eventId,
+    eventType: input.event.type,
+    reason: "observed",
+    strength: "normal",
+    occurredAt: input.event.time.occurredAt,
+    summary: rawEvent.summary,
+    rawEventLine,
+  };
+}
+
+export async function createSegment(input: {
+  paths: SessionMemoryPaths;
+  state: SessionMemoryStateV1;
+  cursor: SessionMemoryCursorV1;
+  reason: SessionMemorySegmentReason;
+  strength: SessionMemorySignalStrength;
+  signals: SessionMemorySignal[];
+  currentLine: number;
+  now: string;
+}): Promise<SessionEvidenceSegmentV1> {
+  const fromLine = (input.cursor.lastCapturedLine ?? 0) + 1;
+  const toLine = input.currentLine;
+  const raw = await readLineRange(input.paths.eventsPath, fromLine, toLine);
+  const truncated = truncateUtf8Tail(raw.content, input.state.policy.maxRawSegmentBytes);
+  const signalSummaries = input.signals.map((signal) => signal.summary).filter(Boolean);
+  const failures = input.signals
+    .filter((signal) => signal.reason === "failure-signal")
+    .map((signal) => signal.summary);
+  const verifications = input.signals
+    .filter(
+      (signal) =>
+        signal.reason === "verification-after-fix" || VERIFICATION_PATTERN.test(signal.summary),
+    )
+    .map((signal) => signal.summary);
+  const segmentId = createStableId("segment", [
+    input.state.projectKey,
+    input.state.sessionKey,
+    input.reason,
+    String(fromLine),
+    String(toLine),
+    input.signals.at(-1)?.eventId ?? input.now,
+  ]);
+  const rawSha256 = sha256Hex(truncated.content);
+  return {
+    schemaVersion: 1,
+    kind: "session-evidence-segment",
+    id: segmentId,
+    projectKey: input.state.projectKey,
+    runId: input.state.runId,
+    roleId: input.state.roleId,
+    sessionKey: input.state.sessionKey,
+    target: input.state.target,
+    createdAt: input.now,
+    reason: input.reason,
+    strength: input.strength,
+    retention: {
+      policyDays: input.state.policy.retentionDays,
+      expiresAt: addDays(input.now, input.state.policy.retentionDays),
+      rawState: "available",
+      rawPurgedAt: null,
+      originalRawSha256: rawSha256,
+    },
+    source: {
+      traceRefId: input.cursor.traceRefId,
+      sourcePath: input.paths.eventsPath,
+      fromOffset: null,
+      toOffset: null,
+      fromLine,
+      toLine,
+      fromEventId: input.signals[0]?.eventId ?? null,
+      toEventId: input.signals.at(-1)?.eventId ?? null,
+    },
+    signals: input.signals,
+    rawExcerpt: {
+      stored: true,
+      encoding: "utf8",
+      content: truncated.content,
+      truncated: raw.truncated || truncated.truncated,
+      byteLength: Buffer.byteLength(truncated.content, "utf8"),
+      sha256: rawSha256,
+    },
+    normalized: {
+      summary: createSegmentSummary(input.reason, input.strength, signalSummaries),
+      userIntent: firstUserPromptSummary(input.signals),
+      intentDelta: intentDeltaSummary(input.signals),
+      decisions: [],
+      failures,
+      verifications,
+      touchedTools: collectTouchedTools(raw.content),
+    },
+    privacy: {
+      localOnly: true,
+      rawPromptStored: input.signals.some((signal) => signal.eventType === "UserPromptSubmit"),
+      rawOutputStored: input.signals.some((signal) =>
+        ["PostToolUse", "PostToolUseFailure", "PostToolBatch"].includes(signal.eventType),
+      ),
+      sourceContentStored: true,
+      secretsDetected: raw.secretsDetected,
+      sensitivity: raw.sensitivity,
+      sensitivityReasons: raw.sensitivityReasons,
+      redactionApplied: raw.redactionApplied,
+      externalUploadAllowed: false,
+    },
+    lifecycle: {
+      status: "captured",
+      reviewState: "not-required",
+      consumedByBatchIds: [],
+    },
+  };
+}
+
+function addDays(value: string, days: number): string {
+  const timestamp = Date.parse(value);
+  if (!Number.isFinite(timestamp)) throw new Error("Invalid Session Evidence retention timestamp.");
+  return new Date(timestamp + days * 24 * 60 * 60 * 1000).toISOString();
+}
+
+export function estimateTokenCount(rawPayloadJson: string, summary: string): number {
+  return Math.max(1, Math.ceil((rawPayloadJson.length + summary.length) / 4));
+}
+
+function createSegmentSummary(
+  reason: SessionMemorySegmentReason,
+  strength: SessionMemorySignalStrength,
+  summaries: string[],
+): string {
+  const suffix = summaries.length === 0 ? "No event summary." : summaries.slice(-3).join(" / ");
+  return sanitizeSummary(`Session segment ${reason} (${strength}): ${suffix}`);
+}
+
+function firstUserPromptSummary(signals: SessionMemorySignal[]): string | null {
+  return signals.find((signal) => signal.eventType === "UserPromptSubmit")?.summary ?? null;
+}
+
+function intentDeltaSummary(signals: SessionMemorySignal[]): string | null {
+  const prompts = signals.filter((signal) => signal.eventType === "UserPromptSubmit");
+  if (prompts.length < 2) return null;
+  return prompts.at(-1)?.summary ?? null;
+}
+
+function collectTouchedTools(rawContent: string): string[] {
+  const tools = new Set<string>();
+  for (const match of rawContent.matchAll(/"tool(?:Name|_name)"\s*:\s*"([^"]+)"/g)) {
+    const tool = match[1]?.trim();
+    if (tool !== undefined && tool !== "") tools.add(sanitizeStorageId(tool, "tool"));
+  }
+  return [...tools].slice(0, 20);
+}
+
+function stringifyPayload(payload: Record<string, unknown>): string {
+  try {
+    return JSON.stringify(payload);
+  } catch {
+    return JSON.stringify({ unserializable: true });
+  }
+}

package/src/evolution/evidence/session-memory/semantic-packet.ts

@@ -0,0 +1,408 @@
+import { sha256Hex } from "../../../utils/index.ts";
+import {
+  detectSessionMemorySensitivity,
+  redactSessionMemoryCredentialText,
+} from "./sensitivity.ts";
+import type {
+  HistoricalImportStoredRecordV1,
+  SemanticEvidencePacketV1,
+  SessionEvidenceSegmentV1,
+} from "./types.ts";
+
+const DEFAULT_MAX_PACKET_CHARS = 48_000;
+const MAX_CONVERSATION_MESSAGES = 12;
+const MAX_TOOL_EVIDENCE = 12;
+const MAX_TOOL_EXCERPT_CHARS = 4_000;
+
+interface ParsedHistoricalRecord {
+  stored: HistoricalImportStoredRecordV1;
+  canonical: Record<string, unknown> | null;
+  redacted: boolean;
+  order: number;
+}
+
+interface ConversationCandidate {
+  role: "user" | "assistant";
+  text: string;
+  recordKeyHash: string;
+  order: number;
+  priority: number;
+}
+
+interface ToolCallCandidate {
+  recordKeyHash: string;
+  callId: string | null;
+  toolName: string;
+  input: string | null;
+  inputTruncated: boolean;
+  order: number;
+}
+
+export function createHistoricalSemanticEvidencePacket(input: {
+  segment: SessionEvidenceSegmentV1;
+  maxChars?: number;
+}): SemanticEvidencePacketV1 {
+  if (input.segment.origin?.kind !== "historical-import") {
+    throw new Error("Semantic historical packet requires historical import origin.");
+  }
+  if (!input.segment.rawExcerpt.stored || input.segment.retention?.rawState === "purged") {
+    throw new Error("Historical raw evidence is no longer available.");
+  }
+  const maxChars = input.maxChars ?? DEFAULT_MAX_PACKET_CHARS;
+  if (!Number.isSafeInteger(maxChars) || maxChars < 1_000) {
+    throw new Error("Semantic evidence packet budget is too small.");
+  }
+
+  const records = parseHistoricalRecords(input.segment.rawExcerpt.content);
+  const conversationCandidates: ConversationCandidate[] = [];
+  const toolCalls = new Map<string, ToolCallCandidate>();
+  const anonymousToolCalls: ToolCallCandidate[] = [];
+  const toolResults: Array<{
+    recordKeyHash: string;
+    callId: string | null;
+    output: string | null;
+    outputTruncated: boolean;
+    status: string | null;
+    exitCode: number | null;
+    order: number;
+  }> = [];
+  let redactionCount = 0;
+  let sourceTruncated = input.segment.rawExcerpt.truncated;
+
+  for (const record of records) {
+    redactionCount += record.redacted ? 1 : 0;
+    sourceTruncated ||= record.stored.canonicalJsonTruncated;
+    const content = readText(record.canonical?.content);
+    if (
+      content !== null &&
+      (record.stored.recordType === "user" || record.stored.recordType === "assistant")
+    ) {
+      const role = record.stored.recordType;
+      conversationCandidates.push({
+        role,
+        text: content,
+        recordKeyHash: record.stored.recordKeyHash,
+        order: record.order,
+        priority: conversationPriority(role, content, record.order, records.length),
+      });
+    }
+    if (record.stored.recordType === "assistant-tool-call") {
+      const calls = Array.isArray(record.canonical?.tool_calls) ? record.canonical.tool_calls : [];
+      for (const value of calls) {
+        const call = asRecord(value);
+        if (call === null) continue;
+        const toolName = readText(call.name) ?? "unknown-tool";
+        const callId = readText(call.id);
+        const inputText = stringifyBoundedToolValue(call.args);
+        const candidate: ToolCallCandidate = {
+          recordKeyHash: record.stored.recordKeyHash,
+          callId,
+          toolName,
+          input: inputText.value,
+          inputTruncated: inputText.truncated,
+          order: record.order,
+        };
+        if (callId === null) anonymousToolCalls.push(candidate);
+        else toolCalls.set(callId, candidate);
+      }
+    }
+    if (record.stored.recordType === "tool") {
+      const output = stringifyBoundedToolValue(record.canonical?.content);
+      toolResults.push({
+        recordKeyHash: record.stored.recordKeyHash,
+        callId: readText(record.canonical?.tool_call_id),
+        output: output.value,
+        outputTruncated: output.truncated,
+        status: readText(record.canonical?.status),
+        exitCode: readInteger(record.canonical?.exit_code, record.canonical?.exitCode),
+        order: record.order,
+      });
+    }
+  }
+
+  const conversation = selectConversation(conversationCandidates, maxChars);
+  const toolEvidence = toolResults.flatMap((result) => {
+    const call =
+      (result.callId === null ? undefined : toolCalls.get(result.callId)) ??
+      anonymousToolCalls.filter((candidate) => candidate.order <= result.order).at(-1);
+    const reason = classifyToolEvidence(call, result);
+    if (reason === null) return [];
+    return [
+      {
+        reason,
+        toolName: call?.toolName ?? "unknown-tool",
+        safeInputExcerpt: call?.input ?? null,
+        outputExcerpt: result.output,
+        status: result.status,
+        exitCode: result.exitCode,
+        inputTruncated: call?.inputTruncated ?? false,
+        outputTruncated: result.outputTruncated,
+        order: result.order,
+        recordKeys: [...(call === undefined ? [] : [call.recordKeyHash]), result.recordKeyHash],
+      },
+    ];
+  });
+  toolEvidence.sort(
+    (left, right) =>
+      toolReasonPriority(left.reason) - toolReasonPriority(right.reason) ||
+      left.order - right.order,
+  );
+
+  const selectedToolEvidence = toolEvidence.slice(0, MAX_TOOL_EVIDENCE);
+  const selectedKeys = new Set(conversation.map((message) => message.recordKeyHash));
+  for (const evidence of selectedToolEvidence) {
+    for (const key of evidence.recordKeys) selectedKeys.add(key);
+  }
+  const touchedPaths = collectTouchedPaths([
+    ...conversation.map((message) => message.text),
+    ...selectedToolEvidence.flatMap((evidence) => [
+      evidence.safeInputExcerpt ?? "",
+      evidence.outputExcerpt ?? "",
+    ]),
+  ]);
+  const packet: SemanticEvidencePacketV1 = {
+    schemaVersion: 1,
+    kind: "semantic-evidence-packet",
+    projectKey: input.segment.projectKey,
+    segmentId: input.segment.id,
+    conversation: conversation.map(({ role, text, recordKeyHash }) => ({
+      role,
+      text,
+      recordKeyHash,
+    })),
+    toolEvidence: selectedToolEvidence.map(
+      ({ order: _order, recordKeys: _recordKeys, ...evidence }) => evidence,
+    ),
+    touchedPaths,
+    selection: {
+      selectedRecords: selectedKeys.size,
+      droppedRecords: Math.max(0, records.length - selectedKeys.size),
+      redactionCount,
+      truncated:
+        sourceTruncated ||
+        conversation.length < conversationCandidates.length ||
+        selectedToolEvidence.length < toolEvidence.length ||
+        selectedToolEvidence.some(
+          (evidence) => evidence.inputTruncated || evidence.outputTruncated,
+        ),
+    },
+    privacy: {
+      containsPrivateContent: true,
+      credentialRedacted: true,
+      rawSegmentIncluded: false,
+    },
+  };
+  enforcePacketBudget(packet, maxChars);
+  if (detectSessionMemorySensitivity(packet).classification === "credential") {
+    throw new Error("Semantic evidence packet still contains a credential.");
+  }
+  return packet;
+}
+
+function parseHistoricalRecords(content: string): ParsedHistoricalRecord[] {
+  return content
+    .split("\n")
+    .filter((line) => line.trim() !== "")
+    .map((line, order) => {
+      let value: unknown;
+      try {
+        value = JSON.parse(line) as unknown;
+      } catch {
+        throw new Error("Historical Session Evidence contains invalid JSONL.");
+      }
+      const record = asRecord(value);
+      if (
+        record === null ||
+        record.schemaVersion !== 1 ||
+        record.kind !== "historical-import-record" ||
+        typeof record.recordKeyHash !== "string" ||
+        typeof record.recordType !== "string" ||
+        typeof record.canonicalJson !== "string" ||
+        typeof record.canonicalJsonTruncated !== "boolean"
+      ) {
+        throw new Error("Historical Session Evidence record is invalid.");
+      }
+      const redacted = redactSessionMemoryCredentialText(record.canonicalJson);
+      let canonical: Record<string, unknown> | null = null;
+      if (!record.canonicalJsonTruncated) {
+        try {
+          canonical = asRecord(JSON.parse(redacted.value) as unknown);
+        } catch {
+          throw new Error("Historical canonical evidence contains invalid JSON.");
+        }
+      }
+      return {
+        stored: {
+          schemaVersion: 1,
+          kind: "historical-import-record",
+          recordKeyHash: record.recordKeyHash,
+          recordType: expectStoredRecordType(record.recordType),
+          canonicalJson: redacted.value,
+          canonicalJsonTruncated: record.canonicalJsonTruncated,
+        },
+        canonical,
+        redacted: redacted.redacted,
+        order,
+      };
+    });
+}
+
+function selectConversation(
+  candidates: ConversationCandidate[],
+  maxChars: number,
+): ConversationCandidate[] {
+  const selected: ConversationCandidate[] = [];
+  let used = 0;
+  for (const candidate of [...candidates].sort(
+    (left, right) => right.priority - left.priority || left.order - right.order,
+  )) {
+    if (selected.length >= MAX_CONVERSATION_MESSAGES) break;
+    const safe = redactSessionMemoryCredentialText(candidate.text);
+    const remaining = Math.max(0, maxChars - used);
+    if (remaining < 64) break;
+    const text = safe.value.slice(0, Math.min(safe.value.length, remaining));
+    selected.push({ ...candidate, text });
+    used += text.length;
+  }
+  return selected.sort((left, right) => left.order - right.order);
+}
+
+function conversationPriority(
+  role: "user" | "assistant",
+  text: string,
+  order: number,
+  total: number,
+): number {
+  let priority = role === "user" ? 20 : 10;
+  if (/\b(?:instead|correction|actually|must|should|do not|don't|wrong|require)\b/iu.test(text)) {
+    priority += 40;
+  }
+  if (order >= total - 2) priority += 15;
+  return priority;
+}
+
+function classifyToolEvidence(
+  call: ToolCallCandidate | undefined,
+  result: {
+    output: string | null;
+    status: string | null;
+    exitCode: number | null;
+  },
+): SemanticEvidencePacketV1["toolEvidence"][number]["reason"] | null {
+  const comparable = `${call?.toolName ?? ""} ${call?.input ?? ""} ${result.output ?? ""} ${
+    result.status ?? ""
+  }`;
+  if (/\b(?:permission|denied|unauthorized|forbidden)\b/iu.test(comparable)) {
+    return "permission";
+  }
+  if (/\b(?:rollback|revert|restore)\b/iu.test(comparable)) return "rollback";
+  const failureComparable = comparable.replace(/\b0\s+fail(?:ed|ure|ures)?\b/giu, "");
+  if (
+    (result.exitCode !== null && result.exitCode !== 0) ||
+    /\b(?:fail(?:ed|ure)?|error|exception|non-zero)\b/iu.test(failureComparable)
+  ) {
+    return "failure";
+  }
+  if (
+    /\b(?:test|lint|typecheck|build|verify|check)\b/iu.test(comparable) &&
+    !/\b(?:fail(?:ed|ure)?|error)\b/iu.test(failureComparable)
+  ) {
+    return "verification";
+  }
+  return null;
+}
+
+function stringifyBoundedToolValue(value: unknown): {
+  value: string | null;
+  truncated: boolean;
+} {
+  if (value === null || value === undefined) return { value: null, truncated: false };
+  const raw = typeof value === "string" ? value : JSON.stringify(value);
+  const safe = redactSessionMemoryCredentialText(raw);
+  if (safe.value.length <= MAX_TOOL_EXCERPT_CHARS) {
+    return { value: safe.value, truncated: false };
+  }
+  return {
+    value: `${safe.value.slice(0, MAX_TOOL_EXCERPT_CHARS - 1)}…`,
+    truncated: true,
+  };
+}
+
+function collectTouchedPaths(values: string[]): string[] {
+  const paths = new Set<string>();
+  for (const value of values) {
+    for (const match of value.matchAll(
+      /(?:^|[\s"'`(])((?:(?:\/|\.{1,2}\/)[A-Za-z0-9._~@%+,:=-]+|[A-Za-z0-9._~@%+=-]+\/[A-Za-z0-9._~@%+,:=-]+)(?:\/[A-Za-z0-9._~@%+,:=-]+)*)/gu,
+    )) {
+      const path = match[1];
+      if (path !== undefined) paths.add(path.slice(0, 500));
+      if (paths.size >= 20) return [...paths];
+    }
+  }
+  return [...paths];
+}
+
+function enforcePacketBudget(packet: SemanticEvidencePacketV1, maxChars: number): void {
+  while (JSON.stringify(packet).length > maxChars && packet.toolEvidence.length > 0) {
+    packet.toolEvidence.pop();
+    packet.selection.truncated = true;
+  }
+  while (JSON.stringify(packet).length > maxChars && packet.conversation.length > 1) {
+    packet.conversation.splice(1, 1);
+    packet.selection.truncated = true;
+  }
+  if (JSON.stringify(packet).length > maxChars) {
+    const message = packet.conversation[0];
+    if (message !== undefined) {
+      const over = JSON.stringify(packet).length - maxChars;
+      message.text = `${message.text.slice(0, Math.max(0, message.text.length - over - 1))}…`;
+      packet.selection.truncated = true;
+    }
+  }
+  if (JSON.stringify(packet).length > maxChars) {
+    throw new Error("Semantic evidence packet could not fit its bounded budget.");
+  }
+}
+
+function toolReasonPriority(
+  reason: SemanticEvidencePacketV1["toolEvidence"][number]["reason"],
+): number {
+  if (reason === "failure") return 0;
+  if (reason === "permission") return 1;
+  if (reason === "rollback") return 2;
+  return 3;
+}
+
+function expectStoredRecordType(value: string): HistoricalImportStoredRecordV1["recordType"] {
+  if (
+    value === "user" ||
+    value === "reasoning" ||
+    value === "assistant" ||
+    value === "assistant-tool-call" ||
+    value === "tool"
+  ) {
+    return value;
+  }
+  throw new Error("Historical Session Evidence record type is invalid.");
+}
+
+function readText(value: unknown): string | null {
+  return typeof value === "string" && value.trim() !== "" ? value : null;
+}
+
+function readInteger(...values: unknown[]): number | null {
+  for (const value of values) {
+    if (typeof value === "number" && Number.isInteger(value)) return value;
+  }
+  return null;
+}
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+  return value !== null && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}
+
+export function createSemanticPacketMessageKey(value: string): string {
+  return sha256Hex(value);
+}