@evo-dev/core 0.0.1-alpha.11 → 0.0.1-alpha.13

package/src/evolution/imports/materialize.ts

@@ -0,0 +1,640 @@
+import {
+  createStableId,
+  sanitizeStorageId,
+  sanitizeSummary,
+  sha256Hex,
+} from "../../utils/index.ts";
+import {
+  type HistoricalImportStoredRecordV1,
+  type SessionEvidenceSegmentV1,
+  type SessionMemoryPolicySnapshot,
+  type SessionMemorySensitivity,
+  type SessionMemorySensitivityReason,
+  type SessionMemorySignal,
+  detectSessionMemorySensitivity,
+  redactSessionMemoryCredentials,
+} from "../evidence/session-memory/index.ts";
+import {
+  createTrajectoryImportConfigHash,
+  createTrajectoryImportPreviewId,
+  createTrajectoryImportStableRecordKeyHash,
+} from "./diff.ts";
+import type {
+  CanonicalImportRecordType,
+  CanonicalImportSnapshot,
+  TrajectoryImportPreviewBuildResult,
+  TrajectoryImportPreviewV1,
+  TrajectoryImportPreviousSnapshotV1,
+  TrajectoryImportSource,
+} from "./types.ts";
+
+export const TRAJECTORY_IMPORT_MATERIALIZER_VERSION = 1 as const;
+const MIN_HISTORICAL_SEGMENT_BYTES = 1_024;
+const MAX_HISTORICAL_SIGNALS = 120;
+
+interface PreparedCanonicalRecord {
+  recordKeyHash: string;
+  recordType: CanonicalImportRecordType;
+  canonicalJson: string;
+  canonicalValue: unknown;
+  credentialRedacted: boolean;
+}
+
+export interface PreparedHistoricalImportSegment {
+  id: string;
+  firstRecordKeyHash: string;
+  lastRecordKeyHash: string;
+  recordCount: number;
+  rawExcerpt: {
+    content: string;
+    byteLength: number;
+    sha256: string;
+    truncated: boolean;
+  };
+  normalized: SessionEvidenceSegmentV1["normalized"];
+  signals: SessionMemorySignal[];
+  sensitivity: SessionMemorySensitivity;
+  sensitivityReasons: SessionMemorySensitivityReason[];
+  redactionApplied: boolean;
+}
+
+export interface TrajectoryImportMaterializationPlan {
+  preview: TrajectoryImportPreviewV1;
+  comparisonSnapshot: TrajectoryImportPreviousSnapshotV1;
+  policy: SessionMemoryPolicySnapshot;
+  preparedSegments: PreparedHistoricalImportSegment[];
+}
+
+export interface CreateTrajectoryImportMaterializationPlanInput {
+  snapshot: CanonicalImportSnapshot;
+  previewBuild: TrajectoryImportPreviewBuildResult;
+  previous?: TrajectoryImportPreviousSnapshotV1 | null;
+  policy: SessionMemoryPolicySnapshot;
+  automation: {
+    knowledge: boolean;
+    semanticKnowledge: boolean;
+  };
+}
+
+export function createTrajectoryImportMaterializationPlan(
+  input: CreateTrajectoryImportMaterializationPlanInput,
+): TrajectoryImportMaterializationPlan {
+  const blockers = new Set(input.previewBuild.preview.blockers);
+  if (!input.policy.enabled) blockers.add("session-memory-disabled");
+  if (!input.policy.storeRawSegments) blockers.add("session-memory-raw-storage-disabled");
+  if (input.policy.maxRawSegmentBytes < MIN_HISTORICAL_SEGMENT_BYTES) {
+    blockers.add("materializer-byte-budget-too-small");
+  }
+
+  const prepared =
+    blockers.size === 0
+      ? prepareHistoricalSegments({
+          snapshot: input.snapshot,
+          comparisonSnapshot: input.previewBuild.comparisonSnapshot,
+          previous: input.previous ?? null,
+          maxSegmentBytes: input.policy.maxRawSegmentBytes,
+          blockers,
+        })
+      : { segments: [], selectedRecords: 0, credentialRedactions: 0 };
+  const automationEligible = input.automation.knowledge && input.automation.semanticKnowledge;
+  const { id: _basePreviewId, ...base } = input.previewBuild.preview;
+  const previewWithoutId: Omit<TrajectoryImportPreviewV1, "id"> = {
+    ...base,
+    contracts: {
+      ...base.contracts,
+      materializerVersion: TRAJECTORY_IMPORT_MATERIALIZER_VERSION,
+      materializerConfigHash: createTrajectoryImportConfigHash({
+        schemaVersion: 1,
+        enabled: input.policy.enabled,
+        storeRawSegments: input.policy.storeRawSegments,
+        maxRawSegmentBytes: input.policy.maxRawSegmentBytes,
+        retentionDays: input.policy.retentionDays,
+      }),
+    },
+    materialization: {
+      selectedRecords: prepared.selectedRecords,
+      estimatedSegments: prepared.segments.length,
+      truncatedSegments: prepared.segments.filter((segment) => segment.rawExcerpt.truncated).length,
+      credentialRedactions: prepared.credentialRedactions,
+      estimatedTriggers: prepared.segments.length,
+      retentionDays: input.policy.retentionDays,
+    },
+    downstream: {
+      semanticProcessing: automationEligible ? "automation-eligible" : "queue-only",
+      automationKnowledge: input.automation.knowledge,
+      automationSemanticKnowledge: input.automation.semanticKnowledge,
+      mayInvokeConfiguredProviderAfterApply: automationEligible,
+    },
+    blockers: [...blockers].sort(),
+    canApply: base.projectKey !== null && blockers.size === 0,
+  };
+  const { schemaVersion, kind, ...previewFields } = previewWithoutId;
+  const preview: TrajectoryImportPreviewV1 = {
+    schemaVersion,
+    kind,
+    id: createTrajectoryImportPreviewId(previewWithoutId),
+    ...previewFields,
+  };
+  return {
+    preview,
+    comparisonSnapshot: input.previewBuild.comparisonSnapshot,
+    policy: input.policy,
+    preparedSegments: prepared.segments,
+  };
+}
+
+export function createHistoricalSessionEvidenceSegments(input: {
+  plan: TrajectoryImportMaterializationPlan;
+  createdAt: string;
+}): SessionEvidenceSegmentV1[] {
+  const projectKey = input.plan.preview.projectKey;
+  if (projectKey === null) {
+    throw new Error("Historical materialization requires a projectKey.");
+  }
+  const createdAt = normalizeIsoTimestamp(input.createdAt);
+  const expiresAt = addDays(createdAt, input.plan.policy.retentionDays);
+  const sessionKey = `historical-${input.plan.preview.sourceKey}`;
+  const target = sourceTarget(input.plan.preview.source);
+  return input.plan.preparedSegments.map((prepared) => ({
+    schemaVersion: 1,
+    kind: "session-evidence-segment",
+    id: prepared.id,
+    projectKey,
+    runId: null,
+    roleId: input.plan.preview.scope.roleId,
+    sessionKey,
+    target,
+    createdAt,
+    reason: "historical-import",
+    strength: "normal",
+    origin: {
+      kind: "historical-import",
+      importId: input.plan.preview.id,
+      generationId: input.plan.preview.generationId,
+      snapshotId: input.plan.preview.snapshotId,
+      sourceKey: input.plan.preview.sourceKey,
+      firstRecordKeyHash: prepared.firstRecordKeyHash,
+      lastRecordKeyHash: prepared.lastRecordKeyHash,
+      recordCount: prepared.recordCount,
+    },
+    retention: {
+      policyDays: input.plan.policy.retentionDays,
+      expiresAt,
+      rawState: "available",
+      rawPurgedAt: null,
+      originalRawSha256: prepared.rawExcerpt.sha256,
+    },
+    source: {
+      traceRefId: null,
+      sourcePath: null,
+      fromOffset: null,
+      toOffset: null,
+      fromLine: null,
+      toLine: null,
+      fromEventId: null,
+      toEventId: null,
+    },
+    signals: prepared.signals,
+    rawExcerpt: {
+      stored: true,
+      encoding: "utf8",
+      content: prepared.rawExcerpt.content,
+      truncated: prepared.rawExcerpt.truncated,
+      byteLength: prepared.rawExcerpt.byteLength,
+      sha256: prepared.rawExcerpt.sha256,
+    },
+    normalized: prepared.normalized,
+    privacy: {
+      localOnly: true,
+      rawPromptStored: prepared.signals.some((signal) => signal.eventType === "UserPromptSubmit"),
+      rawOutputStored: prepared.signals.some((signal) =>
+        ["PostToolUse", "PostToolUseFailure"].includes(signal.eventType),
+      ),
+      sourceContentStored: true,
+      secretsDetected: false,
+      sensitivity: prepared.sensitivity,
+      sensitivityReasons: prepared.sensitivityReasons,
+      redactionApplied: prepared.redactionApplied,
+      externalUploadAllowed: false,
+    },
+    lifecycle: {
+      status: "captured",
+      reviewState: "not-required",
+      consumedByBatchIds: [],
+    },
+  }));
+}
+
+function prepareHistoricalSegments(input: {
+  snapshot: CanonicalImportSnapshot;
+  comparisonSnapshot: TrajectoryImportPreviousSnapshotV1;
+  previous: TrajectoryImportPreviousSnapshotV1 | null;
+  maxSegmentBytes: number;
+  blockers: Set<string>;
+}): {
+  segments: PreparedHistoricalImportSegment[];
+  selectedRecords: number;
+  credentialRedactions: number;
+} {
+  if (input.snapshot.groups.length !== 1) {
+    input.blockers.add("materializer-source-group-mismatch");
+    return { segments: [], selectedRecords: 0, credentialRedactions: 0 };
+  }
+  const byKey = new Map<string, PreparedCanonicalRecord>();
+  for (const record of input.snapshot.groups[0]?.records ?? []) {
+    const recordKeyHash = createTrajectoryImportStableRecordKeyHash(
+      input.comparisonSnapshot.sourceKey,
+      record.recordId,
+    );
+    let canonicalValue: unknown;
+    try {
+      canonicalValue = JSON.parse(record.recordJson) as unknown;
+    } catch {
+      input.blockers.add("materializer-record-json-invalid");
+      continue;
+    }
+    const redacted = redactSessionMemoryCredentials(canonicalValue);
+    const canonicalJson = JSON.stringify(redacted.value);
+    if (detectSessionMemorySensitivity(canonicalJson).classification === "credential") {
+      input.blockers.add("materializer-credential-remains");
+      continue;
+    }
+    byKey.set(recordKeyHash, {
+      recordKeyHash,
+      recordType: record.recordType,
+      canonicalJson,
+      canonicalValue: redacted.value,
+      credentialRedacted: redacted.redacted,
+    });
+  }
+
+  const ordered: PreparedCanonicalRecord[] = [];
+  for (const indexed of input.comparisonSnapshot.records) {
+    const record = byKey.get(indexed.stableRecordKeyHash);
+    if (record === undefined || record.recordType !== indexed.recordType) {
+      input.blockers.add("materializer-record-index-mismatch");
+      continue;
+    }
+    ordered.push(record);
+  }
+  if (input.blockers.size > 0) {
+    return { segments: [], selectedRecords: 0, credentialRedactions: 0 };
+  }
+
+  const previousKeys = new Set(
+    (input.previous?.records ?? []).map((record) => record.stableRecordKeyHash),
+  );
+  const newIndexes = ordered.flatMap((record, index) =>
+    previousKeys.has(record.recordKeyHash) ? [] : [index],
+  );
+  const windows = mergeWindows(
+    contiguousRuns(newIndexes).map(([start, end]) => expandConversationWindow(ordered, start, end)),
+  );
+  const segments: PreparedHistoricalImportSegment[] = [];
+  let selectedRecords = 0;
+  let credentialRedactions = 0;
+
+  for (const [start, end] of windows) {
+    const selected = ordered.slice(start, end + 1).filter((record) => record.recordType !== "meta");
+    const chunks = splitStoredRecords(selected, input.maxSegmentBytes, input.blockers);
+    for (const chunk of chunks) {
+      if (chunk.records.length === 0) continue;
+      const rawContent = chunk.lines.join("\n");
+      const first = chunk.records[0];
+      const last = chunk.records.at(-1);
+      if (first === undefined || last === undefined) continue;
+      const rawSha256 = sha256Hex(rawContent);
+      const id = createStableId("historical-segment", [
+        input.comparisonSnapshot.sourceKey,
+        input.comparisonSnapshot.generationId,
+        input.comparisonSnapshot.snapshotId,
+        first.recordKeyHash,
+        last.recordKeyHash,
+        rawSha256,
+      ]);
+      const sensitivity = detectSessionMemorySensitivity(rawContent);
+      if (sensitivity.classification === "credential") {
+        input.blockers.add("materializer-credential-remains");
+        continue;
+      }
+      const derived = deriveHistoricalMetadata(chunk.records, id);
+      segments.push({
+        id,
+        firstRecordKeyHash: first.recordKeyHash,
+        lastRecordKeyHash: last.recordKeyHash,
+        recordCount: chunk.records.length,
+        rawExcerpt: {
+          content: rawContent,
+          byteLength: Buffer.byteLength(rawContent, "utf8"),
+          sha256: rawSha256,
+          truncated: chunk.truncated,
+        },
+        normalized: derived.normalized,
+        signals: derived.signals,
+        sensitivity: sensitivity.classification,
+        sensitivityReasons: sensitivity.reasons,
+        redactionApplied: chunk.records.some((record) => record.credentialRedacted),
+      });
+      selectedRecords += chunk.records.length;
+      credentialRedactions += chunk.records.filter((record) => record.credentialRedacted).length;
+    }
+  }
+
+  if (input.blockers.size > 0) {
+    return { segments: [], selectedRecords: 0, credentialRedactions: 0 };
+  }
+  return { segments, selectedRecords, credentialRedactions };
+}
+
+function contiguousRuns(indexes: number[]): Array<[number, number]> {
+  const runs: Array<[number, number]> = [];
+  for (const index of indexes) {
+    const last = runs.at(-1);
+    if (last !== undefined && index === last[1] + 1) last[1] = index;
+    else runs.push([index, index]);
+  }
+  return runs;
+}
+
+function expandConversationWindow(
+  records: PreparedCanonicalRecord[],
+  start: number,
+  end: number,
+): [number, number] {
+  let expandedStart = start;
+  for (let index = start; index >= 0; index -= 1) {
+    if (records[index]?.recordType === "user") {
+      expandedStart = index;
+      break;
+    }
+  }
+  let expandedEnd = end;
+  for (let index = end; index < records.length; index += 1) {
+    if (records[index]?.recordType === "assistant") {
+      expandedEnd = index;
+      break;
+    }
+  }
+  return [expandedStart, expandedEnd];
+}
+
+function mergeWindows(windows: Array<[number, number]>): Array<[number, number]> {
+  const merged: Array<[number, number]> = [];
+  for (const window of windows) {
+    const last = merged.at(-1);
+    if (last !== undefined && window[0] <= last[1] + 1) {
+      last[1] = Math.max(last[1], window[1]);
+    } else {
+      merged.push([...window]);
+    }
+  }
+  return merged;
+}
+
+function splitStoredRecords(
+  records: PreparedCanonicalRecord[],
+  maxBytes: number,
+  blockers: Set<string>,
+): Array<{
+  records: PreparedCanonicalRecord[];
+  lines: string[];
+  truncated: boolean;
+}> {
+  const chunks: Array<{
+    records: PreparedCanonicalRecord[];
+    lines: string[];
+    truncated: boolean;
+  }> = [];
+  let current = {
+    records: [] as PreparedCanonicalRecord[],
+    lines: [] as string[],
+    truncated: false,
+  };
+  for (const record of records) {
+    const fitted = fitStoredRecord(record, maxBytes);
+    if (fitted === null) {
+      blockers.add("materializer-byte-budget-too-small");
+      return [];
+    }
+    const separatorBytes = current.lines.length === 0 ? 0 : 1;
+    const currentBytes = Buffer.byteLength(current.lines.join("\n"), "utf8");
+    const lineBytes = Buffer.byteLength(fitted.line, "utf8");
+    if (current.lines.length > 0 && currentBytes + separatorBytes + lineBytes > maxBytes) {
+      chunks.push(current);
+      current = { records: [], lines: [], truncated: false };
+    }
+    current.records.push(record);
+    current.lines.push(fitted.line);
+    current.truncated ||= fitted.truncated;
+  }
+  if (current.lines.length > 0) chunks.push(current);
+  return chunks;
+}
+
+function fitStoredRecord(
+  record: PreparedCanonicalRecord,
+  maxBytes: number,
+): { line: string; truncated: boolean } | null {
+  const recordType = record.recordType;
+  if (recordType === "meta") {
+    throw new Error("Historical materializer cannot persist meta records.");
+  }
+  const createLine = (canonicalJson: string, truncated: boolean): string =>
+    JSON.stringify({
+      schemaVersion: 1,
+      kind: "historical-import-record",
+      recordKeyHash: record.recordKeyHash,
+      recordType,
+      canonicalJson,
+      canonicalJsonTruncated: truncated,
+    } satisfies HistoricalImportStoredRecordV1);
+  const complete = createLine(record.canonicalJson, false);
+  if (Buffer.byteLength(complete, "utf8") <= maxBytes) {
+    return { line: complete, truncated: false };
+  }
+
+  let low = 0;
+  let high = record.canonicalJson.length;
+  let best: string | null = null;
+  while (low <= high) {
+    const middle = Math.floor((low + high) / 2);
+    const excerpt = `${record.canonicalJson.slice(0, middle)}…`;
+    const line = createLine(excerpt, true);
+    if (Buffer.byteLength(line, "utf8") <= maxBytes) {
+      best = line;
+      low = middle + 1;
+    } else {
+      high = middle - 1;
+    }
+  }
+  return best === null ? null : { line: best, truncated: true };
+}
+
+function deriveHistoricalMetadata(
+  records: PreparedCanonicalRecord[],
+  segmentId: string,
+): {
+  normalized: SessionEvidenceSegmentV1["normalized"];
+  signals: SessionMemorySignal[];
+} {
+  const userTexts: string[] = [];
+  const failures: string[] = [];
+  const verifications: string[] = [];
+  const touchedTools = new Set<string>();
+  const signals: SessionMemorySignal[] = [];
+
+  for (const [index, record] of records.entries()) {
+    const parsed = asRecord(record.canonicalValue);
+    const content = extractCanonicalContent(parsed);
+    if (record.recordType === "user" && content !== null) userTexts.push(content);
+    for (const tool of extractToolNames(parsed)) touchedTools.add(tool);
+    if (isFailureRecord(record, parsed, content)) {
+      failures.push(sanitizeSummary(content ?? "Historical tool failure observed."));
+    }
+    if (isVerificationRecord(record, parsed, content)) {
+      verifications.push(sanitizeSummary(content ?? "Historical verification observed."));
+    }
+    signals.push({
+      eventId: createStableId("historical-event", [segmentId, record.recordKeyHash]),
+      eventType: historicalEventType(record, parsed, content),
+      reason: "observed",
+      strength: "normal",
+      occurredAt: readTimestamp(parsed),
+      summary: historicalSignalSummary(record, parsed, content),
+      rawEventLine: index + 1,
+    });
+  }
+  const boundedSignals = signals.slice(0, MAX_HISTORICAL_SIGNALS);
+  if (boundedSignals.length === 0) {
+    boundedSignals.push({
+      eventId: createStableId("historical-event", [segmentId, "summary"]),
+      eventType: "unknown",
+      reason: "historical-import",
+      strength: "normal",
+      occurredAt: null,
+      summary: "Historical transcript evidence imported.",
+      rawEventLine: 0,
+    });
+  } else {
+    const last = boundedSignals.at(-1);
+    if (last !== undefined) last.reason = "historical-import";
+  }
+
+  return {
+    normalized: {
+      summary: sanitizeSummary(
+        `Historical import segment contains ${records.length} bounded canonical record(s).`,
+      ),
+      userIntent: userTexts[0] === undefined ? null : sanitizeSummary(userTexts[0]),
+      intentDelta:
+        userTexts.length < 2 ? null : sanitizeSummary(userTexts.at(-1) ?? userTexts[0] ?? ""),
+      decisions: [],
+      failures: failures.slice(0, 8),
+      verifications: verifications.slice(0, 8),
+      touchedTools: [...touchedTools].slice(0, 20),
+    },
+    signals: boundedSignals,
+  };
+}
+
+function historicalEventType(
+  record: PreparedCanonicalRecord,
+  parsed: Record<string, unknown> | null,
+  content: string | null,
+): string {
+  if (record.recordType === "user") return "UserPromptSubmit";
+  if (record.recordType === "assistant") return "Stop";
+  if (record.recordType === "assistant-tool-call") return "PreToolUse";
+  if (record.recordType === "tool") {
+    return isFailureRecord(record, parsed, content) ? "PostToolUseFailure" : "PostToolUse";
+  }
+  return "unknown";
+}
+
+function historicalSignalSummary(
+  record: PreparedCanonicalRecord,
+  parsed: Record<string, unknown> | null,
+  content: string | null,
+): string {
+  const tools = extractToolNames(parsed);
+  if (tools.length > 0) return sanitizeSummary(`Historical tool evidence: ${tools.join(", ")}.`);
+  if (content !== null) return sanitizeSummary(content);
+  return sanitizeSummary(`Historical ${record.recordType} record observed.`);
+}
+
+function isFailureRecord(
+  record: PreparedCanonicalRecord,
+  parsed: Record<string, unknown> | null,
+  content: string | null,
+): boolean {
+  if (record.recordType !== "tool") return false;
+  const exitCode = readExitCode(parsed);
+  return (
+    (exitCode !== null && exitCode !== 0) ||
+    /\b(?:fail(?:ed|ure)?|error|denied|exception|non-zero)\b/iu.test(content ?? "")
+  );
+}
+
+function isVerificationRecord(
+  record: PreparedCanonicalRecord,
+  parsed: Record<string, unknown> | null,
+  content: string | null,
+): boolean {
+  const comparable = `${extractToolNames(parsed).join(" ")} ${content ?? ""}`;
+  return (
+    (record.recordType === "assistant-tool-call" || record.recordType === "tool") &&
+    /\b(?:test|lint|typecheck|build|verify|check)\b/iu.test(comparable)
+  );
+}
+
+function extractToolNames(parsed: Record<string, unknown> | null): string[] {
+  if (parsed === null || !Array.isArray(parsed.tool_calls)) return [];
+  return parsed.tool_calls.flatMap((value) => {
+    const record = asRecord(value);
+    if (record === null || typeof record.name !== "string" || record.name.trim() === "") return [];
+    return [sanitizeStorageId(record.name, "tool")];
+  });
+}
+
+function extractCanonicalContent(parsed: Record<string, unknown> | null): string | null {
+  if (parsed === null) return null;
+  if (typeof parsed.content === "string" && parsed.content.trim() !== "") return parsed.content;
+  return null;
+}
+
+function readTimestamp(parsed: Record<string, unknown> | null): string | null {
+  if (parsed === null || typeof parsed.timestamp !== "string") return null;
+  return Number.isFinite(Date.parse(parsed.timestamp)) ? parsed.timestamp : null;
+}
+
+function readExitCode(parsed: Record<string, unknown> | null): number | null {
+  if (parsed === null) return null;
+  for (const field of ["exit_code", "exitCode"]) {
+    const value = parsed[field];
+    if (typeof value === "number" && Number.isInteger(value)) return value;
+  }
+  return null;
+}
+
+function asRecord(value: unknown): Record<string, unknown> | null {
+  return value !== null && typeof value === "object" && !Array.isArray(value)
+    ? (value as Record<string, unknown>)
+    : null;
+}
+
+function sourceTarget(source: TrajectoryImportSource): "claude" | "codex" | "unknown" {
+  if (source === "claude-code") return "claude";
+  if (source === "codex") return "codex";
+  return "unknown";
+}
+
+function normalizeIsoTimestamp(value: string): string {
+  if (!Number.isFinite(Date.parse(value))) {
+    throw new Error("Historical materialization requires an ISO timestamp.");
+  }
+  return new Date(value).toISOString();
+}
+
+function addDays(value: string, days: number): string {
+  return new Date(Date.parse(value) + days * 24 * 60 * 60 * 1000).toISOString();
+}