@evo-dev/core 0.0.1-alpha.11 → 0.0.1-alpha.12

package/src/evolution/imports/diff.ts

@@ -0,0 +1,472 @@
+import { sha256Hex } from "../../utils/index.ts";
+import {
+  type CanonicalImportRecordType,
+  type CanonicalImportSnapshot,
+  type CanonicalImportSourceIdentityKind,
+  type CreateTrajectoryImportPreviewInput,
+  TRAJECTORY_IMPORT_DIAGNOSTIC_POLICY_VERSION,
+  TRAJECTORY_IMPORT_PREVIEW_SCHEMA_VERSION,
+  type TrajectoryImportDiagnosticSeverity,
+  type TrajectoryImportDownstreamPreviewV1,
+  type TrajectoryImportGenerationContractV1,
+  type TrajectoryImportIndexRecordV1,
+  type TrajectoryImportPreviewBuildResult,
+  type TrajectoryImportPreviewV1,
+} from "./types.ts";
+
+const SOURCE_KEY_HASH_LENGTH = 24;
+const STATE_ID_HASH_LENGTH = 24;
+const PREVIEW_ID_HASH_LENGTH = 24;
+const MAX_SCOPE_ID_LENGTH = 120;
+const MAX_EXPLICIT_SOURCE_ID_LENGTH = 256;
+
+const DIAGNOSTIC_POLICY = {
+  invalid_json_line: "blocker",
+  non_object_json_line: "blocker",
+  injected_context_dropped: "info",
+  noise_record_dropped: "info",
+  sidechain_record_dropped: "info",
+  tool_call_id_synthesized: "warning",
+  duplicate_tool_call_id: "blocker",
+  orphan_tool_result: "warning",
+  duplicate_tool_result: "blocker",
+  unknown_tool_name: "warning",
+  tool_arguments_reshaped: "warning",
+  tool_arguments_truncated: "warning",
+  tool_result_truncated: "warning",
+  timestamps_synthesized: "info",
+  timestamps_interpolated: "info",
+} as const satisfies Record<string, TrajectoryImportDiagnosticSeverity>;
+
+const EMPTY_MATERIALIZER_CONFIG_HASH = sha256Hex(
+  stableTrajectoryImportJson({ implemented: false }),
+);
+
+const DEFAULT_DOWNSTREAM: TrajectoryImportDownstreamPreviewV1 = {
+  semanticProcessing: "queue-only",
+  automationKnowledge: false,
+  automationSemanticKnowledge: false,
+  mayInvokeConfiguredProviderAfterApply: false,
+};
+
+export function createTrajectoryImportPreview(
+  input: CreateTrajectoryImportPreviewInput,
+): TrajectoryImportPreviewBuildResult {
+  const projectKey = normalizeScopeId(input.projectKey, "projectKey");
+  const roleId = normalizeScopeId(input.roleId, "roleId");
+  if (roleId !== null && projectKey === null) {
+    throw new Error("Trajectory import --role requires --project.");
+  }
+  validateSnapshot(input.snapshot);
+
+  const blockers = new Set<string>();
+  const sourceKey = resolveSourceKey(input.snapshot, blockers);
+  const generationContract = createGenerationContract(input.snapshot);
+  const previous = input.previous ?? null;
+
+  if (previous !== null) {
+    if (previous.sourceKey !== sourceKey) blockers.add("source-key-mismatch");
+    if (!sameGenerationContract(previous.generationContract, generationContract)) {
+      blockers.add("normalizer-contract-changed");
+    }
+  }
+
+  const indexed = createIndex(input.snapshot, sourceKey, blockers);
+  const diff = compareIndexes(indexed, previous?.records ?? [], blockers);
+  const generationId =
+    previous?.generationId ??
+    createStateId("gen", [
+      sourceKey,
+      stableTrajectoryImportJson(generationContract),
+      input.snapshot.inputFingerprint,
+    ]);
+  const snapshotId = createTrajectoryImportSnapshotId(generationId, indexed);
+  const diagnostics = summarizeDiagnostics(input.snapshot.diagnostics, blockers);
+  const previewWithoutId: Omit<TrajectoryImportPreviewV1, "id"> = {
+    schemaVersion: TRAJECTORY_IMPORT_PREVIEW_SCHEMA_VERSION,
+    kind: "trajectory-import-preview",
+    source: input.snapshot.source,
+    sourceKey,
+    projectKey,
+    scope: {
+      roleId,
+    },
+    input: {
+      bytes: input.snapshot.inputBytes,
+      fingerprint: input.snapshot.inputFingerprint,
+      pathHash: input.pathHash,
+    },
+    normalizer: {
+      version: input.snapshot.normalizer.version,
+      canonicalSchemaVersion: input.snapshot.normalizer.canonicalSchemaVersion,
+      canonicalConfigHash: input.snapshot.normalizer.canonicalConfigHash,
+    },
+    contracts: {
+      diagnosticPolicyVersion: TRAJECTORY_IMPORT_DIAGNOSTIC_POLICY_VERSION,
+      materializerVersion: 0,
+      materializerConfigHash: EMPTY_MATERIALIZER_CONFIG_HASH,
+    },
+    generationId,
+    snapshotId,
+    previousSnapshotId: previous?.snapshotId ?? null,
+    records: {
+      total: indexed.length,
+      ...diff,
+      recordTypes: countRecordTypes(indexed),
+      identityKinds: countIdentityKinds(indexed),
+    },
+    diagnostics,
+    materialization: null,
+    downstream: DEFAULT_DOWNSTREAM,
+    blockers: [],
+    canApply: false,
+    rawContentStored: false,
+  };
+  previewWithoutId.blockers = [...blockers].sort();
+
+  const previewId = createTrajectoryImportPreviewId(previewWithoutId);
+  const { schemaVersion, kind, ...previewFields } = previewWithoutId;
+  const preview: TrajectoryImportPreviewV1 = {
+    schemaVersion,
+    kind,
+    id: previewId,
+    ...previewFields,
+  };
+
+  return {
+    preview,
+    comparisonSnapshot: {
+      sourceKey,
+      generationId,
+      initialInputFingerprint: previous?.initialInputFingerprint ?? input.snapshot.inputFingerprint,
+      generationContract,
+      snapshotId,
+      records: indexed,
+    },
+  };
+}
+
+export function createTrajectoryImportConfigHash(value: unknown): string {
+  return sha256Hex(stableTrajectoryImportJson(value));
+}
+
+export function createTrajectoryImportIndexDigest(
+  records: TrajectoryImportIndexRecordV1[],
+): string {
+  return sha256Hex(
+    stableTrajectoryImportJson(
+      records.map((record) => ({
+        schemaVersion: record.schemaVersion,
+        kind: record.kind,
+        stableRecordKeyHash: record.stableRecordKeyHash,
+        contentDigest: record.contentDigest,
+        ordinal: record.ordinal,
+        sourceIdentityKind: record.sourceIdentityKind,
+        recordType: record.recordType,
+      })),
+    ),
+  );
+}
+
+export function createTrajectoryImportSnapshotId(
+  generationId: string,
+  records: TrajectoryImportIndexRecordV1[],
+): string {
+  return createStateId("snap", [generationId, createTrajectoryImportIndexDigest(records)]);
+}
+
+export function createTrajectoryImportStableRecordKeyHash(
+  sourceKey: string,
+  recordId: string,
+): string {
+  return sha256Hex(`${sourceKey}\0${recordId}`);
+}
+
+export function createTrajectoryImportContentDigest(
+  sourceKey: string,
+  contentHash: string,
+): string {
+  return sha256Hex(`${sourceKey}\0${contentHash}`);
+}
+
+export function createTrajectoryImportPreviewId(
+  preview: Omit<TrajectoryImportPreviewV1, "id">,
+): string {
+  const previewIdentity = {
+    ...preview,
+    input: {
+      bytes: preview.input.bytes,
+      fingerprint: preview.input.fingerprint,
+    },
+  };
+  return `preview-${sha256Hex(stableTrajectoryImportJson(previewIdentity)).slice(
+    0,
+    PREVIEW_ID_HASH_LENGTH,
+  )}`;
+}
+
+export function stableTrajectoryImportJson(value: unknown): string {
+  if (value === null || typeof value !== "object") {
+    const serialized = JSON.stringify(value);
+    return serialized === undefined ? "null" : serialized;
+  }
+  if (Array.isArray(value)) {
+    return `[${value.map(stableTrajectoryImportJson).join(",")}]`;
+  }
+  const entries = Object.entries(value as Record<string, unknown>)
+    .filter(([, child]) => child !== undefined)
+    .sort(([left], [right]) => left.localeCompare(right));
+  return `{${entries
+    .map(([key, child]) => `${JSON.stringify(key)}:${stableTrajectoryImportJson(child)}`)
+    .join(",")}}`;
+}
+
+function validateSnapshot(snapshot: CanonicalImportSnapshot): void {
+  if (snapshot.inputBytes < 0 || !Number.isSafeInteger(snapshot.inputBytes)) {
+    throw new Error("Trajectory import inputBytes must be a non-negative safe integer.");
+  }
+  assertNonEmpty("inputFingerprint", snapshot.inputFingerprint);
+  assertNonEmpty("normalizer.version", snapshot.normalizer.version);
+  assertNonEmpty("normalizer.canonicalConfigHash", snapshot.normalizer.canonicalConfigHash);
+  if (
+    !Number.isInteger(snapshot.normalizer.canonicalSchemaVersion) ||
+    snapshot.normalizer.canonicalSchemaVersion < 1
+  ) {
+    throw new Error("Trajectory import canonicalSchemaVersion must be a positive integer.");
+  }
+  if (snapshot.explicitSourceId !== null) {
+    const sourceId = snapshot.explicitSourceId;
+    if (
+      sourceId.trim() === "" ||
+      sourceId.length > MAX_EXPLICIT_SOURCE_ID_LENGTH ||
+      containsControlCharacter(sourceId)
+    ) {
+      throw new Error("Trajectory import --source-id must be a bounded printable value.");
+    }
+  }
+  for (const group of snapshot.groups) {
+    assertNonEmpty("sourceGroupId", group.sourceGroupId);
+    for (const record of group.records) {
+      assertNonEmpty("recordId", record.recordId);
+      assertNonEmpty("contentHash", record.contentHash);
+      assertNonEmpty("sourceOrderId", record.sourceOrderId);
+      if (!Number.isSafeInteger(record.componentIndex) || record.componentIndex < 0) {
+        throw new Error("Trajectory import componentIndex must be a non-negative safe integer.");
+      }
+    }
+  }
+}
+
+function resolveSourceKey(snapshot: CanonicalImportSnapshot, blockers: Set<string>): string {
+  if (snapshot.groups.length !== 1) {
+    blockers.add(snapshot.groups.length === 0 ? "source-group-missing" : "multiple-source-groups");
+    return `src-unresolved-${sha256Hex(`${snapshot.source}\0${snapshot.inputFingerprint}`).slice(
+      0,
+      SOURCE_KEY_HASH_LENGTH,
+    )}`;
+  }
+  const groupId = snapshot.groups[0]?.sourceGroupId ?? "";
+  let locator: string;
+  if (groupId === "default") {
+    if (snapshot.explicitSourceId === null) {
+      blockers.add("explicit-source-id-required");
+      locator = `unresolved:${snapshot.inputFingerprint}`;
+    } else {
+      locator = `explicit:${snapshot.explicitSourceId}`;
+    }
+  } else {
+    locator = `group:${groupId}`;
+  }
+  return `src-${sha256Hex(`${snapshot.source}\0${locator}`).slice(0, SOURCE_KEY_HASH_LENGTH)}`;
+}
+
+function createGenerationContract(
+  snapshot: CanonicalImportSnapshot,
+): TrajectoryImportGenerationContractV1 {
+  return {
+    normalizerVersion: snapshot.normalizer.version,
+    canonicalSchemaVersion: snapshot.normalizer.canonicalSchemaVersion,
+    canonicalConfigHash: snapshot.normalizer.canonicalConfigHash,
+  };
+}
+
+function createIndex(
+  snapshot: CanonicalImportSnapshot,
+  sourceKey: string,
+  blockers: Set<string>,
+): TrajectoryImportIndexRecordV1[] {
+  const candidates = snapshot.groups.flatMap((group) =>
+    group.records.map((record) => ({
+      stableRecordKeyHash: createTrajectoryImportStableRecordKeyHash(sourceKey, record.recordId),
+      contentDigest: createTrajectoryImportContentDigest(sourceKey, record.contentHash),
+      sourceOrderId: record.sourceOrderId,
+      componentIndex: record.componentIndex,
+      sourceIdentityKind: record.sourceIdentityKind,
+      recordType: record.recordType,
+    })),
+  );
+  candidates.sort(
+    (left, right) =>
+      left.sourceOrderId.localeCompare(right.sourceOrderId) ||
+      left.componentIndex - right.componentIndex ||
+      left.stableRecordKeyHash.localeCompare(right.stableRecordKeyHash) ||
+      left.contentDigest.localeCompare(right.contentDigest) ||
+      left.sourceIdentityKind.localeCompare(right.sourceIdentityKind) ||
+      left.recordType.localeCompare(right.recordType),
+  );
+
+  const unique = new Map<string, (typeof candidates)[number]>();
+  for (const candidate of candidates) {
+    const existing = unique.get(candidate.stableRecordKeyHash);
+    if (existing === undefined) {
+      unique.set(candidate.stableRecordKeyHash, candidate);
+      continue;
+    }
+    if (
+      existing.contentDigest !== candidate.contentDigest ||
+      existing.sourceIdentityKind !== candidate.sourceIdentityKind ||
+      existing.recordType !== candidate.recordType
+    ) {
+      blockers.add("duplicate-record-identity-conflict");
+    }
+  }
+
+  return [...unique.values()].map((record, ordinal) => ({
+    schemaVersion: 1,
+    kind: "trajectory-import-index-record",
+    stableRecordKeyHash: record.stableRecordKeyHash,
+    contentDigest: record.contentDigest,
+    ordinal,
+    sourceIdentityKind: record.sourceIdentityKind,
+    recordType: record.recordType,
+  }));
+}
+
+function compareIndexes(
+  current: TrajectoryImportIndexRecordV1[],
+  previous: TrajectoryImportIndexRecordV1[],
+  blockers: Set<string>,
+): Pick<TrajectoryImportPreviewV1["records"], "new" | "unchanged" | "changed" | "missing"> {
+  const currentByKey = new Map(current.map((record) => [record.stableRecordKeyHash, record]));
+  const previousByKey = new Map(previous.map((record) => [record.stableRecordKeyHash, record]));
+  let newRecords = 0;
+  let unchanged = 0;
+  let changed = 0;
+
+  for (const record of current) {
+    const old = previousByKey.get(record.stableRecordKeyHash);
+    if (old === undefined) {
+      newRecords += 1;
+      continue;
+    }
+    if (old.sourceIdentityKind !== "content" && record.sourceIdentityKind === "content") {
+      blockers.add("source-identity-confidence-regressed");
+    }
+    if (old.contentDigest === record.contentDigest) unchanged += 1;
+    else changed += 1;
+  }
+
+  let missing = 0;
+  for (const record of previous) {
+    if (!currentByKey.has(record.stableRecordKeyHash)) missing += 1;
+  }
+  if (changed > 0) blockers.add("changed-records");
+  if (missing > 0) blockers.add("missing-records");
+  return { new: newRecords, unchanged, changed, missing };
+}
+
+function summarizeDiagnostics(
+  diagnostics: Array<{ code: string }>,
+  blockers: Set<string>,
+): TrajectoryImportPreviewV1["diagnostics"] {
+  const counts = new Map<string, number>();
+  for (const diagnostic of diagnostics) {
+    const code = normalizeDiagnosticCode(diagnostic.code);
+    counts.set(code, (counts.get(code) ?? 0) + 1);
+  }
+  return [...counts.entries()]
+    .sort(([left], [right]) => left.localeCompare(right))
+    .map(([code, count]) => {
+      const severity = diagnosticSeverity(code);
+      if (severity === "blocker") blockers.add(`diagnostic:${code}`);
+      return { code, severity, count };
+    });
+}
+
+function diagnosticSeverity(code: string): TrajectoryImportDiagnosticSeverity {
+  return DIAGNOSTIC_POLICY[code as keyof typeof DIAGNOSTIC_POLICY] ?? "blocker";
+}
+
+function normalizeDiagnosticCode(code: string): string {
+  return /^[a-z][a-z0-9_-]{0,79}$/u.test(code)
+    ? code
+    : `unrecognized-${sha256Hex(code).slice(0, 16)}`;
+}
+
+function countRecordTypes(
+  records: TrajectoryImportIndexRecordV1[],
+): Record<CanonicalImportRecordType, number> {
+  const counts: Record<CanonicalImportRecordType, number> = {
+    meta: 0,
+    user: 0,
+    reasoning: 0,
+    assistant: 0,
+    "assistant-tool-call": 0,
+    tool: 0,
+  };
+  for (const record of records) counts[record.recordType] += 1;
+  return counts;
+}
+
+function countIdentityKinds(
+  records: TrajectoryImportIndexRecordV1[],
+): Record<CanonicalImportSourceIdentityKind, number> {
+  const counts: Record<CanonicalImportSourceIdentityKind, number> = {
+    native: 0,
+    location: 0,
+    content: 0,
+    synthetic: 0,
+  };
+  for (const record of records) counts[record.sourceIdentityKind] += 1;
+  return counts;
+}
+
+function sameGenerationContract(
+  left: TrajectoryImportGenerationContractV1,
+  right: TrajectoryImportGenerationContractV1,
+): boolean {
+  return (
+    left.normalizerVersion === right.normalizerVersion &&
+    left.canonicalSchemaVersion === right.canonicalSchemaVersion &&
+    left.canonicalConfigHash === right.canonicalConfigHash
+  );
+}
+
+function normalizeScopeId(value: string | null | undefined, field: string): string | null {
+  if (value === null || value === undefined) return null;
+  if (
+    value.length > MAX_SCOPE_ID_LENGTH ||
+    !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/u.test(value) ||
+    value === "." ||
+    value === ".."
+  ) {
+    throw new Error(`Invalid trajectory import ${field}.`);
+  }
+  return value;
+}
+
+function containsControlCharacter(value: string): boolean {
+  for (const character of value) {
+    const code = character.codePointAt(0) ?? 0;
+    if (code <= 31 || code === 127) return true;
+  }
+  return false;
+}
+
+function createStateId(prefix: string, parts: string[]): string {
+  return `${prefix}-${sha256Hex(parts.join("\0")).slice(0, STATE_ID_HASH_LENGTH)}`;
+}
+
+function assertNonEmpty(field: string, value: string): void {
+  if (value.trim() === "") throw new Error(`Trajectory import ${field} must not be empty.`);
+}

package/src/evolution/imports/index.ts

@@ -0,0 +1,7 @@
+export * from "./types.ts";
+export * from "./diff.ts";
+export * from "./paths.ts";
+export * from "./storage.ts";
+export * from "./stage.ts";
+export * from "./materialize.ts";
+export * from "./apply.ts";