@evo-dev/core 0.0.1-alpha.11 → 0.0.1-alpha.12

package/src/evolution/imports/paths.ts

@@ -0,0 +1,129 @@
+import { isAbsolute, join, relative, resolve, sep } from "node:path";
+import { resolveEvoDevPaths } from "../../config/paths.ts";
+
+export interface TrajectoryImportSourcePaths {
+  rootDir: string;
+  projectDir: string;
+  sourceDir: string;
+  manifestPath: string;
+  lockDir: string;
+  lockOwnerPath: string;
+  lockRecoveryPath: string;
+  generationsDir: string;
+  receiptsDir: string;
+  materializationsDir: string;
+}
+
+export interface TrajectoryImportPaths extends TrajectoryImportSourcePaths {
+  generationDir: string;
+  snapshotsDir: string;
+  snapshotDir: string;
+  recordIndexPath: string;
+  previewReceiptDir: string;
+  stageReceiptPath: string;
+  materializeReceiptPath: string;
+  materializationPath: string;
+}
+
+export interface ResolveTrajectoryImportSourcePathsInput {
+  homeDir: string;
+  projectKey: string;
+  sourceKey: string;
+}
+
+export interface ResolveTrajectoryImportPathsInput extends ResolveTrajectoryImportSourcePathsInput {
+  generationId: string;
+  snapshotId: string;
+  previewId: string;
+}
+
+export function resolveTrajectoryImportSourcePaths(
+  input: ResolveTrajectoryImportSourcePathsInput,
+): TrajectoryImportSourcePaths {
+  assertProjectKey(input.projectKey);
+  assertStateId("sourceKey", input.sourceKey, /^src-[a-f0-9]{24}$/u);
+
+  const rootDir = resolve(resolveEvoDevPaths(input.homeDir).stateDir, "trajectory-imports");
+  const projectDir = join(rootDir, input.projectKey);
+  const sourceDir = join(projectDir, input.sourceKey);
+  const lockDir = join(sourceDir, "lock");
+  const generationsDir = join(sourceDir, "generations");
+  const receiptsDir = join(sourceDir, "receipts");
+  const materializationsDir = join(sourceDir, "materializations");
+
+  for (const candidate of [
+    projectDir,
+    sourceDir,
+    lockDir,
+    generationsDir,
+    receiptsDir,
+    materializationsDir,
+  ]) {
+    assertDescendant(rootDir, candidate);
+  }
+
+  return {
+    rootDir,
+    projectDir,
+    sourceDir,
+    manifestPath: join(sourceDir, "manifest.json"),
+    lockDir,
+    lockOwnerPath: join(lockDir, "owner.json"),
+    lockRecoveryPath: join(lockDir, "recovery.json"),
+    generationsDir,
+    receiptsDir,
+    materializationsDir,
+  };
+}
+
+export function resolveTrajectoryImportPaths(
+  input: ResolveTrajectoryImportPathsInput,
+): TrajectoryImportPaths {
+  assertStateId("generationId", input.generationId, /^gen-[a-f0-9]{24}$/u);
+  assertStateId("snapshotId", input.snapshotId, /^snap-[a-f0-9]{24}$/u);
+  assertStateId("previewId", input.previewId, /^preview-[a-f0-9]{24}$/u);
+
+  const sourcePaths = resolveTrajectoryImportSourcePaths(input);
+  const generationDir = join(sourcePaths.generationsDir, input.generationId);
+  const snapshotsDir = join(generationDir, "snapshots");
+  const snapshotDir = join(snapshotsDir, input.snapshotId);
+  const previewReceiptDir = join(sourcePaths.receiptsDir, input.previewId);
+
+  for (const candidate of [generationDir, snapshotDir, previewReceiptDir]) {
+    assertDescendant(sourcePaths.rootDir, candidate);
+  }
+
+  return {
+    ...sourcePaths,
+    generationDir,
+    snapshotsDir,
+    snapshotDir,
+    recordIndexPath: join(snapshotDir, "record-index.jsonl"),
+    previewReceiptDir,
+    stageReceiptPath: join(previewReceiptDir, "stage.json"),
+    materializeReceiptPath: join(previewReceiptDir, "materialize.json"),
+    materializationPath: join(sourcePaths.materializationsDir, `${input.snapshotId}.json`),
+  };
+}
+
+function assertProjectKey(value: string): void {
+  if (
+    value.length > 120 ||
+    !/^[a-zA-Z0-9][a-zA-Z0-9._-]*$/u.test(value) ||
+    value === "." ||
+    value === ".."
+  ) {
+    throw new Error("Invalid trajectory import projectKey.");
+  }
+}
+
+function assertStateId(field: string, value: string, pattern: RegExp): void {
+  if (!pattern.test(value)) throw new Error(`Invalid trajectory import ${field}.`);
+}
+
+function assertDescendant(rootDir: string, candidate: string): void {
+  const child = relative(rootDir, candidate);
+  if (child === "" || child === ".." || child.startsWith(`..${sep}`) || isAbsolute(child)) {
+    throw new Error("Trajectory import path escaped its private state root.");
+  }
+}

package/src/evolution/imports/stage.ts

@@ -0,0 +1,414 @@
+import {
+  createTrajectoryImportIndexDigest,
+  createTrajectoryImportPreviewId,
+  createTrajectoryImportSnapshotId,
+} from "./diff.ts";
+import { resolveTrajectoryImportPaths, resolveTrajectoryImportSourcePaths } from "./paths.ts";
+import {
+  type TrajectoryImportSourceLock,
+  assertTrajectoryImportSourceLockOwned,
+  ensureTrajectoryImportStageDirectories,
+  readTrajectoryImportIndex,
+  readTrajectoryImportManifestAtPath,
+  readTrajectoryImportReceipt,
+  withTrajectoryImportSourceLock,
+  writeImmutableTrajectoryImportIndex,
+  writeImmutableTrajectoryImportReceipt,
+  writeTrajectoryImportManifest,
+} from "./storage.ts";
+import type {
+  StageTrajectoryImportInput,
+  StageTrajectoryImportResult,
+  TrajectoryImportGenerationContractV1,
+  TrajectoryImportIndexRecordV1,
+  TrajectoryImportManifestV1,
+  TrajectoryImportReceiptV1,
+} from "./types.ts";
+
+export async function stageTrajectoryImport(
+  input: StageTrajectoryImportInput,
+): Promise<StageTrajectoryImportResult> {
+  const { sourcePaths } = validateAndResolveStageInput(input);
+  return withTrajectoryImportSourceLock(sourcePaths, async (lock) =>
+    stageTrajectoryImportWithLock(input, lock),
+  );
+}
+
+export async function stageTrajectoryImportWithLock(
+  input: StageTrajectoryImportInput,
+  lock: TrajectoryImportSourceLock,
+): Promise<StageTrajectoryImportResult> {
+  const { projectKey, sourcePaths } = validateAndResolveStageInput(input);
+  if (lock.ownerPath !== sourcePaths.lockOwnerPath) {
+    throw new Error("Trajectory import stage lock does not match its source.");
+  }
+  await assertTrajectoryImportSourceLockOwned(lock);
+  const manifest = await readTrajectoryImportManifestAtPath(sourcePaths.manifestPath);
+  validateManifestTransition(input, manifest);
+  await validateCurrentIndexTransition(input, manifest);
+
+  if (manifest !== null && manifest.currentSnapshotId === input.preview.snapshotId) {
+    return readUnchangedResult(input, manifest);
+  }
+  if (manifest !== null && input.preview.records.new === 0) {
+    return readUnchangedResult(input, manifest);
+  }
+
+  const paths = resolveTrajectoryImportPaths({
+    homeDir: input.homeDir,
+    projectKey,
+    sourceKey: input.preview.sourceKey,
+    generationId: input.preview.generationId,
+    snapshotId: input.preview.snapshotId,
+    previewId: input.preview.id,
+  });
+  await ensureTrajectoryImportStageDirectories(paths);
+  await assertTrajectoryImportSourceLockOwned(lock);
+
+  const indexWrite = await writeImmutableTrajectoryImportIndex(
+    paths,
+    input.comparisonSnapshot.records,
+  );
+  if (indexWrite.digest !== createTrajectoryImportIndexDigest(input.comparisonSnapshot.records)) {
+    throw new Error("Trajectory import staged index digest changed during validation.");
+  }
+  await assertTrajectoryImportSourceLockOwned(lock);
+
+  const existingReceipt = await readTrajectoryImportReceipt(paths.stageReceiptPath);
+  const receipt = resolveStageReceipt(input, existingReceipt);
+  const receiptWrite = await writeImmutableTrajectoryImportReceipt(paths.stageReceiptPath, receipt);
+  await assertTrajectoryImportSourceLockOwned(lock);
+
+  const nextManifest: TrajectoryImportManifestV1 = {
+    schemaVersion: 1,
+    kind: "trajectory-import-manifest",
+    source: input.preview.source,
+    sourceKey: input.preview.sourceKey,
+    projectKey,
+    scope: {
+      roleId: input.preview.scope.roleId,
+    },
+    currentGenerationId: input.preview.generationId,
+    initialInputFingerprint: input.comparisonSnapshot.initialInputFingerprint,
+    generationContract: input.comparisonSnapshot.generationContract,
+    currentSnapshotId: input.preview.snapshotId,
+    previousSnapshotId: input.preview.previousSnapshotId,
+    currentPreviewId: input.preview.id,
+    currentInputFingerprint: input.preview.input.fingerprint,
+    currentInputBytes: input.preview.input.bytes,
+    currentIndexDigest: indexWrite.digest,
+    stagedAt: receipt.createdAt,
+    applyState: "staged",
+    latestMaterializedSnapshotId: manifest?.latestMaterializedSnapshotId ?? null,
+    rawContentStored: false,
+  };
+  await writeTrajectoryImportManifest(sourcePaths, nextManifest);
+  await assertTrajectoryImportSourceLockOwned(lock);
+
+  return {
+    status: "staged",
+    manifest: nextManifest,
+    receipt,
+    stateWrites: [
+      ...(indexWrite.created
+        ? [
+            `generations/${input.preview.generationId}/snapshots/${input.preview.snapshotId}/record-index.jsonl`,
+          ]
+        : []),
+      ...(receiptWrite.created ? [`receipts/${input.preview.id}/stage.json`] : []),
+      "manifest.json",
+    ],
+  };
+}
+
+function validateAndResolveStageInput(input: StageTrajectoryImportInput): {
+  projectKey: string;
+  sourcePaths: ReturnType<typeof resolveTrajectoryImportSourcePaths>;
+} {
+  validateStageInput(input);
+  const projectKey = input.preview.projectKey;
+  if (projectKey === null) {
+    throw new Error("Trajectory import stage requires a projectKey.");
+  }
+  return {
+    projectKey,
+    sourcePaths: resolveTrajectoryImportSourcePaths({
+      homeDir: input.homeDir,
+      projectKey,
+      sourceKey: input.preview.sourceKey,
+    }),
+  };
+}
+
+async function readUnchangedResult(
+  input: StageTrajectoryImportInput,
+  manifest: TrajectoryImportManifestV1,
+): Promise<StageTrajectoryImportResult> {
+  const projectKey = input.preview.projectKey;
+  if (projectKey === null) {
+    throw new Error("Trajectory import stage requires a projectKey.");
+  }
+  const currentPaths = resolveTrajectoryImportPaths({
+    homeDir: input.homeDir,
+    projectKey,
+    sourceKey: manifest.sourceKey,
+    generationId: manifest.currentGenerationId,
+    snapshotId: manifest.currentSnapshotId,
+    previewId: manifest.currentPreviewId,
+  });
+  const records = await readTrajectoryImportIndex(currentPaths.recordIndexPath);
+  const digest = createTrajectoryImportIndexDigest(records);
+  if (digest !== manifest.currentIndexDigest) {
+    throw new Error("Trajectory import current index digest does not match its manifest.");
+  }
+  const receipt = await readTrajectoryImportReceipt(currentPaths.stageReceiptPath);
+  if (receipt === null) {
+    throw new Error("Trajectory import manifest is missing its immutable stage receipt.");
+  }
+  assertReceiptMatchesManifest(receipt, manifest);
+  return {
+    status: "unchanged",
+    manifest,
+    receipt,
+    stateWrites: [],
+  };
+}
+
+function validateStageInput(input: StageTrajectoryImportInput): void {
+  const { preview, comparisonSnapshot } = input;
+  if (preview.schemaVersion !== 1 || preview.kind !== "trajectory-import-preview") {
+    throw new Error("Trajectory import stage requires a v1 preview.");
+  }
+  const { id: previewId, ...previewWithoutId } = preview;
+  if (createTrajectoryImportPreviewId(previewWithoutId) !== previewId) {
+    throw new Error("Trajectory import preview id does not match its content.");
+  }
+  if (preview.rawContentStored !== false) {
+    throw new Error("Trajectory import stage cannot persist raw content.");
+  }
+  if (preview.blockers.length > 0) {
+    throw new Error(`Trajectory import stage is blocked: ${preview.blockers.join(", ")}.`);
+  }
+  if (preview.records.changed > 0 || preview.records.missing > 0) {
+    throw new Error("Trajectory import stage is append-only.");
+  }
+  if (
+    preview.records.total !== comparisonSnapshot.records.length ||
+    preview.records.total !==
+      preview.records.new + preview.records.unchanged + preview.records.changed
+  ) {
+    throw new Error("Trajectory import preview counts do not match its index.");
+  }
+  if (
+    comparisonSnapshot.sourceKey !== preview.sourceKey ||
+    comparisonSnapshot.generationId !== preview.generationId ||
+    comparisonSnapshot.snapshotId !== preview.snapshotId
+  ) {
+    throw new Error("Trajectory import preview does not match its comparison snapshot.");
+  }
+  if (
+    createTrajectoryImportSnapshotId(
+      comparisonSnapshot.generationId,
+      comparisonSnapshot.records,
+    ) !== preview.snapshotId
+  ) {
+    throw new Error("Trajectory import snapshot id does not match its index.");
+  }
+  if (
+    !sameGenerationContract(comparisonSnapshot.generationContract, {
+      normalizerVersion: preview.normalizer.version,
+      canonicalSchemaVersion: preview.normalizer.canonicalSchemaVersion,
+      canonicalConfigHash: preview.normalizer.canonicalConfigHash,
+    })
+  ) {
+    throw new Error("Trajectory import generation contract does not match its preview.");
+  }
+  if (
+    input.now !== undefined &&
+    (!Number.isFinite(Date.parse(input.now)) || new Date(input.now).toISOString() !== input.now)
+  ) {
+    throw new Error("Trajectory import stage now must be an ISO timestamp.");
+  }
+}
+
+async function validateCurrentIndexTransition(
+  input: StageTrajectoryImportInput,
+  manifest: TrajectoryImportManifestV1 | null,
+): Promise<void> {
+  if (manifest === null) return;
+  const projectKey = input.preview.projectKey;
+  if (projectKey === null) {
+    throw new Error("Trajectory import stage requires a projectKey.");
+  }
+  const currentPaths = resolveTrajectoryImportPaths({
+    homeDir: input.homeDir,
+    projectKey,
+    sourceKey: manifest.sourceKey,
+    generationId: manifest.currentGenerationId,
+    snapshotId: manifest.currentSnapshotId,
+    previewId: manifest.currentPreviewId,
+  });
+  const previousRecords = await readTrajectoryImportIndex(currentPaths.recordIndexPath);
+  const previousDigest = createTrajectoryImportIndexDigest(previousRecords);
+  if (
+    previousDigest !== manifest.currentIndexDigest ||
+    createTrajectoryImportSnapshotId(manifest.currentGenerationId, previousRecords) !==
+      manifest.currentSnapshotId
+  ) {
+    throw new Error("Trajectory import current index does not match its manifest.");
+  }
+  if (manifest.currentSnapshotId === input.preview.snapshotId) {
+    if (createTrajectoryImportIndexDigest(input.comparisonSnapshot.records) !== previousDigest) {
+      throw new Error("Trajectory import repeated snapshot does not match staged state.");
+    }
+    return;
+  }
+
+  const actual = compareAppendOnlyIndexes(input.comparisonSnapshot.records, previousRecords);
+  if (
+    actual.new !== input.preview.records.new ||
+    actual.unchanged !== input.preview.records.unchanged ||
+    actual.changed !== input.preview.records.changed ||
+    actual.missing !== input.preview.records.missing
+  ) {
+    throw new Error("Trajectory import preview diff does not match current staged state.");
+  }
+  if (actual.changed > 0 || actual.missing > 0 || actual.identityRegressed) {
+    throw new Error("Trajectory import stage is append-only.");
+  }
+}
+
+function compareAppendOnlyIndexes(
+  current: TrajectoryImportIndexRecordV1[],
+  previous: TrajectoryImportIndexRecordV1[],
+): {
+  new: number;
+  unchanged: number;
+  changed: number;
+  missing: number;
+  identityRegressed: boolean;
+} {
+  const currentByKey = new Map(current.map((record) => [record.stableRecordKeyHash, record]));
+  const previousByKey = new Map(previous.map((record) => [record.stableRecordKeyHash, record]));
+  let newRecords = 0;
+  let unchanged = 0;
+  let changed = 0;
+  let identityRegressed = false;
+  for (const record of current) {
+    const old = previousByKey.get(record.stableRecordKeyHash);
+    if (old === undefined) {
+      newRecords += 1;
+      continue;
+    }
+    if (old.sourceIdentityKind !== "content" && record.sourceIdentityKind === "content") {
+      identityRegressed = true;
+    }
+    if (old.contentDigest === record.contentDigest) unchanged += 1;
+    else changed += 1;
+  }
+  let missing = 0;
+  for (const record of previous) {
+    if (!currentByKey.has(record.stableRecordKeyHash)) missing += 1;
+  }
+  return { new: newRecords, unchanged, changed, missing, identityRegressed };
+}
+
+function validateManifestTransition(
+  input: StageTrajectoryImportInput,
+  manifest: TrajectoryImportManifestV1 | null,
+): void {
+  const { preview, comparisonSnapshot } = input;
+  if (manifest === null) {
+    if (preview.previousSnapshotId !== null) {
+      throw new Error("Trajectory import initial stage cannot reference a previous snapshot.");
+    }
+    if (preview.records.new !== preview.records.total || preview.records.unchanged !== 0) {
+      throw new Error("Trajectory import initial stage must contain only new records.");
+    }
+    return;
+  }
+
+  if (
+    manifest.source !== preview.source ||
+    manifest.sourceKey !== preview.sourceKey ||
+    manifest.projectKey !== preview.projectKey ||
+    manifest.scope.roleId !== preview.scope.roleId
+  ) {
+    throw new Error("Trajectory import stage scope does not match its manifest.");
+  }
+  if (
+    manifest.currentGenerationId !== preview.generationId ||
+    manifest.initialInputFingerprint !== comparisonSnapshot.initialInputFingerprint ||
+    !sameGenerationContract(manifest.generationContract, comparisonSnapshot.generationContract)
+  ) {
+    throw new Error("Trajectory import generation contract changed.");
+  }
+  if (manifest.currentSnapshotId === preview.snapshotId) return;
+  if (preview.previousSnapshotId !== manifest.currentSnapshotId) {
+    throw new Error("Trajectory import preview is stale for the current manifest.");
+  }
+  if (
+    manifest.applyState !== "completed" ||
+    manifest.latestMaterializedSnapshotId !== manifest.currentSnapshotId
+  ) {
+    throw new Error("Trajectory import cannot append until the current snapshot is materialized.");
+  }
+}
+
+function resolveStageReceipt(
+  input: StageTrajectoryImportInput,
+  existing: TrajectoryImportReceiptV1 | null,
+): TrajectoryImportReceiptV1 {
+  const expected: TrajectoryImportReceiptV1 = {
+    schemaVersion: 1,
+    kind: "trajectory-import-receipt",
+    previewId: input.preview.id,
+    sourceKey: input.preview.sourceKey,
+    generationId: input.preview.generationId,
+    snapshotId: input.preview.snapshotId,
+    operation: "stage",
+    outcome: "staged",
+    counts: {
+      total: input.preview.records.total,
+      new: input.preview.records.new,
+      unchanged: input.preview.records.unchanged,
+      changed: input.preview.records.changed,
+      missing: input.preview.records.missing,
+    },
+    createdAt: input.now ?? new Date().toISOString(),
+    rawContentStored: false,
+  };
+  if (existing === null) return expected;
+  const existingWithoutTime = { ...existing, createdAt: expected.createdAt };
+  if (JSON.stringify(existingWithoutTime) !== JSON.stringify(expected)) {
+    throw new Error("Trajectory import immutable stage receipt does not match its preview.");
+  }
+  return existing;
+}
+
+function assertReceiptMatchesManifest(
+  receipt: TrajectoryImportReceiptV1,
+  manifest: TrajectoryImportManifestV1,
+): void {
+  if (
+    receipt.operation !== "stage" ||
+    receipt.outcome !== "staged" ||
+    receipt.sourceKey !== manifest.sourceKey ||
+    receipt.generationId !== manifest.currentGenerationId ||
+    receipt.snapshotId !== manifest.currentSnapshotId ||
+    receipt.previewId !== manifest.currentPreviewId
+  ) {
+    throw new Error("Trajectory import stage receipt does not match its manifest.");
+  }
+}
+
+function sameGenerationContract(
+  left: TrajectoryImportGenerationContractV1,
+  right: TrajectoryImportGenerationContractV1,
+): boolean {
+  return (
+    left.normalizerVersion === right.normalizerVersion &&
+    left.canonicalSchemaVersion === right.canonicalSchemaVersion &&
+    left.canonicalConfigHash === right.canonicalConfigHash
+  );
+}