@forgeax/engine-ddc 0.0.0-dev.8d955ade1c79

package/src/entry-store.ts

@@ -0,0 +1,317 @@
+import { createHash, randomUUID } from 'node:crypto';
+import { mkdir, readdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises';
+import { join } from 'node:path';
+import { DdcStoreError } from './errors.js';
+import { canonicalDdcJson } from './key.js';
+
+export { DdcStoreError } from './errors.js';
+
+export interface DdcArtifact {
+  readonly mediaType: string;
+  readonly bytes: Uint8Array;
+}
+
+export interface DdcReceipt {
+  readonly guid: string;
+  readonly key: string;
+  readonly producer: string;
+  readonly inputFingerprint: string;
+  readonly outputDigest: string;
+}
+
+export interface DdcEntry {
+  readonly key: string;
+  readonly guid: string;
+  readonly payload: unknown;
+  readonly refs: readonly string[];
+  readonly artifacts: Readonly<Record<string, DdcArtifact>>;
+  readonly receipt: DdcReceipt;
+}
+
+export interface StagedDdcEntry {
+  readonly key: string;
+  readonly path: string;
+}
+
+export interface PublishDdcEntryResult {
+  readonly result: 'published' | 'existing' | 'conflict';
+  readonly key: string;
+}
+
+interface ArtifactIntegrity {
+  readonly mediaType: string;
+  readonly byteLength: number;
+  readonly digest: string;
+}
+
+interface EntryIntegrity {
+  readonly payload: string;
+  readonly refs: string;
+  readonly receipt: string;
+  readonly artifacts: Readonly<Record<string, ArtifactIntegrity>>;
+  readonly entry: string;
+}
+
+function digest(value: string | Uint8Array): string {
+  return createHash('sha256').update(value).digest('hex');
+}
+
+function artifactFile(key: string): string {
+  if (/^[A-Za-z0-9._-]+$/.test(key)) return key;
+  return Buffer.from(key, 'utf8').toString('base64url');
+}
+
+function validateKey(key: string): void {
+  if (!/^[a-f0-9]{64}$/.test(key)) {
+    throw new DdcStoreError('ddc-entry-invalid', `invalid semantic key: ${key}`);
+  }
+}
+
+function errorCode(error: unknown): string | undefined {
+  if (error !== null && typeof error === 'object' && 'code' in error) {
+    const code = (error as { readonly code?: unknown }).code;
+    return typeof code === 'string' ? code : undefined;
+  }
+  return undefined;
+}
+
+function sortedArtifacts(entry: DdcEntry): readonly [string, DdcArtifact][] {
+  return Object.entries(entry.artifacts).sort(([left], [right]) => left.localeCompare(right));
+}
+
+/** The one output identity shared by receipt validation and DDC publication. */
+export function ddcOutputDigest(
+  entry: Pick<DdcEntry, 'guid' | 'payload' | 'refs' | 'artifacts'>,
+): string {
+  return `sha256:${digest(
+    canonicalDdcJson({
+      guid: entry.guid,
+      payload: entry.payload,
+      refs: entry.refs,
+      artifacts: Object.fromEntries(
+        sortedArtifacts(entry as DdcEntry).map(([key, artifact]) => [
+          key,
+          {
+            mediaType: artifact.mediaType,
+            bytes: artifact.bytes,
+          },
+        ]),
+      ),
+    }),
+  )}`;
+}
+
+function existingResult(
+  existing: DdcEntry,
+  candidate: DdcEntry,
+  key: string,
+): PublishDdcEntryResult {
+  return canonicalDdcJson(existing) === canonicalDdcJson(candidate)
+    ? { result: 'existing', key }
+    : { result: 'conflict', key };
+}
+
+function entryShape(entry: DdcEntry): Record<string, unknown> {
+  return {
+    key: entry.key,
+    guid: entry.guid,
+    payload: entry.payload,
+    refs: entry.refs,
+    receipt: entry.receipt,
+    artifacts: Object.fromEntries(
+      sortedArtifacts(entry).map(([key, artifact]) => [
+        key,
+        { mediaType: artifact.mediaType, byteLength: artifact.bytes.byteLength },
+      ]),
+    ),
+  };
+}
+
+function integrityFor(entry: DdcEntry): EntryIntegrity {
+  const artifacts = Object.fromEntries(
+    sortedArtifacts(entry).map(([key, artifact]) => [
+      key,
+      {
+        mediaType: artifact.mediaType,
+        byteLength: artifact.bytes.byteLength,
+        digest: digest(artifact.bytes),
+      },
+    ]),
+  );
+  return {
+    payload: digest(canonicalDdcJson(entry.payload)),
+    refs: digest(canonicalDdcJson(entry.refs)),
+    receipt: digest(canonicalDdcJson(entry.receipt)),
+    artifacts,
+    entry: digest(canonicalDdcJson(entryShape(entry))),
+  };
+}
+
+async function readJson<T>(path: string): Promise<T> {
+  return JSON.parse(await readFile(path, 'utf8')) as T;
+}
+
+export class DdcEntryStore {
+  private readonly entries: string;
+  private readonly staging: string;
+
+  public constructor(root: string) {
+    this.entries = join(root, 'entries');
+    this.staging = join(root, 'staging');
+  }
+
+  public async stage(entry: DdcEntry): Promise<StagedDdcEntry> {
+    validateKey(entry.key);
+    if (entry.receipt.key !== entry.key || entry.receipt.guid !== entry.guid) {
+      throw new DdcStoreError('ddc-entry-invalid', 'receipt identity does not match entry');
+    }
+    if (entry.receipt.outputDigest !== ddcOutputDigest(entry)) {
+      throw new DdcStoreError('ddc-entry-invalid', 'receipt output digest does not match entry');
+    }
+    const path = join(this.staging, `${entry.key}-${randomUUID()}`);
+    try {
+      await mkdir(join(path, 'artifacts'), { recursive: true });
+      await writeFile(join(path, 'payload.json'), canonicalDdcJson(entry.payload));
+      await writeFile(join(path, 'refs.json'), canonicalDdcJson(entry.refs));
+      await writeFile(join(path, 'receipt.json'), canonicalDdcJson(entry.receipt));
+      const artifacts: Record<string, { mediaType: string; file: string }> = {};
+      for (const [key, artifact] of sortedArtifacts(entry)) {
+        const file = artifactFile(key);
+        artifacts[key] = { mediaType: artifact.mediaType, file };
+        await writeFile(join(path, 'artifacts', `${file}.bin`), artifact.bytes);
+      }
+      await writeFile(join(path, 'artifacts.json'), canonicalDdcJson(artifacts));
+      await writeFile(join(path, 'integrity.json'), canonicalDdcJson(integrityFor(entry)));
+      return { key: entry.key, path };
+    } catch (error) {
+      await rm(path, { recursive: true, force: true }).catch(() => {});
+      throw error;
+    }
+  }
+
+  public async publish(staged: StagedDdcEntry): Promise<PublishDdcEntryResult> {
+    validateKey(staged.key);
+    const candidate = await this.readDirectory(staged.path, true);
+    const target = join(this.entries, staged.key);
+    await mkdir(this.entries, { recursive: true });
+    const useExisting = async (): Promise<PublishDdcEntryResult> => {
+      const existing = await this.readDirectory(target, false);
+      await rm(staged.path, { recursive: true, force: true });
+      return existingResult(existing, candidate, staged.key);
+    };
+    try {
+      await stat(target);
+      return useExisting();
+    } catch (error) {
+      if (error instanceof DdcStoreError) throw error;
+      if (errorCode(error) !== 'ENOENT') throw error;
+    }
+    try {
+      await rename(staged.path, target);
+      return { result: 'published', key: staged.key };
+    } catch (error) {
+      if (!['EEXIST', 'ENOTEMPTY', 'EISDIR'].includes(errorCode(error) ?? '')) throw error;
+      return useExisting();
+    }
+  }
+
+  public async discard(staged: StagedDdcEntry): Promise<void> {
+    await rm(staged.path, { recursive: true, force: true });
+  }
+
+  public async write(entry: DdcEntry): Promise<PublishDdcEntryResult> {
+    const staged = await this.stage(entry);
+    return this.publish(staged);
+  }
+
+  public async read(key: string): Promise<DdcEntry | null> {
+    try {
+      validateKey(key);
+      return await this.readDirectory(join(this.entries, key), false);
+    } catch {
+      return null;
+    }
+  }
+
+  public async listKeys(): Promise<readonly string[]> {
+    const entries = await readdir(this.entries, { withFileTypes: true }).catch(() => []);
+    return entries
+      .filter((entry) => entry.isDirectory())
+      .map((entry) => entry.name)
+      .sort();
+  }
+
+  public async remove(key: string): Promise<void> {
+    validateKey(key);
+    await rm(join(this.entries, key), { recursive: true, force: false });
+  }
+
+  /** Read with a machine-readable corruption result for recovery tooling. */
+  public async readChecked(
+    key: string,
+  ): Promise<
+    | { readonly ok: true; readonly value: DdcEntry | null }
+    | { readonly ok: false; readonly error: DdcStoreError }
+  > {
+    try {
+      validateKey(key);
+      try {
+        await stat(join(this.entries, key));
+      } catch (error) {
+        if (errorCode(error) === 'ENOENT') return { ok: true, value: null };
+        throw error;
+      }
+      try {
+        return { ok: true, value: await this.readDirectory(join(this.entries, key), false) };
+      } catch (error) {
+        if (error instanceof DdcStoreError && error.code === 'ddc-entry-incomplete') {
+          return { ok: true, value: null };
+        }
+        throw error;
+      }
+    } catch (error) {
+      const normalized =
+        error instanceof DdcStoreError
+          ? error
+          : new DdcStoreError('ddc-entry-invalid', 'entry is unreadable');
+      return { ok: false, error: normalized };
+    }
+  }
+
+  private async readDirectory(path: string, incompleteIsError: boolean): Promise<DdcEntry> {
+    try {
+      const payload = await readJson<unknown>(join(path, 'payload.json'));
+      const refs = await readJson<readonly string[]>(join(path, 'refs.json'));
+      const receipt = await readJson<DdcReceipt>(join(path, 'receipt.json'));
+      const manifest = await readJson<
+        Readonly<Record<string, { mediaType: string; file: string }>>
+      >(join(path, 'artifacts.json'));
+      const integrity = await readJson<EntryIntegrity>(join(path, 'integrity.json'));
+      const artifacts: Record<string, DdcArtifact> = {};
+      for (const [key, descriptor] of Object.entries(manifest).sort(([left], [right]) =>
+        left.localeCompare(right),
+      )) {
+        const bytes = new Uint8Array(
+          await readFile(join(path, 'artifacts', `${descriptor.file}.bin`)),
+        );
+        artifacts[key] = { mediaType: descriptor.mediaType, bytes };
+      }
+      const entry = { key: receipt.key, guid: receipt.guid, payload, refs, artifacts, receipt };
+      if (entry.receipt.outputDigest !== ddcOutputDigest(entry)) {
+        throw new DdcStoreError('ddc-entry-invalid', 'receipt output digest does not match entry');
+      }
+      const actual = integrityFor(entry);
+      if (canonicalDdcJson(actual) !== canonicalDdcJson(integrity)) {
+        throw new DdcStoreError('ddc-entry-invalid', 'entry integrity mismatch');
+      }
+      return entry;
+    } catch (error) {
+      if (incompleteIsError) {
+        if (error instanceof DdcStoreError) throw error;
+        throw new DdcStoreError('ddc-entry-incomplete', 'staged entry is incomplete');
+      }
+      if (error instanceof DdcStoreError) throw error;
+      throw new DdcStoreError('ddc-entry-invalid', 'entry is unreadable');
+    }
+  }
+}

package/src/errors.ts

@@ -0,0 +1,164 @@
+export const DDC_ERROR_CODES = [
+  'ddc-project-root-required',
+  'ddc-object-conflict',
+  'ddc-head-conflict',
+  'ddc-generation-conflict',
+  'ddc-scope-mismatch',
+  'ddc-lease-expired',
+] as const;
+
+export type DdcContractErrorCode = (typeof DDC_ERROR_CODES)[number];
+export type DdcEntryErrorCode = 'ddc-entry-incomplete' | 'ddc-entry-invalid' | 'ddc-entry-conflict';
+export type DdcErrorCode = DdcContractErrorCode | DdcEntryErrorCode;
+
+export type DdcErrorRootKind = 'build-cache' | 'project-ddc' | 'runtime';
+
+export interface DdcRecoveryAction {
+  readonly kind:
+    | 'inspect'
+    | 'allocate-generation'
+    | 'retry'
+    | 'cold-rebuild'
+    | 'prune'
+    | 'release-lease';
+  readonly executable: boolean;
+  readonly exactTarget?: string;
+  readonly reason?: string;
+}
+
+export interface DdcStoreErrorInit {
+  readonly code: DdcErrorCode;
+  readonly detail: string;
+  readonly hint?: string;
+  readonly expected?: unknown;
+  readonly actual?: unknown;
+  readonly owner?: string;
+  readonly rootKind?: DdcErrorRootKind;
+  readonly scope?: string;
+  readonly generation?: number;
+  readonly lease?: string;
+  readonly revision?: number;
+  readonly recoveryActions?: readonly DdcRecoveryAction[];
+  readonly hostPath?: string;
+}
+
+export interface DdcBrowserError {
+  readonly code: DdcErrorCode;
+  readonly hint: string;
+  readonly expected: unknown;
+  readonly actual?: unknown;
+  readonly owner?: string;
+  readonly rootKind?: DdcErrorRootKind;
+  readonly scope?: string;
+  readonly generation?: number;
+  readonly lease?: string;
+  readonly revision?: number;
+  readonly recoveryActions: readonly Omit<DdcRecoveryAction, 'exactTarget'>[];
+}
+
+const defaultHints: Record<DdcErrorCode, string> = {
+  'ddc-project-root-required': 'inject the canonical projectDdcRoot before serving or publishing',
+  'ddc-object-conflict': 'keep the verified object and reject the competing publication',
+  'ddc-head-conflict': 'inspect the current head and retry with a fresh revision',
+  'ddc-generation-conflict': 'allocate a new persistent generation; do not edit the counter',
+  'ddc-scope-mismatch': 'reinspect the canonical root and allocate a new scope',
+  'ddc-lease-expired': 'discard the stale commit and acquire a new lease',
+  'ddc-entry-incomplete': 'discard the partial entry and cold-cook from author authority',
+  'ddc-entry-invalid': 'preserve last-known-good, inspect the entry, then cold-cook',
+  'ddc-entry-conflict': 'keep the verified entry and reject the competing publication',
+};
+
+function defaultExpected(code: DdcErrorCode): unknown {
+  if (code === 'ddc-project-root-required') return 'projectDdcRoot';
+  if (code === 'ddc-entry-incomplete' || code === 'ddc-entry-invalid') {
+    return 'a complete DDC entry whose receipt and integrity digests validate';
+  }
+  return 'a valid DDC owner, scope, generation, revision, and lease';
+}
+
+function defaultActions(code: DdcErrorCode): readonly DdcRecoveryAction[] {
+  switch (code) {
+    case 'ddc-project-root-required':
+    case 'ddc-scope-mismatch':
+      return [
+        { kind: 'inspect', executable: true },
+        { kind: 'retry', executable: true },
+      ];
+    case 'ddc-generation-conflict':
+      return [{ kind: 'allocate-generation', executable: true }];
+    case 'ddc-lease-expired':
+      return [
+        { kind: 'release-lease', executable: true },
+        { kind: 'retry', executable: true },
+      ];
+    case 'ddc-object-conflict':
+    case 'ddc-head-conflict':
+      return [
+        { kind: 'inspect', executable: true },
+        { kind: 'retry', executable: true },
+      ];
+    case 'ddc-entry-incomplete':
+    case 'ddc-entry-invalid':
+      return [{ kind: 'cold-rebuild', executable: true }];
+    case 'ddc-entry-conflict':
+      return [{ kind: 'inspect', executable: true }];
+  }
+}
+
+export class DdcStoreError extends Error {
+  public readonly code: DdcErrorCode;
+  public readonly detail: string;
+  public readonly hint: string;
+  public readonly expected: unknown;
+  public readonly actual: unknown;
+  public readonly owner: string | undefined;
+  public readonly rootKind: DdcErrorRootKind | undefined;
+  public readonly scope: string | undefined;
+  public readonly generation: number | undefined;
+  public readonly lease: string | undefined;
+  public readonly revision: number | undefined;
+  public readonly recoveryActions: readonly DdcRecoveryAction[];
+  public readonly hostPath: string | undefined;
+
+  public constructor(code: DdcErrorCode, detail: string);
+  public constructor(input: DdcStoreErrorInit);
+  public constructor(inputOrCode: DdcStoreErrorInit | DdcErrorCode, legacyDetail?: string) {
+    const input: DdcStoreErrorInit =
+      typeof inputOrCode === 'string'
+        ? { code: inputOrCode, detail: legacyDetail ?? inputOrCode }
+        : inputOrCode;
+    super(input.detail);
+    this.name = 'DdcStoreError';
+    this.code = input.code;
+    this.detail = input.detail;
+    this.hint = input.hint ?? defaultHints[input.code];
+    this.expected = input.expected ?? defaultExpected(input.code);
+    this.actual = input.actual;
+    this.owner = input.owner;
+    this.rootKind = input.rootKind;
+    this.scope = input.scope;
+    this.generation = input.generation;
+    this.lease = input.lease;
+    this.revision = input.revision;
+    this.recoveryActions = input.recoveryActions ?? defaultActions(input.code);
+    this.hostPath = input.hostPath;
+  }
+
+  public toBrowserProjection(): DdcBrowserError {
+    return {
+      code: this.code,
+      hint: this.hint,
+      expected: this.expected,
+      ...(this.actual === undefined ? {} : { actual: this.actual }),
+      ...(this.owner === undefined ? {} : { owner: this.owner }),
+      ...(this.rootKind === undefined ? {} : { rootKind: this.rootKind }),
+      ...(this.scope === undefined ? {} : { scope: this.scope }),
+      ...(this.generation === undefined ? {} : { generation: this.generation }),
+      ...(this.lease === undefined ? {} : { lease: this.lease }),
+      ...(this.revision === undefined ? {} : { revision: this.revision }),
+      recoveryActions: this.recoveryActions.map(
+        ({ exactTarget: _exactTarget, ...action }) => action,
+      ),
+    };
+  }
+}

package/src/gc.ts

@@ -0,0 +1,34 @@
+import { readdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import { withDdcLock } from './lifecycle.js';
+
+export interface DdcGcInput {
+  readonly currentKeys: readonly string[];
+  readonly lastKnownGoodKeys: readonly string[];
+  readonly activeLeaseKeys: readonly string[];
+  readonly scopeIds?: readonly string[];
+}
+
+export interface DdcGcResult {
+  readonly deleted: readonly string[];
+  readonly protected: readonly string[];
+}
+
+/** Mark current/LKG/lease roots, then sweep only the same project root. */
+export async function collectDdcGarbage(root: string, input: DdcGcInput): Promise<DdcGcResult> {
+  return withDdcLock(root, 'gc', async () => {
+    const protectedKeys = new Set([
+      ...input.currentKeys,
+      ...input.lastKnownGoodKeys,
+      ...input.activeLeaseKeys,
+    ]);
+    const entries = await readdir(join(root, 'entries'), { withFileTypes: true }).catch(() => []);
+    const deleted: string[] = [];
+    for (const entry of entries) {
+      if (!entry.isDirectory() || protectedKeys.has(entry.name)) continue;
+      await rm(join(root, 'entries', entry.name), { recursive: true, force: false });
+      deleted.push(entry.name);
+    }
+    return { deleted, protected: [...protectedKeys].sort() };
+  });
+}

package/src/index.ts

@@ -0,0 +1,92 @@
+export {
+  type BuildDdcInput,
+  resolveDdcRoot,
+  semanticBuildKey,
+} from './build-cache.js';
+export {
+  type DdcArtifact,
+  type DdcEntry,
+  DdcEntryStore,
+  type DdcReceipt,
+  DdcStoreError,
+  ddcOutputDigest,
+  type PublishDdcEntryResult,
+  type StagedDdcEntry,
+} from './entry-store.js';
+export {
+  DDC_ERROR_CODES,
+  type DdcBrowserError,
+  type DdcContractErrorCode,
+  type DdcEntryErrorCode,
+  type DdcErrorCode,
+  type DdcErrorRootKind,
+  type DdcRecoveryAction as DdcErrorRecoveryAction,
+  type DdcStoreErrorInit,
+} from './errors.js';
+export { collectDdcGarbage, type DdcGcInput, type DdcGcResult } from './gc.js';
+export { semanticDdcKey } from './key.js';
+export {
+  DDC_LAYOUT_VERSION,
+  type DdcBuildLayout,
+  type DdcBuildLayoutResult,
+  type DdcBuildOptions,
+  type DdcLayout,
+  type DdcLayoutError,
+  type DdcLayoutFailure,
+  type DdcLayoutResult,
+  type DdcProjectLayout,
+  type DdcServeOptions,
+  isDdcLayout,
+  resolveBuildDdcLayout,
+  resolveDdcLayout,
+} from './layout.js';
+export {
+  type DdcBeginResult,
+  type DdcCommitResult,
+  type DdcHead,
+  type DdcLease,
+  DdcLifecycle,
+  type DdcLifecycleState,
+  type DdcRestoreFence,
+  type DdcRestoreResult,
+  type DdcRollbackSnapshot,
+} from './lifecycle.js';
+export {
+  type AcceptedPublicationCandidate,
+  type AcceptedPublicationStore,
+  createAcceptedPublication,
+  createAcceptedPublicationStore,
+  type ScriptablePackPublicationBuildInput,
+  type ScriptablePackPublicationError,
+  type ScriptablePackPublicationSnapshot,
+  scriptablePackOutputSetDigest,
+  scriptablePackPublicationGeneration,
+} from './publication.js';
+export {
+  assertRuntimeScope,
+  createRuntimeScope,
+  type DdcRuntimeScope,
+  runtimeScopeHash,
+} from './runtime-scope.js';
+export {
+  type DdcGenerationCandidate,
+  type DdcGenerationEntryCandidate,
+  DdcGenerationSession,
+  type DdcSessionMetrics,
+  type DdcSessionOptions,
+} from './session.js';
+export {
+  type BrowserDdcRecoveryAction,
+  type BrowserDdcStatus,
+  type BrowserDdcStatusLayer,
+  DDC_STATUS_SCHEMA,
+  type DdcRecoveryAction,
+  type DdcStatus,
+  type DdcStatusHealth,
+  type DdcStatusLayer,
+  type DdcStatusLayerKind,
+  type DdcStatusRootKind,
+  projectDdcStatusForBrowser,
+  serializeDdcStatus,
+  toBrowserDdcStatus,
+} from './status.js';

package/src/key.ts

@@ -0,0 +1,46 @@
+import { createHash } from 'node:crypto';
+
+export interface SemanticDdcInput {
+  readonly schemaVersion: string;
+  readonly importer: string;
+  readonly codec: string;
+  readonly settings: unknown;
+  readonly sourceBytes: readonly Uint8Array[];
+  readonly declaredGuids: readonly string[];
+  readonly targetProfile: string;
+  readonly producer: string;
+}
+
+export function canonicalDdcJson(value: unknown): string {
+  const sorted = sortValue(value);
+  return JSON.stringify(sorted) ?? 'null';
+}
+
+function sortValue(value: unknown): unknown {
+  if (value instanceof Uint8Array) {
+    return { encoding: 'base64', bytes: Buffer.from(value).toString('base64') };
+  }
+  if (Array.isArray(value)) return value.map(sortValue);
+  if (value !== null && typeof value === 'object') {
+    const result: Record<string, unknown> = {};
+    for (const key of Object.keys(value as Record<string, unknown>).sort()) {
+      result[key] = sortValue((value as Record<string, unknown>)[key]);
+    }
+    return result;
+  }
+  return value;
+}
+
+export function semanticDdcKey(input: SemanticDdcInput): string {
+  const semantic = {
+    schemaVersion: input.schemaVersion,
+    importer: input.importer,
+    codec: input.codec,
+    settings: input.settings,
+    sourceBytes: input.sourceBytes,
+    declaredGuids: [...input.declaredGuids].sort(),
+    targetProfile: input.targetProfile,
+    producer: input.producer,
+  };
+  return createHash('sha256').update(canonicalDdcJson(semantic)).digest('hex');
+}