@ankhorage/infra 5.1.6 → 6.0.0

package/CHANGELOG.md

@@ -1,5 +1,11 @@
 # @ankhorage/infra
 
+## 6.0.0
+
+### Major Changes
+
+- 99a0dbd: Align Infra with Contracts 20 and add production S3 backup/recovery acceptance for the Supabase production slice.
+
 ## 5.1.6
 
 ### Patch Changes

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@ankhorage/infra",
   "type": "module",
-  "version": "5.1.6",
+  "version": "6.0.0",
   "description": "Project-aware infrastructure generation library and executable Ankh provider for Ankhorage.",
   "homepage": "https://github.com/ankhorage/infra#readme",
   "bugs": {
@@ -35,7 +35,7 @@
     "openapi"
   ],
   "dependencies": {
-    "@ankhorage/contracts": "^19.4.0"
+    "@ankhorage/contracts": "^20.0.0"
   },
   "files": [
     "dist",
@@ -102,6 +102,7 @@
     "test:e2e": "ANKH_INFRA_COMPOSITION_E2E=1 bun test src/infraComposition.integration.test.ts",
     "test:e2e:docker-compose-providers": "ANKH_INFRA_DOCKER_COMPOSE_PROVIDERS_E2E=1 bun test src/localDockerComposeProviders.integration.test.ts",
     "test:e2e:minikube-supabase": "ANKH_INFRA_MINIKUBE_SUPABASE_E2E=1 bun test src/localMinikubeSupabase.integration.test.ts",
+    "test:e2e:minikube-supabase-recovery": "ANKH_INFRA_MINIKUBE_SUPABASE_RECOVERY_E2E=1 bun test src/localMinikubeSupabaseRecovery.integration.test.ts",
     "typecheck": "bun x tsc --noEmit -p tsconfig.json",
     "check-types": "bun run typecheck",
     "version-packages": "ankhorage-changeset version",
@@ -116,10 +117,10 @@
     "@ankhorage/hetzner": "^0.3.0",
     "@ankhorage/k3s": "^0.4.1",
     "@ankhorage/local": "^0.3.0",
-    "@ankhorage/minikube": "^0.4.2",
+    "@ankhorage/minikube": "^0.4.4",
     "@ankhorage/paradox": "^0.1.24",
     "@ankhorage/r2": "^0.2.0",
-    "@ankhorage/supabase": "^0.5.5",
+    "@ankhorage/supabase": "^0.5.9",
     "@ankhorage/supabase-vault": "^0.3.1",
     "@types/bun": "^1.4.1",
     "@types/node": "^25.6.0",

package/src/hetznerK3sProduction.test.ts

@@ -40,6 +40,13 @@ import type { InfraOrchestrationDependencies } from './types/infraOrchestration.
 const projectId = 'infra145-production';
 const publicBaseUrl = 'https://api.production.example.test';
 const domain = 'api.production.example.test';
+const persistenceTarget = {
+  endpoint: 'https://s3.production.example.test',
+  region: 'eu-central-1',
+  bucket: 'infra145-database-backups',
+  credentials: { source: 'control-plane', name: 'S3_PERSISTENCE' },
+  forcePathStyle: true,
+} as const;
 
 const manifest = {
   environments: {
@@ -54,8 +61,17 @@ const manifest = {
         compute: { provider: 'hetzner', location: 'fsn1', serverType: 'cx23' },
         runtime: { provider: 'k3s', topology: { servers: 1, agents: 0 } },
       },
-      database: { provider: 'supabase', tier: 'prod' },
+      database: {
+        provider: 'supabase',
+        tier: 'prod',
+        backup: { mode: 'scheduled', target: persistenceTarget, intervalHours: 24 },
+      },
       auth: { provider: 'supabase' },
+      objectStorage: {
+        provider: 'supabase',
+        buckets: ['media'],
+        backend: { ...persistenceTarget, bucket: 'infra145-storage' },
+      },
       networking: {
         domain,
         publicBaseUrl,
@@ -106,10 +122,15 @@ test('composes Hetzner, k3s, Kubernetes and Supabase while retaining compute for
   expect(access?.transport.kind === 'ssh' && access.transport.host).toBe('203.0.113.10');
   expect(JSON.stringify(firstUp)).not.toContain('phase10-private-key');
   expect(JSON.stringify(firstUp)).not.toContain('phase10-service-role-key');
+  expect(JSON.stringify(firstUp)).not.toContain('phase10-s3-secret-key');
   expect(fixture.kubernetes.serializedResources()).toContain(`"host":"${domain}"`);
   expect(fixture.kubernetes.serializedResources()).toContain('PersistentVolumeClaim');
   expect(fixture.kubernetes.serializedResources()).toContain('/etc/postgresql-custom');
   expect(fixture.kubernetes.serializedResources()).toContain('.ankhorage-image-seeded');
+  expect(fixture.kubernetes.serializedResources()).toContain('supabase-db-backup');
+  expect(
+    fixture.kubernetes.resources.filter(({ kind }) => kind === 'PersistentVolumeClaim'),
+  ).toHaveLength(2);
   expect(firstUp.resources.some(({ persistent }) => persistent)).toBe(true);
   const urlOutput = firstUp.outputs.find(({ name }) => name === 'url');
   expect(urlOutput?.value).toBe(publicBaseUrl);
@@ -232,6 +253,9 @@ function resolveCredential(name: string): InfraResult<Readonly<Record<string, st
       privateKey: 'phase10-private-key',
     });
   }
+  if (name === 'S3_PERSISTENCE') {
+    return success({ accessKeyId: 'phase10-s3-access', secretAccessKey: 'phase10-s3-secret-key' });
+  }
   if (name === 'SUPABASE_BOOTSTRAP') {
     return success({
       postgresPassword: 'phase10-postgres-password',

package/src/localMinikubeSupabaseRecovery.integration.test.ts

@@ -0,0 +1,527 @@
+import { createHmac } from 'node:crypto';
+
+import type {
+  InfraLedger,
+  InfraManifest,
+  InfraResourceIdentity,
+  InfraResult,
+} from '@ankhorage/contracts/infra';
+import { expect, test } from 'bun:test';
+
+import { createNodeInfraAdapterPackageResolver } from './features/environment-lifecycle/adapters/outbound/createNodeInfraAdapterPackageResolver.js';
+import { destroyInfraEnvironmentAsync } from './features/environment-lifecycle/application/use-cases/destroyInfraEnvironmentAsync.js';
+import { upInfraEnvironmentAsync } from './features/environment-lifecycle/application/use-cases/upInfraEnvironmentAsync.js';
+import type { InfraOrchestrationDependencies } from './types/infraOrchestration.js';
+
+const projectId = 'infra145-minikube-supabase-recovery';
+const profile = projectId;
+const namespace = `${projectId}-local`;
+const baseUrl = 'http://127.0.0.1:54322';
+const logicalBucket = 'phase10-objects';
+const physicalBucket = 'phase10-persistence';
+const minioContainer = 'infra145-minio-recovery';
+const minioHostEndpoint = 'http://127.0.0.1:19000';
+const minioPodEndpoint = 'http://host.minikube.internal:19000';
+const minioImage = 'quay.io/minio/minio:RELEASE.2025-09-07T16-13-09Z';
+const s3AccessKey = 'phase10-access';
+const s3SecretKey = 'phase10-secret-key';
+const databaseMarker = 'database-survived-from-s3-backup';
+const objectMarker = 'storage-object-survived-off-cluster';
+const jwtSecret = 'phase10-jwt-secret-that-is-at-least-thirty-two-characters';
+const serviceRoleKey = createJwt('service_role');
+const persistenceTarget = {
+  endpoint: minioPodEndpoint,
+  region: 'us-east-1',
+  bucket: physicalBucket,
+  credentials: { source: 'control-plane', name: 'S3_PERSISTENCE' },
+  forcePathStyle: true,
+} as const;
+const manifest = {
+  environments: {
+    local: {
+      deployment: {
+        compute: { provider: 'local' },
+        runtime: {
+          provider: 'minikube',
+          profile,
+          driver: 'docker',
+          cpus: 2,
+          memoryMiB: 5_120,
+        },
+      },
+      database: {
+        provider: 'supabase',
+        tier: 'prod',
+        backup: { mode: 'scheduled', target: persistenceTarget, intervalHours: 24 },
+      },
+      auth: { provider: 'supabase' },
+      objectStorage: {
+        provider: 'supabase',
+        buckets: [logicalBucket],
+        backend: persistenceTarget,
+      },
+      networking: { publicBaseUrl: baseUrl },
+    },
+  },
+  modules: [],
+} as const satisfies InfraManifest;
+
+test.skipIf(process.env.ANKH_INFRA_MINIKUBE_SUPABASE_RECOVERY_E2E !== '1')(
+  'recovers a fresh Minikube Supabase database while S3-backed Storage remains off-cluster',
+  async () => {
+    await stopMinioAsync();
+    await startMinioAsync();
+    try {
+      const dependencies = createDependencies();
+      const firstUp = requireSuccess(
+        await upInfraEnvironmentAsync({ projectId, manifest }, dependencies),
+      );
+      await writeDatabaseMarkerAsync();
+      await uploadStorageObjectAsync();
+      await forceFreshBackupAsync();
+      expect(await readLatestBackupPointerAsync()).toMatch(/^database\/\d{8}T\d{6}Z$/);
+
+      const destroyed = requireSuccess(
+        await destroyInfraEnvironmentAsync(
+          createDestroyRequest(firstUp.ledger, clusterPersistentIdentities(firstUp.ledger)),
+          dependencies,
+        ),
+      );
+      expectRetainedStorageBucket(destroyed.ledger);
+
+      const recovered = requireSuccess(
+        await upInfraEnvironmentAsync({ projectId, manifest }, dependencies),
+      );
+      expect(await readDatabaseMarkerAsync()).toBe(databaseMarker);
+      expect(await downloadStorageObjectAsync()).toBe(objectMarker);
+
+      const finalDestroy = requireSuccess(
+        await destroyInfraEnvironmentAsync(
+          createDestroyRequest(recovered.ledger, clusterPersistentIdentities(recovered.ledger)),
+          dependencies,
+        ),
+      );
+      expectRetainedStorageBucket(finalDestroy.ledger);
+    } catch (error) {
+      const diagnostics = await collectFailureDiagnosticsAsync();
+      const message = error instanceof Error ? error.message : String(error);
+      throw new Error(`${message}\nRecovery diagnostics: ${diagnostics}`, { cause: error });
+    } finally {
+      await runAllowFailureAsync(['minikube', 'delete', '-p', profile]);
+      await stopMinioAsync();
+    }
+  },
+  1_200_000,
+);
+
+async function startMinioAsync(): Promise<void> {
+  await runAsync(
+    [
+      'docker',
+      'run',
+      '--detach',
+      '--rm',
+      '--name',
+      minioContainer,
+      '--publish',
+      '19000:9000',
+      '--env',
+      `MINIO_ROOT_USER=${s3AccessKey}`,
+      '--env',
+      `MINIO_ROOT_PASSWORD=${s3SecretKey}`,
+      minioImage,
+      'server',
+      '/data',
+      '--address',
+      ':9000',
+    ],
+    'start MinIO',
+  );
+  await waitForMinioAsync();
+  await runAsync(
+    [
+      'curl',
+      '--fail',
+      '--silent',
+      '--show-error',
+      '--request',
+      'PUT',
+      '--aws-sigv4',
+      'aws:amz:us-east-1:s3',
+      '--user',
+      `${s3AccessKey}:${s3SecretKey}`,
+      `${minioHostEndpoint}/${physicalBucket}`,
+    ],
+    'create MinIO bucket',
+  );
+}
+
+async function waitForMinioAsync(): Promise<void> {
+  for (const _attempt of Array.from({ length: 60 }, (_, index) => index)) {
+    const response = await fetch(`${minioHostEndpoint}/minio/health/live`).catch(() => undefined);
+    if (response?.ok === true) return;
+    await Bun.sleep(500);
+  }
+  throw new Error('MinIO did not become ready.');
+}
+
+async function stopMinioAsync(): Promise<void> {
+  await runAllowFailureAsync(['docker', 'rm', '--force', minioContainer]);
+}
+
+async function writeDatabaseMarkerAsync(): Promise<void> {
+  const pod = await findPodAsync('supabase-db-', 'supabase-db-backup-');
+  await runAsync(
+    [
+      'kubectl',
+      'exec',
+      '--namespace',
+      namespace,
+      pod,
+      '--',
+      'psql',
+      '-U',
+      'postgres',
+      '-d',
+      'postgres',
+      '-v',
+      'ON_ERROR_STOP=1',
+      '-c',
+      `CREATE TABLE public.phase10_recovery_marker(value text NOT NULL); INSERT INTO public.phase10_recovery_marker(value) VALUES ('${databaseMarker}');`,
+    ],
+    'write database recovery marker',
+  );
+}
+
+async function readDatabaseMarkerAsync(): Promise<string> {
+  const pod = await findPodAsync('supabase-db-', 'supabase-db-backup-');
+  return runAsync(
+    [
+      'kubectl',
+      'exec',
+      '--namespace',
+      namespace,
+      pod,
+      '--',
+      'psql',
+      '-U',
+      'postgres',
+      '-d',
+      'postgres',
+      '-Atc',
+      'SELECT value FROM public.phase10_recovery_marker LIMIT 1;',
+    ],
+    'read database recovery marker',
+  );
+}
+
+async function uploadStorageObjectAsync(): Promise<void> {
+  const response = await fetch(`${baseUrl}/storage/v1/object/${logicalBucket}/recovery.txt`, {
+    method: 'POST',
+    headers: {
+      Authorization: `Bearer ${serviceRoleKey}`,
+      apikey: serviceRoleKey,
+      'Content-Type': 'text/plain',
+    },
+    body: objectMarker,
+  });
+  if (!response.ok) throw new Error(`Storage upload failed with HTTP ${response.status}.`);
+}
+
+async function downloadStorageObjectAsync(): Promise<string> {
+  const response = await fetch(`${baseUrl}/storage/v1/object/${logicalBucket}/recovery.txt`, {
+    headers: { Authorization: `Bearer ${serviceRoleKey}`, apikey: serviceRoleKey },
+  });
+  if (!response.ok) throw new Error(`Storage download failed with HTTP ${response.status}.`);
+  return response.text();
+}
+
+async function forceFreshBackupAsync(): Promise<void> {
+  await runAsync(
+    ['kubectl', 'rollout', 'restart', 'deployment/supabase-db-backup', '--namespace', namespace],
+    'restart database backup workload',
+  );
+  await runAsync(
+    [
+      'kubectl',
+      'rollout',
+      'status',
+      'deployment/supabase-db-backup',
+      '--namespace',
+      namespace,
+      '--timeout=180s',
+    ],
+    'wait for database backup workload',
+  );
+}
+
+async function readLatestBackupPointerAsync(): Promise<string> {
+  return runAsync(
+    [
+      'curl',
+      '--fail',
+      '--silent',
+      '--show-error',
+      '--aws-sigv4',
+      'aws:amz:us-east-1:s3',
+      '--user',
+      `${s3AccessKey}:${s3SecretKey}`,
+      `${minioHostEndpoint}/${physicalBucket}/database/latest`,
+    ],
+    'read latest database backup pointer',
+  );
+}
+
+async function findPodAsync(prefix: string, excludedPrefix?: string): Promise<string> {
+  const names = (
+    await runAsync(
+      [
+        'kubectl',
+        'get',
+        'pods',
+        '--namespace',
+        namespace,
+        '-o',
+        'jsonpath={.items[*].metadata.name}',
+      ],
+      'list Supabase pods',
+    )
+  ).split(' ');
+  const name = names.find(
+    (candidate) => candidate.startsWith(prefix) && !candidate.startsWith(excludedPrefix ?? '\0'),
+  );
+  if (name === undefined) throw new Error(`No pod matched ${prefix}.`);
+  return name;
+}
+
+async function collectFailureDiagnosticsAsync(): Promise<string> {
+  const [
+    auth,
+    storageStatus,
+    buckets,
+    databaseState,
+    databaseLogs,
+    previousDatabaseLogs,
+    dataRestoreState,
+    dataRestoreLogs,
+    previousDataRestoreLogs,
+  ] = await Promise.all([
+    readEndpointDiagnosticAsync('/auth/v1/health', false),
+    readEndpointDiagnosticAsync('/storage/v1/status', false),
+    readEndpointDiagnosticAsync('/storage/v1/bucket', true),
+    readDatabaseStateAsync(),
+    readDatabaseLogsAsync(false),
+    readDatabaseLogsAsync(true),
+    readDataRestoreStateAsync(),
+    readDataRestoreLogsAsync(false),
+    readDataRestoreLogsAsync(true),
+  ]);
+  return [
+    auth,
+    storageStatus,
+    buckets,
+    databaseState,
+    databaseLogs,
+    previousDatabaseLogs,
+    dataRestoreState,
+    dataRestoreLogs,
+    previousDataRestoreLogs,
+  ].join(' | ');
+}
+
+async function readDatabaseStateAsync(): Promise<string> {
+  const pod = await findPodAsync('supabase-db-', 'supabase-db-backup-').catch(() => undefined);
+  if (pod === undefined) return 'database-state=unavailable';
+  const state = await runAsync(
+    [
+      'kubectl',
+      'get',
+      'pod',
+      pod,
+      '--namespace',
+      namespace,
+      '-o',
+      'jsonpath={.status.containerStatuses[?(@.name=="supabase-db")].restartCount}',
+    ],
+    'read database restart state',
+  ).catch((error: unknown) => (error instanceof Error ? error.message : String(error)));
+  return `database-restarts=${redactDiagnosticText(state)}`;
+}
+
+async function readDatabaseLogsAsync(previous: boolean): Promise<string> {
+  const pod = await findPodAsync('supabase-db-', 'supabase-db-backup-').catch(() => undefined);
+  const label = previous ? 'database-previous-logs' : 'database-logs';
+  if (pod === undefined) return `${label}=unavailable`;
+  const logs = await runAsync(
+    [
+      'kubectl',
+      'logs',
+      '--namespace',
+      namespace,
+      pod,
+      '--container',
+      'supabase-db',
+      ...(previous ? ['--previous'] : []),
+      '--tail=240',
+    ],
+    `read ${label}`,
+  ).catch((error: unknown) => (error instanceof Error ? error.message : String(error)));
+  return `${label}=${redactDiagnosticText(logs).slice(-6_000)}`;
+}
+
+async function readDataRestoreStateAsync(): Promise<string> {
+  const pod = await findPodAsync('supabase-db-data-restore-').catch(() => undefined);
+  if (pod === undefined) return 'data-restore-state=unavailable';
+  const state = await runAsync(
+    [
+      'kubectl',
+      'get',
+      'pod',
+      pod,
+      '--namespace',
+      namespace,
+      '-o',
+      'jsonpath={.status.containerStatuses[?(@.name=="supabase-db-data-restore")].restartCount}',
+    ],
+    'read data restore restart state',
+  ).catch((error: unknown) => (error instanceof Error ? error.message : String(error)));
+  return `data-restore-restarts=${redactDiagnosticText(state)}`;
+}
+
+async function readDataRestoreLogsAsync(previous: boolean): Promise<string> {
+  const pod = await findPodAsync('supabase-db-data-restore-').catch(() => undefined);
+  const label = previous ? 'data-restore-previous-logs' : 'data-restore-logs';
+  if (pod === undefined) return `${label}=unavailable`;
+  const logs = await runAsync(
+    [
+      'kubectl',
+      'logs',
+      '--namespace',
+      namespace,
+      pod,
+      '--container',
+      'supabase-db-data-restore',
+      ...(previous ? ['--previous'] : []),
+      '--tail=240',
+    ],
+    `read ${label}`,
+  ).catch((error: unknown) => (error instanceof Error ? error.message : String(error)));
+  return `${label}=${redactDiagnosticText(logs).slice(-6_000)}`;
+}
+
+async function readEndpointDiagnosticAsync(path: string, authenticated: boolean): Promise<string> {
+  const response = await fetch(`${baseUrl}${path}`, {
+    headers: authenticated
+      ? { Authorization: `Bearer ${serviceRoleKey}`, apikey: serviceRoleKey }
+      : undefined,
+  }).catch(() => undefined);
+  if (response === undefined) return `${path}=unreachable`;
+  const body = redactDiagnosticText((await response.text()).slice(0, 500));
+  return `${path}=${response.status}:${body}`;
+}
+
+function redactDiagnosticText(value: string): string {
+  return value
+    .replaceAll(s3SecretKey, '[redacted]')
+    .replaceAll(serviceRoleKey, '[redacted]')
+    .replaceAll(jwtSecret, '[redacted]');
+}
+
+function createDependencies(): InfraOrchestrationDependencies {
+  return {
+    adapterResolver: createNodeInfraAdapterPackageResolver(),
+    credentials: {
+      resolveAsync: ({ name }) =>
+        Promise.resolve(
+          name === 'SUPABASE_BOOTSTRAP'
+            ? success(supabaseCredentials())
+            : name === 'S3_PERSISTENCE'
+              ? success({ accessKeyId: s3AccessKey, secretAccessKey: s3SecretKey })
+              : failure('unexpected-credential', `Unexpected credential reference: ${name}`),
+        ),
+    },
+    secrets: {
+      resolveAsync: ({ ref, key }) =>
+        Promise.resolve(failure('unexpected-secret', `Unexpected managed secret ${ref}/${key}.`)),
+    },
+  };
+}
+
+function createDestroyRequest(
+  previous: InfraLedger,
+  confirmedResources: readonly InfraResourceIdentity[],
+) {
+  return {
+    projectId,
+    manifest,
+    environment: 'local' as const,
+    previous,
+    confirmation: { projectId, environment: 'local' as const },
+    persistence: { policy: 'delete' as const, confirmedResources },
+  };
+}
+
+function clusterPersistentIdentities(ledger: InfraLedger): readonly InfraResourceIdentity[] {
+  return ledger.resources
+    .filter(({ identity, persistent }) => persistent && identity.adapter !== 'supabase')
+    .map(({ identity }) => identity);
+}
+
+function expectRetainedStorageBucket(ledger: InfraLedger | null): void {
+  expect(
+    ledger?.resources.map(({ identity }) => `${identity.adapter}:${identity.resourceId}`),
+  ).toEqual([`supabase:bucket/${logicalBucket}`]);
+}
+
+function supabaseCredentials() {
+  return {
+    postgresPassword: 'phase10-postgres-password',
+    jwtSecret,
+    anonKey: createJwt('anon'),
+    serviceRoleKey,
+    realtimeSecretKeyBase: 'r'.repeat(64),
+    realtimeDatabaseEncryptionKey: '0123456789abcdef',
+    pgMetaCryptoKey: 'phase10-meta-crypto-key-that-is-at-least-32-characters',
+  } as const;
+}
+
+function createJwt(role: 'anon' | 'service_role'): string {
+  const header = encodeJwtPart({ alg: 'HS256', typ: 'JWT' });
+  const payload = encodeJwtPart({ role, iss: 'supabase', iat: 1_700_000_000, exp: 4_102_444_800 });
+  const unsigned = `${header}.${payload}`;
+  const signature = createHmac('sha256', jwtSecret).update(unsigned).digest('base64url');
+  return `${unsigned}.${signature}`;
+}
+
+function encodeJwtPart(value: Readonly<Record<string, string | number>>): string {
+  return Buffer.from(JSON.stringify(value)).toString('base64url');
+}
+
+async function runAsync(command: readonly string[], label: string): Promise<string> {
+  const process = Bun.spawn(command, { stdout: 'pipe', stderr: 'pipe' });
+  const [exitCode, stdout, stderr] = await Promise.all([
+    process.exited,
+    new Response(process.stdout).text(),
+    new Response(process.stderr).text(),
+  ]);
+  if (exitCode !== 0) throw new Error(`${label} failed: ${stderr.trim()}`);
+  return stdout.trim();
+}
+
+async function runAllowFailureAsync(command: readonly string[]): Promise<void> {
+  const process = Bun.spawn(command, { stdout: 'ignore', stderr: 'ignore' });
+  await process.exited;
+}
+
+function requireSuccess<T>(result: InfraResult<T>): T {
+  if (!result.ok) throw new Error(result.diagnostics.map(({ message }) => message).join(' '));
+  return result.value;
+}
+
+function success<T>(value: T): InfraResult<T> {
+  return { ok: true, value, diagnostics: [] };
+}
+
+function failure(code: string, message: string): InfraResult<never> {
+  return { ok: false, diagnostics: [{ severity: 'error', code, message }] };
+}