@bygghammar/cms-platform 0.3.1 → 0.5.0

package/src/backup/export-core.ts

@@ -0,0 +1,350 @@
+/**
+ * Pure helpers for building a self-contained migration package.
+ *
+ * A migration package is a superset of a backup archive: same portable table
+ * data, plus a versioned manifest with per-file checksums, expected row counts
+ * and provenance. Secrets never enter it.
+ */
+import { createHash } from "node:crypto";
+
+import { ARCHIVE_FORMAT_VERSION, SCHEMA_VERSION, type ArchiveManifest } from "./datasets";
+import {
+  verifyMediaFile,
+  isSafeArchivePath,
+  sanitizeSecrets,
+  isPlainObject,
+  toJsonSafe,
+} from "./safety";
+
+/** Bump when the package layout changes in a way older tools cannot read. */
+export const MIGRATION_PACKAGE_VERSION = 2;
+
+export type MediaChecksum = { path: string; bytes: number; sha256: string };
+
+export type MigrationManifest = {
+  packageVersion: number;
+  archiveFormat: number;
+  sourceSchemaVersion: number;
+  exportedAt: string;
+  source: string | null;
+  groups: string[];
+  /** Expected row count per table, asserted by `verify`. */
+  counts: Record<string, number>;
+  users: number;
+  media: { files: number; bytes: number };
+  checksums: MediaChecksum[];
+  /** Names only — values are never exported. */
+  secretRefs: string[];
+  /** Fields whose credential identifiers were redacted and must be reconfigured. */
+  redacted?: string[];
+  rejected: string[];
+};
+
+export function sha256(bytes: Uint8Array): string {
+  return createHash("sha256").update(Buffer.from(bytes)).digest("hex");
+}
+
+/**
+ * Keys whose values must never leave the source system. Matching values are
+ * replaced by a `secret:NAME` reference so the target knows what to configure.
+ */
+export const SECRET_KEY_RE =
+  /(password|passwd|secret|token|api[_-]?key|apikey|private[_-]?key|credential|service[_-]?role|smtp_pass|authorization|bearer|cookie|hash)/i;
+
+/**
+ * Keys that hold only the *name* of a secret (an environment-variable name),
+ * never its value. These must survive the export unchanged, otherwise the
+ * self-hosted mail system cannot find its credentials.
+ */
+export const SECRET_NAME_KEY_RE =
+  /(secret_name|secret_ref_name|_secret_ref|ref_name|key_id|_name)$/i;
+
+/** Translation metadata (`svHash`, `enHash`, …) is content, not a credential. */
+export const TRANSLATION_HASH_KEY_RE = /^[a-z]{2}(-[A-Za-z]{2,4})?hash$/i;
+
+export function isSecretNameKey(key: string): boolean {
+  return SECRET_NAME_KEY_RE.test(key) || TRANSLATION_HASH_KEY_RE.test(key);
+}
+
+export type SecretScanResult<T> = { value: T; refs: string[] };
+
+/** Recursively replace secret-like values with reference names. */
+export function stripSecrets<T>(value: T, path: string[] = []): SecretScanResult<T> {
+  const refs: string[] = [];
+
+  const walk = (v: unknown, keys: string[]): unknown => {
+    if (v instanceof Date) return v.toISOString();
+    if (Array.isArray(v)) return v.map((item, i) => walk(item, [...keys, String(i)]));
+    if (v && typeof v === "object") {
+      if (!isPlainObject(v)) return toJsonSafe(v);
+      const out: Record<string, unknown> = {};
+      for (const [k, item] of Object.entries(v)) {
+        if (
+          SECRET_KEY_RE.test(k) &&
+          !isSecretNameKey(k) &&
+          item != null &&
+          !isPlainObject(item) &&
+          !Array.isArray(item)
+        ) {
+          const ref = [...keys, k].join(".");
+          refs.push(ref);
+          out[k] = `secret:${k.toUpperCase()}`;
+          continue;
+        }
+        out[k] = walk(item, [...keys, k]);
+      }
+      return out;
+    }
+    return toJsonSafe(v);
+  };
+
+  return { value: walk(sanitizeSecrets(value), path) as T, refs };
+}
+
+/* ------------------------------------------------- credential surrogates */
+
+/** Prefix of every non-live client-token surrogate written by the exporter. */
+export const CLIENT_TOKEN_SURROGATE_PREFIX = "migrated:";
+
+/** Deterministic, unique, non-live replacement for a live client token. */
+export function surrogateClientToken(rowId: unknown, fallback: number): string {
+  const id = typeof rowId === "string" && rowId.trim() ? rowId.trim() : `row-${fallback}`;
+  return `${CLIENT_TOKEN_SURROGATE_PREFIX}${id}`;
+}
+
+/**
+ * Replace live partial-submission client tokens with surrogates. Every row is
+ * preserved; the values stay NOT NULL and unique per (form_key, client_token).
+ */
+export function sanitizePartialTokens(rows: Record<string, unknown>[]): Record<string, unknown>[] {
+  return rows.map((row, i) => ({ ...row, client_token: surrogateClientToken(row["id"], i) }));
+}
+
+/** Placeholder written where a credential identifier used to be. */
+export const REDACTED = "redacted:configure-on-server";
+
+/** AWS access-key-ID shaped identifiers (SES SMTP usernames included). */
+export const ACCESS_KEY_ID_RE =
+  /^(?:AKIA|ASIA|AIDA|AROA|AGPA|AIPA|ANPA|ANVA|ABIA|ACCA)[A-Z0-9]{12,}$/;
+
+export function looksLikeCredentialIdentifier(value: unknown): boolean {
+  if (typeof value !== "string") return false;
+  const v = value.trim();
+  if (!v) return false;
+  if (ACCESS_KEY_ID_RE.test(v)) return true;
+  // Opaque machine credential: long, no spaces, no @ (so real usernames and
+  // e-mail addresses are kept).
+  return v.length >= 20 && !/[\s@]/.test(v) && /[A-Z0-9]/.test(v) && !/^[a-z._-]+$/.test(v);
+}
+
+/**
+ * Redact credential identifiers (e.g. SES SMTP usernames) from email provider
+ * configuration. Values are never logged — only the field path is reported.
+ */
+export function redactProviderCredentials(rows: Record<string, unknown>[]): {
+  rows: Record<string, unknown>[];
+  redacted: string[];
+} {
+  const redacted: string[] = [];
+  const out = rows.map((row) => {
+    const config = row["config"];
+    if (!isPlainObject(config)) return row;
+    const next: Record<string, unknown> = { ...config };
+    for (const key of ["username", "user", "smtp_username", "access_key_id", "accessKeyId"]) {
+      if (key in next && looksLikeCredentialIdentifier(next[key])) {
+        next[key] = REDACTED;
+        redacted.push(`email_providers.config.${key}`);
+      }
+    }
+    return { ...row, config: next };
+  });
+  return { rows: out, redacted: [...new Set(redacted)] };
+}
+
+/* ------------------------------------------------------ credential audit */
+
+const PRIVATE_KEY_RE = /-----BEGIN [A-Z ]*PRIVATE KEY-----/;
+const BCRYPT_RE = /^\$2[aby]?\$\d{2}\$[./A-Za-z0-9]{53}$/;
+const ARGON_RE = /^\$argon2[a-z]{0,2}\$/;
+const JWT_RE = /^ey[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}$/;
+const CREDENTIALED_URL_RE = /\b[a-z+]{2,12}:\/\/[^\s/:@]+:[^\s/@]+@/i;
+const API_KEY_RE =
+  /\b(sk_live_[A-Za-z0-9]{10,}|rk_live_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|sb_secret_[A-Za-z0-9_-]{10,}|xox[baprs]-[A-Za-z0-9-]{10,})\b/;
+
+/**
+ * Scan the data about to be written (or just read back) for values that must
+ * never appear in a migration package. Returns human-readable problems; the
+ * offending values themselves are never included.
+ */
+export function auditExportedData(data: Map<string, Record<string, unknown>[]>): string[] {
+  const problems: string[] = [];
+
+  const partials = data.get("form_partials") ?? [];
+  partials.forEach((row, i) => {
+    const token = row["client_token"];
+    if (typeof token !== "string" || !token.startsWith(CLIENT_TOKEN_SURROGATE_PREFIX)) {
+      problems.push(
+        `form_partials[${i}].client_token is a live token — it must be replaced by a surrogate.`,
+      );
+    }
+  });
+
+  for (const [table, rows] of data) {
+    rows.forEach((row, i) => {
+      const walk = (value: unknown, path: string): void => {
+        if (Array.isArray(value)) {
+          value.forEach((item, j) => walk(item, `${path}[${j}]`));
+          return;
+        }
+        if (value && typeof value === "object") {
+          if (!isPlainObject(value)) return;
+          for (const [k, item] of Object.entries(value)) {
+            const child = `${path}.${k}`;
+            if (isSecretNameKey(k)) continue;
+            if (
+              typeof item === "string" &&
+              item &&
+              !item.startsWith("secret:") &&
+              !item.startsWith(REDACTED) &&
+              !item.startsWith(CLIENT_TOKEN_SURROGATE_PREFIX) &&
+              SECRET_KEY_RE.test(k)
+            ) {
+              problems.push(`${child} contains a credential value.`);
+              continue;
+            }
+            if (
+              table === "email_providers" &&
+              /(^|\.)config\./.test(child) &&
+              /^(username|user|smtp_username|access_key_id|accessKeyId)$/.test(k) &&
+              looksLikeCredentialIdentifier(item)
+            ) {
+              problems.push(`${child} looks like a credential identifier.`);
+              continue;
+            }
+            walk(item, child);
+          }
+          return;
+        }
+        if (typeof value !== "string" || !value) return;
+        if (PRIVATE_KEY_RE.test(value)) problems.push(`${path} contains a private key.`);
+        else if (BCRYPT_RE.test(value) || ARGON_RE.test(value))
+          problems.push(`${path} contains a password hash.`);
+        else if (JWT_RE.test(value)) problems.push(`${path} contains a JSON Web Token.`);
+        else if (API_KEY_RE.test(value)) problems.push(`${path} contains an API key.`);
+        else if (CREDENTIALED_URL_RE.test(value))
+          problems.push(`${path} contains a credentialed URL.`);
+      };
+      walk(row, `${table}[${i}]`);
+    });
+  }
+
+  return [...new Set(problems)];
+}
+
+/** Tables that must never be exported, whatever a caller asks for. */
+export const EXCLUDED_TABLE_RE =
+  /^(auth|storage|realtime|vault|supabase_functions|pgsodium|net)[._]/i;
+
+export function isExportableTable(table: string): boolean {
+  return !EXCLUDED_TABLE_RE.test(table);
+}
+
+export type MediaCandidate = { path: string; bytes: Uint8Array };
+
+export type MediaScan = {
+  accepted: { path: string; bytes: Uint8Array; sha256: string }[];
+  rejected: string[];
+  totalBytes: number;
+};
+
+/**
+ * Validate media before it enters the package: safe path, allowed extension,
+ * real file signature (no HTML disguised as an image), no duplicate paths.
+ */
+export function scanMedia(files: MediaCandidate[]): MediaScan {
+  const accepted: MediaScan["accepted"] = [];
+  const rejected: string[] = [];
+  const seen = new Set<string>();
+  let totalBytes = 0;
+
+  for (const file of files) {
+    if (!isSafeArchivePath(file.path)) {
+      rejected.push(`${file.path} (unsafe path)`);
+      continue;
+    }
+    if (seen.has(file.path)) {
+      rejected.push(`${file.path} (duplicate path)`);
+      continue;
+    }
+    const verdict = verifyMediaFile(file.path, file.bytes);
+    if (!verdict.ok) {
+      rejected.push(`${file.path} (${verdict.reason})`);
+      continue;
+    }
+    seen.add(file.path);
+    accepted.push({ path: file.path, bytes: file.bytes, sha256: sha256(file.bytes) });
+    totalBytes += file.bytes.byteLength;
+  }
+
+  return { accepted, rejected, totalBytes };
+}
+
+export type BuildManifestInput = {
+  source: string | null;
+  groups: string[];
+  counts: Record<string, number>;
+  users: number;
+  media: MediaScan;
+  secretRefs: string[];
+  redacted?: string[];
+  archiveManifest?: Pick<ArchiveManifest, "schemaVersion"> | null;
+  now?: () => Date;
+};
+
+export function buildMigrationManifest(input: BuildManifestInput): MigrationManifest {
+  const now = (input.now ?? (() => new Date()))().toISOString();
+  return {
+    packageVersion: MIGRATION_PACKAGE_VERSION,
+    archiveFormat: ARCHIVE_FORMAT_VERSION,
+    sourceSchemaVersion: input.archiveManifest?.schemaVersion ?? SCHEMA_VERSION,
+    exportedAt: now,
+    source: input.source,
+    groups: [...input.groups],
+    counts: { ...input.counts },
+    users: input.users,
+    media: { files: input.media.accepted.length, bytes: input.media.totalBytes },
+    checksums: input.media.accepted.map((f) => ({
+      path: f.path,
+      bytes: f.bytes.byteLength,
+      sha256: f.sha256,
+    })),
+    secretRefs: [...new Set(input.secretRefs)].sort(),
+    redacted: [...new Set(input.redacted ?? [])].sort(),
+    rejected: [...input.media.rejected],
+  };
+}
+
+/** Verify the checksums in a manifest against the files actually present. */
+export function checkChecksums(
+  manifest: MigrationManifest,
+  files: Map<string, Uint8Array>,
+): string[] {
+  const problems: string[] = [];
+  for (const entry of manifest.checksums) {
+    const bytes = files.get(entry.path);
+    if (!bytes) {
+      problems.push(`missing media file: ${entry.path}`);
+      continue;
+    }
+    if (bytes.byteLength !== entry.bytes) {
+      problems.push(`size mismatch: ${entry.path}`);
+      continue;
+    }
+    if (sha256(bytes) !== entry.sha256) problems.push(`checksum mismatch: ${entry.path}`);
+  }
+  for (const path of files.keys()) {
+    if (!manifest.checksums.some((c) => c.path === path))
+      problems.push(`unlisted media file: ${path}`);
+  }
+  return problems;
+}

package/src/backup/media-export.ts

@@ -0,0 +1,24 @@
+import {
+  buildBrowserMediaArchive as buildSharedBrowserMediaArchive,
+  type MediaExportProgress,
+} from "./browser-media";
+
+import { ARCHIVE_FORMAT_VERSION, SCHEMA_VERSION, type ArchiveManifest } from "./datasets";
+import type { MediaStorage } from "../media/storage";
+
+export type { MediaExportProgress };
+
+type MediaExportOptions = {
+  baseArchive: Uint8Array;
+  groups: string[];
+  storage: Pick<MediaStorage, "list" | "download">;
+  onProgress?: (progress: MediaExportProgress) => void;
+};
+
+export function buildBrowserMediaArchive(options: MediaExportOptions) {
+  return buildSharedBrowserMediaArchive<ArchiveManifest>({
+    ...options,
+    expectedFormat: ARCHIVE_FORMAT_VERSION,
+    expectedSchemaVersion: SCHEMA_VERSION,
+  });
+}