@rehearsal-db/core 0.1.0-beta.12 → 0.1.0-beta.14

package/src/source/asset_transfer.mjs

@@ -2,6 +2,8 @@
 
 const BUCKET = /^[a-z0-9][a-z0-9.-]{0,99}$/u;
 const LOOPBACK = new Set(["127.0.0.1", "::1", "localhost"]);
+const MAXIMUM_STORAGE_DEPTH = 64;
+const MAXIMUM_STORAGE_PATH_BYTES = 4096;
 
 const safeBaseUrl = (value) => {
   let url;
@@ -34,14 +36,40 @@ const safePath = (value, label) => {
   ) {
     throw new Error(`${label} is unsafe.`);
   }
+  if (Buffer.byteLength(value) > MAXIMUM_STORAGE_PATH_BYTES) {
+    throw new Error(`${label} is too long.`);
+  }
   return value;
 };
 
+const safeInventoryName = (value) => {
+  const name = safePath(value, "Storage inventory name");
+  if (name.includes("/") || name.includes("\\")) {
+    throw new Error("Storage inventory name is not a single path segment.");
+  }
+  return name;
+};
+
 const readError = (label, response) =>
   new Error(
     `${label} failed with HTTP ${response.status}; response content was withheld.`,
   );
 
+const canonicalContentType = (...values) => {
+  for (const value of values) {
+    if (typeof value !== "string") continue;
+    const mediaType = value.split(";", 1)[0].trim().toLowerCase();
+    if (
+      /^[a-z0-9][a-z0-9!#$&^_.+-]*\/[a-z0-9][a-z0-9!#$&^_.+-]*$/u.test(
+        mediaType,
+      )
+    ) {
+      return mediaType;
+    }
+  }
+  return "application/octet-stream";
+};
+
 const responseBody = async function* ({
   response,
   expectedBytes,
@@ -69,6 +97,7 @@ export const streamApprovedSupabaseAssets = async function* ({
   maximumObjects = 10_000,
   maximumObjectBytes = 50 * 1024 * 1024,
   maximumTotalBytes = 2 * 1024 * 1024 * 1024,
+  pathMapper,
 }) {
   const endpoint = safeBaseUrl(baseUrl);
   if (typeof token !== "string" || token.length < 16) {
@@ -77,101 +106,154 @@ export const streamApprovedSupabaseAssets = async function* ({
   if (!Array.isArray(declarations) || declarations.length === 0) return;
   let objectCount = 0;
   let totalBytes = 0;
+  const inventoryPaths = new Set();
   for (const declaration of declarations) {
     if (!BUCKET.test(declaration.bucket ?? "")) {
       throw new Error("Approved Storage bucket is invalid.");
     }
-    const prefix = safePath(
+    const approvedPrefix = safePath(
       declaration.prefix.replace(/\/$/u, ""),
       "Approved Storage prefix",
     );
-    let offset = 0;
-    while (true) {
-      const response = await fetchImplementation(
-        `${endpoint}/storage/v1/object/list/${encodeURIComponent(declaration.bucket)}`,
-        {
-          method: "POST",
-          headers: {
-            authorization: `Bearer ${token}`,
-            "content-type": "application/json",
-          },
-          body: JSON.stringify({
-            prefix,
-            limit: 100,
-            offset,
-            sortBy: { column: "name", order: "asc" },
-          }),
-        },
-      );
-      if (!response.ok) throw readError("Storage inventory", response);
-      const inventory = await response.json();
-      if (!Array.isArray(inventory))
-        throw new Error("Storage inventory response is invalid.");
-      for (const object of inventory) {
-        const name = safePath(
-          `${prefix}/${object.name}`,
-          "Storage object path",
-        );
-        if (!name.startsWith(`${prefix}/`))
-          throw new Error("Storage inventory escaped its approved prefix.");
-        const expectedBytes = Number(
-          object.metadata?.size ?? object.metadata?.contentLength,
-        );
-        if (!Number.isSafeInteger(expectedBytes) || expectedBytes < 0) {
-          throw new Error("Storage inventory omitted an exact object size.");
-        }
-        if (expectedBytes > maximumObjectBytes) {
-          throw new Error("Storage object exceeds its reviewed size limit.");
-        }
-        objectCount += 1;
-        totalBytes += expectedBytes;
-        if (objectCount > maximumObjects || totalBytes > maximumTotalBytes) {
-          throw new Error(
-            "Storage inventory exceeds its reviewed transfer boundary.",
-          );
-        }
-        const encodedPath = name.split("/").map(encodeURIComponent).join("/");
-        const download = await fetchImplementation(
-          `${endpoint}/storage/v1/object/authenticated/${encodeURIComponent(declaration.bucket)}/${encodedPath}`,
+    const prefixes = [{ path: approvedPrefix, depth: 0 }];
+    const scheduledPrefixes = new Set([approvedPrefix]);
+    for (let prefixIndex = 0; prefixIndex < prefixes.length; prefixIndex += 1) {
+      const prefix = prefixes[prefixIndex];
+      let offset = 0;
+      while (true) {
+        const response = await fetchImplementation(
+          `${endpoint}/storage/v1/object/list/${encodeURIComponent(declaration.bucket)}`,
           {
+            method: "POST",
             headers: {
               authorization: `Bearer ${token}`,
-              ...(object.metadata?.eTag
-                ? { "if-match": object.metadata.eTag }
-                : {}),
+              "content-type": "application/json",
             },
+            body: JSON.stringify({
+              prefix: prefix.path,
+              limit: 100,
+              offset,
+              sortBy: { column: "name", order: "asc" },
+            }),
           },
         );
-        if (!download.ok) throw readError("Storage object transfer", download);
-        const responseEtag = download.headers.get("etag");
-        if (object.metadata?.eTag) {
-          if (!responseEtag) {
+        if (!response.ok) throw readError("Storage inventory", response);
+        const inventory = await response.json();
+        if (!Array.isArray(inventory))
+          throw new Error("Storage inventory response is invalid.");
+        for (const object of inventory) {
+          if (
+            object === null ||
+            typeof object !== "object" ||
+            Array.isArray(object)
+          ) {
+            throw new Error("Storage inventory entry is invalid.");
+          }
+          const entryName = safeInventoryName(object.name);
+          const name = safePath(
+            `${prefix.path}/${entryName}`,
+            "Storage object path",
+          );
+          if (!name.startsWith(`${approvedPrefix}/`))
+            throw new Error("Storage inventory escaped its approved prefix.");
+          const inventoryKey = `${declaration.bucket}\0${name}`;
+          if (inventoryPaths.has(inventoryKey)) {
+            throw new Error("Storage inventory returned a duplicate path.");
+          }
+          inventoryPaths.add(inventoryKey);
+          objectCount += 1;
+          if (objectCount > maximumObjects) {
+            throw new Error(
+              "Storage inventory exceeds its reviewed transfer boundary.",
+            );
+          }
+          const directory = object.id === null && object.metadata === null;
+          if (directory) {
+            if (prefix.depth >= MAXIMUM_STORAGE_DEPTH) {
+              throw new Error(
+                "Storage inventory exceeds its safe folder depth.",
+              );
+            }
+            if (scheduledPrefixes.has(name)) {
+              throw new Error("Storage inventory returned a duplicate folder.");
+            }
+            scheduledPrefixes.add(name);
+            prefixes.push({ path: name, depth: prefix.depth + 1 });
+            continue;
+          }
+          const expectedBytes = Number(
+            object.metadata?.size ?? object.metadata?.contentLength,
+          );
+          if (!Number.isSafeInteger(expectedBytes) || expectedBytes < 0) {
+            throw new Error("Storage inventory omitted an exact object size.");
+          }
+          if (expectedBytes > maximumObjectBytes) {
+            throw new Error("Storage object exceeds its reviewed size limit.");
+          }
+          totalBytes += expectedBytes;
+          if (totalBytes > maximumTotalBytes) {
             throw new Error(
-              "Storage transfer omitted the reviewed object version.",
+              "Storage inventory exceeds its reviewed transfer boundary.",
             );
           }
-          if (responseEtag !== object.metadata.eTag) {
+          const encodedPath = name.split("/").map(encodeURIComponent).join("/");
+          const download = await fetchImplementation(
+            `${endpoint}/storage/v1/object/authenticated/${encodeURIComponent(declaration.bucket)}/${encodedPath}`,
+            {
+              headers: {
+                authorization: `Bearer ${token}`,
+                ...(object.metadata?.eTag
+                  ? { "if-match": object.metadata.eTag }
+                  : {}),
+              },
+            },
+          );
+          if (!download.ok)
+            throw readError("Storage object transfer", download);
+          const responseEtag = download.headers.get("etag");
+          if (object.metadata?.eTag) {
+            if (!responseEtag) {
+              throw new Error(
+                "Storage transfer omitted the reviewed object version.",
+              );
+            }
+            if (responseEtag !== object.metadata.eTag) {
+              throw new Error(
+                "Storage object changed between inventory and transfer.",
+              );
+            }
+          }
+          if (declaration.pathMapping && typeof pathMapper !== "function") {
             throw new Error(
-              "Storage object changed between inventory and transfer.",
+              "Storage path mapping requires the package-owned privacy mapper.",
             );
           }
+          const objectPath = declaration.pathMapping
+            ? safePath(
+                pathMapper({
+                  mapping: declaration.pathMapping,
+                  value: name,
+                }),
+                "Mapped Storage object path",
+              )
+            : name;
+          yield {
+            bucket: declaration.bucket,
+            objectPath,
+            contentType: canonicalContentType(
+              object.metadata?.mimetype,
+              download.headers.get("content-type"),
+            ),
+            content: responseBody({
+              response: download,
+              expectedBytes,
+              maximumBytes: maximumObjectBytes,
+            }),
+          };
         }
-        yield {
-          bucket: declaration.bucket,
-          objectPath: name,
-          contentType:
-            download.headers.get("content-type") ??
-            object.metadata?.mimetype ??
-            "application/octet-stream",
-          content: responseBody({
-            response: download,
-            expectedBytes,
-            maximumBytes: maximumObjectBytes,
-          }),
-        };
+        if (inventory.length < 100) break;
+        offset += inventory.length;
       }
-      if (inventory.length < 100) break;
-      offset += inventory.length;
     }
   }
 };

package/src/source/baseline.mjs

@@ -24,6 +24,7 @@ import {
   validateExecutablePrivacyPolicy,
 } from "../baseline/privacy_engine.mjs";
 import {
+  sourceAccessPolicyFingerprint,
   sourceTargetFingerprint,
   validateSourceAccessPolicy,
 } from "./access.mjs";
@@ -45,6 +46,15 @@ const quoteIdentifier = (value) => {
 
 const canonicalJson = (value) => `${JSON.stringify(value, null, "\t")}\n`;
 const sha256 = (value) => createHash("sha256").update(value).digest("hex");
+const parseEstimateValue = (value) => {
+  if (
+    (typeof value !== "string" && typeof value !== "number") ||
+    (typeof value === "string" && !/^(?:0|[1-9]\d*)$/u.test(value))
+  ) {
+    return Number.NaN;
+  }
+  return Number(value);
+};
 
 const normalizeLimits = (limits = {}) => {
   const normalized = { ...DEFAULT_LIMITS, ...limits };
@@ -132,10 +142,20 @@ const estimateExport = async (client, policy) => {
               coalesce(sum(pg_column_size(export_row)), 0)::bigint as bytes
        from ${quoteIdentifier(policy.exportSchema)}.${quoteIdentifier(relation.view)} export_row`,
     );
-    const tableRows = Number(result.rows[0].rows);
-    const tableBytes = Number(result.rows[0].bytes);
-    if (!Number.isSafeInteger(tableRows) || !Number.isSafeInteger(tableBytes)) {
-      throw new Error("Source export estimate exceeds safe numeric limits.");
+    const estimate = result.rows?.[0];
+    const tableRows = parseEstimateValue(estimate?.rows);
+    const tableBytes = parseEstimateValue(estimate?.bytes);
+    if (
+      !Number.isSafeInteger(tableRows) ||
+      !Number.isSafeInteger(tableBytes) ||
+      tableRows < 0 ||
+      tableBytes < 0 ||
+      !Number.isSafeInteger(rows + tableRows) ||
+      !Number.isSafeInteger(bytes + tableBytes)
+    ) {
+      throw new Error(
+        "Source export estimate is missing, negative, or exceeds safe numeric limits.",
+      );
     }
     rows += tableRows;
     bytes += tableBytes;
@@ -238,6 +258,14 @@ const streamSanitizedRecords = async function* ({
         const batch = await client.query(
           `fetch forward ${limits.batchRows} from ${quoteIdentifier(cursor)}`,
         );
+        if (
+          !Array.isArray(batch?.rows) ||
+          batch.rows.length > limits.batchRows
+        ) {
+          throw new Error(
+            "Source reader returned a malformed or oversized cursor batch.",
+          );
+        }
         if (batch.rows.length === 0) break;
         for (const row of batch.rows) {
           const sanitized = privacyEngine.sanitize({
@@ -295,6 +323,7 @@ export const refreshPostgresqlBaseline = async ({
     );
   }
   const expectedReceiptShape = {
+    accessMode: sourcePolicy.reader.mode,
     exportSchema: sourcePolicy.exportSchema,
     views: sourcePolicy.relations.map((relation) => relation.view).sort(),
     relations: sourcePolicy.relations.map((relation) => ({
@@ -303,19 +332,25 @@ export const refreshPostgresqlBaseline = async ({
     })),
     migrationLedger: sourcePolicy.migrationLedger,
     readerRole: sourcePolicy.reader.role,
-    ownerRole: sourcePolicy.reader.ownerRole,
+    ownerRole: sourcePolicy.reader.ownerRole ?? null,
   };
   const actualReceiptShape = {
+    accessMode: sourceAccessReceipt?.accessMode ?? "managed",
     exportSchema: sourceAccessReceipt?.exportSchema,
     views: [...(sourceAccessReceipt?.views ?? [])].sort(),
     relations: sourceAccessReceipt?.relations,
     migrationLedger: sourceAccessReceipt?.migrationLedger,
     readerRole: sourceAccessReceipt?.readerRole,
-    ownerRole: sourceAccessReceipt?.ownerRole,
+    ownerRole: sourceAccessReceipt?.ownerRole ?? null,
   };
+  const policyFingerprint = sourceAccessPolicyFingerprint(sourcePolicy);
   if (
     sourceAccessReceipt?.targetFingerprint !== sourcePolicy.targetFingerprint ||
     !/^[a-f0-9]{64}$/u.test(sourceAccessReceipt?.planDigest ?? "") ||
+    (sourcePolicy.reader.mode === "external" &&
+      sourceAccessReceipt?.policyFingerprint !== policyFingerprint) ||
+    (sourceAccessReceipt?.policyFingerprint !== undefined &&
+      sourceAccessReceipt.policyFingerprint !== policyFingerprint) ||
     new Date(sourceAccessReceipt.expiresAt).valueOf() <= Date.now() ||
     JSON.stringify(actualReceiptShape) !== JSON.stringify(expectedReceiptShape)
   ) {
@@ -388,6 +423,7 @@ export const refreshPostgresqlBaseline = async ({
           canonicalJson({
             targetFingerprint: sourceAccessReceipt.targetFingerprint,
             planDigest: sourceAccessReceipt.planDigest,
+            policyFingerprint: sourceAccessReceipt.policyFingerprint ?? null,
             expiresAt: sourceAccessReceipt.expiresAt,
           }),
         ),