@adcp/sdk 15.9.0 → 16.0.0-beta.1

package/dist/lib/reporting/ledger/row-storage.js

@@ -55,12 +55,14 @@ class ReportingRowStorageV1 {
       if (!provider) throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", `row binding ${id} names an unknown provider`);
       provider.validateLocation(binding.location);
       (0, import_row_storage_object.assertReportingRowPrefixV1)(binding.prefix);
-      (0, import_row_storage_object.assertReportingRowKeyTemplateV1)(binding.keyTemplate ?? import_row_storage_object.REPORTING_ROW_DEFAULT_KEY_TEMPLATE);
       if (binding.credentialRef !== void 0 && !CREDENTIAL_REF.test(binding.credentialRef)) {
         throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", `row binding ${id} has an invalid credentialRef`);
       }
     }
-    this.bindings = Object.freeze({ ...bindings });
+    this.bindings = Object.freeze(
+      Object.fromEntries(Object.entries(bindings).map(([id, binding]) => [id, Object.freeze({ ...binding })]))
+    );
+    this.assertBindingKeyTemplates();
     this.deploymentNamespace = options.deploymentNamespace ?? "default";
     if (!this.deploymentNamespace || this.deploymentNamespace.length > 256) {
       throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "deploymentNamespace must be 1-256 characters");
@@ -81,12 +83,34 @@ class ReportingRowStorageV1 {
   get writeDeadlineMilliseconds() {
     return this.writeDeadline;
   }
+  /** Every writable object binding's key template must include `{revision_key}`; throws `INVALID_INPUT`. */
+  assertBindingKeyTemplates() {
+    for (const [id, binding] of Object.entries(this.bindings)) {
+      if (binding.kind !== "object") continue;
+      try {
+        (0, import_row_storage_object.assertReportingRowKeyTemplateV1)(binding.keyTemplate ?? import_row_storage_object.REPORTING_ROW_DEFAULT_KEY_TEMPLATE, {
+          requireRevisionKey: binding.readOnly !== true
+        });
+      } catch (error) {
+        if ((0, import_row_storage_errors.isReportingRowStoreError)(error)) {
+          throw new import_row_storage_errors.ReportingRowStoreError(
+            "INVALID_INPUT",
+            `row binding ${id}: ${error.message.replace(/^Reporting row store: INVALID_INPUT: /, "")}`
+          );
+        }
+        throw error;
+      }
+    }
+  }
   /**
    * Refuse a configuration in which compaction could delete an object a concurrent commit is
    * adopting. An object binding whose key template omits `{revision_key}` can map two revisions
    * with identical chunk content to one object key, and a create-only write adopts an existing
    * object with identical bytes. Compaction deleting that object at the same moment would leave
-   * the newer revision without its rows. The default template includes `{revision_key}`.
+   * the newer revision without its rows. Every writable binding is already required to include
+   * `{revision_key}`, so this cannot fail for a validated configuration. A `readOnly` binding is
+   * skipped: it never takes commits, so nothing can adopt through it, and every delete is
+   * reference-checked.
    *
    * Since the reference-checked, per-object-locked delete this is no longer the only guard: every
    * delete path now skips an object another row set still references. It is kept as defense in
@@ -94,7 +118,7 @@ class ReportingRowStorageV1 {
    */
   assertCompactionSafeKeyTemplates() {
     const unsafe = Object.entries(this.bindings).filter(
-      ([, binding]) => binding.kind === "object" && !(binding.keyTemplate ?? import_row_storage_object.REPORTING_ROW_DEFAULT_KEY_TEMPLATE).includes("{revision_key}")
+      ([, binding]) => binding.kind === "object" && binding.readOnly !== true && !(binding.keyTemplate ?? import_row_storage_object.REPORTING_ROW_DEFAULT_KEY_TEMPLATE).includes("{revision_key}")
     ).map(([id]) => id);
     if (unsafe.length > 0) {
       throw new import_row_storage_errors.ReportingRowStoreError(
@@ -136,30 +160,46 @@ class ReportingRowStorageV1 {
     if (digest.sha256 !== input.binding.sha256 || digest.byteCount !== input.binding.byteCount || input.binding.rowCount !== input.rows.length) {
       throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "rows do not reproduce their committed binding");
     }
-    const ids = Object.keys(this.bindings);
+    const writable = Object.keys(this.bindings).filter((id) => {
+      const candidate = this.bindings[id];
+      return candidate.kind !== "object" || candidate.readOnly !== true;
+    });
+    if (writable.length === 0 && !input.bindingId && !this.options.selectRowBinding) {
+      throw new import_row_storage_errors.ReportingRowStoreError(
+        "UNSAFE_BINDING",
+        "no writable row binding is configured: every binding is readOnly"
+      );
+    }
     const bindingId = input.bindingId ? input.bindingId : this.options.selectRowBinding ? await this.options.selectRowBinding({
       account_id: input.accountId,
       reporting_obligation_id: input.obligationId,
       row_set_kind: input.rowSetKind,
       ...input.finality ? { finality: input.finality } : {},
       row_count: input.rows.length,
-      canonical_byte_count: digest.byteCount
-    }) : ids.length === 1 ? ids[0] : DEFAULT_BINDING_ID;
+      content_size_bytes: digest.byteCount
+    }) : writable.length === 1 ? writable[0] : DEFAULT_BINDING_ID;
     const binding = this.bindings[bindingId];
     if (!binding) throw new import_row_storage_errors.ReportingRowStoreError("STATE_UNAVAILABLE", `row binding ${bindingId} is not configured`);
+    if (binding.kind === "object" && binding.readOnly === true) {
+      throw new import_row_storage_errors.ReportingRowStoreError(
+        "UNSAFE_BINDING",
+        `row binding ${bindingId} is read-only and cannot take new row sets`
+      );
+    }
     if (binding.kind === "postgres") return Object.freeze({ input, encoded, bindingId, binding, digestProfile });
     const compression = binding.compression ?? "gzip";
     const namespaceKey = this.installation.namespaceKey;
     const objects = encoded.chunks.map((chunk) => {
       const physicalBytes = compression === "gzip" ? (0, import_zlib.gzipSync)(chunk.bytes, { level: 6 }) : chunk.bytes;
       return Object.freeze({
-        key: (0, import_row_storage_object.renderReportingRowObjectKeyV1)({
+        objectRef: (0, import_row_storage_object.renderReportingRowObjectKeyV1)({
           template: binding.keyTemplate ?? import_row_storage_object.REPORTING_ROW_DEFAULT_KEY_TEMPLATE,
           prefix: binding.prefix,
           namespaceKey,
           accountId: input.accountId,
           rowSetId: input.rowSetId,
           ...input.periodStart ? { periodStart: input.periodStart } : {},
+          ...input.sourceTimezone ? { sourceTimezone: input.sourceTimezone } : {},
           ...input.finality ? { finality: input.finality } : {},
           contentSha256: input.binding.sha256,
           chunkIndex: chunk.manifest.chunk_index,
@@ -194,7 +234,7 @@ class ReportingRowStorageV1 {
         () => provider.putIfAbsent(
           {
             ...location,
-            key: object.key,
+            objectRef: object.objectRef,
             bytes: object.physicalBytes,
             contentType: object.compression === "gzip" ? "application/gzip" : "application/x-ndjson",
             metadata: {
@@ -206,20 +246,27 @@ class ReportingRowStorageV1 {
         )
       );
       if (put.created) {
-        await onCreated({ objectKey: object.key, nativeVersion: put.nativeVersion });
+        await onCreated({ objectRef: object.objectRef, nativeVersionRef: put.nativeVersionRef });
+        assertNativeVersionRef(put.nativeVersionRef);
         locators.push({
-          objectKey: object.key,
-          nativeVersion: put.nativeVersion,
+          objectRef: object.objectRef,
+          nativeVersionRef: put.nativeVersionRef,
           compression: object.compression,
           physicalSha256: object.physicalSha256,
-          physicalByteCount: object.physicalBytes.byteLength,
+          sizeBytes: object.physicalBytes.byteLength,
           created: true
         });
         continue;
       }
+      assertNativeVersionRef(put.nativeVersionRef);
       const existing = await this.provider(
         () => provider.get(
-          { ...location, key: object.key, nativeVersion: put.nativeVersion, maxBytes: MAX_ADOPTED_OBJECT_BYTES },
+          {
+            ...location,
+            objectRef: object.objectRef,
+            nativeVersionRef: put.nativeVersionRef,
+            maxBytes: MAX_ADOPTED_OBJECT_BYTES
+          },
           context
         )
       );
@@ -228,11 +275,11 @@ class ReportingRowStorageV1 {
         throw new import_row_storage_errors.ReportingRowStoreError("CONTENT_CONFLICT", `object for chunk ${index} holds different bytes`);
       }
       locators.push({
-        objectKey: object.key,
-        nativeVersion: put.nativeVersion,
+        objectRef: object.objectRef,
+        nativeVersionRef: put.nativeVersionRef,
         compression: object.compression,
         physicalSha256: sha256Hex(existing),
-        physicalByteCount: existing.byteLength,
+        sizeBytes: existing.byteLength,
         created: false
       });
     }
@@ -251,17 +298,18 @@ class ReportingRowStorageV1 {
     }
     const inserted = await transaction.query(
       `INSERT INTO adcp_reporting_row_sets
-         (row_set_id, row_set_kind, account_id, obligation_id, encoding, digest_profile, content_sha256,
-          canonical_byte_count, row_count, row_manifest_sha256, chunk_count, row_binding_id,
+         (row_set_id, row_set_kind, account_id, obligation_id, format, algorithm, digest_profile, content_sha256,
+          content_size_bytes, row_count, row_manifest_sha256, chunk_count, row_binding_id,
           rows_shared_from_row_set_id)
-       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)
+       VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
        ON CONFLICT (row_set_id) DO NOTHING`,
       [
         input.rowSetId,
         input.rowSetKind,
         input.accountId,
         input.obligationId,
-        import_row_encoding.REPORTING_ROW_ENCODING_V1,
+        import_row_encoding.REPORTING_ROW_FORMAT_V1,
+        import_row_encoding.REPORTING_ROW_ALGORITHM_V1,
         prepared.digestProfile,
         input.binding.sha256,
         input.binding.byteCount,
@@ -295,8 +343,8 @@ class ReportingRowStorageV1 {
       }
       await transaction.query(
         `INSERT INTO adcp_reporting_row_chunks
-           (row_set_id, chunk_index, account_id, obligation_id, first_ordinal, row_count, byte_count, sha256, segments,
-            object_key, native_version, compression, physical_sha256, physical_byte_count)
+           (row_set_id, chunk_index, account_id, obligation_id, first_ordinal, row_count, canonical_size_bytes, sha256,
+            segments, object_ref, native_version_ref, compression, physical_sha256, size_bytes)
          VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9::jsonb, $10, $11, $12, $13, $14)`,
         [
           input.rowSetId,
@@ -308,11 +356,11 @@ class ReportingRowStorageV1 {
           manifest.byte_count,
           manifest.sha256,
           JSON.stringify(manifest.segments),
-          locator?.objectKey ?? null,
-          locator?.nativeVersion ?? null,
+          locator?.objectRef ?? null,
+          locator?.nativeVersionRef ?? null,
           locator?.compression ?? "none",
           locator?.physicalSha256 ?? null,
-          locator?.physicalByteCount ?? null
+          locator?.sizeBytes ?? null
         ]
       );
     }
@@ -326,7 +374,7 @@ class ReportingRowStorageV1 {
   async find(rowSetId, accountId, queryable = this.pool) {
     const found = await queryable.query(
       `SELECT row_set.row_set_id, row_set.row_set_kind, row_set.account_id, row_set.obligation_id,
-              row_set.digest_profile, row_set.content_sha256, row_set.canonical_byte_count, row_set.row_count,
+              row_set.digest_profile, row_set.content_sha256, row_set.content_size_bytes, row_set.row_count,
               row_set.row_manifest_sha256, row_set.chunk_count, row_set.row_binding_id,
               row_set.row_location_version, row_set.rows_state, binding.kind, binding.identity_sha256
          FROM adcp_reporting_row_sets row_set
@@ -341,8 +389,8 @@ class ReportingRowStorageV1 {
     }
     if (record.rows_state === "unavailable") throw new import_row_storage_errors.ReportingRowStoreError("ROWS_UNAVAILABLE");
     const chunks = await queryable.query(
-      `SELECT chunk_index, first_ordinal, row_count, byte_count, sha256, segments,
-              object_key, native_version, compression, physical_sha256, physical_byte_count
+      `SELECT chunk_index, first_ordinal, row_count, canonical_size_bytes, sha256, segments,
+              object_ref, native_version_ref, compression, physical_sha256, size_bytes
          FROM adcp_reporting_row_chunks WHERE row_set_id = $1 ORDER BY chunk_index`,
       [rowSetId]
     );
@@ -350,7 +398,7 @@ class ReportingRowStorageV1 {
       chunk_index: chunk.chunk_index,
       first_ordinal: Number(chunk.first_ordinal),
       row_count: chunk.row_count,
-      byte_count: Number(chunk.byte_count),
+      byte_count: Number(chunk.canonical_size_bytes),
       sha256: chunk.sha256,
       segments: chunk.segments.map((segment) => ({
         first_ordinal: Number(segment.first_ordinal),
@@ -371,16 +419,16 @@ class ReportingRowStorageV1 {
       throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", "chunk count does not match the row set");
     }
     const locators = chunks.rows.map((chunk) => {
-      const hasLocation = chunk.object_key !== null;
+      const hasLocation = chunk.object_ref !== null;
       if (hasLocation !== (record.kind === "object")) {
         throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", "chunk location does not match its binding kind");
       }
       return hasLocation ? Object.freeze({
-        objectKey: chunk.object_key,
-        nativeVersion: chunk.native_version,
+        objectRef: chunk.object_ref,
+        nativeVersionRef: chunk.native_version_ref,
         compression: chunk.compression,
         physicalSha256: chunk.physical_sha256,
-        physicalByteCount: Number(chunk.physical_byte_count)
+        sizeBytes: Number(chunk.size_bytes)
       }) : void 0;
     });
     return Object.freeze({
@@ -390,7 +438,7 @@ class ReportingRowStorageV1 {
       obligationId: record.obligation_id,
       digestProfile: record.digest_profile,
       contentSha256: record.content_sha256,
-      canonicalByteCount: Number(record.canonical_byte_count),
+      contentSizeBytes: Number(record.content_size_bytes),
       rowCount,
       rowManifestSha256: record.row_manifest_sha256,
       bindingId: record.row_binding_id,
@@ -438,7 +486,7 @@ class ReportingRowStorageV1 {
    * `controlTotals` are required for revision row sets.
    */
   async readAll(handle, controlTotals, options = {}) {
-    if (options.maxBytes !== void 0 && handle.canonicalByteCount > options.maxBytes) {
+    if (options.maxBytes !== void 0 && handle.contentSizeBytes > options.maxBytes) {
       throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", "row set exceeds the hydration byte limit");
     }
     const hasher = handle.digestProfile === "revision_envelope_v1" ? (0, import_row_encoding.createReportingRevisionEnvelopeHasherV1)({
@@ -457,7 +505,7 @@ class ReportingRowStorageV1 {
       );
     }
     const digest = this.decode(() => hasher.digest());
-    if (digest.sha256 !== handle.contentSha256 || digest.byteCount !== handle.canonicalByteCount) {
+    if (digest.sha256 !== handle.contentSha256 || digest.byteCount !== handle.contentSizeBytes) {
       throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", "rows do not reproduce the committed binding");
     }
     return rows;
@@ -476,22 +524,28 @@ class ReportingRowStorageV1 {
     const ownedPrefix = `${binding.prefix}/${this.installation.namespaceKey}/`;
     const context = { signal: deadlineSignal(this.writeDeadline, signal) };
     for (const object of objects) {
-      if (!object.objectKey.startsWith(ownedPrefix)) {
+      if (!object.objectRef.startsWith(ownedPrefix)) {
         throw new import_row_storage_errors.ReportingRowStoreError("UNSAFE_BINDING", "refusing to delete an object outside the owned prefix");
       }
       await this.provider(
         () => provider.delete(
-          { ...providerLocation(binding), key: object.objectKey, nativeVersion: object.nativeVersion },
+          { ...providerLocation(binding), objectRef: object.objectRef, nativeVersionRef: object.nativeVersionRef },
           context
         )
       );
     }
   }
   /**
-   * Prove each object is still present at its recorded version with the recorded bytes.
+   * Prove each object is still present at its recorded version with the recorded size.
    * A commit that adopted an object another row set created calls this under the object's
    * lock, so a concurrent delete either finished first (this throws, the commit fails and
    * the writer re-uploads) or sees the commit's chunk reference and skips the delete.
+   *
+   * Native versions are write-unique (S3 VersionId, Azure versionId, GCS generation, filesystem
+   * `ino:birthtime:mtime:size`), so a version-pinned metadata check, "this exact version exists at the
+   * recorded size", proves the object unchanged without downloading it while the commit holds its
+   * locks. Bytes were verified when the object was uploaded or adopted. A custom provider without
+   * `stat` falls back to reading the whole object and comparing its physical digest.
    */
   async assertObjectsPresent(bindingId, objects, signal) {
     await this.ready();
@@ -501,24 +555,24 @@ class ReportingRowStorageV1 {
     }
     const provider = this.providers[binding.provider];
     const context = { signal: deadlineSignal(this.writeDeadline, signal) };
+    const location = providerLocation(binding);
+    const changed = () => new import_row_storage_errors.ReportingRowStoreError(
+      "STATE_UNAVAILABLE",
+      "an adopted row object was deleted or replaced before its commit; retry the upload"
+    );
     for (const object of objects) {
+      const identity = { ...location, objectRef: object.objectRef, nativeVersionRef: object.nativeVersionRef };
+      if (provider.stat) {
+        const stat = await this.provider(() => provider.stat(identity, context));
+        if (!stat || stat.nativeVersionRef !== object.nativeVersionRef || !Number.isSafeInteger(stat.sizeBytes) || stat.sizeBytes !== object.sizeBytes) {
+          throw changed();
+        }
+        continue;
+      }
       const bytes = await this.provider(
-        () => provider.get(
-          {
-            ...providerLocation(binding),
-            key: object.objectKey,
-            nativeVersion: object.nativeVersion,
-            maxBytes: MAX_ADOPTED_OBJECT_BYTES
-          },
-          context
-        )
+        () => provider.get({ ...identity, maxBytes: MAX_ADOPTED_OBJECT_BYTES }, context)
       );
-      if (!bytes || sha256Hex(bytes) !== object.physicalSha256) {
-        throw new import_row_storage_errors.ReportingRowStoreError(
-          "STATE_UNAVAILABLE",
-          "an adopted row object was deleted or replaced before its commit; retry the upload"
-        );
-      }
+      if (!bytes || sha256Hex(bytes) !== object.physicalSha256) throw changed();
     }
   }
   /**
@@ -545,21 +599,21 @@ class ReportingRowStorageV1 {
     const rowSetId = found.rows[0]?.row_set_id;
     if (!rowSetId) return void 0;
     const chunks = await queryable.query(
-      `SELECT object_key, native_version, compression, physical_sha256, physical_byte_count
+      `SELECT object_ref, native_version_ref, compression, physical_sha256, size_bytes
          FROM adcp_reporting_row_chunks WHERE row_set_id = $1 ORDER BY chunk_index`,
       [rowSetId]
     );
     if (chunks.rows.length !== prepared.encoded.chunks.length) return void 0;
     if (prepared.binding.kind === "postgres") return { rowSetId, locators: [] };
-    if (chunks.rows.some((chunk) => chunk.object_key === null)) return void 0;
+    if (chunks.rows.some((chunk) => chunk.object_ref === null)) return void 0;
     return {
       rowSetId,
       locators: chunks.rows.map((chunk) => ({
-        objectKey: chunk.object_key,
-        nativeVersion: chunk.native_version,
+        objectRef: chunk.object_ref,
+        nativeVersionRef: chunk.native_version_ref,
         compression: chunk.compression,
         physicalSha256: chunk.physical_sha256,
-        physicalByteCount: Number(chunk.physical_byte_count),
+        sizeBytes: Number(chunk.size_bytes),
         created: false
       }))
     };
@@ -594,8 +648,8 @@ class ReportingRowStorageV1 {
         () => provider.get(
           {
             ...location,
-            key: locator.objectKey,
-            nativeVersion: locator.nativeVersion,
+            objectRef: locator.objectRef,
+            nativeVersionRef: locator.nativeVersionRef,
             range: { offset: byteOffset, length: byteCount },
             maxBytes: byteCount
           },
@@ -613,9 +667,9 @@ class ReportingRowStorageV1 {
       () => provider.get(
         {
           ...location,
-          key: locator.objectKey,
-          nativeVersion: locator.nativeVersion,
-          maxBytes: locator.physicalByteCount
+          objectRef: locator.objectRef,
+          nativeVersionRef: locator.nativeVersionRef,
+          maxBytes: locator.sizeBytes
         },
         context
       )
@@ -623,7 +677,7 @@ class ReportingRowStorageV1 {
     if (!stored)
       throw new import_row_storage_errors.ReportingRowStoreError("ROWS_UNAVAILABLE", `chunk ${manifest.chunk_index} object is missing`);
     const physical = Buffer.from(stored);
-    if (physical.byteLength !== locator.physicalByteCount || sha256Hex(physical) !== locator.physicalSha256) {
+    if (physical.byteLength !== locator.sizeBytes || sha256Hex(physical) !== locator.physicalSha256) {
       throw new import_row_storage_errors.ReportingRowStoreError("ROWS_INTEGRITY_FAILED", `chunk ${manifest.chunk_index} stored bytes differ`);
     }
     const logical = this.inflate(physical, locator.compression, manifest.byte_count);
@@ -662,6 +716,7 @@ class ReportingRowStorageV1 {
     }
   }
   async register() {
+    this.assertBindingKeyTemplates();
     const { installationId } = await (0, import_row_storage_migration.probeReportingRowStorageSchemaV1)(this.pool);
     const namespaceKey = sha256Hex((0, import_jcs.canonicalize)(["adcp.rows.v1", this.deploymentNamespace, installationId]));
     this.installation = { installationId, namespaceKey };
@@ -731,6 +786,11 @@ function deadlineSignal(milliseconds, signal) {
   const timeout = AbortSignal.timeout(milliseconds);
   return signal ? AbortSignal.any([signal, timeout]) : timeout;
 }
+function assertNativeVersionRef(value) {
+  if (typeof value !== "string" || value.length < 1 || value.length > import_row_storage_object.REPORTING_ROW_NATIVE_VERSION_REF_MAX_LENGTH) {
+    throw new import_row_storage_errors.ReportingRowStoreError("PROVIDER_UNAVAILABLE", "provider returned an invalid native version reference");
+  }
+}
 function positiveMilliseconds(value, name) {
   if (!Number.isSafeInteger(value) || value < 1e3 || value > 6e5) {
     throw new import_row_storage_errors.ReportingRowStoreError("INVALID_INPUT", `${name} must be between 1000 and 600000`);