@adcp/sdk 15.7.1 → 15.8.0

package/dist/lib/reporting/ledger/postgres.js

@@ -444,6 +444,14 @@ class PostgresReportingLedgerStore {
       [accountId, kind, recordId, obligationId]
     );
   }
+  /**
+   * Throws `INVALID_INPUT` when any configured object row binding's key template omits
+   * `{revision_key}`, which makes snapshot compaction unsafe (see `compactSupersededSnapshots`).
+   * Called by `compactSupersededSnapshots` and by service construction when compaction is enabled.
+   */
+  assertCompactionSafeRowBindings() {
+    this.rowStorage.assertCompactionSafeKeyTemplates();
+  }
   /** Probe and register row storage when enabled. Safe to call repeatedly. */
   async readyRowStorage() {
     if (this.rowStorageWrites) await this.rowStorage.ready();
@@ -954,7 +962,14 @@ ${managedDueArm}       )
       [obligationId]
     );
     const revisions = [];
-    for (const row of result.rows) revisions.push(await this.hydrateRevision(clone(row.data), row.account_id));
+    for (const row of result.rows) {
+      try {
+        revisions.push(await this.hydrateRevision(clone(row.data), row.account_id));
+      } catch (error) {
+        if ((0, import_row_storage.isReportingRowStoreError)(error) && error.code === "ROWS_EXPIRED") continue;
+        throw error;
+      }
+    }
     return revisions;
   }
   async listRevisionMetadata(obligationId, accountId) {
@@ -1425,7 +1440,9 @@ ${managedDueArm}       )
         list.push({ objectKey: row.object_key, nativeVersion: row.native_version });
         byBinding.set(row.row_binding_id, list);
       }
-      for (const [bindingId, list] of byBinding) await this.rowStorage.deleteObjects(bindingId, list, signal);
+      for (const [bindingId, list] of byBinding) {
+        await this.deleteUnreferencedObjects(bindingId, list, { obligationId }, signal);
+      }
     }
     await this.transaction(
       async (client) => {
@@ -1461,6 +1478,228 @@ ${managedDueArm}       )
       { preBeginAdvisoryLock: accountLock(accountId) }
     );
   }
+  /**
+   * Compact superseded snapshot revisions to metadata only, as adcontextprotocol/adcp#8100
+   * permits (`reporting-revision.json` `content_retention`). Opt-in: nothing calls this
+   * unless the host enables `compaction` in maintenance, and it must not be enabled in
+   * production before the AdCP release that takes compaction into effect (the next
+   * eligible minor after November 20, 2026).
+   *
+   * A snapshot revision is compacted only when every one of these holds:
+   *
+   * - it is superseded: a later revision names it in `supersedes_reporting_revision_id`
+   *   (so a current revision, and every official revision, is never compacted);
+   * - it is not the revision the current, unsuperseded revision names;
+   * - its recovery grace has passed: `max(automatedRecoveryWindowSeconds, 3600)` seconds
+   *   since its successor's `created_at`;
+   * - while its window is open (`statusRetentionDays` since its successor's `created_at`),
+   *   no caller's reporting receipt (any status) names it and no caller's current consumer
+   *   status statement of `received`, `unreadable` or `content_mismatch` names it. The
+   *   check spans every caller, so content any caller is entitled to is kept;
+   * - it has rows (nothing to reclaim otherwise, and an empty exact read is still correct);
+   * - no live obligation lease, no pending materialization of it, and no live change-feed
+   *   consumer that has not yet passed its change row (the holds retirement applies,
+   *   narrowed to this revision).
+   *
+   * Only row content is dropped: the revision header, its manifest and every metadata
+   * listing stay. The row set is marked `pruned`, so an exact read answers
+   * `ROWS_EXPIRED`, which the handlers surface as the same nondisclosing
+   * `REFERENCE_NOT_FOUND` as an unknown ID. No change-feed row is recorded: the revision's
+   * metadata is unchanged, and a sink still catching up holds compaction back.
+   *
+   * Compaction is resumable and runs in the same three phases as retirement, under the
+   * account lock where eligibility is decided:
+   *
+   * 1. re-check eligibility and mark the row set `pruning`, so reads answer as expired and
+   *    no new revision can share its chunks;
+   * 2. delete external objects at their recorded versions, skipping any object another
+   *    row set that is not being pruned still references (chunk sharing);
+   * 3. delete the chunk manifest rows and any PostgreSQL chunk body nothing else
+   *    references, and mark the row set `pruned`.
+   *
+   * A crash after phase 1 leaves a `pruning` row set that the next pass resumes.
+   */
+  async compactSupersededSnapshots(input) {
+    this.assertCompactionSafeRowBindings();
+    positiveInteger(input.statusRetentionDays, "statusRetentionDays");
+    positiveInteger(input.automatedRecoveryWindowSeconds, "automatedRecoveryWindowSeconds");
+    const limit = input.limit ?? 100;
+    positiveInteger(limit, "limit");
+    const holdDays = input.maxFeedHoldDays ?? 7;
+    positiveInteger(holdDays, "maxFeedHoldDays");
+    const tables = await this.optionalTables();
+    if (!tables.has("adcp_reporting_row_sets")) return { compacted: 0, failed: [], failures: [] };
+    const graceSeconds = Math.max(input.automatedRecoveryWindowSeconds, 3600);
+    const eligible = `
+      FROM adcp_reporting_revisions revision
+      JOIN adcp_reporting_obligations obligation ON obligation.obligation_id = revision.obligation_id
+      JOIN adcp_reporting_row_sets row_set
+        ON row_set.row_set_id = revision.revision_id AND row_set.row_set_kind = 'revision'
+      JOIN LATERAL (
+        SELECT MAX(later.created_at) AS created_at FROM adcp_reporting_revisions later
+         WHERE later.supersedes_revision_id = revision.revision_id
+      ) successor ON successor.created_at IS NOT NULL
+      WHERE revision.finality = 'snapshot' AND revision.kind = 'snapshot'
+        AND ($4::text IS NULL OR obligation.account_id = $4)
+        AND ($5::text IS NULL OR revision.revision_id = $5)
+        AND $3::integer > 0
+        AND row_set.row_count > 0
+        AND row_set.rows_state IN ('live', 'unavailable')
+        AND successor.created_at + ($2::bigint * INTERVAL '1 second') <= clock_timestamp()
+        AND NOT EXISTS (
+          SELECT 1 FROM adcp_reporting_revisions current_revision
+           WHERE current_revision.supersedes_revision_id = revision.revision_id
+             AND NOT EXISTS (
+               SELECT 1 FROM adcp_reporting_revisions next_revision
+                WHERE next_revision.supersedes_revision_id = current_revision.revision_id
+             )
+        )
+        AND (obligation.lease_expires_at IS NULL OR obligation.lease_expires_at <= clock_timestamp())
+        AND NOT EXISTS (SELECT 1 FROM adcp_reporting_obligation_tombstones tombstone
+                         WHERE tombstone.obligation_id = obligation.obligation_id)
+        AND (
+          successor.created_at + ($1::integer * INTERVAL '1 day') <= clock_timestamp()
+          OR (
+            NOT EXISTS (
+              SELECT 1 FROM adcp_reporting_consumer_statuses statement
+               WHERE statement.revision_id = revision.revision_id AND statement.is_current
+                 AND COALESCE(statement.data->>'consumer_status', statement.data->'status'->>'consumer_status')
+                     IN ('received', 'unreadable', 'content_mismatch')
+            )
+            ${tables.has("adcp_reporting_receipts") ? `AND NOT EXISTS (
+                     SELECT 1 FROM adcp_reporting_receipts receipt
+                      WHERE receipt.receipt_kind = 'revision' AND receipt.subject_id = revision.revision_id
+                   )` : ""}
+          )
+        )
+        ${tables.has("adcp_reporting_materializations") ? `AND NOT EXISTS (
+                 SELECT 1 FROM adcp_reporting_materializations hold
+                  WHERE hold.revision_id = revision.revision_id AND hold.status = 'pending'
+               )` : ""}
+        ${tables.has("adcp_reporting_changes") && tables.has("adcp_reporting_feed_consumers") ? `AND NOT EXISTS (
+                 SELECT 1 FROM adcp_reporting_changes change
+                   JOIN adcp_reporting_feed_consumers consumer
+                     ON consumer.updated_at > clock_timestamp() - ($3::integer * INTERVAL '1 day')
+                  WHERE change.record_kind = 'revision' AND change.record_id = revision.revision_id
+                    AND ((consumer.account_id IS NULL
+                          AND (change.xid, change.seq) > (consumer.cursor_xid, consumer.cursor_seq))
+                      OR (consumer.account_id = change.account_id AND change.seq > consumer.cursor_seq)))` : ""}`;
+    const baseValues = [input.statusRetentionDays, graceSeconds, holdDays, input.account_id ?? null];
+    const candidates = await this.query(
+      `SELECT revision_id, obligation_id, account_id FROM (
+         SELECT row_set.row_set_id AS revision_id, row_set.obligation_id, row_set.account_id, 0 AS resume,
+                row_set.recorded_at
+           FROM adcp_reporting_row_sets row_set
+          WHERE row_set.row_set_kind = 'revision' AND row_set.rows_state = 'pruning'
+            AND ($4::text IS NULL OR row_set.account_id = $4)
+            AND EXISTS (SELECT 1 FROM adcp_reporting_revisions revision
+                         WHERE revision.revision_id = row_set.row_set_id)
+            AND NOT EXISTS (SELECT 1 FROM adcp_reporting_obligation_tombstones tombstone
+                             WHERE tombstone.obligation_id = row_set.obligation_id)
+         UNION ALL
+         SELECT revision.revision_id, revision.obligation_id, obligation.account_id, 1, row_set.recorded_at
+           ${eligible}
+       ) candidate
+       ORDER BY resume, recorded_at, revision_id
+       LIMIT $6`,
+      [...baseValues, null, limit]
+    );
+    let compacted = 0;
+    const failed = [];
+    const failures = [];
+    for (const candidate of candidates.rows) {
+      input.signal?.throwIfAborted();
+      try {
+        const done = await this.compactSnapshot(
+          candidate.revision_id,
+          candidate.obligation_id,
+          candidate.account_id,
+          eligible,
+          baseValues,
+          input.signal
+        );
+        if (done) compacted += 1;
+      } catch (error) {
+        failed.push(candidate.revision_id);
+        failures.push({
+          reporting_obligation_id: candidate.obligation_id,
+          reporting_revision_id: candidate.revision_id,
+          cause: (0, import_row_storage.isReportingRowStoreError)(error) ? error.code : error instanceof Error ? error.name : "Error"
+        });
+      }
+    }
+    return { compacted, failed, failures };
+  }
+  async compactSnapshot(revisionId, obligationId, accountId, eligible, baseValues, signal) {
+    const claimed = await this.transaction(
+      async (client) => {
+        const marked = await client.query(
+          `UPDATE adcp_reporting_row_sets SET rows_state = 'pruning', rows_state_changed_at = clock_timestamp()
+            WHERE row_set_id = $5 AND rows_state IN ('live', 'unavailable')
+              AND row_set_id IN (SELECT revision.revision_id ${eligible})`,
+          [...baseValues, revisionId]
+        );
+        if (marked.rowCount === 1) return true;
+        const resumed = await client.query(
+          `SELECT 1 FROM adcp_reporting_row_sets row_set
+            WHERE row_set.row_set_id = $1 AND row_set.rows_state = 'pruning'
+              AND NOT EXISTS (SELECT 1 FROM adcp_reporting_obligation_tombstones tombstone
+                               WHERE tombstone.obligation_id = row_set.obligation_id)`,
+          [revisionId]
+        );
+        return resumed.rows.length === 1;
+      },
+      { preBeginAdvisoryLock: accountLock(accountId) }
+    );
+    if (!claimed) return false;
+    const binding = await this.query(
+      `SELECT row_binding_id FROM adcp_reporting_row_sets WHERE row_set_id = $1`,
+      [revisionId]
+    );
+    if (binding.rows.length === 0) return false;
+    const objects = await this.query(
+      `SELECT DISTINCT chunk.object_key, chunk.native_version
+         FROM adcp_reporting_row_chunks chunk
+        WHERE chunk.row_set_id = $1 AND chunk.object_key IS NOT NULL`,
+      [revisionId]
+    );
+    if (objects.rows.length > 0) {
+      await this.deleteUnreferencedObjects(
+        binding.rows[0].row_binding_id,
+        objects.rows.map((row) => ({ objectKey: row.object_key, nativeVersion: row.native_version })),
+        { rowSetId: revisionId },
+        signal
+      );
+    }
+    await this.transaction(
+      async (client) => {
+        const dropped = await client.query(
+          `DELETE FROM adcp_reporting_row_chunks WHERE row_set_id = $1 RETURNING sha256, object_key`,
+          [revisionId]
+        );
+        const bodies = [...new Set(dropped.rows.filter((row) => row.object_key === null).map((row) => row.sha256))];
+        if (bodies.length > 0) {
+          await client.query(
+            `DELETE FROM adcp_reporting_chunk_bodies body
+              WHERE body.account_id = $1 AND body.obligation_id = $2 AND body.sha256 = ANY($3::text[])
+                AND NOT EXISTS (
+                  SELECT 1 FROM adcp_reporting_row_chunks other
+                   WHERE other.account_id = body.account_id AND other.obligation_id = body.obligation_id
+                     AND other.sha256 = body.sha256 AND other.object_key IS NULL
+                )`,
+            [accountId, obligationId, bodies]
+          );
+        }
+        await client.query(
+          `UPDATE adcp_reporting_row_sets SET rows_state = 'pruned', rows_state_changed_at = clock_timestamp()
+            WHERE row_set_id = $1 AND rows_state = 'pruning'`,
+          [revisionId]
+        );
+      },
+      { preBeginAdvisoryLock: accountLock(accountId) }
+    );
+    return true;
+  }
   /**
    * Resolve abandoned object uploads (spec §4.3). An expired `open` write
    * intent whose ledger commit never happened is moved to `sweeping` under
@@ -1502,35 +1741,85 @@ ${managedDueArm}       )
           { preBeginAdvisoryLock: accountLock(intent.account_id) }
         );
         if (!claimed) continue;
-        const orphaned = [];
-        for (const object of claimed) {
-          const referenced = await this.query(
-            `SELECT 1 FROM adcp_reporting_row_chunks WHERE object_key = $1 AND native_version = $2 LIMIT 1`,
-            [object.objectKey, object.nativeVersion]
-          );
-          if (referenced.rows.length === 0) orphaned.push(object);
-        }
-        if (orphaned.length > 0) await this.rowStorage.deleteObjects(intent.row_binding_id, orphaned, input.signal);
+        const deleted = await this.deleteUnreferencedObjects(intent.row_binding_id, claimed, {}, input.signal);
         await this.query(
           `DELETE FROM adcp_reporting_row_write_intents
             WHERE row_binding_id = $1 AND row_set_id = $2 AND content_sha256 = $3 AND state = 'sweeping'`,
           key
         );
         swept += 1;
-        objectsDeleted += orphaned.length;
+        objectsDeleted += deleted;
       } catch {
         failed += 1;
       }
     }
     return { swept, objectsDeleted, failed };
   }
+  /**
+   * Delete external row objects, skipping any object another row set still references.
+   *
+   * A custom `keyTemplate` need only include `{content_sha256}` and `{chunk_index}`, so row sets
+   * from different obligations or accounts with identical chunk content can share one object (a
+   * create-only write adopts an existing object). An object is therefore deleted only when no
+   * chunk of a live (or unavailable) row set outside this operation references the same binding,
+   * key and native version, and no other open write intent created it. Row sets already `pruning`
+   * or `pruned` do not count: their own operation deletes the object, and counting them would let
+   * two concurrent retirements each skip the object and leak it. `scope` names what this
+   * operation is deleting: an obligation (retirement) or a row set (compaction).
+   *
+   * The check and the provider delete run in one transaction under a per-object advisory lock.
+   * A commit that adopts an existing object takes the same lock before it inserts its chunk rows
+   * and re-verifies the object under it (`writePreparedRowSet`), and holds it until commit. So
+   * either the delete wins the lock and the adopting commit then fails cleanly, or the commit wins
+   * and the delete sees its chunk rows and skips. A skipped object stays for the row set that owns
+   * it: that row set's own retirement, compaction or sweep deletes it, because by then nothing
+   * else references it. Locks are taken one object at a time, never while waiting on the account
+   * lock, so commits (account lock, then object locks) cannot deadlock with these deletes.
+   */
+  async deleteUnreferencedObjects(bindingId, objects, scope, signal) {
+    let deleted = 0;
+    for (const object of objects) {
+      signal?.throwIfAborted();
+      const removed = await this.transaction(async (client) => {
+        await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [
+          objectLock(bindingId, object.objectKey)
+        ]);
+        const referenced = await client.query(
+          `(SELECT 1 FROM adcp_reporting_row_chunks chunk
+              JOIN adcp_reporting_row_sets row_set ON row_set.row_set_id = chunk.row_set_id
+             WHERE row_set.row_binding_id = $1 AND chunk.object_key = $2 AND chunk.native_version = $3
+               AND row_set.rows_state IN ('live', 'unavailable')
+               AND ($4::text IS NULL OR chunk.obligation_id <> $4)
+               AND ($5::text IS NULL OR chunk.row_set_id <> $5)
+             LIMIT 1)
+           UNION ALL
+           (SELECT 1 FROM adcp_reporting_row_write_intents intent
+             WHERE intent.row_binding_id = $1 AND intent.state = 'open' AND intent.created_objects @> $6::jsonb
+             LIMIT 1)`,
+          [
+            bindingId,
+            object.objectKey,
+            object.nativeVersion,
+            scope.obligationId ?? null,
+            scope.rowSetId ?? null,
+            JSON.stringify([{ objectKey: object.objectKey, nativeVersion: object.nativeVersion }])
+          ]
+        );
+        if (referenced.rows.length > 0) return false;
+        await this.rowStorage.deleteObjects(bindingId, [object], signal);
+        return true;
+      });
+      if (removed) deleted += 1;
+    }
+    return deleted;
+  }
   async optionalTables() {
     const result = await this.query(
       `SELECT table_name AS name FROM information_schema.tables
         WHERE table_schema = current_schema()
           AND table_name IN ('adcp_reporting_row_sets', 'adcp_reporting_row_write_intents',
                              'adcp_reporting_materializations', 'adcp_reporting_changes',
-                             'adcp_reporting_feed_consumers')`
+                             'adcp_reporting_feed_consumers', 'adcp_reporting_receipts')`
     );
     return new Set(result.rows.map((row) => row.name));
   }
@@ -1634,6 +1923,15 @@ ${managedDueArm}       )
         throw new import_row_storage.ReportingRowStoreError("STATE_UNAVAILABLE", "the row upload intent was taken over or swept");
       }
     }
+    if (!upload.sharedFrom) {
+      const adopted = upload.locators.filter((locator) => !locator.created).sort((left, right) => left.objectKey < right.objectKey ? -1 : left.objectKey > right.objectKey ? 1 : 0);
+      for (const locator of adopted) {
+        await client.query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))", [
+          objectLock(prepared.bindingId, locator.objectKey)
+        ]);
+      }
+      if (adopted.length > 0) await this.rowStorage.assertObjectsPresent(prepared.bindingId, adopted);
+    }
     await this.rowStorage.writeInTransaction(client, prepared, upload.locators, upload.sharedFrom);
   }
   /** Attach verified rows to a revision document whose rows live in row storage. */
@@ -3822,6 +4120,9 @@ function rfc3339Microseconds(expression) {
 function accountLock(accountId) {
   return `adcp-reporting-account:${accountId}`;
 }
+function objectLock(bindingId, objectKey) {
+  return `adcp-reporting-object:${bindingId}:${objectKey}`;
+}
 async function resolveStoredFinalityBaseline(transaction, obligationId) {
   const latest = await transaction.query(
     `SELECT transition_id, data->>'finality' AS finality