@adcp/sdk 17.0.0-beta.1 → 17.0.0-beta.3

package/dist/lib/reporting/ledger/producer.js

@@ -54,7 +54,10 @@ function createReportingProducer(options) {
     subscribersByAccount.set(subscriber.account_id, count);
   }
   return {
-    async installConfiguration(input) {
+    async installConfiguration(input, installOptions = {}) {
+      if (Object.hasOwn(input, "historyStart") || Object.hasOwn(input, "historyStartOrdinal")) {
+        throw new TypeError("Reporting history must be supplied through trusted install options");
+      }
       const normalizedInput = {
         ...structuredClone(input),
         schedule: {
@@ -81,6 +84,19 @@ function createReportingProducer(options) {
         if (!replayFingerprints.includes(replay.semanticFingerprint)) {
           throw new Error("Reporting configuration generation is immutable");
         }
+        if (installOptions.historyStart !== void 0) {
+          const requested = historyInstant(installOptions.historyStart);
+          const ownedStart = replay.historyStart !== void 0 ? Date.parse(replay.historyStart) : void 0;
+          if (ownedStart !== void 0 && requested === ownedStart) return replay;
+          if (requested > Date.now()) throw new RangeError("Reporting historyStart must not be in the future");
+          if (ownedStart !== void 0 && requested >= ownedStart) return replay;
+          const schedule = (0, import_schedule.reportingPeriodSchedule)(replay);
+          const start = schedule.boundary(schedule.ceil(requested));
+          const previous = ownedStart ?? schedule.boundary(Math.max(0, schedule.ceil(Date.parse(replay.installedAt))));
+          if (start < previous) {
+            throw new Error("Use extendConfigurationHistory to change an installed history boundary");
+          }
+        }
         return replay;
       }
       validateConfigurationAgainstOffering(normalizedInput, requiredOffering(offeringById, normalizedInput.offeringId));
@@ -113,12 +129,59 @@ function createReportingProducer(options) {
         installedAt,
         semanticFingerprint: calendarRules ? (0, import_schedule.reportingCalendarDayFingerprint)(normalizedInput, calendarRules) : semanticFingerprint
       };
-      return (await options.store.putConfiguration(configuration)).value;
+      if (installOptions.historyStart !== void 0) {
+        Object.assign(
+          configuration,
+          normalizeConfigurationHistory(
+            configuration,
+            installOptions.historyStart,
+            requiredOffering(offeringById, configuration.offeringId),
+            Date.parse(installedAt)
+          )
+        );
+      }
+      (0, import_coverage.assertReportingLedgerHistoryDoesNotOverlap)(configuration, existing);
+      const stored = (await options.store.putConfiguration(configuration)).value;
+      if (configuration.historyStart !== void 0) {
+        const storedSchedule = stored.historyStart === void 0 ? (0, import_schedule.reportingPeriodSchedule)(stored) : void 0;
+        const storedStart = stored.historyStart !== void 0 ? Date.parse(stored.historyStart) : storedSchedule.boundary(Math.max(0, storedSchedule.ceil(Date.parse(stored.installedAt))));
+        if (storedStart > Date.parse(configuration.historyStart)) {
+          throw new Error("Use extendConfigurationHistory to change an installed history boundary");
+        }
+      }
+      return stored;
+    },
+    async extendConfigurationHistory(input) {
+      if (!options.store.extendConfigurationHistory) {
+        throw new TypeError("Reporting ledger store does not support history extension");
+      }
+      const configuration = (await options.store.listConfigurations()).find(
+        (value) => value.configurationId === input.configurationId
+      );
+      if (!configuration) throw new Error("Unknown reporting configuration");
+      if (configuration.historyStart !== void 0) {
+        const requested = historyInstant(input.historyStart);
+        if (requested === Date.parse(configuration.historyStart)) return configuration;
+        const schedule = (0, import_schedule.reportingPeriodSchedule)(configuration);
+        if (schedule.boundary(schedule.ceil(requested)) === Date.parse(configuration.historyStart)) {
+          return configuration;
+        }
+      }
+      const history = normalizeConfigurationHistory(
+        configuration,
+        input.historyStart,
+        requiredOffering(offeringById, configuration.offeringId),
+        Date.now()
+      );
+      return options.store.extendConfigurationHistory({ configurationId: input.configurationId, ...history });
     },
     async planObligations(now = (/* @__PURE__ */ new Date()).toISOString(), planOptions = {}) {
       const nowMs = instant(now, "now");
       const maxObligations = planOptions.maxObligations ?? 1e3;
       positiveInteger(maxObligations, "maxObligations");
+      const maxHistory = planOptions.maxHistoryObligationsPerSource ?? maxObligations;
+      nonnegativeInteger(maxHistory, "maxHistoryObligationsPerSource");
+      const historyBySource = /* @__PURE__ */ new Map();
       const created = [];
       let attempted = 0;
       const configurations = (await options.store.listConfigurations(planOptions.account_id)).sort(
@@ -126,74 +189,86 @@ function createReportingProducer(options) {
       );
       const obligationsByAccount = /* @__PURE__ */ new Map();
       const retiredByAccount = /* @__PURE__ */ new Map();
-      for (const configuration of configurations) {
-        const successor = (0, import_coverage.reportingLedgerSuccessor)(configuration, configurations);
-        const anchor = instant(configuration.schedule.anchor, "schedule.anchor");
-        const effectiveFrom = Math.max(anchor, instant(configuration.installedAt, "installedAt"));
-        let schedule;
-        try {
-          schedule = (0, import_schedule.reportingPeriodSchedule)(configuration);
-        } catch (error) {
-          const predecessor = configurations.find(
-            (candidate) => (0, import_coverage.reportingLedgerSuccessor)(candidate, configurations)?.configurationId === configuration.configurationId
-          );
-          if ((0, import_schedule.isFrozenCalendarRulesMismatch)(error) && !successor && predecessor && runtimeCanResolveCalendarGeneration(predecessor)) {
+      for (const historyPass of [false, true]) {
+        for (const configuration of configurations) {
+          const successor = (0, import_coverage.reportingLedgerSuccessor)(configuration, configurations);
+          const anchor = instant(configuration.schedule.anchor, "schedule.anchor");
+          const effectiveFrom = configuration.historyStart ? instant(configuration.historyStart, "historyStart") : Math.max(anchor, instant(configuration.installedAt, "installedAt"));
+          let schedule;
+          try {
+            schedule = (0, import_schedule.reportingPeriodSchedule)(configuration);
+          } catch (error) {
+            const predecessor = configurations.find(
+              (candidate) => (0, import_coverage.reportingLedgerSuccessor)(candidate, configurations)?.configurationId === configuration.configurationId
+            );
+            if ((0, import_schedule.isFrozenCalendarRulesMismatch)(error) && !successor && predecessor && runtimeCanResolveCalendarGeneration(predecessor)) {
+              continue;
+            }
+            if (!(0, import_schedule.isFrozenCalendarRulesMismatch)(error) || !await supersededCalendarGenerationIsDurablyCovered(options.store, configuration, successor)) {
+              throw error;
+            }
             continue;
           }
-          if (!(0, import_schedule.isFrozenCalendarRulesMismatch)(error) || !await supersededCalendarGenerationIsDurablyCovered(options.store, configuration, successor)) {
-            throw error;
+          const generationEndValue = Math.min(
+            successor ? instant(successor.installedAt, "successor.installedAt") : Number.POSITIVE_INFINITY,
+            configuration.supersededAt ? instant(configuration.supersededAt, "supersededAt") : Number.POSITIVE_INFINITY
+          );
+          const generationEnd = Number.isFinite(generationEndValue) ? generationEndValue : void 0;
+          const effectiveUntil = Math.min(nowMs, generationEnd ?? nowMs);
+          const ownedFirst = configuration.historyStartOrdinal ?? Math.max(0, schedule.ceil(effectiveFrom));
+          const liveFirst = Math.max(0, schedule.ceil(instant(configuration.installedAt, "installedAt")));
+          const first = historyPass ? ownedFirst : Math.max(ownedFirst, liveFirst);
+          const ownershipLast = generationEnd ? schedule.ceil(effectiveUntil) - 1 : Number.POSITIVE_INFINITY;
+          if (ownershipLast < first) continue;
+          const latestClosed = schedule.floor(
+            Number.isFinite(ownershipLast) ? Math.min(nowMs, schedule.boundary(ownershipLast + 1)) : nowMs
+          ) - 1;
+          const last = Math.min(ownershipLast, latestClosed, historyPass ? liveFirst - 1 : Number.POSITIVE_INFINITY);
+          let accountObligations = obligationsByAccount.get(configuration.account.account_id);
+          if (!accountObligations) {
+            accountObligations = await options.store.listObligations(configuration.account.account_id);
+            obligationsByAccount.set(configuration.account.account_id, accountObligations);
           }
-          continue;
-        }
-        const generationEndValue = Math.min(
-          successor ? instant(successor.installedAt, "successor.installedAt") : Number.POSITIVE_INFINITY,
-          configuration.supersededAt ? instant(configuration.supersededAt, "supersededAt") : Number.POSITIVE_INFINITY
-        );
-        const generationEnd = Number.isFinite(generationEndValue) ? generationEndValue : void 0;
-        const effectiveUntil = Math.min(nowMs, generationEnd ?? nowMs);
-        const first = Math.max(0, schedule.ceil(effectiveFrom));
-        const ownershipLast = generationEnd ? schedule.ceil(effectiveUntil) - 1 : Number.POSITIVE_INFINITY;
-        if (ownershipLast < first) continue;
-        const latestClosed = schedule.floor(
-          Number.isFinite(ownershipLast) ? Math.min(nowMs, schedule.boundary(ownershipLast + 1)) : nowMs
-        ) - 1;
-        const last = Math.min(ownershipLast, latestClosed);
-        let accountObligations = obligationsByAccount.get(configuration.account.account_id);
-        if (!accountObligations) {
-          accountObligations = await options.store.listObligations(configuration.account.account_id);
-          obligationsByAccount.set(configuration.account.account_id, accountObligations);
-        }
-        let accountRetired = retiredByAccount.get(configuration.account.account_id);
-        if (!accountRetired) {
-          accountRetired = await options.store.listRetiredObligationOrdinals?.(configuration.account.account_id) ?? [];
-          retiredByAccount.set(configuration.account.account_id, accountRetired);
-        }
-        const existingOrdinals = [
-          ...accountObligations.filter((value) => value.configurationId === configuration.configurationId).map((value) => value.periodOrdinal),
-          ...accountRetired.filter((value) => value.configurationId === configuration.configurationId).map((value) => value.periodOrdinal)
-        ];
-        const candidates = missingOrdinals(first, last, existingOrdinals, maxObligations - attempted);
-        for (const ordinal of candidates) {
-          attempted += 1;
-          const obligation = planObligation(configuration, ordinal, now);
-          const written = await options.store.putObligation(obligation);
-          if (written.inserted) {
-            created.push(written.value);
-            await (0, import_lifecycle.reconcileReportingStatusLifecycleV1)({
-              store: options.store,
-              reporting_obligation_id: written.value.reporting_obligation_id,
-              // A fallback clock, never a pinned cutoff. `now` is this host's
-              // instant: a host running fast pinned a cutoff ahead of the
-              // database, the watermark was stamped with it, and every
-              // database-timestamped change inside that skew — a revocation,
-              // a receipt — landed behind the watermark and never made the
-              // obligation due again, so a `complete` transition and its
-              // webhook stood over state that had already contradicted it.
-              now: () => new Date(now),
-              subscribers: options.subscribers
-            });
+          let accountRetired = retiredByAccount.get(configuration.account.account_id);
+          if (!accountRetired) {
+            accountRetired = await options.store.listRetiredObligationOrdinals?.(configuration.account.account_id) ?? [];
+            retiredByAccount.set(configuration.account.account_id, accountRetired);
+          }
+          const existingOrdinals = [
+            ...accountObligations.filter((value) => value.configurationId === configuration.configurationId).map((value) => value.periodOrdinal),
+            ...accountRetired.filter((value) => value.configurationId === configuration.configurationId).map((value) => value.periodOrdinal)
+          ];
+          const historyRemaining = maxHistory - (historyBySource.get(configuration.offeringId) ?? 0);
+          const candidates = missingOrdinals(
+            first,
+            last,
+            existingOrdinals,
+            Math.min(maxObligations - attempted, historyPass ? historyRemaining : maxObligations)
+          );
+          for (const ordinal of candidates) {
+            attempted += 1;
+            if (historyPass)
+              historyBySource.set(configuration.offeringId, (historyBySource.get(configuration.offeringId) ?? 0) + 1);
+            const obligation = planObligation(configuration, ordinal, now);
+            const written = await options.store.putObligation(obligation);
+            if (written.inserted) {
+              created.push(written.value);
+              await (0, import_lifecycle.reconcileReportingStatusLifecycleV1)({
+                store: options.store,
+                reporting_obligation_id: written.value.reporting_obligation_id,
+                // A fallback clock, never a pinned cutoff. `now` is this host's
+                // instant: a host running fast pinned a cutoff ahead of the
+                // database, the watermark was stamped with it, and every
+                // database-timestamped change inside that skew — a revocation,
+                // a receipt — landed behind the watermark and never made the
+                // obligation due again, so a `complete` transition and its
+                // webhook stood over state that had already contradicted it.
+                now: () => new Date(now),
+                subscribers: options.subscribers
+              });
+            }
+            if (attempted >= maxObligations) return created;
           }
-          if (attempted >= maxObligations) return created;
         }
       }
       return created;
@@ -201,6 +276,9 @@ function createReportingProducer(options) {
     async runWorker(workerOptions = {}) {
       const now = workerOptions.now ?? (() => /* @__PURE__ */ new Date());
       const maxIterations = workerOptions.maxIterations ?? 25;
+      const maxHistory = workerOptions.maxHistorySlicesPerSource ?? 10;
+      nonnegativeInteger(maxHistory, "maxHistorySlicesPerSource");
+      const historyBySource = /* @__PURE__ */ new Map();
       const retryDelay = workerOptions.retryDelayMilliseconds ?? 3e4;
       const executionDeadline = workerOptions.executionDeadlineMilliseconds ?? EXECUTION_DEADLINE_MS;
       const settlementGrace = workerOptions.settlementGraceMilliseconds ?? DEFAULT_SETTLEMENT_GRACE_MS;
@@ -214,6 +292,10 @@ function createReportingProducer(options) {
       if (settlementGrace > MAX_SETTLEMENT_GRACE_MS) {
         throw new RangeError(`settlementGraceMilliseconds must not exceed ${MAX_SETTLEMENT_GRACE_MS}`);
       }
+      const historyOfferingIds = /* @__PURE__ */ new Set([
+        ...offeringById.keys(),
+        ...(await options.store.listConfigurations(workerOptions.account_id)).map((value) => value.offeringId)
+      ]);
       const owner = `reporting-worker-${(0, import_node_crypto.randomUUID)()}`;
       const counts = { claimed: 0, revisionsCommitted: 0, notReady: 0, failed: 0, reconcilesDeferred: 0 };
       const reconcileQuietly = async (reporting_obligation_id, nowAt) => {
@@ -254,14 +336,35 @@ function createReportingProducer(options) {
           owner,
           now: nowValue.toISOString(),
           leaseMilliseconds: DEFAULT_LEASE_MS,
+          excludeHistoryOfferingIds: [.../* @__PURE__ */ new Set([...historyOfferingIds, ...historyBySource.keys()])].filter(
+            (id) => (historyBySource.get(id) ?? 0) >= maxHistory
+          ),
           ...workerOptions.account_id ? { account_id: workerOptions.account_id } : {}
         });
         if (!lease) break;
+        if (lease.obligation.history && (historyBySource.get(lease.obligation.offeringId) ?? 0) >= maxHistory) {
+          await options.store.releaseObligationLease(lease);
+          throw new TypeError("Reporting ledger store must honor excludeHistoryOfferingIds");
+        }
         counts.claimed += 1;
+        if (lease.obligation.history) {
+          const sourceId = lease.obligation.offeringId;
+          historyBySource.set(sourceId, (historyBySource.get(sourceId) ?? 0) + 1);
+        }
         let executionDeadlineTimer;
-        let activeRecoveryDeadlineAt = lease.obligation.recoveryDeadlineAt;
+        let activeRecoveryDeadlineAt = lease.obligation.historyRecoveryDeadlineAt ?? lease.obligation.recoveryDeadlineAt;
         try {
           const obligation = lease.obligation;
+          if (obligation.history && !obligation.historyRecoveryDeadlineAt) {
+            obligation.historyRecoveryDeadlineAt = new Date(
+              Math.max(
+                Date.parse(obligation.recoveryDeadlineAt),
+                nowValue.getTime() + obligation.schedule.recoveryWindowMilliseconds
+              )
+            ).toISOString();
+            activeRecoveryDeadlineAt = obligation.historyRecoveryDeadlineAt;
+            await options.store.updateObligation(obligation, lease);
+          }
           const offering = requiredOffering(offeringById, obligation.offeringId);
           if ((0, import_schedule.isReportingCalendarDay)(obligation.schedule, obligation.period.sourceTimezone)) {
             try {
@@ -287,7 +390,10 @@ function createReportingProducer(options) {
             if (scheduledAt) {
               const recoveryWindow = instant(obligation.recoveryDeadlineAt, "recoveryDeadlineAt") - instant(obligation.expectedAt, "expectedAt");
               activeRecoveryDeadlineAt = new Date(
-                instant(scheduledAt, "nextPublicationAt") + recoveryWindow
+                Math.max(
+                  obligation.history ? Date.parse(activeRecoveryDeadlineAt) : Number.NEGATIVE_INFINITY,
+                  instant(scheduledAt, "nextPublicationAt") + recoveryWindow
+                )
               ).toISOString();
             }
           }
@@ -458,6 +564,42 @@ async function supersededCalendarGenerationIsDurablyCovered(store, configuration
     await store.listObligations(configuration.account.account_id)
   );
 }
+function historyInstant(value) {
+  const parsed = import_source.ReportingInstantV1Schema.safeParse(value);
+  if (!parsed.success) {
+    throw new TypeError("historyStart must be a valid RFC 3339 instant with at most millisecond precision");
+  }
+  return instant(parsed.data, "historyStart");
+}
+function normalizeConfigurationHistory(configuration, requested, offering, now) {
+  const requestedMs = historyInstant(requested);
+  if (requestedMs < now - offering.retentionDays * 864e5) {
+    throw new RangeError("Reporting historyStart is outside source retentionDays");
+  }
+  if (requestedMs > now) throw new RangeError("Reporting historyStart must not be in the future");
+  const schedule = (0, import_schedule.reportingPeriodSchedule)(configuration);
+  const ordinal = schedule.ceil(requestedMs);
+  const start = schedule.boundary(ordinal);
+  const previousStart = configuration.historyStart ? Date.parse(configuration.historyStart) : schedule.boundary(
+    Math.max(
+      0,
+      schedule.ceil(Math.max(Date.parse(configuration.schedule.anchor), Date.parse(configuration.installedAt)))
+    )
+  );
+  if (requestedMs > previousStart || start > previousStart) {
+    throw new RangeError("Reporting historyStart may only move earlier");
+  }
+  if ((0, import_schedule.isReportingCalendarDay)(configuration.schedule, configuration.sourceTimezone)) {
+    (0, import_schedule.assertReportingCalendarDaySource)(configuration.schedule, configuration.sourceTimezone, offering, now, start);
+    const previousFirst = configuration.historyStartOrdinal ?? Math.max(0, schedule.ceil(previousStart));
+    for (let selected = ordinal; selected < previousFirst; selected += 1) schedule.period(selected);
+  } else if (configuration.schedule.alignment && configuration.schedule.alignment !== "billing_cycle") {
+    if ((0, import_source.reportingUtcOffsetChangesV1)(configuration.sourceTimezone, start, Math.max(now, start))) {
+      throw new Error("Reporting history crosses a timezone offset change; install a calendar-day generation");
+    }
+  }
+  return { historyStart: new Date(start).toISOString(), historyStartOrdinal: ordinal };
+}
 function planObligation(configuration, periodOrdinal, createdAt) {
   const schedule = (0, import_schedule.reportingPeriodSchedule)(configuration);
   const { start, end } = schedule.period(periodOrdinal);
@@ -486,6 +628,7 @@ function planObligation(configuration, periodOrdinal, createdAt) {
     ...configuration.finalityPolicy ? { finalityPolicy: structuredClone(configuration.finalityPolicy) } : {},
     ...configuration.canonicalization ? { canonicalization: structuredClone(configuration.canonicalization) } : {},
     periodOrdinal,
+    ...configuration.historyStart !== void 0 && periodOrdinal < Math.max(0, schedule.ceil(Date.parse(configuration.installedAt))) ? { history: true } : {},
     period: {
       start: new Date(start).toISOString(),
       end: new Date(end).toISOString(),
@@ -552,7 +695,10 @@ function sourceRequest(obligation, offering, previous, adjustments, now) {
       windowing: structuredClone(offering.windowing)
     },
     finality: revisionKind === "correction" ? { revisionKind, supersedesPublicationId: latest.sourcePublicationId } : { revisionKind },
-    trigger: { kind: obligation.attemptCount ? "retry" : "scheduled_poll", id: `trigger-${obligation.attemptCount}` },
+    trigger: {
+      kind: obligation.attemptCount ? "retry" : obligation.history ? "backfill" : "scheduled_poll",
+      id: `trigger-${obligation.attemptCount}`
+    },
     sourceRequest: { groupIds: [] },
     coverage: {
       expected: "full",