@centerforagenticai/pi-multi-account 0.1.2 → 0.1.5

package/src/usage-fetch.ts

@@ -6,8 +6,10 @@ import {
 	type MachineLeaseHandle,
 } from "./machine-lease.js";
 import {
+	MAX_USAGE_ATTEMPT_DELAY_MS,
 	normalizeUsageEndpointPercent,
 	type SharedUsageAttemptRecord,
+	type SharedUsageFailureReason,
 	type SharedUsageStore,
 	type UsageFailureDetail,
 } from "./shared-usage.js";
@@ -39,9 +41,8 @@ export const USAGE_FETCH_TIMEOUT_MS = 10_000;
 const MAX_RESPONSE_BYTES = 128 * 1024;
 const MAX_ERROR_CHARS = 512;
 const BASE_BACKOFF_MS = 30_000;
-const MAX_BACKOFF_MS = 15 * 60_000;
 const BACKOFF_LADDER_STEPS =
-	Math.ceil(Math.log2(MAX_BACKOFF_MS / BASE_BACKOFF_MS)) + 1;
+	Math.ceil(Math.log2(MAX_USAGE_ATTEMPT_DELAY_MS / BASE_BACKOFF_MS)) + 1;
 /** Disable only after the capped rung has failed once more. */
 export const USAGE_FETCH_DISABLE_AFTER_FAILURES = BACKOFF_LADDER_STEPS + 1;
 
@@ -104,12 +105,7 @@ export interface UsageFetchStatus {
 	readonly disabled: boolean;
 	readonly failureCount: number;
 	readonly nextAttemptAtMs?: number;
-	readonly disabledReason?:
-		| "rate-limit"
-		| "server-error"
-		| "credential-unavailable"
-		| "malformed-response"
-		| "network-error";
+	readonly disabledReason?: SharedUsageFailureReason;
 }
 
 export type UsageFetchResultStatus =
@@ -217,7 +213,10 @@ function retryAfterMs(response: UsageFetchResponse): number | undefined {
 	}
 	const seconds = Number(value);
 	if (!Number.isFinite(seconds) || seconds < 0) return undefined;
-	return Math.min(MAX_BACKOFF_MS, Math.max(0, Math.ceil(seconds * 1_000)));
+	return Math.min(
+		MAX_USAGE_ATTEMPT_DELAY_MS,
+		Math.max(0, Math.ceil(seconds * 1_000)),
+	);
 }
 
 /** Redacts bearer and access-token values before an error can escape this module. */
@@ -642,14 +641,28 @@ async function queryEndpoint(
 		: normalizeAnthropicUsagePayload(payload);
 }
 
+// Every attempt a reader here sees comes from `SharedUsageStore.latestAttempt`
+// called with that reader's own clock, and that is what keeps a far-future
+// deadline (internal issue #108) from suppressing polling:
+//
+//  - the store refuses, on append and read, any attempt whose
+//    `nextAttemptAtMs - observedAtMs` exceeds MAX_USAGE_ATTEMPT_DELAY_MS; and
+//  - `latestAttempt` skips any attempt observed after `nowMs` (#133).
+//
+// Together these give `nextAttemptAtMs <= nowMs + MAX_USAGE_ATTEMPT_DELAY_MS`
+// for every record a reader receives. A per-reader "plausible deadline" check
+// used to restate that bound; once the origin filter moved into the store it
+// could no longer fail on any reachable input and was removed (#133,
+// CR-REFRESH-READER-BOUND-UNREACHABLE). Read attempts only through
+// `latestAttempt(providerId, family, nowMs)` with the clock the decision uses.
+
 function statusFromAttempt(
 	attempt: SharedUsageAttemptRecord | undefined,
 	enabled: boolean,
 ): UsageFetchStatus {
 	const nextAttemptAtMs =
 		attempt?.failureCount === 0 ? undefined : attempt?.nextAttemptAtMs;
-	const disabledReason =
-		attempt?.failureReason as UsageFetchStatus["disabledReason"];
+	const disabledReason = attempt?.failureReason;
 	return {
 		enabled,
 		disabled: attempt?.disabled ?? false,
@@ -831,7 +844,7 @@ export class UsageFetcher {
 	): UsageFetchStatus {
 		const enabled = config.usageFetchEnabled?.[family] ?? true;
 		return statusFromAttempt(
-			this.#sharedStore.latestAttempt(providerId, family),
+			this.#sharedStore.latestAttempt(providerId, family, this.#now()),
 			enabled,
 		);
 	}
@@ -994,18 +1007,16 @@ export class UsageFetcher {
 		// the debounce clause, so `already-answered` and `backoff` returned
 		// before it ran: an attempt with `observedAtMs` centuries ahead is
 		// trivially `>= failedAtMs`, and suppressed every failure refresh
-		// forever. The bound belongs here, on the record, before any clause
-		// reads it.
+		// forever. `latestAttempt(..., nowMs)` now refuses such a record at the
+		// source (#133), which also bounds `nextAttemptAtMs` to one capped rung
+		// past `nowMs` (see the note above `statusFromAttempt`).
 		//
-		// `nextAttemptAtMs` is allowed one debounce interval of headroom because
-		// Only the ORIGIN timestamps are bounded, not `nextAttemptAtMs`: a real
-		// failure ladder legitimately schedules its next rung far ahead, up to
-		// MAX_BACKOFF_MS, and bounding that would break genuine backoff
-		// suppression. An honest record cannot have been OBSERVED in the future.
+		// The failure trigger is not tied to `observedAtMs` by the store, so a
+		// record observed in the past can still name a failure in the future;
+		// such a marker cannot be honest and must not arm the debounce.
 		if (
-			attempt.observedAtMs > nowMs ||
-			(attempt.failureTriggeredAtMs !== undefined &&
-				attempt.failureTriggeredAtMs > nowMs)
+			attempt.failureTriggeredAtMs !== undefined &&
+			attempt.failureTriggeredAtMs > nowMs
 		) {
 			return undefined;
 		}
@@ -1044,15 +1055,17 @@ export class UsageFetcher {
 		}
 
 		// Step 4/5: durable checks before the lease.
+		const beforeLeaseNowMs = this.#now();
 		const beforeLease = this.#sharedStore.latestAttempt(
 			account.providerId,
 			account.family,
+			beforeLeaseNowMs,
 		);
 		if (
 			this.#failureRefreshSuppressedBy(
 				beforeLease,
 				failedAtMs,
-				this.#now(),
+				beforeLeaseNowMs,
 			) !== undefined
 		) {
 			return { providerId: account.providerId, status: "not-due" };
@@ -1072,15 +1085,17 @@ export class UsageFetcher {
 		const leaseGuard: AntigravityLeaseGuard = { lease, handedOff: false };
 		try {
 			// Step 7: the same checks again, now that nobody else can write.
+			const underLeaseNowMs = this.#now();
 			const underLease = this.#sharedStore.latestAttempt(
 				account.providerId,
 				account.family,
+				underLeaseNowMs,
 			);
 			if (
 				this.#failureRefreshSuppressedBy(
 					underLease,
 					failedAtMs,
-					this.#now(),
+					underLeaseNowMs,
 				) !== undefined
 			) {
 				return { providerId: account.providerId, status: "not-due" };
@@ -1169,6 +1184,7 @@ export class UsageFetcher {
 		const prior = this.#sharedStore.latestAttempt(
 			account.providerId,
 			account.family,
+			nowMs,
 		);
 		// Backoff is checked BEFORE `disabled`, deliberately.
 		//
@@ -1190,7 +1206,7 @@ export class UsageFetcher {
 		// Once the recorded backoff has elapsed, the account is retried whether or
 		// not it was disabled. A retry that fails again simply re-arms the ladder at
 		// its capped rung, so a genuinely dead endpoint is polled at most once per
-		// MAX_BACKOFF_MS rather than hammered.
+		// MAX_USAGE_ATTEMPT_DELAY_MS rather than hammered.
 		if (prior !== undefined && prior.nextAttemptAtMs > nowMs) {
 			await this.#persistWindowGap(
 				account,
@@ -1215,15 +1231,17 @@ export class UsageFetcher {
 		}
 		const leaseGuard: AntigravityLeaseGuard = { lease, handedOff: false };
 		try {
-			const current = this.#sharedStore.latestAttempt(
-				account.providerId,
-				account.family,
-			);
 			// Same ordering as the pre-lease check above: an elapsed backoff wins
 			// over a stale `disabled` flag, so a recovered account can be observed
 			// again. Re-read under the lease because a peer may have attempted in
 			// between.
-			if (current !== undefined && current.nextAttemptAtMs > this.#now()) {
+			const currentNowMs = this.#now();
+			const current = this.#sharedStore.latestAttempt(
+				account.providerId,
+				account.family,
+				currentNowMs,
+			);
+			if (current !== undefined && current.nextAttemptAtMs > currentNowMs) {
 				await this.#persistWindowGap(
 					account,
 					this.#now(),
@@ -1555,7 +1573,7 @@ export class UsageFetcher {
 					: new UsageEndpointError("usage endpoint failed", "network-error");
 			const failureCount = (prior?.failureCount ?? 0) + 1;
 			const backoff = Math.min(
-				MAX_BACKOFF_MS,
+				MAX_USAGE_ATTEMPT_DELAY_MS,
 				BASE_BACKOFF_MS * 2 ** Math.max(0, failureCount - 1),
 			);
 			const disabled = failureCount >= USAGE_FETCH_DISABLE_AFTER_FAILURES;
@@ -1617,13 +1635,15 @@ export class UsageFetcher {
 			}
 			// REQ-WINDOW-CADENCE: enforce cross-process 15-minute floor.
 			// Check durable attempt record to prevent peer-process fetch within the window.
+			const cadenceNowMs = this.#now();
 			const prior = this.#sharedStore.latestAttempt(
 				account.providerId,
 				account.family,
+				cadenceNowMs,
 			);
 			if (
 				prior !== undefined &&
-				this.#now() - prior.observedAtMs < WINDOW_SAMPLE_INTERVAL_MS
+				cadenceNowMs - prior.observedAtMs < WINDOW_SAMPLE_INTERVAL_MS
 			) {
 				continue;
 			}

package/src/usage.ts

@@ -337,8 +337,8 @@ export class UsageLedger {
 	 * exhaustion hold is one. But a hold is machine-global: deleting the record
 	 * would revoke it for every other session on the machine, and those peers
 	 * may still be refusing work on that account. So the override is
-	 * PROCESS-LOCAL -- this session stops honouring holds installed up to the
-	 * moment of the clear, and peers keep theirs.
+	 * PROCESS-LOCAL -- this session stops honouring holds whose failure happened
+	 * at or before the clear, and peers keep theirs.
 	 *
 	 * Stored as the clear time rather than a boolean so a LATER failure still
 	 * installs a hold this session honours. A boolean would make one `clear`
@@ -346,6 +346,13 @@ export class UsageLedger {
 	 * which is a permanent effect from a transient instruction.
 	 */
 	readonly #holdOverrides = new Map<string, number>();
+	/**
+	 * When `clear()` with no account last ran. Clear-all is one point in time
+	 * for EVERY account, including ones with no hold visible at that instant:
+	 * enumerating the holds present then let a peer publish a pre-clear hold a
+	 * moment later and still have it honoured (internal issue #110).
+	 */
+	#clearAllAtMs: number | undefined;
 	readonly #sharedStore: SharedUsageStore | undefined;
 	readonly #now: () => number;
 
@@ -714,29 +721,29 @@ export class UsageLedger {
 		family: AllowedFamily,
 		nowMs = this.#now(),
 	): number | undefined {
-		const holdUntilMs = this.#sharedStore?.activeExhaustionHoldUntilMs(
+		// An operator `clear` in THIS process stops us honouring holds whose
+		// failure happened at or before it. A hold from a LATER failure is
+		// honoured again: the override is a point in time, not a permanent
+		// exemption. Peers keep honouring the record either way, because it is
+		// not deleted.
+		//
+		// The store compares each hold's own validated `failedAtMs`. Deriving
+		// the failure time from the deadline (`holdUntilMs - EXHAUSTION_HOLD_MS`)
+		// is only right for an exact one-hour hold, and the store accepts every
+		// shorter span (internal issue #110).
+		const accountClearedAtMs = this.#holdOverrides.get(providerId);
+		const clearedAtMs =
+			accountClearedAtMs === undefined
+				? this.#clearAllAtMs
+				: this.#clearAllAtMs === undefined
+					? accountClearedAtMs
+					: Math.max(accountClearedAtMs, this.#clearAllAtMs);
+		return this.#sharedStore?.activeExhaustionHoldUntilMs(
 			providerId,
 			family,
 			nowMs,
+			clearedAtMs,
 		);
-		if (holdUntilMs === undefined) return undefined;
-		// An operator `clear` in THIS process stops us honouring holds that were
-		// already installed when it ran. A hold from a LATER failure is honoured
-		// again: the override is a point in time, not a permanent exemption.
-		// Peers keep honouring the record either way, because it is not deleted.
-		//
-		// `holdUntilMs - EXHAUSTION_HOLD_MS` recovers the failure time from the
-		// reported deadline without a second store read. The writer always sets
-		// exactly that span (asserted by the hold-duration test), and the
-		// store's own read bound refuses any record claiming more.
-		const clearedAtMs = this.#holdOverrides.get(providerId);
-		if (
-			clearedAtMs !== undefined &&
-			holdUntilMs - EXHAUSTION_HOLD_MS <= clearedAtMs
-		) {
-			return undefined;
-		}
-		return holdUntilMs;
 	}
 
 	/**
@@ -811,11 +818,7 @@ export class UsageLedger {
 		if (providerId === undefined) {
 			this.#snapshots.clear();
 			this.#quotaObservations.clear();
-			for (const id of this.#sharedStore
-				?.readExhaustionHolds()
-				.map((hold) => hold.providerId) ?? []) {
-				this.#holdOverrides.set(id, nowMs);
-			}
+			this.#clearAllAtMs = Math.max(this.#clearAllAtMs ?? nowMs, nowMs);
 		} else {
 			this.#snapshots.delete(providerId);
 			this.#quotaObservations.delete(providerId);