@teamvibe/poller 0.8.7 → 0.8.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -28,6 +28,8 @@ export interface SpawnResult {
28
28
  poolAttempts?: readonly PoolAttempt[];
29
29
  quota?: ClaudeQuotaFailure;
30
30
  rateLimitInfo?: ClaudeRateLimitInfo;
31
+ /** Rejections anywhere in this attempt survive a later allowed stream event. */
32
+ rateLimitRejections?: readonly ClaudeRateLimitInfo[];
31
33
  /** Actual ID passed to this CLI attempt, including a failed first turn. */
32
34
  attemptedSessionId?: string;
33
35
  success: boolean;
@@ -20,7 +20,7 @@ import { fetchSecretsForSpawn, buildSecretsReminder } from './secrets-client.js'
20
20
  import { fetchConfigForSpawn } from './config-client.js';
21
21
  import { parseClaudeRateLimitInfo } from './claude-rate-limit.js';
22
22
  import { parseClaudeQuotaFailure } from './claude-quota.js';
23
- import { abortLocalProbe, accountPin, preparePool, fetchRuntimePool, validateAccount, poolDeferredError, PoolDeferredError } from './subscription-runtime.js';
23
+ import { abortLocalProbe, accountPin, preparePool, fetchRuntimePool, validateAccount, subscriptionUsageStore, poolDeferredError, PoolDeferredError } from './subscription-runtime.js';
24
24
  import { selectPoolForResolvedModel } from './subscription-pool.js';
25
25
  import { isQuotaPaused, QuotaPausedError } from './quota-pause.js';
26
26
  import { redactStartupText } from './startup-auth.js';
@@ -911,6 +911,14 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
911
911
  });
912
912
  if (runtime?.account)
913
913
  await validateAccount(runtime.account, runtime.resolved);
914
+ if (runtime?.account && !runtime.shouldStop()) {
915
+ const store = subscriptionUsageStore();
916
+ const attemptAt = Date.now();
917
+ const windows = await store.beginVerification(runtime.account, runtime.resolved.model, attemptAt);
918
+ if (windows === undefined)
919
+ throw await poolDeferredError('usage', { kind: 'ready', accounts: [runtime.account] }, runtime.resolved, store);
920
+ runtime.verificationAttemptAt = windows.length ? attemptAt : undefined;
921
+ }
914
922
  return new Promise((resolve) => {
915
923
  // Preparation above awaits remote config/context. Another thread may
916
924
  // have exhausted the shared account meanwhile. No awaits before spawn.
@@ -1083,6 +1091,16 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
1083
1091
  // Retain CLI error/result evidence, not the full parsed assistant/tool stream.
1084
1092
  const failureEvents = [];
1085
1093
  let rateLimitInfo;
1094
+ const rateLimitRejections = new Map();
1095
+ const captureRateLimit = (value) => {
1096
+ rateLimitInfo = parseClaudeRateLimitInfo(value);
1097
+ if (!runtime?.account || rateLimitInfo?.status !== 'rejected')
1098
+ return;
1099
+ for (const w of normalizePassiveUsage(rateLimitInfo, Date.now()).windows) {
1100
+ if (w.status === 'rejected')
1101
+ rateLimitRejections.set(`${w.window}:${w.scope}`, rateLimitInfo);
1102
+ }
1103
+ };
1086
1104
  let metrics;
1087
1105
  let lastAssistant;
1088
1106
  // teamvibe.ai#284 (poller-brain#235): end-of-turn guard state — tracked
@@ -1143,7 +1161,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
1143
1161
  // (including metadata keys) to logs, telemetry, typing, or Slack.
1144
1162
  const event = safeEvent(rawEvent);
1145
1163
  if (rawEvent?.type === 'rate_limit_event') {
1146
- rateLimitInfo = parseClaudeRateLimitInfo(event.rate_limit_info);
1164
+ captureRateLimit(event.rate_limit_info);
1147
1165
  }
1148
1166
  if (rawEvent?.type === 'result' || rawEvent?.type === 'error' || (rawEvent?.type === 'assistant' && rawEvent.error)) {
1149
1167
  failureEvents.push(rawEvent);
@@ -1367,7 +1385,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
1367
1385
  const rawEvent = JSON.parse(stdoutBuffer);
1368
1386
  const event = safeEvent(rawEvent);
1369
1387
  if (event?.type === 'rate_limit_event') {
1370
- rateLimitInfo = parseClaudeRateLimitInfo(event.rate_limit_info);
1388
+ captureRateLimit(event.rate_limit_info);
1371
1389
  }
1372
1390
  failureEvents.push(rawEvent);
1373
1391
  if (event?.type === 'assistant') {
@@ -1386,6 +1404,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
1386
1404
  attemptedSessionId: sessionId,
1387
1405
  ...(quota ? { quota } : {}),
1388
1406
  ...(rateLimitInfo ? { rateLimitInfo } : {}),
1407
+ ...(rateLimitRejections.size ? { rateLimitRejections: [...rateLimitRejections.values()] } : {}),
1389
1408
  output: redactOutput(stdout),
1390
1409
  exitCode: code,
1391
1410
  metrics,
@@ -1610,7 +1629,7 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1610
1629
  const modelNormalizedFrom = scoped?.model === resolved.model && scoped?.modelSource === resolved.modelSource
1611
1630
  ? scoped?.modelNormalizedFrom : undefined;
1612
1631
  const runtime = poolRuntime ? {
1613
- resolved, modelNormalizedFrom, claudeConfigDir, account: prepared?.account,
1632
+ resolved, modelNormalizedFrom, claudeConfigDir, account: prepared?.account, verificationAttemptAt: undefined,
1614
1633
  shouldStop: poolRuntime.shouldStop,
1615
1634
  resumePending: fetchedPool?.kind !== 'no_accounts_configured' && poolRuntime.pin?.resumePending === true,
1616
1635
  onSpawn: () => {
@@ -1649,10 +1668,12 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1649
1668
  if (!runtime?.resumePending)
1650
1669
  sawSendMessageCall = false;
1651
1670
  const spawned = await runClaudeCode(msg, sessionLog, effectiveCwd, directory, id, runtime?.resumePending ? false : first, lastMessageTs, onMessageSent, typingIndicator, runtime, gitDelivery);
1671
+ const rejected = spawned.rateLimitRejections?.find(rejectedForModel);
1652
1672
  const result = prepared?.account
1653
- ? { ...spawned, quota: quotaForModel(spawned.quota), success: rejectedForModel(spawned.rateLimitInfo) ? false : spawned.success } : spawned;
1673
+ ? { ...spawned, ...(rejected ? { rateLimitInfo: rejected } : {}), quota: quotaForModel(spawned.quota), success: rejected || rejectedForModel(spawned.rateLimitInfo) ? false : spawned.success } : spawned;
1654
1674
  sawSendMessageCall ||= result.sawSendMessageCall === true;
1655
1675
  if (prepared?.account) {
1676
+ const observedAt = Date.now();
1656
1677
  if (poolAttempts.length >= 1000)
1657
1678
  poolAttempts.shift();
1658
1679
  const selection = prepared.selection;
@@ -1664,7 +1685,8 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1664
1685
  accountId: prepared.account.accountId,
1665
1686
  credentialVersion: prepared.account.credentialVersion,
1666
1687
  accountLabel: prepared.account.label ?? prepared.account.accountId,
1667
- observedAt: Date.now(),
1688
+ observedAt,
1689
+ verificationAttemptAt: runtime?.verificationAttemptAt,
1668
1690
  rateLimitInfo: result.rateLimitInfo,
1669
1691
  sessionId: id,
1670
1692
  model: resolved.model ?? result.metrics?.model,
@@ -1675,13 +1697,19 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1675
1697
  metrics: result.metrics,
1676
1698
  });
1677
1699
  emitEvent('pool.account_attempted', msg.teamvibe.channelId, { workspaceId: msg.teamvibe.workspaceId, threadId: msg.thread_id, ...poolRunTelemetry(poolAttempts, poolRuntime?.quotaDeferEpisodes) });
1700
+ // Retain every rejected scope before completion can clear a verified
1701
+ // hold or return early. A failed write must not skip another scope.
1702
+ const rejectionOutcomes = await Promise.allSettled((result.rateLimitRejections ?? []).map(info => logPoolUsageWrite(sessionLog, prepared.account, info, () => prepared.store.recordPassive(prepared.account, info, observedAt, { expireFutureObservations: true }))));
1678
1703
  if (result.success && id && poolRuntime) {
1679
- // Complete the inbox before any fallible post-inference store writes.
1680
- // The loop retains bookkeeping independently of the original prompt.
1704
+ // Complete the inbox even if rejection bookkeeping failed. The loop
1705
+ // retains completion independently of the original prompt.
1681
1706
  const bookkeepingComplete = await poolRuntime.complete(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: false }, poolAttempts, result.rateLimitInfo);
1682
1707
  if (bookkeepingComplete !== false)
1683
1708
  await emitCandidateUsage(msg, [prepared.account], prepared.store);
1684
1709
  retainWorktree = false;
1710
+ const failed = rejectionOutcomes.find((outcome) => outcome.status === 'rejected');
1711
+ if (failed)
1712
+ throw new PoolDeferredError('infrastructure', { error: failed.reason });
1685
1713
  return result;
1686
1714
  }
1687
1715
  if ((result.quota || rejectedForModel(result.rateLimitInfo)) && !result.killedByPoller) {
@@ -1692,15 +1720,17 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1692
1720
  // retaining the attempt. Selection waits for all writes before switching.
1693
1721
  const writes = [];
1694
1722
  if (result.quota && !result.killedByPoller)
1695
- writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.quota.window ? { rateLimitType: result.quota.window } : undefined, () => prepared.store.recordQuota(prepared.account, result.quota, Date.now(), { expireFutureObservations: true }), 'quota'));
1696
- if (result.rateLimitInfo)
1697
- writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.rateLimitInfo, () => prepared.store.recordPassive(prepared.account, result.rateLimitInfo, Date.now(), { expireFutureObservations: true })));
1723
+ writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.quota.window ? { rateLimitType: result.quota.window } : undefined, () => prepared.store.recordQuota(prepared.account, result.quota, observedAt, { expireFutureObservations: true }), 'quota'));
1724
+ if (!result.rateLimitRejections?.length && result.rateLimitInfo)
1725
+ writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.rateLimitInfo, () => prepared.store.recordPassive(prepared.account, result.rateLimitInfo, observedAt, { expireFutureObservations: true })));
1698
1726
  if (id && poolRuntime)
1699
1727
  writes.push(poolRuntime.persist(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: runtime?.resumePending, sawSendMessageCall }, poolAttempts));
1700
- const outcomes = await Promise.allSettled(writes);
1728
+ const outcomes = [...rejectionOutcomes, ...await Promise.allSettled(writes)];
1701
1729
  const failed = outcomes.find((outcome) => outcome.status === 'rejected');
1702
1730
  if (failed)
1703
1731
  throw new PoolDeferredError('infrastructure', { error: failed.reason });
1732
+ if (runtime?.verificationAttemptAt !== undefined && !result.killedByPoller && result.exitCode !== null)
1733
+ await prepared.store.completeVerification(prepared.account, runtime.verificationAttemptAt, observedAt);
1704
1734
  }
1705
1735
  return result;
1706
1736
  };
package/dist/poller.js CHANGED
@@ -323,8 +323,10 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
323
323
  const lastAttempt = state.attempts?.at(-1);
324
324
  // Existing telemetry falls back to account IDs. Local pool logs use labels only.
325
325
  const accountLabel = lastAttempt?.accountLabel !== state.pin.accountId ? lastAttempt?.accountLabel : undefined;
326
+ // Reuse the spawn's observation clock: the spawner already recorded all
327
+ // rejected scopes, and replaying its final event is the same evidence.
326
328
  if (state.completed.rateLimitInfo)
327
- writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
329
+ writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, lastAttempt?.observedAt ?? state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
328
330
  writes.push(persistSessionId(state.sessionId, state.pin));
329
331
  const outcomes = await Promise.allSettled(writes);
330
332
  if (outcomes.some(outcome => outcome.status === 'rejected' || outcome.value === false)) {
@@ -337,6 +339,21 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
337
339
  }
338
340
  return false;
339
341
  }
342
+ if (lastAttempt?.verificationAttemptAt !== undefined) {
343
+ try {
344
+ await subscriptionUsageStore().completeVerification(state.pin, lastAttempt.verificationAttemptAt, state.completed.observedAt);
345
+ }
346
+ catch {
347
+ sessionLog.warn('[Loop] Verification bookkeeping failed and will retry without running the agent');
348
+ try {
349
+ saveInboxRuntime(kbPath, threadId, { ...state, completed: { ...state.completed, retryAt: Date.now() + POOL_RECHECK_MS } });
350
+ }
351
+ catch {
352
+ sessionLog.warn('[Loop] Could not save bookkeeping retry; completion remains recorded');
353
+ }
354
+ return false;
355
+ }
356
+ }
340
357
  currentSessionId = state.sessionId;
341
358
  currentPin = state.pin;
342
359
  // The spawner publishes live completions only after bookkeeping succeeds.
@@ -625,7 +642,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
625
642
  quotaPaused = true;
626
643
  deferred = true;
627
644
  // Revisit held work hourly so background probes can refresh the account.
628
- // Selection still enforces each window's real reset before inference.
645
+ // Selection enforces resets or a contradicted rejection's verification backoff.
629
646
  const recheckAt = error.cause === 'usage' ? Math.min(error.eligibleAt, Date.now() + HELD_PROBE_INTERVAL_MS) : error.eligibleAt;
630
647
  const pause = deferPool(threadId, recheckAt, error.resetAt, inboxQueueMessageIds(kbPath, threadId));
631
648
  const retained = readInboxRuntime(kbPath, threadId);
@@ -644,7 +661,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
644
661
  cause: error.cause,
645
662
  thresholdDeferred: error.notice?.thresholdDeferred && !error.attempts?.some(a => a.quota),
646
663
  resetAt: pause.resetAt ? Date.parse(pause.resetAt) : error.notice?.resetAt,
647
- timeZone: config.POLLER_TZ,
664
+ timeZone: deliveryMsg.teamvibe.timeZone ?? error.notice?.timeZone,
648
665
  });
649
666
  await sendSlackMessage(deliveryMsg, notice, { clientMessageId: pause.episodeId, ...(reconcile ? { reconcile: true } : {}) });
650
667
  setPoolNotice(threadId, pause.episodeId, 'delivered');
@@ -10,6 +10,8 @@ import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
10
10
  import type { ClaudeQuotaFailure } from './claude-quota.js';
11
11
  import type { KilledByPoller } from './classify-failed-iteration.js';
12
12
  export interface PoolAttempt extends Partial<HeadroomScore> {
13
+ /** Local completion bookkeeping token; omitted from event transport DTOs. */
14
+ verificationAttemptAt?: number;
13
15
  workspaceId: string;
14
16
  accountId: string;
15
17
  accountLabel?: string;
@@ -205,6 +205,8 @@ async function poolUsageNotice(pool, resolved, store) {
205
205
  return {
206
206
  label: account.label, useFullQuota: account.useFullQuota, windows: windows.filter(w => binding.some(b => b.window === w.window && b.scope === w.scope)), rejected,
207
207
  thresholdBlocked: blockers.length > 0,
208
+ blockedWindows: binding.map(w => ({ window: w.window, scope: w.scope })),
209
+ providerResetAt: binding.length && binding.every(w => w.resetAt !== undefined) ? Math.max(...binding.map(w => w.resetAt)) : undefined,
208
210
  resetAt: resets.length && resets.every((r) => r !== undefined) ? Math.max(...resets) : undefined,
209
211
  };
210
212
  }));
@@ -212,7 +214,8 @@ async function poolUsageNotice(pool, resolved, store) {
212
214
  // Snapshot expiry is only a recheck. Use actual window resets for the notice.
213
215
  const resets = accounts.map(a => a.resetAt);
214
216
  const resetAt = thresholdDeferred && resets.every((r) => r !== undefined) ? Math.min(...resets) : undefined;
215
- return { accounts, thresholdDeferred, resetAt };
217
+ const providerResets = accounts.map(a => a.providerResetAt);
218
+ return { accounts, thresholdDeferred, resetAt: resetAt ?? (providerResets.length && providerResets.every((r) => r !== undefined) ? Math.min(...providerResets) : undefined), blockedWindows: accounts.flatMap(a => a.blockedWindows) };
216
219
  }
217
220
  catch {
218
221
  return {};
@@ -226,8 +229,9 @@ export async function poolDeferredError(cause, pool, resolved, store, options =
226
229
  return new PoolDeferredError(cause, { error: options.error ?? (pool.kind === 'fetch_unavailable' && pool.cause ? { code: pool.cause } : undefined) });
227
230
  const notice = await poolUsageNotice(pool, resolved, store);
228
231
  const deferredError = (resetAt) => {
232
+ resetAt ??= !notice.thresholdDeferred && notice.resetAt !== undefined ? new Date(notice.resetAt).toISOString() : undefined;
229
233
  const error = new PoolDeferredError('usage', { eligibleAt, resetAt });
230
- error.notice = { ...notice, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
234
+ error.notice = { ...notice, eligibleAt: error.eligibleAt, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
231
235
  return error;
232
236
  };
233
237
  if (eligibleAt === undefined) {
@@ -10,6 +10,13 @@ export declare const PROBE_USAGE_TTL_MS = 60000;
10
10
  export declare const POOL_RECHECK_MS: number;
11
11
  /** Safety hold from the rejection observation when the provider gives no reset. */
12
12
  export declare const REJECTED_BLOCK_MS: number;
13
+ export declare const VERIFICATION_MAX_BACKOFF_MS: number;
14
+ export interface VerificationEvidence {
15
+ observedAt: number;
16
+ source: 'passive' | 'probe';
17
+ status: 'allowed' | 'allowed_warning';
18
+ percent: number;
19
+ }
13
20
  export interface AccountIdentity {
14
21
  workspaceId: string;
15
22
  accountId: string;
@@ -47,6 +54,11 @@ export interface UsageWindow {
47
54
  source?: 'passive' | 'probe';
48
55
  percent?: number;
49
56
  resetAt?: number;
57
+ /** Missing on legacy rejections means the first consecutive rejection. */
58
+ rejectionCount?: number;
59
+ lastVerificationAttemptAt?: number;
60
+ /** A genuine same-window measurement; never inferred from an omitted meter. */
61
+ verificationEvidence?: VerificationEvidence;
50
62
  }
51
63
  export interface UsageObservation {
52
64
  pollerVersion?: string;
@@ -67,11 +79,11 @@ export declare function projectLegacyFableUsage<T extends UsageObservation | und
67
79
  * Deduplicate uncertain windows before splitting measurements from rejections.
68
80
  */
69
81
  export declare function supersedeLegacyFableProjections(observations: readonly (UsageObservation | undefined)[], now: number): (UsageObservation | undefined)[];
70
- /** Latest measurements plus durable genuine rejections; omission or success never clears a genuine hold. */
82
+ /** Latest measurements plus durable rejections; only a claimed real spawn can clear a hold. */
71
83
  export declare function mergeUsageObservations(observations: readonly (UsageObservation | undefined)[], now: number): UsageObservation | undefined;
72
84
  /** Preserve last-known meters, projecting legacy identity before its global companion is replaced. */
73
85
  export declare function mergePersistedUsageObservations(previous: UsageObservation | undefined, incoming: UsageObservation | undefined, now: number): UsageObservation | undefined;
74
- export interface ReactiveExhaustion {
86
+ export interface ReactiveExhaustion extends Pick<UsageWindow, 'rejectionCount' | 'lastVerificationAttemptAt' | 'verificationEvidence'> {
75
87
  observedAt: number;
76
88
  quota: ClaudeQuotaFailure;
77
89
  }
@@ -106,6 +118,13 @@ export declare function record(value: unknown): Record<string, unknown>;
106
118
  export declare function normalizePassiveUsage(value: unknown, observedAt: number): UsageObservation;
107
119
  export declare function normalizeProbeUsage(value: unknown, observedAt: number): UsageObservation;
108
120
  export declare function family(model: string | undefined): 'sonnet' | 'opus' | 'haiku' | 'fable' | undefined;
121
+ export declare function rejectionResetAt(window: UsageWindow, observedAt: number): number;
122
+ export declare function verificationEligibleAt(window: UsageWindow, observedAt: number): number;
123
+ export declare function rejectionEvidence(usage: UsageObservation | undefined, window: UsageWindow, now: number): VerificationEvidence | undefined;
124
+ export declare function rejectionEligibleAt(usage: UsageObservation | undefined, window: UsageWindow, account: AccountIdentity, now: number): number;
125
+ export declare function reactiveUsageWindow(reactive: ReactiveExhaustion): UsageWindow;
126
+ /** Due windows are claimed durably at the real spawn boundary, never by pure selection. */
127
+ export declare function verificationWindows(account: AccountIdentity, state: AccountUsageState, model: string | undefined, now: number): UsageWindow[];
109
128
  export declare function reactiveEligibleAt(reactive: ReactiveExhaustion): number;
110
129
  /** Pure policy. Unknown is eligible after every known/reactive blocker is ruled out. */
111
130
  export declare function selectSubscriptionAccount<A extends AccountIdentity>(input: {
@@ -8,6 +8,7 @@ export const PROBE_USAGE_TTL_MS = 60_000;
8
8
  export const POOL_RECHECK_MS = 15 * 60_000;
9
9
  /** Safety hold from the rejection observation when the provider gives no reset. */
10
10
  export const REJECTED_BLOCK_MS = 15 * 60_000;
11
+ export const VERIFICATION_MAX_BACKOFF_MS = 8 * 60 * 60_000;
11
12
  /** Passive and probe percentages remain lower bounds until each window resets. */
12
13
  export function usageWindowExpiresAt(usage, window) {
13
14
  const ttlExpiry = (window.observedAt ?? usage.observedAt) + ((window.source ?? usage.source) === 'passive' ? PASSIVE_USAGE_TTL_MS : PROBE_USAGE_TTL_MS);
@@ -116,7 +117,7 @@ export function supersedeLegacyFableProjections(observations, now) {
116
117
  }),
117
118
  }));
118
119
  }
119
- /** Latest measurements plus durable genuine rejections; omission or success never clears a genuine hold. */
120
+ /** Latest measurements plus durable rejections; only a claimed real spawn can clear a hold. */
120
121
  export function mergeUsageObservations(observations, now) {
121
122
  let latest;
122
123
  const rejected = new Map();
@@ -139,13 +140,24 @@ export function mergeUsageObservations(observations, now) {
139
140
  if (w.status !== 'rejected')
140
141
  continue;
141
142
  const previous = rejected.get(key);
142
- if (!previous || observedAt >= previous.observedAt)
143
- rejected.set(key, { ...w, observedAt, source: w.source ?? usage.source });
143
+ if (!previous || observedAt >= previous.observedAt) {
144
+ const consecutive = previous && observedAt > previous.observedAt && observedAt < rejectionResetAt(previous, previous.observedAt);
145
+ rejected.set(key, {
146
+ ...(previous?.observedAt === observedAt ? previous : {}),
147
+ ...w, observedAt, source: w.source ?? usage.source,
148
+ ...(consecutive && w.resetAt === undefined && previous.resetAt !== undefined ? { resetAt: previous.resetAt } : {}),
149
+ ...(consecutive || w.rejectionCount !== undefined || previous?.observedAt === observedAt && previous.rejectionCount !== undefined
150
+ ? { rejectionCount: Math.max(w.rejectionCount ?? 1, consecutive ? (previous.rejectionCount ?? 1) + 1 : previous?.rejectionCount ?? 1) } : {}),
151
+ });
152
+ }
144
153
  }
145
154
  }
146
155
  if (!latest)
147
156
  return undefined;
148
- const active = [...rejected.values()].filter(w => now < (w.resetAt ?? w.observedAt + REJECTED_BLOCK_MS));
157
+ const active = [...rejected.values()].filter(w => now < rejectionResetAt(w, w.observedAt)).map(w => {
158
+ const evidence = latestVerificationEvidence(observations, w, w.observedAt, now);
159
+ return evidence ? { ...w, verificationEvidence: evidence } : w;
160
+ });
149
161
  const retained = [...scopedExhausted.values()].filter(w => isScopedExhaustion(w) &&
150
162
  !latest.windows.some(other => other.window === w.window && other.scope === w.scope) &&
151
163
  !active.some(other => other.window === w.window && other.scope === w.scope) &&
@@ -281,7 +293,9 @@ export function normalizeProbeUsage(value, observedAt) {
281
293
  if (raw == null)
282
294
  continue;
283
295
  const item = record(raw);
284
- windows.push({ window, scope, percent: percent(item['utilization'], 100), resetAt: date(item['resets_at']) });
296
+ windows.push({ window, scope, percent: percent(item['utilization'], 100), resetAt: date(item['resets_at']),
297
+ ...(percent(item['utilization'], 100) !== undefined ? { status: 'allowed' } : {}),
298
+ });
285
299
  }
286
300
  // The optional endpoint is undocumented. Accept only the explicit window /
287
301
  // model form below; arbitrary internal codenames never map to a family.
@@ -295,7 +309,9 @@ export function normalizeProbeUsage(value, observedAt) {
295
309
  }
296
310
  const model = item['model'];
297
311
  const scope = model === 'global' ? 'global' : typeof model === 'string' ? family(model) ?? 'uncertain' : 'uncertain';
298
- windows.push({ window, scope, percent: percent(item['utilization'], 100), resetAt: date(item['resets_at']) });
312
+ windows.push({ window, scope, percent: percent(item['utilization'], 100), resetAt: date(item['resets_at']),
313
+ ...(percent(item['utilization'], 100) !== undefined ? { status: 'allowed' } : {}),
314
+ });
299
315
  }
300
316
  if (info['limits'].length > 128)
301
317
  windows.push({ window: 'seven_day', scope: 'uncertain' });
@@ -308,6 +324,63 @@ export function family(model) {
308
324
  function applicable(scope, model) {
309
325
  return scope === 'global' || scope === 'uncertain' || family(model) === undefined || scope === family(model);
310
326
  }
327
+ export function rejectionResetAt(window, observedAt) {
328
+ return window.resetAt ?? observedAt + REJECTED_BLOCK_MS;
329
+ }
330
+ export function verificationEligibleAt(window, observedAt) {
331
+ return Math.min(rejectionResetAt(window, observedAt), observedAt +
332
+ Math.min(VERIFICATION_MAX_BACKOFF_MS, REJECTED_BLOCK_MS * 2 ** Math.min(5, Math.max(0, (window.rejectionCount ?? 1) - 1))));
333
+ }
334
+ function latestVerificationEvidence(observations, rejected, rejectedAt, now) {
335
+ let evidence = rejected.verificationEvidence;
336
+ if (evidence && (evidence.observedAt <= rejectedAt || evidence.observedAt > now))
337
+ evidence = undefined;
338
+ // Uncertain/projected identity cannot prove that the provider's exact quota recovered.
339
+ if (rejected.scope === 'uncertain' || rejected.kind === 'legacy-uncertain-seven-day')
340
+ return undefined;
341
+ for (const usage of observations) {
342
+ if (!usage || usage.observedAt > now)
343
+ continue;
344
+ for (const w of usage.windows) {
345
+ const observedAt = w.observedAt ?? usage.observedAt;
346
+ if (w.window !== rejected.window || w.scope !== rejected.scope || w.kind === 'legacy-uncertain-seven-day' ||
347
+ observedAt <= rejectedAt || observedAt > now || w.percent === undefined ||
348
+ (w.status !== 'allowed' && w.status !== 'allowed_warning'))
349
+ continue;
350
+ if (!evidence || observedAt > evidence.observedAt || observedAt === evidence.observedAt && w.percent > evidence.percent)
351
+ evidence = { observedAt, source: w.source ?? usage.source, status: w.status, percent: w.percent };
352
+ }
353
+ }
354
+ return evidence;
355
+ }
356
+ export function rejectionEvidence(usage, window, now) {
357
+ return latestVerificationEvidence([usage], window, window.observedAt ?? usage?.observedAt ?? now, now);
358
+ }
359
+ export function rejectionEligibleAt(usage, window, account, now) {
360
+ const observedAt = window.observedAt ?? usage?.observedAt ?? now;
361
+ const resetAt = rejectionResetAt(window, observedAt);
362
+ const evidence = rejectionEvidence(usage, window, now);
363
+ return evidence && evidence.percent < (account.useFullQuota ? 100 : USAGE_THRESHOLD_PERCENT) &&
364
+ !(window.lastVerificationAttemptAt !== undefined && window.lastVerificationAttemptAt >= observedAt)
365
+ ? verificationEligibleAt(window, observedAt) : resetAt;
366
+ }
367
+ export function reactiveUsageWindow(reactive) {
368
+ return {
369
+ window: reactive.quota.window ?? 'seven_day',
370
+ scope: reactive.quota.modelScope ? family(reactive.quota.modelScope) ?? 'uncertain' : 'global',
371
+ status: 'rejected', observedAt: reactive.observedAt, resetAt: reactiveEligibleAt(reactive),
372
+ rejectionCount: reactive.rejectionCount, lastVerificationAttemptAt: reactive.lastVerificationAttemptAt,
373
+ verificationEvidence: reactive.verificationEvidence,
374
+ };
375
+ }
376
+ /** Due windows are claimed durably at the real spawn boundary, never by pure selection. */
377
+ export function verificationWindows(account, state, model, now) {
378
+ return [...(state.usage?.windows ?? []), ...(state.reactive ?? []).filter(r => r.quota.window !== undefined).map(reactiveUsageWindow)].filter(w => {
379
+ const observedAt = w.observedAt ?? state.usage?.observedAt ?? now;
380
+ return w.status === 'rejected' && observedAt <= now && applicable(w.scope, model) &&
381
+ now < rejectionResetAt(w, observedAt) && now >= rejectionEligibleAt(state.usage, w, account, now);
382
+ });
383
+ }
311
384
  export function reactiveEligibleAt(reactive) {
312
385
  const parsed = reactive.quota.resetAt ? Date.parse(reactive.quota.resetAt) : NaN;
313
386
  return Number.isFinite(parsed) && parsed > reactive.observedAt ? parsed : reactive.observedAt + POOL_RECHECK_MS;
@@ -342,6 +415,7 @@ export function selectSubscriptionAccount(input) {
342
415
  continue;
343
416
  const state = input.states[account.accountId];
344
417
  const blockers = [];
418
+ const usage = supersedeLegacyFableProjections([state?.usage], input.now)[0];
345
419
  for (const reactive of state?.reactive ?? []) {
346
420
  // Like durable pause createdAt, future observations expire on clock rollback.
347
421
  if (input.now < reactive.observedAt)
@@ -349,12 +423,11 @@ export function selectSubscriptionAccount(input) {
349
423
  const scope = reactive.quota.modelScope;
350
424
  if (scope && family(scope) && family(input.model) && family(scope) !== family(input.model))
351
425
  continue;
352
- const until = reactiveEligibleAt(reactive);
426
+ const until = reactive.quota.window ? rejectionEligibleAt(usage, reactiveUsageWindow(reactive), account, input.now) : reactiveEligibleAt(reactive);
353
427
  if (input.now < until)
354
428
  blockers.push(until);
355
429
  }
356
- const usage = supersedeLegacyFableProjections([state?.usage], input.now)[0];
357
- // A provider rejection outlives snapshot freshness. Never score it as headroom.
430
+ // A contradicted rejection permits one claimed verification after backoff.
358
431
  if (usage && usage.observedAt <= input.now) {
359
432
  for (const w of usage.windows) {
360
433
  if (w.status !== 'rejected' || !applicable(w.scope, input.model))
@@ -362,7 +435,7 @@ export function selectSubscriptionAccount(input) {
362
435
  const observedAt = w.observedAt ?? usage.observedAt;
363
436
  if (observedAt > input.now)
364
437
  continue;
365
- const until = w.resetAt ?? observedAt + REJECTED_BLOCK_MS;
438
+ const until = rejectionEligibleAt(usage, w, account, input.now);
366
439
  if (input.now < until)
367
440
  blockers.push(until);
368
441
  }
@@ -414,14 +487,18 @@ export function blockingUsageWindows(account, state, model, now) {
414
487
  const usage = supersedeLegacyFableProjections([state?.usage], now)[0];
415
488
  const windows = [];
416
489
  for (const window of usage?.windows ?? []) {
417
- const result = selectSubscriptionAccount({ pool, states: { [account.accountId]: { usage: { ...usage, windows: [window] } } }, model, now });
490
+ const evidence = window.status === 'rejected' ? rejectionEvidence(usage, window, now) : undefined;
491
+ const result = selectSubscriptionAccount({ pool, states: { [account.accountId]: { usage: { ...usage, windows: [{ ...window, ...(evidence ? { verificationEvidence: evidence } : {}) }] } } }, model, now });
418
492
  if (result.kind === 'deferred')
419
493
  windows.push({ ...window, eligibleAt: result.eligibleAt });
420
494
  }
421
495
  for (const reactive of state?.reactive ?? []) {
422
- const result = selectSubscriptionAccount({ pool, states: { [account.accountId]: { reactive: [reactive] } }, model, now });
423
- if (result.kind === 'deferred')
424
- windows.push({ observedAt: reactive.observedAt, window: reactive.quota.window ?? 'seven_day', scope: reactive.quota.modelScope ? family(reactive.quota.modelScope) ?? 'uncertain' : 'global', status: 'rejected', resetAt: reactive.quota.resetAt ? Date.parse(reactive.quota.resetAt) : undefined, eligibleAt: result.eligibleAt });
496
+ const window = reactiveUsageWindow(reactive);
497
+ if (applicable(window.scope, model) && reactive.observedAt <= now) {
498
+ const eligibleAt = reactive.quota.window ? rejectionEligibleAt(usage, window, account, now) : reactiveEligibleAt(reactive);
499
+ if (now < eligibleAt)
500
+ windows.push({ ...window, resetAt: reactive.quota.resetAt ? Date.parse(reactive.quota.resetAt) : undefined, eligibleAt });
501
+ }
425
502
  }
426
503
  return windows;
427
504
  }
@@ -1,6 +1,6 @@
1
1
  import { type ClaudeRateLimitInfo } from './claude-rate-limit.js';
2
2
  import type { ClaudeQuotaFailure } from './claude-quota.js';
3
- import { type AccountIdentity, type AccountUsageState, type ReactiveExhaustion, type UsageObservation } from './subscription-selection.js';
3
+ import { type AccountIdentity, type AccountUsageState, type ReactiveExhaustion, type UsageObservation, type UsageWindow } from './subscription-selection.js';
4
4
  export type UsageKey = Pick<AccountIdentity, 'workspaceId' | 'accountId' | 'credentialVersion'>;
5
5
  export type ProbeStatus = 'attempted' | 'ok' | 'unsupported' | 'unauthorized' | 'unavailable' | 'cancelled';
6
6
  export interface ProbeObservation {
@@ -34,6 +34,9 @@ export interface SubscriptionUsageStore {
34
34
  claimLocalProbe(key: UsageKey, observedAt: number, minIntervalMs: number): Promise<boolean>;
35
35
  claimProbeRequest(requestId: string, expiresAt: number): Promise<boolean>;
36
36
  finishProbe(key: UsageKey, observation: ProbeObservation): Promise<void>;
37
+ /** Atomic revalidation and one durable claim immediately before a real spawn. */
38
+ beginVerification(key: AccountIdentity, model: string | undefined, observedAt: number): Promise<UsageWindow[] | undefined>;
39
+ completeVerification(key: UsageKey, attemptAt: number, observedAt: number): Promise<void>;
37
40
  }
38
41
  export declare function usageStateForSelection(state: StoredAccountUsage, now: number): AccountUsageState;
39
42
  export declare class SubscriptionUsageStoreError extends Error {
@@ -54,4 +57,6 @@ export declare class FileSubscriptionUsageStore implements SubscriptionUsageStor
54
57
  /** Request claims live in the local data directory, alongside usage, never in a brain. */
55
58
  claimProbeRequest(requestId: string, expiresAt: number): Promise<boolean>;
56
59
  finishProbe(key: UsageKey, observation: ProbeObservation): Promise<void>;
60
+ beginVerification(key: AccountIdentity, model: string | undefined, observedAt: number): Promise<UsageWindow[] | undefined>;
61
+ completeVerification(key: UsageKey, attemptAt: number, observedAt: number): Promise<void>;
57
62
  }
@@ -1,9 +1,10 @@
1
1
  import { redactStartupText } from './redact-text.js';
2
+ import { logger } from './logger.js';
2
3
  import { createHash, randomUUID } from 'node:crypto';
3
- import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
4
+ import { mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
4
5
  import { join, resolve } from 'node:path';
5
6
  import { parseClaudeRateLimitInfo } from './claude-rate-limit.js';
6
- import { normalizePassiveUsage, mergePersistedUsageObservations, mergeUsageObservations, record, reactiveEligibleAt, family } from './subscription-selection.js';
7
+ import { normalizePassiveUsage, mergePersistedUsageObservations, mergeUsageObservations, record, reactiveEligibleAt, reactiveUsageWindow, rejectionEvidence, rejectionResetAt, verificationEligibleAt, verificationWindows, selectSubscriptionAccount, family, USAGE_THRESHOLD_PERCENT } from './subscription-selection.js';
7
8
  export function usageStateForSelection(state, now) {
8
9
  return {
9
10
  usage: mergeUsageObservations([
@@ -80,12 +81,31 @@ function safeUsage(value, source) {
80
81
  if (observedAt !== undefined)
81
82
  validTime(observedAt);
82
83
  const source = w['source'] === 'passive' || w['source'] === 'probe' ? w['source'] : undefined;
83
- return { window, scope, ...((w['kind'] === 'seven_day_overage_included' || w['kind'] === 'legacy-uncertain-seven-day') ? { kind: w['kind'] } : {}), percent, resetAt, ...(status ? { status } : {}), ...(observedAt ? { observedAt } : {}), ...(source ? { source } : {}) };
84
+ return { window, scope, ...((w['kind'] === 'seven_day_overage_included' || w['kind'] === 'legacy-uncertain-seven-day') ? { kind: w['kind'] } : {}), percent, resetAt, ...(status ? { status } : {}), ...(observedAt ? { observedAt } : {}), ...(source ? { source } : {}), ...safeVerificationFields(w) };
84
85
  });
85
86
  if (usage['windows'].length > 133)
86
87
  windows.push({ window: 'seven_day', scope: 'uncertain' });
87
88
  return { source, observedAt: usage['observedAt'], windows };
88
89
  }
90
+ function safeVerificationFields(value) {
91
+ const result = {};
92
+ const count = value['rejectionCount'];
93
+ if (typeof count === 'number' && Number.isSafeInteger(count) && count >= 1)
94
+ result.rejectionCount = count;
95
+ const attemptAt = value['lastVerificationAttemptAt'];
96
+ if (typeof attemptAt === 'number') {
97
+ validTime(attemptAt);
98
+ result.lastVerificationAttemptAt = attemptAt;
99
+ }
100
+ const evidence = record(value['verificationEvidence']);
101
+ if (typeof evidence['observedAt'] === 'number' && (evidence['source'] === 'probe' || evidence['source'] === 'passive') &&
102
+ (evidence['status'] === 'allowed' || evidence['status'] === 'allowed_warning') &&
103
+ typeof evidence['percent'] === 'number' && Number.isFinite(evidence['percent']) && evidence['percent'] >= 0 && evidence['percent'] <= 100) {
104
+ validTime(evidence['observedAt']);
105
+ result.verificationEvidence = { observedAt: evidence['observedAt'], source: evidence['source'], status: evidence['status'], percent: evidence['percent'] };
106
+ }
107
+ return result;
108
+ }
89
109
  function safeProbe(value) {
90
110
  const obj = record(value);
91
111
  const status = obj['status'];
@@ -126,7 +146,7 @@ function safeReactive(value) {
126
146
  if (typeof obj['observedAt'] !== 'number' || !obj['quota'] || typeof obj['quota'] !== 'object')
127
147
  throw new SubscriptionUsageStoreError();
128
148
  validTime(obj['observedAt']);
129
- return { observedAt: obj['observedAt'], quota: safeQuota(obj['quota']) };
149
+ return { observedAt: obj['observedAt'], quota: safeQuota(obj['quota']), ...safeVerificationFields(obj) };
130
150
  });
131
151
  }
132
152
  /** One file per version + one reactive file per workspace/account. No credentials or raw provider bodies. */
@@ -139,6 +159,84 @@ export class FileSubscriptionUsageStore {
139
159
  throw new SubscriptionUsageStoreError();
140
160
  return join(this.#directory, `${createHash('sha256').update(JSON.stringify(parts)).digest('hex')}.json`);
141
161
  }
162
+ #claimPath(key, window, fallbackAt) {
163
+ // The epoch identifies the backoff step, so processes with different wall
164
+ // clocks still compete for the same exclusive marker.
165
+ const epoch = verificationEligibleAt(window, window.observedAt ?? fallbackAt);
166
+ return join(this.#directory, 'claims', `${encodeURIComponent(key.accountId)}.${window.scope}.${window.window}.${epoch}.claim`);
167
+ }
168
+ async #claimMarkers(key) {
169
+ const directory = join(this.#directory, 'claims');
170
+ const claims = new Map();
171
+ try {
172
+ const names = await readdir(directory);
173
+ const now = Date.now();
174
+ for (const name of names) {
175
+ if (!name.startsWith(`${encodeURIComponent(key.accountId)}.`) || !name.endsWith('.claim'))
176
+ continue;
177
+ const path = join(directory, name);
178
+ try {
179
+ const raw = await readFile(path, 'utf8');
180
+ let marker;
181
+ try {
182
+ marker = JSON.parse(raw);
183
+ }
184
+ catch {
185
+ // Exclusive creation already owns the step, including the interval
186
+ // before metadata is written or a crash during that write.
187
+ if (now - (await stat(path)).mtimeMs > 24 * 3600_000) {
188
+ await rm(path, { force: true });
189
+ continue;
190
+ }
191
+ const epoch = Number(name.split('.').at(-2));
192
+ validTime(epoch);
193
+ claims.set(path, epoch);
194
+ continue;
195
+ }
196
+ validTime(marker.attemptAt);
197
+ validTime(marker.resetAt);
198
+ if (now - marker.attemptAt > 24 * 3600_000 || marker.resetAt <= now) {
199
+ await rm(path, { force: true });
200
+ continue;
201
+ }
202
+ claims.set(path, marker.attemptAt);
203
+ }
204
+ catch (error) {
205
+ if (error.code !== 'ENOENT')
206
+ throw error;
207
+ }
208
+ }
209
+ return claims;
210
+ }
211
+ catch (error) {
212
+ if (error.code === 'ENOENT')
213
+ return claims;
214
+ throw new SubscriptionUsageStoreError();
215
+ }
216
+ }
217
+ async #claimVerification(key, windows, observedAt) {
218
+ try {
219
+ await mkdir(join(this.#directory, 'claims'), { recursive: true, mode: 0o700 });
220
+ const markers = new Map(windows.map(w => [this.#claimPath(key, w, observedAt), w]));
221
+ for (const [path, window] of [...markers].sort(([a], [b]) => a.localeCompare(b))) {
222
+ const file = await open(path, 'wx', 0o600);
223
+ try {
224
+ await file.writeFile(JSON.stringify({ attemptAt: observedAt, resetAt: rejectionResetAt(window, window.observedAt ?? observedAt) }));
225
+ }
226
+ finally {
227
+ await file.close();
228
+ }
229
+ }
230
+ return true;
231
+ }
232
+ catch (error) {
233
+ if (error.code === 'EEXIST')
234
+ return false;
235
+ // A marker remains spent even if the JSON write fails. Readers consult
236
+ // markers as well, so restarting cannot launch the same verification.
237
+ throw new SubscriptionUsageStoreError();
238
+ }
239
+ }
142
240
  async #load(path, fallback) {
143
241
  try {
144
242
  const raw = await readFile(path, 'utf8');
@@ -169,6 +267,49 @@ export class FileSubscriptionUsageStore {
169
267
  await rm(temporary, { force: true }).catch(() => { });
170
268
  }
171
269
  }
270
+ #log(message) { logger.info(`[Pool] ${message}`); }
271
+ #logEvidence(previous, window, usage, now, threshold) {
272
+ const evidence = rejectionEvidence(usage, window, now);
273
+ if (evidence && evidence.percent < threshold && evidence.observedAt !== previous?.verificationEvidence?.observedAt &&
274
+ !(window.lastVerificationAttemptAt !== undefined && window.lastVerificationAttemptAt >= (window.observedAt ?? now))) {
275
+ this.#log(`rejection marked verifiable window=${window.window}/${window.scope} count=${window.rejectionCount ?? 1} evidence=${evidence.source}:${evidence.status}:${evidence.percent}% observedAt=${new Date(evidence.observedAt).toISOString()} eligibleAt=${new Date(verificationEligibleAt(window, window.observedAt ?? now)).toISOString()}`);
276
+ }
277
+ }
278
+ async #mergeUsage(key, state, previous, incoming, now) {
279
+ const path = this.#path(key, false);
280
+ const account = await this.#load(path, { reactive: [] });
281
+ account.reactive = safeReactive(account.reactive);
282
+ if (incoming)
283
+ incoming = { ...incoming, windows: incoming.windows.map(w => {
284
+ if (w.status !== 'rejected')
285
+ return w;
286
+ const at = w.observedAt ?? incoming.observedAt;
287
+ const prior = account.reactive.map(reactiveUsageWindow).find(r => r.window === w.window && r.scope === w.scope && r.observedAt <= at && r.resetAt > at);
288
+ return prior ? { ...w, rejectionCount: Math.max(w.rejectionCount ?? 1, (prior.rejectionCount ?? 1) + Number(prior.observedAt < at)) } : w;
289
+ }) };
290
+ state.usage = mergePersistedUsageObservations(previous, incoming, now);
291
+ const threshold = key.useFullQuota ? 100 : USAGE_THRESHOLD_PERCENT;
292
+ if (state.usage)
293
+ state.usage = { ...state.usage, windows: state.usage.windows.map(w => w.status === 'rejected' ? { ...w, rejectionCount: w.rejectionCount ?? 1 } : w) };
294
+ for (const window of state.usage?.windows ?? []) {
295
+ if (window.status !== 'rejected')
296
+ continue;
297
+ const old = previous?.windows.find(w => w.status === 'rejected' && w.window === window.window && w.scope === window.scope);
298
+ this.#logEvidence(old, window, state.usage, now, threshold);
299
+ if (old?.lastVerificationAttemptAt !== undefined && window.observedAt > old.observedAt)
300
+ this.#log(`verification re-rejected window=${window.window}/${window.scope} count=${window.rejectionCount ?? 1} eligibleAt=${new Date(verificationEligibleAt(window, window.observedAt)).toISOString()}`);
301
+ }
302
+ const reactive = safeReactive(account.reactive).map(r => {
303
+ if (!r.quota.window)
304
+ return r;
305
+ const window = reactiveUsageWindow(r);
306
+ const verificationEvidence = rejectionEvidence(state.usage, window, now);
307
+ this.#logEvidence(window, { ...window, verificationEvidence }, state.usage, now, threshold);
308
+ return { ...r, verificationEvidence };
309
+ });
310
+ if (JSON.stringify(reactive) !== JSON.stringify(account.reactive))
311
+ await this.#save(path, { reactive });
312
+ }
172
313
  async read(key) {
173
314
  const [version, account] = await Promise.all([
174
315
  this.#load(this.#path(key, true), {}),
@@ -177,19 +318,32 @@ export class FileSubscriptionUsageStore {
177
318
  // Corrupt reactive data must never silently resurrect an exhausted account.
178
319
  if (!Array.isArray(account.reactive) || account.reactive.some(r => !r || !Number.isFinite(r.observedAt) || !r.quota))
179
320
  throw new SubscriptionUsageStoreError();
180
- return { ...safeVersion(version), reactive: safeReactive(account.reactive) };
321
+ const state = { ...safeVersion(version), reactive: safeReactive(account.reactive) };
322
+ const claims = await this.#claimMarkers(key);
323
+ const claimedWindow = (w, fallbackAt) => {
324
+ const attemptAt = claims.get(this.#claimPath(key, w, fallbackAt));
325
+ return w.status === 'rejected' && attemptAt !== undefined ? { ...w, lastVerificationAttemptAt: Math.max(w.lastVerificationAttemptAt ?? 0, attemptAt) } : w;
326
+ };
327
+ if (state.usage)
328
+ state.usage = { ...state.usage, windows: state.usage.windows.map(w => claimedWindow(w, state.usage.observedAt)) };
329
+ state.reactive = state.reactive.map(r => ({ ...r, lastVerificationAttemptAt: claimedWindow(reactiveUsageWindow(r), r.observedAt).lastVerificationAttemptAt }));
330
+ return state;
181
331
  }
182
332
  async recordPassive(key, rateLimitInfo, observedAt, options) {
183
333
  validTime(observedAt);
184
334
  const path = this.#path(key, true);
185
- await serial(path, async () => {
335
+ await serial(this.#path(key, false), async () => {
186
336
  const state = safeVersion(await this.#load(path, {}));
187
337
  if (state.passive && state.passive.observedAt > observedAt && !options?.expireFutureObservations)
188
338
  return;
189
339
  // Retain Fable observation clocks even when the latest report omits its meter.
190
340
  const previous = state.usage ?? usageStateForSelection({ ...state, reactive: [] }, observedAt).usage;
191
341
  state.passive = { rateLimitInfo: safeRateLimitInfo(rateLimitInfo), observedAt };
192
- state.usage = mergePersistedUsageObservations(previous, normalizePassiveUsage(state.passive.rateLimitInfo, observedAt), observedAt);
342
+ const incoming = normalizePassiveUsage(state.passive.rateLimitInfo, observedAt);
343
+ const status = state.passive.rateLimitInfo.status;
344
+ if (status === 'allowed' || status === 'allowed_warning')
345
+ incoming.windows = incoming.windows.map(w => ({ ...w, status: w.status ?? status }));
346
+ await this.#mergeUsage(key, state, previous, incoming, observedAt);
193
347
  await this.#save(path, state);
194
348
  });
195
349
  }
@@ -199,24 +353,41 @@ export class FileSubscriptionUsageStore {
199
353
  await serial(path, async () => {
200
354
  const state = await this.#load(path, { reactive: [] });
201
355
  state.reactive = safeReactive(state.reactive).filter(r => !options?.expireFutureObservations || r.observedAt <= observedAt);
202
- const next = { observedAt, quota: safeQuota(quota) };
356
+ const next = { observedAt, quota: safeQuota(quota), rejectionCount: 1 };
203
357
  const until = reactiveEligibleAt;
204
358
  const sameScope = (r) => r.quota.modelScope === next.quota.modelScope && r.quota.window === next.quota.window;
205
359
  const previous = state.reactive.find(sameScope);
360
+ const version = safeVersion(await this.#load(this.#path(key, true), {}));
361
+ const window = reactiveUsageWindow(next);
362
+ const passive = version.usage?.windows.find(w => w.status === 'rejected' && w.window === window.window && w.scope === window.scope &&
363
+ (w.observedAt ?? version.usage.observedAt) <= observedAt && rejectionResetAt(w, w.observedAt ?? version.usage.observedAt) > observedAt);
206
364
  const reactive = state.reactive.filter(r => !sameScope(r) && until(r) > observedAt);
207
365
  const hasExplicitReset = (r) => r.quota.resetAt !== undefined && Date.parse(r.quota.resetAt) > r.observedAt;
208
366
  // A recheck is not a reset. New matching quota evidence can replace that
209
367
  // provisional deadline with its explicit reset, without clearing a known
210
368
  // active reset or accepting an older observation as revalidation.
211
369
  const revalidated = previous && next.observedAt >= previous.observedAt && !hasExplicitReset(previous) && hasExplicitReset(next);
212
- reactive.push(previous && !revalidated && until(previous) > until(next) ? previous : next);
370
+ if (previous && next.observedAt < previous.observedAt)
371
+ reactive.push(previous);
372
+ else if (previous && next.observedAt === previous.observedAt && !revalidated)
373
+ reactive.push(previous);
374
+ else {
375
+ next.rejectionCount = Math.max(previous && until(previous) > observedAt ? (previous.rejectionCount ?? 1) + Number(previous.observedAt < observedAt) : 1, passive ? (passive.rejectionCount ?? 1) + Number((passive.observedAt ?? version.usage.observedAt) < observedAt) : 1);
376
+ if (previous && !next.quota.resetAt && until(previous) > until(next))
377
+ next.quota.resetAt = previous.quota.resetAt;
378
+ if (!next.quota.resetAt && passive?.resetAt)
379
+ next.quota.resetAt = new Date(passive.resetAt).toISOString();
380
+ reactive.push(next);
381
+ if (previous?.lastVerificationAttemptAt !== undefined || passive?.lastVerificationAttemptAt !== undefined)
382
+ this.#log(`verification re-rejected window=${next.quota.window ?? 'seven_day'}/${next.quota.modelScope ?? 'global'} count=${next.rejectionCount} eligibleAt=${new Date(verificationEligibleAt(reactiveUsageWindow(next), observedAt)).toISOString()}`);
383
+ }
213
384
  await this.#save(path, { reactive });
214
385
  });
215
386
  }
216
387
  async beginProbe(key, observedAt) {
217
388
  validTime(observedAt);
218
389
  const path = this.#path(key, true);
219
- return serial(path, async () => {
390
+ return serial(this.#path(key, false), async () => {
220
391
  const state = safeVersion(await this.#load(path, {}));
221
392
  if (state.probe)
222
393
  return false;
@@ -229,7 +400,7 @@ export class FileSubscriptionUsageStore {
229
400
  async claimLocalProbe(key, observedAt, minIntervalMs) {
230
401
  validTime(observedAt);
231
402
  const path = this.#path(key, true);
232
- return serial(path, async () => {
403
+ return serial(this.#path(key, false), async () => {
233
404
  const state = safeVersion(await this.#load(path, {}));
234
405
  if (state.localProbeClaimedAt !== undefined && observedAt >= state.localProbeClaimedAt &&
235
406
  observedAt - state.localProbeClaimedAt < minIntervalMs)
@@ -259,15 +430,77 @@ export class FileSubscriptionUsageStore {
259
430
  async finishProbe(key, observation) {
260
431
  validTime(observation.observedAt);
261
432
  const path = this.#path(key, true);
262
- await serial(path, async () => {
433
+ await serial(this.#path(key, false), async () => {
263
434
  const state = safeVersion(await this.#load(path, {}));
264
435
  // Network completion order must not discard a newer stored snapshot.
265
436
  const mergeAt = Math.max(observation.observedAt, state.usage?.observedAt ?? 0, state.passive?.observedAt ?? 0);
266
437
  // Retain Fable observation clocks even when the latest report omits its meter.
267
438
  const previous = state.usage ?? usageStateForSelection({ ...state, reactive: [] }, mergeAt).usage;
268
439
  state.probe = safeProbe(observation);
269
- state.usage = mergePersistedUsageObservations(previous, state.probe.usage, mergeAt);
440
+ await this.#mergeUsage(key, state, previous, state.probe.usage, mergeAt);
270
441
  await this.#save(path, state);
271
442
  });
272
443
  }
444
+ async beginVerification(key, model, observedAt) {
445
+ validTime(observedAt);
446
+ return serial(this.#path(key, false), async () => {
447
+ const stored = await this.read(key);
448
+ const state = usageStateForSelection(stored, observedAt);
449
+ const selected = selectSubscriptionAccount({ pool: { kind: 'ready', accounts: [key] }, states: { [key.accountId]: state }, model, now: observedAt });
450
+ if (selected.kind !== 'selected')
451
+ return undefined;
452
+ const windows = verificationWindows(key, state, model, observedAt);
453
+ if (!windows.length)
454
+ return [];
455
+ if (!await this.#claimVerification(key, windows, observedAt))
456
+ return undefined;
457
+ const matches = (w) => windows.some(v => v.window === w.window && v.scope === w.scope && v.observedAt === (w.observedAt ?? stored.usage?.observedAt ?? state.usage?.observedAt));
458
+ const version = safeVersion(await this.#load(this.#path(key, true), {}));
459
+ version.usage ??= state.usage;
460
+ if (version.usage)
461
+ version.usage = { ...version.usage, windows: version.usage.windows.map(w => matches(w) ? { ...w, lastVerificationAttemptAt: observedAt } : w) };
462
+ const reactive = stored.reactive.map(r => matches(reactiveUsageWindow(r)) ? { ...r, lastVerificationAttemptAt: observedAt } : r);
463
+ await this.#save(this.#path(key, true), version);
464
+ await this.#save(this.#path(key, false), { reactive });
465
+ for (const w of windows)
466
+ this.#log(`verification attempt started window=${w.window}/${w.scope} count=${w.rejectionCount ?? 1} attemptAt=${new Date(observedAt).toISOString()}`);
467
+ return windows;
468
+ });
469
+ }
470
+ async completeVerification(key, attemptAt, observedAt) {
471
+ validTime(attemptAt);
472
+ validTime(observedAt);
473
+ await serial(this.#path(key, false), async () => {
474
+ const stored = await this.read(key);
475
+ const cleared = [];
476
+ const canClear = (w) => w.status === 'rejected' && w.lastVerificationAttemptAt === attemptAt &&
477
+ (w.observedAt ?? stored.usage?.observedAt ?? observedAt) <= attemptAt && attemptAt <= observedAt;
478
+ const version = safeVersion(await this.#load(this.#path(key, true), {}));
479
+ const windows = version.usage?.windows.filter(w => { if (!canClear(w))
480
+ return true; cleared.push({ ...w, observedAt: w.observedAt ?? version.usage.observedAt }); return false; });
481
+ if (version.usage && windows)
482
+ version.usage = { ...version.usage, windows };
483
+ const reactive = stored.reactive.filter(r => { const w = reactiveUsageWindow(r); if (!canClear(w))
484
+ return true; cleared.push(w); return false; });
485
+ // Raw snapshots are also selection inputs; remove the cleared rejection there.
486
+ const matches = (w, fallback) => cleared.some(c => c.window === w.window && c.scope === w.scope && (w.observedAt ?? fallback) === c.observedAt);
487
+ if (version.passive && normalizePassiveUsage(version.passive.rateLimitInfo, version.passive.observedAt).windows.some(w => w.status === 'rejected' && matches(w, version.passive.observedAt))) {
488
+ // The rejected percentage is not a successful measurement. Keeping it
489
+ // as status-less usage could immediately re-block the cleared account.
490
+ delete version.passive;
491
+ }
492
+ if (version.probe?.usage)
493
+ version.probe.usage = { ...version.probe.usage, windows: version.probe.usage.windows.filter(w => w.status !== 'rejected' || !matches(w, version.probe.usage.observedAt)) };
494
+ await this.#save(this.#path(key, true), version);
495
+ await this.#save(this.#path(key, false), { reactive });
496
+ const markers = await this.#claimMarkers(key);
497
+ await Promise.all([...markers.keys()].filter(path => cleared.some(w => {
498
+ const prefix = join(this.#directory, 'claims', `${encodeURIComponent(key.accountId)}.${w.scope}.${w.window}.`);
499
+ const epoch = Number(path.slice(prefix.length, -'.claim'.length));
500
+ return path.startsWith(prefix) && epoch <= verificationEligibleAt(w, w.observedAt ?? observedAt);
501
+ })).map(path => rm(path, { force: true })));
502
+ for (const w of cleared)
503
+ this.#log(`verification cleared rejection window=${w.window}/${w.scope} count=${w.rejectionCount ?? 1}`);
504
+ });
505
+ }
273
506
  }
package/dist/types.d.ts CHANGED
@@ -56,6 +56,8 @@ export interface TeamVibeQueueMessage {
56
56
  values?: Record<string, string>;
57
57
  };
58
58
  teamvibe: {
59
+ /** Workspace/channel IANA timezone, when supplied by message routing. */
60
+ timeZone?: string;
59
61
  workspaceId: string;
60
62
  botId: string;
61
63
  botToken: string;
@@ -1,3 +1,4 @@
1
+ import { type UsageWindow } from './subscription-selection.js';
1
2
  /** Owner-facing account policy location. */
2
3
  export declare const FULL_QUOTA_SETTINGS_PATH = "Connections (account menu \u2192 Use until exhausted)";
3
4
  export interface UsageResetTiming {
@@ -18,6 +19,8 @@ export interface PoolUsageLimitNotice extends UsageResetTiming {
18
19
  cause: PoolDeferCause;
19
20
  thresholdDeferred?: boolean;
20
21
  accounts?: readonly UsageLimitAccount[];
22
+ eligibleAt?: number;
23
+ blockedWindows?: readonly Pick<UsageWindow, 'window' | 'scope'>[];
21
24
  }
22
25
  export declare function formatUsageReset(input: UsageResetTiming): string | undefined;
23
26
  export declare function formatPoolUsageLimitMessage(input: PoolUsageLimitNotice): string;
@@ -1,6 +1,17 @@
1
1
  import { USAGE_THRESHOLD_PERCENT } from './subscription-selection.js';
2
2
  /** Owner-facing account policy location. */
3
3
  export const FULL_QUOTA_SETTINGS_PATH = 'Connections (account menu → Use until exhausted)';
4
+ /** Quota notices include a calendar date and zone across multi-day holds. */
5
+ function formatQuotaTime(at, timeZone) {
6
+ try {
7
+ const parts = new Intl.DateTimeFormat('en-GB', { timeZone: timeZone || 'UTC', weekday: 'short', day: '2-digit', month: 'short', hour: '2-digit', minute: '2-digit', hourCycle: 'h23', timeZoneName: 'short' }).formatToParts(at);
8
+ const part = (type) => parts.find(p => p.type === type)?.value;
9
+ return `${part('weekday')} ${part('day')} ${part('month')} ${part('hour')}:${part('minute')} ${part('timeZoneName')}`;
10
+ }
11
+ catch {
12
+ return formatQuotaTime(at, 'UTC');
13
+ }
14
+ }
4
15
  export function formatUsageReset(input) {
5
16
  if (input.resetAt === undefined || !Number.isFinite(input.resetAt))
6
17
  return undefined;
@@ -43,6 +54,17 @@ export function formatPoolUsageLimitMessage(input) {
43
54
  const labels = headroom.filter(a => a.label && a.remaining > 0).map(a => `${a.label} ${Number(a.remaining.toFixed(2))} %`).join(', ');
44
55
  return `⏳ All Claude accounts in this workspace are above ${USAGE_THRESHOLD_PERCENT} % of their usage limit, so I'm waiting for the next reset${reset ? ` at ${reset}` : ''}. Your message is saved and will run automatically.\n💡 There is still some capacity left${labels ? ` (${labels})` : ''}. A workspace owner can allow the full quota per account in ${FULL_QUOTA_SETTINGS_PATH}.`;
45
56
  }
57
+ if (input.blockedWindows?.length) {
58
+ const windows = [...new Map(input.blockedWindows.map(w => [`${w.window}:${w.scope}`, w])).values()];
59
+ const names = windows.map(w => `${w.scope === 'global' ? 'global' : w.scope === 'uncertain' ? 'Claude' : w.scope[0].toUpperCase() + w.scope.slice(1)} ${w.window === 'seven_day' ? 'weekly' : 'five-hour'} quota`);
60
+ const global = windows.some(w => w.scope === 'global' || w.scope === 'uncertain');
61
+ const resets = input.resetAt !== undefined ? `resets ${formatQuotaTime(input.resetAt, input.timeZone)}` : 'reset time unavailable';
62
+ const retry = input.eligibleAt !== undefined ? `I'll retry automatically at ${formatQuotaTime(input.eligibleAt, input.timeZone)}` : "I'll retry automatically after the next reset";
63
+ const blocked = windows.length === 1 && !global
64
+ ? `${names[0]} is exhausted on every account in this workspace (${resets}).`
65
+ : `All Claude accounts in this workspace have hit their usage limit (${names.join(', ')}; ${resets}).`;
66
+ return `⏳ ${blocked} Your message is saved. ${retry}${global ? '.' : '; other models keep working.'}`;
67
+ }
46
68
  return `⏳ All Claude accounts in this workspace have hit their usage limit. Your message is saved and will run automatically after ${reset ? `the earliest reset at ${reset}` : 'the next reset'}.`;
47
69
  }
48
70
  export function formatLegacyUsageLimitMessage(input) {
@@ -2,7 +2,7 @@ import { spawn } from 'node:child_process';
2
2
  import { mkdtemp, rm, mkdir } from 'node:fs/promises';
3
3
  import { tmpdir } from 'node:os';
4
4
  import { join } from 'node:path';
5
- import { selectSubscriptionAccount, normalizePassiveUsage } from './subscription-selection.js';
5
+ import { selectSubscriptionAccount, normalizePassiveUsage, reactiveUsageWindow } from './subscription-selection.js';
6
6
  import { safeRateLimitInfo, usageStateForSelection, } from './subscription-usage-store.js';
7
7
  import { parseClaudeRateLimitInfo, } from './claude-rate-limit.js';
8
8
  import { formatPoolUsageWindows, poolInfo, poolLabel } from './pool-logging.js';
@@ -210,18 +210,46 @@ export async function runUsageProbe(account, options, context = {}) {
210
210
  return result;
211
211
  const now = Date.now();
212
212
  const safe = safeRateLimitInfo(info);
213
+ const normalized = normalizePassiveUsage(safe, now);
213
214
  const usage = {
214
- ...normalizePassiveUsage(safe, now),
215
+ ...normalized,
215
216
  source: 'probe',
217
+ windows: normalized.windows.map(w => ({ ...w,
218
+ ...(w.status === undefined && w.percent !== undefined && (safe.status === 'allowed' || safe.status === 'allowed_warning') ? { status: safe.status } : {}),
219
+ })),
216
220
  };
217
221
  measuredWindows = formatPoolUsageWindows(usage.windows);
222
+ const before = usageStateForSelection(await options.store.read(account), now);
218
223
  await options.store.finishProbe(account, {
219
224
  status: 'ok',
220
225
  observedAt: now,
221
226
  usage,
222
227
  });
223
228
  const stored = await options.store.read(account);
224
- if (selectSubscriptionAccount({ pool: { kind: 'ready', accounts: [account] }, states: { [account.accountId]: usageStateForSelection(stored, now) }, model: undefined, now }).kind === 'selected') {
229
+ const after = usageStateForSelection(stored, now);
230
+ // Keep the combined comparison, and compare each rejected scope/window
231
+ // independently so an unrelated, later hold cannot hide an improvement.
232
+ const pairs = new Map();
233
+ for (const state of [before, after])
234
+ for (const w of [...(state.usage?.windows ?? []), ...(state.reactive ?? []).map(reactiveUsageWindow)]) {
235
+ if (w.status === 'rejected')
236
+ pairs.set(`${w.scope}.${w.window}`, w);
237
+ }
238
+ const eligibility = (state, pair) => {
239
+ const matches = (w) => !pair || w.scope === pair.scope && w.window === pair.window;
240
+ const scoped = pair ? {
241
+ usage: state.usage ? { ...state.usage, windows: state.usage.windows.filter(matches) } : undefined,
242
+ reactive: state.reactive?.filter(r => matches(reactiveUsageWindow(r))),
243
+ } : state;
244
+ return selectSubscriptionAccount({ pool: { kind: 'ready', accounts: [account] }, states: { [account.accountId]: scoped }, model: pair?.scope, now });
245
+ };
246
+ const improved = [undefined, ...pairs.values()].some(pair => {
247
+ const previous = eligibility(before, pair);
248
+ const current = eligibility(after, pair);
249
+ return current.kind === 'selected' && (!pair || previous.kind === 'deferred') ||
250
+ current.kind === 'deferred' && previous.kind === 'deferred' && current.eligibleAt < previous.eligibleAt;
251
+ });
252
+ if (improved) {
225
253
  try {
226
254
  options.onHeadroom?.(account.workspaceId);
227
255
  }
@@ -1,12 +1,12 @@
1
1
  {
2
2
  "name": "@teamvibe/poller",
3
- "version": "0.8.7",
3
+ "version": "0.8.8",
4
4
  "lockfileVersion": 3,
5
5
  "requires": true,
6
6
  "packages": {
7
7
  "": {
8
8
  "name": "@teamvibe/poller",
9
- "version": "0.8.7",
9
+ "version": "0.8.8",
10
10
  "dependencies": {
11
11
  "@aws-sdk/client-dynamodb": "3.1108.0",
12
12
  "@aws-sdk/client-sqs": "3.1108.0",
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@teamvibe/poller",
3
- "version": "0.8.7",
3
+ "version": "0.8.8",
4
4
  "type": "module",
5
5
  "main": "./dist/index.js",
6
6
  "bin": {