@teamvibe/poller 0.8.7 → 0.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/claude-spawner.d.ts +2 -0
- package/dist/claude-spawner.js +42 -12
- package/dist/poller.js +20 -3
- package/dist/subscription-runtime.d.ts +2 -0
- package/dist/subscription-runtime.js +6 -2
- package/dist/subscription-selection.d.ts +21 -2
- package/dist/subscription-selection.js +91 -14
- package/dist/subscription-usage-store.d.ts +6 -1
- package/dist/subscription-usage-store.js +246 -13
- package/dist/types.d.ts +2 -0
- package/dist/usage-limit-messages.d.ts +3 -0
- package/dist/usage-limit-messages.js +22 -0
- package/dist/usage-probe-job.js +31 -3
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/claude-spawner.d.ts
CHANGED
|
@@ -28,6 +28,8 @@ export interface SpawnResult {
|
|
|
28
28
|
poolAttempts?: readonly PoolAttempt[];
|
|
29
29
|
quota?: ClaudeQuotaFailure;
|
|
30
30
|
rateLimitInfo?: ClaudeRateLimitInfo;
|
|
31
|
+
/** Rejections anywhere in this attempt survive a later allowed stream event. */
|
|
32
|
+
rateLimitRejections?: readonly ClaudeRateLimitInfo[];
|
|
31
33
|
/** Actual ID passed to this CLI attempt, including a failed first turn. */
|
|
32
34
|
attemptedSessionId?: string;
|
|
33
35
|
success: boolean;
|
package/dist/claude-spawner.js
CHANGED
|
@@ -20,7 +20,7 @@ import { fetchSecretsForSpawn, buildSecretsReminder } from './secrets-client.js'
|
|
|
20
20
|
import { fetchConfigForSpawn } from './config-client.js';
|
|
21
21
|
import { parseClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
22
22
|
import { parseClaudeQuotaFailure } from './claude-quota.js';
|
|
23
|
-
import { abortLocalProbe, accountPin, preparePool, fetchRuntimePool, validateAccount, poolDeferredError, PoolDeferredError } from './subscription-runtime.js';
|
|
23
|
+
import { abortLocalProbe, accountPin, preparePool, fetchRuntimePool, validateAccount, subscriptionUsageStore, poolDeferredError, PoolDeferredError } from './subscription-runtime.js';
|
|
24
24
|
import { selectPoolForResolvedModel } from './subscription-pool.js';
|
|
25
25
|
import { isQuotaPaused, QuotaPausedError } from './quota-pause.js';
|
|
26
26
|
import { redactStartupText } from './startup-auth.js';
|
|
@@ -911,6 +911,14 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
911
911
|
});
|
|
912
912
|
if (runtime?.account)
|
|
913
913
|
await validateAccount(runtime.account, runtime.resolved);
|
|
914
|
+
if (runtime?.account && !runtime.shouldStop()) {
|
|
915
|
+
const store = subscriptionUsageStore();
|
|
916
|
+
const attemptAt = Date.now();
|
|
917
|
+
const windows = await store.beginVerification(runtime.account, runtime.resolved.model, attemptAt);
|
|
918
|
+
if (windows === undefined)
|
|
919
|
+
throw await poolDeferredError('usage', { kind: 'ready', accounts: [runtime.account] }, runtime.resolved, store);
|
|
920
|
+
runtime.verificationAttemptAt = windows.length ? attemptAt : undefined;
|
|
921
|
+
}
|
|
914
922
|
return new Promise((resolve) => {
|
|
915
923
|
// Preparation above awaits remote config/context. Another thread may
|
|
916
924
|
// have exhausted the shared account meanwhile. No awaits before spawn.
|
|
@@ -1083,6 +1091,16 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
1083
1091
|
// Retain CLI error/result evidence, not the full parsed assistant/tool stream.
|
|
1084
1092
|
const failureEvents = [];
|
|
1085
1093
|
let rateLimitInfo;
|
|
1094
|
+
const rateLimitRejections = new Map();
|
|
1095
|
+
const captureRateLimit = (value) => {
|
|
1096
|
+
rateLimitInfo = parseClaudeRateLimitInfo(value);
|
|
1097
|
+
if (!runtime?.account || rateLimitInfo?.status !== 'rejected')
|
|
1098
|
+
return;
|
|
1099
|
+
for (const w of normalizePassiveUsage(rateLimitInfo, Date.now()).windows) {
|
|
1100
|
+
if (w.status === 'rejected')
|
|
1101
|
+
rateLimitRejections.set(`${w.window}:${w.scope}`, rateLimitInfo);
|
|
1102
|
+
}
|
|
1103
|
+
};
|
|
1086
1104
|
let metrics;
|
|
1087
1105
|
let lastAssistant;
|
|
1088
1106
|
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard state — tracked
|
|
@@ -1143,7 +1161,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
1143
1161
|
// (including metadata keys) to logs, telemetry, typing, or Slack.
|
|
1144
1162
|
const event = safeEvent(rawEvent);
|
|
1145
1163
|
if (rawEvent?.type === 'rate_limit_event') {
|
|
1146
|
-
|
|
1164
|
+
captureRateLimit(event.rate_limit_info);
|
|
1147
1165
|
}
|
|
1148
1166
|
if (rawEvent?.type === 'result' || rawEvent?.type === 'error' || (rawEvent?.type === 'assistant' && rawEvent.error)) {
|
|
1149
1167
|
failureEvents.push(rawEvent);
|
|
@@ -1367,7 +1385,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
1367
1385
|
const rawEvent = JSON.parse(stdoutBuffer);
|
|
1368
1386
|
const event = safeEvent(rawEvent);
|
|
1369
1387
|
if (event?.type === 'rate_limit_event') {
|
|
1370
|
-
|
|
1388
|
+
captureRateLimit(event.rate_limit_info);
|
|
1371
1389
|
}
|
|
1372
1390
|
failureEvents.push(rawEvent);
|
|
1373
1391
|
if (event?.type === 'assistant') {
|
|
@@ -1386,6 +1404,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
1386
1404
|
attemptedSessionId: sessionId,
|
|
1387
1405
|
...(quota ? { quota } : {}),
|
|
1388
1406
|
...(rateLimitInfo ? { rateLimitInfo } : {}),
|
|
1407
|
+
...(rateLimitRejections.size ? { rateLimitRejections: [...rateLimitRejections.values()] } : {}),
|
|
1389
1408
|
output: redactOutput(stdout),
|
|
1390
1409
|
exitCode: code,
|
|
1391
1410
|
metrics,
|
|
@@ -1610,7 +1629,7 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1610
1629
|
const modelNormalizedFrom = scoped?.model === resolved.model && scoped?.modelSource === resolved.modelSource
|
|
1611
1630
|
? scoped?.modelNormalizedFrom : undefined;
|
|
1612
1631
|
const runtime = poolRuntime ? {
|
|
1613
|
-
resolved, modelNormalizedFrom, claudeConfigDir, account: prepared?.account,
|
|
1632
|
+
resolved, modelNormalizedFrom, claudeConfigDir, account: prepared?.account, verificationAttemptAt: undefined,
|
|
1614
1633
|
shouldStop: poolRuntime.shouldStop,
|
|
1615
1634
|
resumePending: fetchedPool?.kind !== 'no_accounts_configured' && poolRuntime.pin?.resumePending === true,
|
|
1616
1635
|
onSpawn: () => {
|
|
@@ -1649,10 +1668,12 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1649
1668
|
if (!runtime?.resumePending)
|
|
1650
1669
|
sawSendMessageCall = false;
|
|
1651
1670
|
const spawned = await runClaudeCode(msg, sessionLog, effectiveCwd, directory, id, runtime?.resumePending ? false : first, lastMessageTs, onMessageSent, typingIndicator, runtime, gitDelivery);
|
|
1671
|
+
const rejected = spawned.rateLimitRejections?.find(rejectedForModel);
|
|
1652
1672
|
const result = prepared?.account
|
|
1653
|
-
? { ...spawned, quota: quotaForModel(spawned.quota), success: rejectedForModel(spawned.rateLimitInfo) ? false : spawned.success } : spawned;
|
|
1673
|
+
? { ...spawned, ...(rejected ? { rateLimitInfo: rejected } : {}), quota: quotaForModel(spawned.quota), success: rejected || rejectedForModel(spawned.rateLimitInfo) ? false : spawned.success } : spawned;
|
|
1654
1674
|
sawSendMessageCall ||= result.sawSendMessageCall === true;
|
|
1655
1675
|
if (prepared?.account) {
|
|
1676
|
+
const observedAt = Date.now();
|
|
1656
1677
|
if (poolAttempts.length >= 1000)
|
|
1657
1678
|
poolAttempts.shift();
|
|
1658
1679
|
const selection = prepared.selection;
|
|
@@ -1664,7 +1685,8 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1664
1685
|
accountId: prepared.account.accountId,
|
|
1665
1686
|
credentialVersion: prepared.account.credentialVersion,
|
|
1666
1687
|
accountLabel: prepared.account.label ?? prepared.account.accountId,
|
|
1667
|
-
observedAt
|
|
1688
|
+
observedAt,
|
|
1689
|
+
verificationAttemptAt: runtime?.verificationAttemptAt,
|
|
1668
1690
|
rateLimitInfo: result.rateLimitInfo,
|
|
1669
1691
|
sessionId: id,
|
|
1670
1692
|
model: resolved.model ?? result.metrics?.model,
|
|
@@ -1675,13 +1697,19 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1675
1697
|
metrics: result.metrics,
|
|
1676
1698
|
});
|
|
1677
1699
|
emitEvent('pool.account_attempted', msg.teamvibe.channelId, { workspaceId: msg.teamvibe.workspaceId, threadId: msg.thread_id, ...poolRunTelemetry(poolAttempts, poolRuntime?.quotaDeferEpisodes) });
|
|
1700
|
+
// Retain every rejected scope before completion can clear a verified
|
|
1701
|
+
// hold or return early. A failed write must not skip another scope.
|
|
1702
|
+
const rejectionOutcomes = await Promise.allSettled((result.rateLimitRejections ?? []).map(info => logPoolUsageWrite(sessionLog, prepared.account, info, () => prepared.store.recordPassive(prepared.account, info, observedAt, { expireFutureObservations: true }))));
|
|
1678
1703
|
if (result.success && id && poolRuntime) {
|
|
1679
|
-
// Complete the inbox
|
|
1680
|
-
//
|
|
1704
|
+
// Complete the inbox even if rejection bookkeeping failed. The loop
|
|
1705
|
+
// retains completion independently of the original prompt.
|
|
1681
1706
|
const bookkeepingComplete = await poolRuntime.complete(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: false }, poolAttempts, result.rateLimitInfo);
|
|
1682
1707
|
if (bookkeepingComplete !== false)
|
|
1683
1708
|
await emitCandidateUsage(msg, [prepared.account], prepared.store);
|
|
1684
1709
|
retainWorktree = false;
|
|
1710
|
+
const failed = rejectionOutcomes.find((outcome) => outcome.status === 'rejected');
|
|
1711
|
+
if (failed)
|
|
1712
|
+
throw new PoolDeferredError('infrastructure', { error: failed.reason });
|
|
1685
1713
|
return result;
|
|
1686
1714
|
}
|
|
1687
1715
|
if ((result.quota || rejectedForModel(result.rateLimitInfo)) && !result.killedByPoller) {
|
|
@@ -1692,15 +1720,17 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1692
1720
|
// retaining the attempt. Selection waits for all writes before switching.
|
|
1693
1721
|
const writes = [];
|
|
1694
1722
|
if (result.quota && !result.killedByPoller)
|
|
1695
|
-
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.quota.window ? { rateLimitType: result.quota.window } : undefined, () => prepared.store.recordQuota(prepared.account, result.quota,
|
|
1696
|
-
if (result.rateLimitInfo)
|
|
1697
|
-
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.rateLimitInfo, () => prepared.store.recordPassive(prepared.account, result.rateLimitInfo,
|
|
1723
|
+
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.quota.window ? { rateLimitType: result.quota.window } : undefined, () => prepared.store.recordQuota(prepared.account, result.quota, observedAt, { expireFutureObservations: true }), 'quota'));
|
|
1724
|
+
if (!result.rateLimitRejections?.length && result.rateLimitInfo)
|
|
1725
|
+
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.rateLimitInfo, () => prepared.store.recordPassive(prepared.account, result.rateLimitInfo, observedAt, { expireFutureObservations: true })));
|
|
1698
1726
|
if (id && poolRuntime)
|
|
1699
1727
|
writes.push(poolRuntime.persist(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: runtime?.resumePending, sawSendMessageCall }, poolAttempts));
|
|
1700
|
-
const outcomes = await Promise.allSettled(writes);
|
|
1728
|
+
const outcomes = [...rejectionOutcomes, ...await Promise.allSettled(writes)];
|
|
1701
1729
|
const failed = outcomes.find((outcome) => outcome.status === 'rejected');
|
|
1702
1730
|
if (failed)
|
|
1703
1731
|
throw new PoolDeferredError('infrastructure', { error: failed.reason });
|
|
1732
|
+
if (runtime?.verificationAttemptAt !== undefined && !result.killedByPoller && result.exitCode !== null)
|
|
1733
|
+
await prepared.store.completeVerification(prepared.account, runtime.verificationAttemptAt, observedAt);
|
|
1704
1734
|
}
|
|
1705
1735
|
return result;
|
|
1706
1736
|
};
|
package/dist/poller.js
CHANGED
|
@@ -323,8 +323,10 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
323
323
|
const lastAttempt = state.attempts?.at(-1);
|
|
324
324
|
// Existing telemetry falls back to account IDs. Local pool logs use labels only.
|
|
325
325
|
const accountLabel = lastAttempt?.accountLabel !== state.pin.accountId ? lastAttempt?.accountLabel : undefined;
|
|
326
|
+
// Reuse the spawn's observation clock: the spawner already recorded all
|
|
327
|
+
// rejected scopes, and replaying its final event is the same evidence.
|
|
326
328
|
if (state.completed.rateLimitInfo)
|
|
327
|
-
writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
|
|
329
|
+
writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, lastAttempt?.observedAt ?? state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
|
|
328
330
|
writes.push(persistSessionId(state.sessionId, state.pin));
|
|
329
331
|
const outcomes = await Promise.allSettled(writes);
|
|
330
332
|
if (outcomes.some(outcome => outcome.status === 'rejected' || outcome.value === false)) {
|
|
@@ -337,6 +339,21 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
337
339
|
}
|
|
338
340
|
return false;
|
|
339
341
|
}
|
|
342
|
+
if (lastAttempt?.verificationAttemptAt !== undefined) {
|
|
343
|
+
try {
|
|
344
|
+
await subscriptionUsageStore().completeVerification(state.pin, lastAttempt.verificationAttemptAt, state.completed.observedAt);
|
|
345
|
+
}
|
|
346
|
+
catch {
|
|
347
|
+
sessionLog.warn('[Loop] Verification bookkeeping failed and will retry without running the agent');
|
|
348
|
+
try {
|
|
349
|
+
saveInboxRuntime(kbPath, threadId, { ...state, completed: { ...state.completed, retryAt: Date.now() + POOL_RECHECK_MS } });
|
|
350
|
+
}
|
|
351
|
+
catch {
|
|
352
|
+
sessionLog.warn('[Loop] Could not save bookkeeping retry; completion remains recorded');
|
|
353
|
+
}
|
|
354
|
+
return false;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
340
357
|
currentSessionId = state.sessionId;
|
|
341
358
|
currentPin = state.pin;
|
|
342
359
|
// The spawner publishes live completions only after bookkeeping succeeds.
|
|
@@ -625,7 +642,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
625
642
|
quotaPaused = true;
|
|
626
643
|
deferred = true;
|
|
627
644
|
// Revisit held work hourly so background probes can refresh the account.
|
|
628
|
-
// Selection
|
|
645
|
+
// Selection enforces resets or a contradicted rejection's verification backoff.
|
|
629
646
|
const recheckAt = error.cause === 'usage' ? Math.min(error.eligibleAt, Date.now() + HELD_PROBE_INTERVAL_MS) : error.eligibleAt;
|
|
630
647
|
const pause = deferPool(threadId, recheckAt, error.resetAt, inboxQueueMessageIds(kbPath, threadId));
|
|
631
648
|
const retained = readInboxRuntime(kbPath, threadId);
|
|
@@ -644,7 +661,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
644
661
|
cause: error.cause,
|
|
645
662
|
thresholdDeferred: error.notice?.thresholdDeferred && !error.attempts?.some(a => a.quota),
|
|
646
663
|
resetAt: pause.resetAt ? Date.parse(pause.resetAt) : error.notice?.resetAt,
|
|
647
|
-
timeZone:
|
|
664
|
+
timeZone: deliveryMsg.teamvibe.timeZone ?? error.notice?.timeZone,
|
|
648
665
|
});
|
|
649
666
|
await sendSlackMessage(deliveryMsg, notice, { clientMessageId: pause.episodeId, ...(reconcile ? { reconcile: true } : {}) });
|
|
650
667
|
setPoolNotice(threadId, pause.episodeId, 'delivered');
|
|
@@ -10,6 +10,8 @@ import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
|
10
10
|
import type { ClaudeQuotaFailure } from './claude-quota.js';
|
|
11
11
|
import type { KilledByPoller } from './classify-failed-iteration.js';
|
|
12
12
|
export interface PoolAttempt extends Partial<HeadroomScore> {
|
|
13
|
+
/** Local completion bookkeeping token; omitted from event transport DTOs. */
|
|
14
|
+
verificationAttemptAt?: number;
|
|
13
15
|
workspaceId: string;
|
|
14
16
|
accountId: string;
|
|
15
17
|
accountLabel?: string;
|
|
@@ -205,6 +205,8 @@ async function poolUsageNotice(pool, resolved, store) {
|
|
|
205
205
|
return {
|
|
206
206
|
label: account.label, useFullQuota: account.useFullQuota, windows: windows.filter(w => binding.some(b => b.window === w.window && b.scope === w.scope)), rejected,
|
|
207
207
|
thresholdBlocked: blockers.length > 0,
|
|
208
|
+
blockedWindows: binding.map(w => ({ window: w.window, scope: w.scope })),
|
|
209
|
+
providerResetAt: binding.length && binding.every(w => w.resetAt !== undefined) ? Math.max(...binding.map(w => w.resetAt)) : undefined,
|
|
208
210
|
resetAt: resets.length && resets.every((r) => r !== undefined) ? Math.max(...resets) : undefined,
|
|
209
211
|
};
|
|
210
212
|
}));
|
|
@@ -212,7 +214,8 @@ async function poolUsageNotice(pool, resolved, store) {
|
|
|
212
214
|
// Snapshot expiry is only a recheck. Use actual window resets for the notice.
|
|
213
215
|
const resets = accounts.map(a => a.resetAt);
|
|
214
216
|
const resetAt = thresholdDeferred && resets.every((r) => r !== undefined) ? Math.min(...resets) : undefined;
|
|
215
|
-
|
|
217
|
+
const providerResets = accounts.map(a => a.providerResetAt);
|
|
218
|
+
return { accounts, thresholdDeferred, resetAt: resetAt ?? (providerResets.length && providerResets.every((r) => r !== undefined) ? Math.min(...providerResets) : undefined), blockedWindows: accounts.flatMap(a => a.blockedWindows) };
|
|
216
219
|
}
|
|
217
220
|
catch {
|
|
218
221
|
return {};
|
|
@@ -226,8 +229,9 @@ export async function poolDeferredError(cause, pool, resolved, store, options =
|
|
|
226
229
|
return new PoolDeferredError(cause, { error: options.error ?? (pool.kind === 'fetch_unavailable' && pool.cause ? { code: pool.cause } : undefined) });
|
|
227
230
|
const notice = await poolUsageNotice(pool, resolved, store);
|
|
228
231
|
const deferredError = (resetAt) => {
|
|
232
|
+
resetAt ??= !notice.thresholdDeferred && notice.resetAt !== undefined ? new Date(notice.resetAt).toISOString() : undefined;
|
|
229
233
|
const error = new PoolDeferredError('usage', { eligibleAt, resetAt });
|
|
230
|
-
error.notice = { ...notice, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
|
|
234
|
+
error.notice = { ...notice, eligibleAt: error.eligibleAt, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
|
|
231
235
|
return error;
|
|
232
236
|
};
|
|
233
237
|
if (eligibleAt === undefined) {
|
|
@@ -10,6 +10,13 @@ export declare const PROBE_USAGE_TTL_MS = 60000;
|
|
|
10
10
|
export declare const POOL_RECHECK_MS: number;
|
|
11
11
|
/** Safety hold from the rejection observation when the provider gives no reset. */
|
|
12
12
|
export declare const REJECTED_BLOCK_MS: number;
|
|
13
|
+
export declare const VERIFICATION_MAX_BACKOFF_MS: number;
|
|
14
|
+
export interface VerificationEvidence {
|
|
15
|
+
observedAt: number;
|
|
16
|
+
source: 'passive' | 'probe';
|
|
17
|
+
status: 'allowed' | 'allowed_warning';
|
|
18
|
+
percent: number;
|
|
19
|
+
}
|
|
13
20
|
export interface AccountIdentity {
|
|
14
21
|
workspaceId: string;
|
|
15
22
|
accountId: string;
|
|
@@ -47,6 +54,11 @@ export interface UsageWindow {
|
|
|
47
54
|
source?: 'passive' | 'probe';
|
|
48
55
|
percent?: number;
|
|
49
56
|
resetAt?: number;
|
|
57
|
+
/** Missing on legacy rejections means the first consecutive rejection. */
|
|
58
|
+
rejectionCount?: number;
|
|
59
|
+
lastVerificationAttemptAt?: number;
|
|
60
|
+
/** A genuine same-window measurement; never inferred from an omitted meter. */
|
|
61
|
+
verificationEvidence?: VerificationEvidence;
|
|
50
62
|
}
|
|
51
63
|
export interface UsageObservation {
|
|
52
64
|
pollerVersion?: string;
|
|
@@ -67,11 +79,11 @@ export declare function projectLegacyFableUsage<T extends UsageObservation | und
|
|
|
67
79
|
* Deduplicate uncertain windows before splitting measurements from rejections.
|
|
68
80
|
*/
|
|
69
81
|
export declare function supersedeLegacyFableProjections(observations: readonly (UsageObservation | undefined)[], now: number): (UsageObservation | undefined)[];
|
|
70
|
-
/** Latest measurements plus durable
|
|
82
|
+
/** Latest measurements plus durable rejections; only a claimed real spawn can clear a hold. */
|
|
71
83
|
export declare function mergeUsageObservations(observations: readonly (UsageObservation | undefined)[], now: number): UsageObservation | undefined;
|
|
72
84
|
/** Preserve last-known meters, projecting legacy identity before its global companion is replaced. */
|
|
73
85
|
export declare function mergePersistedUsageObservations(previous: UsageObservation | undefined, incoming: UsageObservation | undefined, now: number): UsageObservation | undefined;
|
|
74
|
-
export interface ReactiveExhaustion {
|
|
86
|
+
export interface ReactiveExhaustion extends Pick<UsageWindow, 'rejectionCount' | 'lastVerificationAttemptAt' | 'verificationEvidence'> {
|
|
75
87
|
observedAt: number;
|
|
76
88
|
quota: ClaudeQuotaFailure;
|
|
77
89
|
}
|
|
@@ -106,6 +118,13 @@ export declare function record(value: unknown): Record<string, unknown>;
|
|
|
106
118
|
export declare function normalizePassiveUsage(value: unknown, observedAt: number): UsageObservation;
|
|
107
119
|
export declare function normalizeProbeUsage(value: unknown, observedAt: number): UsageObservation;
|
|
108
120
|
export declare function family(model: string | undefined): 'sonnet' | 'opus' | 'haiku' | 'fable' | undefined;
|
|
121
|
+
export declare function rejectionResetAt(window: UsageWindow, observedAt: number): number;
|
|
122
|
+
export declare function verificationEligibleAt(window: UsageWindow, observedAt: number): number;
|
|
123
|
+
export declare function rejectionEvidence(usage: UsageObservation | undefined, window: UsageWindow, now: number): VerificationEvidence | undefined;
|
|
124
|
+
export declare function rejectionEligibleAt(usage: UsageObservation | undefined, window: UsageWindow, account: AccountIdentity, now: number): number;
|
|
125
|
+
export declare function reactiveUsageWindow(reactive: ReactiveExhaustion): UsageWindow;
|
|
126
|
+
/** Due windows are claimed durably at the real spawn boundary, never by pure selection. */
|
|
127
|
+
export declare function verificationWindows(account: AccountIdentity, state: AccountUsageState, model: string | undefined, now: number): UsageWindow[];
|
|
109
128
|
export declare function reactiveEligibleAt(reactive: ReactiveExhaustion): number;
|
|
110
129
|
/** Pure policy. Unknown is eligible after every known/reactive blocker is ruled out. */
|
|
111
130
|
export declare function selectSubscriptionAccount<A extends AccountIdentity>(input: {
|
|
@@ -8,6 +8,7 @@ export const PROBE_USAGE_TTL_MS = 60_000;
|
|
|
8
8
|
export const POOL_RECHECK_MS = 15 * 60_000;
|
|
9
9
|
/** Safety hold from the rejection observation when the provider gives no reset. */
|
|
10
10
|
export const REJECTED_BLOCK_MS = 15 * 60_000;
|
|
11
|
+
export const VERIFICATION_MAX_BACKOFF_MS = 8 * 60 * 60_000;
|
|
11
12
|
/** Passive and probe percentages remain lower bounds until each window resets. */
|
|
12
13
|
export function usageWindowExpiresAt(usage, window) {
|
|
13
14
|
const ttlExpiry = (window.observedAt ?? usage.observedAt) + ((window.source ?? usage.source) === 'passive' ? PASSIVE_USAGE_TTL_MS : PROBE_USAGE_TTL_MS);
|
|
@@ -116,7 +117,7 @@ export function supersedeLegacyFableProjections(observations, now) {
|
|
|
116
117
|
}),
|
|
117
118
|
}));
|
|
118
119
|
}
|
|
119
|
-
/** Latest measurements plus durable
|
|
120
|
+
/** Latest measurements plus durable rejections; only a claimed real spawn can clear a hold. */
|
|
120
121
|
export function mergeUsageObservations(observations, now) {
|
|
121
122
|
let latest;
|
|
122
123
|
const rejected = new Map();
|
|
@@ -139,13 +140,24 @@ export function mergeUsageObservations(observations, now) {
|
|
|
139
140
|
if (w.status !== 'rejected')
|
|
140
141
|
continue;
|
|
141
142
|
const previous = rejected.get(key);
|
|
142
|
-
if (!previous || observedAt >= previous.observedAt)
|
|
143
|
-
|
|
143
|
+
if (!previous || observedAt >= previous.observedAt) {
|
|
144
|
+
const consecutive = previous && observedAt > previous.observedAt && observedAt < rejectionResetAt(previous, previous.observedAt);
|
|
145
|
+
rejected.set(key, {
|
|
146
|
+
...(previous?.observedAt === observedAt ? previous : {}),
|
|
147
|
+
...w, observedAt, source: w.source ?? usage.source,
|
|
148
|
+
...(consecutive && w.resetAt === undefined && previous.resetAt !== undefined ? { resetAt: previous.resetAt } : {}),
|
|
149
|
+
...(consecutive || w.rejectionCount !== undefined || previous?.observedAt === observedAt && previous.rejectionCount !== undefined
|
|
150
|
+
? { rejectionCount: Math.max(w.rejectionCount ?? 1, consecutive ? (previous.rejectionCount ?? 1) + 1 : previous?.rejectionCount ?? 1) } : {}),
|
|
151
|
+
});
|
|
152
|
+
}
|
|
144
153
|
}
|
|
145
154
|
}
|
|
146
155
|
if (!latest)
|
|
147
156
|
return undefined;
|
|
148
|
-
const active = [...rejected.values()].filter(w => now < (w
|
|
157
|
+
const active = [...rejected.values()].filter(w => now < rejectionResetAt(w, w.observedAt)).map(w => {
|
|
158
|
+
const evidence = latestVerificationEvidence(observations, w, w.observedAt, now);
|
|
159
|
+
return evidence ? { ...w, verificationEvidence: evidence } : w;
|
|
160
|
+
});
|
|
149
161
|
const retained = [...scopedExhausted.values()].filter(w => isScopedExhaustion(w) &&
|
|
150
162
|
!latest.windows.some(other => other.window === w.window && other.scope === w.scope) &&
|
|
151
163
|
!active.some(other => other.window === w.window && other.scope === w.scope) &&
|
|
@@ -281,7 +293,9 @@ export function normalizeProbeUsage(value, observedAt) {
|
|
|
281
293
|
if (raw == null)
|
|
282
294
|
continue;
|
|
283
295
|
const item = record(raw);
|
|
284
|
-
windows.push({ window, scope, percent: percent(item['utilization'], 100), resetAt: date(item['resets_at'])
|
|
296
|
+
windows.push({ window, scope, percent: percent(item['utilization'], 100), resetAt: date(item['resets_at']),
|
|
297
|
+
...(percent(item['utilization'], 100) !== undefined ? { status: 'allowed' } : {}),
|
|
298
|
+
});
|
|
285
299
|
}
|
|
286
300
|
// The optional endpoint is undocumented. Accept only the explicit window /
|
|
287
301
|
// model form below; arbitrary internal codenames never map to a family.
|
|
@@ -295,7 +309,9 @@ export function normalizeProbeUsage(value, observedAt) {
|
|
|
295
309
|
}
|
|
296
310
|
const model = item['model'];
|
|
297
311
|
const scope = model === 'global' ? 'global' : typeof model === 'string' ? family(model) ?? 'uncertain' : 'uncertain';
|
|
298
|
-
windows.push({ window, scope, percent: percent(item['utilization'], 100), resetAt: date(item['resets_at'])
|
|
312
|
+
windows.push({ window, scope, percent: percent(item['utilization'], 100), resetAt: date(item['resets_at']),
|
|
313
|
+
...(percent(item['utilization'], 100) !== undefined ? { status: 'allowed' } : {}),
|
|
314
|
+
});
|
|
299
315
|
}
|
|
300
316
|
if (info['limits'].length > 128)
|
|
301
317
|
windows.push({ window: 'seven_day', scope: 'uncertain' });
|
|
@@ -308,6 +324,63 @@ export function family(model) {
|
|
|
308
324
|
function applicable(scope, model) {
|
|
309
325
|
return scope === 'global' || scope === 'uncertain' || family(model) === undefined || scope === family(model);
|
|
310
326
|
}
|
|
327
|
+
export function rejectionResetAt(window, observedAt) {
|
|
328
|
+
return window.resetAt ?? observedAt + REJECTED_BLOCK_MS;
|
|
329
|
+
}
|
|
330
|
+
export function verificationEligibleAt(window, observedAt) {
|
|
331
|
+
return Math.min(rejectionResetAt(window, observedAt), observedAt +
|
|
332
|
+
Math.min(VERIFICATION_MAX_BACKOFF_MS, REJECTED_BLOCK_MS * 2 ** Math.min(5, Math.max(0, (window.rejectionCount ?? 1) - 1))));
|
|
333
|
+
}
|
|
334
|
+
function latestVerificationEvidence(observations, rejected, rejectedAt, now) {
|
|
335
|
+
let evidence = rejected.verificationEvidence;
|
|
336
|
+
if (evidence && (evidence.observedAt <= rejectedAt || evidence.observedAt > now))
|
|
337
|
+
evidence = undefined;
|
|
338
|
+
// Uncertain/projected identity cannot prove that the provider's exact quota recovered.
|
|
339
|
+
if (rejected.scope === 'uncertain' || rejected.kind === 'legacy-uncertain-seven-day')
|
|
340
|
+
return undefined;
|
|
341
|
+
for (const usage of observations) {
|
|
342
|
+
if (!usage || usage.observedAt > now)
|
|
343
|
+
continue;
|
|
344
|
+
for (const w of usage.windows) {
|
|
345
|
+
const observedAt = w.observedAt ?? usage.observedAt;
|
|
346
|
+
if (w.window !== rejected.window || w.scope !== rejected.scope || w.kind === 'legacy-uncertain-seven-day' ||
|
|
347
|
+
observedAt <= rejectedAt || observedAt > now || w.percent === undefined ||
|
|
348
|
+
(w.status !== 'allowed' && w.status !== 'allowed_warning'))
|
|
349
|
+
continue;
|
|
350
|
+
if (!evidence || observedAt > evidence.observedAt || observedAt === evidence.observedAt && w.percent > evidence.percent)
|
|
351
|
+
evidence = { observedAt, source: w.source ?? usage.source, status: w.status, percent: w.percent };
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
return evidence;
|
|
355
|
+
}
|
|
356
|
+
export function rejectionEvidence(usage, window, now) {
|
|
357
|
+
return latestVerificationEvidence([usage], window, window.observedAt ?? usage?.observedAt ?? now, now);
|
|
358
|
+
}
|
|
359
|
+
export function rejectionEligibleAt(usage, window, account, now) {
|
|
360
|
+
const observedAt = window.observedAt ?? usage?.observedAt ?? now;
|
|
361
|
+
const resetAt = rejectionResetAt(window, observedAt);
|
|
362
|
+
const evidence = rejectionEvidence(usage, window, now);
|
|
363
|
+
return evidence && evidence.percent < (account.useFullQuota ? 100 : USAGE_THRESHOLD_PERCENT) &&
|
|
364
|
+
!(window.lastVerificationAttemptAt !== undefined && window.lastVerificationAttemptAt >= observedAt)
|
|
365
|
+
? verificationEligibleAt(window, observedAt) : resetAt;
|
|
366
|
+
}
|
|
367
|
+
export function reactiveUsageWindow(reactive) {
|
|
368
|
+
return {
|
|
369
|
+
window: reactive.quota.window ?? 'seven_day',
|
|
370
|
+
scope: reactive.quota.modelScope ? family(reactive.quota.modelScope) ?? 'uncertain' : 'global',
|
|
371
|
+
status: 'rejected', observedAt: reactive.observedAt, resetAt: reactiveEligibleAt(reactive),
|
|
372
|
+
rejectionCount: reactive.rejectionCount, lastVerificationAttemptAt: reactive.lastVerificationAttemptAt,
|
|
373
|
+
verificationEvidence: reactive.verificationEvidence,
|
|
374
|
+
};
|
|
375
|
+
}
|
|
376
|
+
/** Due windows are claimed durably at the real spawn boundary, never by pure selection. */
|
|
377
|
+
export function verificationWindows(account, state, model, now) {
|
|
378
|
+
return [...(state.usage?.windows ?? []), ...(state.reactive ?? []).filter(r => r.quota.window !== undefined).map(reactiveUsageWindow)].filter(w => {
|
|
379
|
+
const observedAt = w.observedAt ?? state.usage?.observedAt ?? now;
|
|
380
|
+
return w.status === 'rejected' && observedAt <= now && applicable(w.scope, model) &&
|
|
381
|
+
now < rejectionResetAt(w, observedAt) && now >= rejectionEligibleAt(state.usage, w, account, now);
|
|
382
|
+
});
|
|
383
|
+
}
|
|
311
384
|
export function reactiveEligibleAt(reactive) {
|
|
312
385
|
const parsed = reactive.quota.resetAt ? Date.parse(reactive.quota.resetAt) : NaN;
|
|
313
386
|
return Number.isFinite(parsed) && parsed > reactive.observedAt ? parsed : reactive.observedAt + POOL_RECHECK_MS;
|
|
@@ -342,6 +415,7 @@ export function selectSubscriptionAccount(input) {
|
|
|
342
415
|
continue;
|
|
343
416
|
const state = input.states[account.accountId];
|
|
344
417
|
const blockers = [];
|
|
418
|
+
const usage = supersedeLegacyFableProjections([state?.usage], input.now)[0];
|
|
345
419
|
for (const reactive of state?.reactive ?? []) {
|
|
346
420
|
// Like durable pause createdAt, future observations expire on clock rollback.
|
|
347
421
|
if (input.now < reactive.observedAt)
|
|
@@ -349,12 +423,11 @@ export function selectSubscriptionAccount(input) {
|
|
|
349
423
|
const scope = reactive.quota.modelScope;
|
|
350
424
|
if (scope && family(scope) && family(input.model) && family(scope) !== family(input.model))
|
|
351
425
|
continue;
|
|
352
|
-
const until = reactiveEligibleAt(reactive);
|
|
426
|
+
const until = reactive.quota.window ? rejectionEligibleAt(usage, reactiveUsageWindow(reactive), account, input.now) : reactiveEligibleAt(reactive);
|
|
353
427
|
if (input.now < until)
|
|
354
428
|
blockers.push(until);
|
|
355
429
|
}
|
|
356
|
-
|
|
357
|
-
// A provider rejection outlives snapshot freshness. Never score it as headroom.
|
|
430
|
+
// A contradicted rejection permits one claimed verification after backoff.
|
|
358
431
|
if (usage && usage.observedAt <= input.now) {
|
|
359
432
|
for (const w of usage.windows) {
|
|
360
433
|
if (w.status !== 'rejected' || !applicable(w.scope, input.model))
|
|
@@ -362,7 +435,7 @@ export function selectSubscriptionAccount(input) {
|
|
|
362
435
|
const observedAt = w.observedAt ?? usage.observedAt;
|
|
363
436
|
if (observedAt > input.now)
|
|
364
437
|
continue;
|
|
365
|
-
const until = w
|
|
438
|
+
const until = rejectionEligibleAt(usage, w, account, input.now);
|
|
366
439
|
if (input.now < until)
|
|
367
440
|
blockers.push(until);
|
|
368
441
|
}
|
|
@@ -414,14 +487,18 @@ export function blockingUsageWindows(account, state, model, now) {
|
|
|
414
487
|
const usage = supersedeLegacyFableProjections([state?.usage], now)[0];
|
|
415
488
|
const windows = [];
|
|
416
489
|
for (const window of usage?.windows ?? []) {
|
|
417
|
-
const
|
|
490
|
+
const evidence = window.status === 'rejected' ? rejectionEvidence(usage, window, now) : undefined;
|
|
491
|
+
const result = selectSubscriptionAccount({ pool, states: { [account.accountId]: { usage: { ...usage, windows: [{ ...window, ...(evidence ? { verificationEvidence: evidence } : {}) }] } } }, model, now });
|
|
418
492
|
if (result.kind === 'deferred')
|
|
419
493
|
windows.push({ ...window, eligibleAt: result.eligibleAt });
|
|
420
494
|
}
|
|
421
495
|
for (const reactive of state?.reactive ?? []) {
|
|
422
|
-
const
|
|
423
|
-
if (
|
|
424
|
-
|
|
496
|
+
const window = reactiveUsageWindow(reactive);
|
|
497
|
+
if (applicable(window.scope, model) && reactive.observedAt <= now) {
|
|
498
|
+
const eligibleAt = reactive.quota.window ? rejectionEligibleAt(usage, window, account, now) : reactiveEligibleAt(reactive);
|
|
499
|
+
if (now < eligibleAt)
|
|
500
|
+
windows.push({ ...window, resetAt: reactive.quota.resetAt ? Date.parse(reactive.quota.resetAt) : undefined, eligibleAt });
|
|
501
|
+
}
|
|
425
502
|
}
|
|
426
503
|
return windows;
|
|
427
504
|
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type ClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
2
2
|
import type { ClaudeQuotaFailure } from './claude-quota.js';
|
|
3
|
-
import { type AccountIdentity, type AccountUsageState, type ReactiveExhaustion, type UsageObservation } from './subscription-selection.js';
|
|
3
|
+
import { type AccountIdentity, type AccountUsageState, type ReactiveExhaustion, type UsageObservation, type UsageWindow } from './subscription-selection.js';
|
|
4
4
|
export type UsageKey = Pick<AccountIdentity, 'workspaceId' | 'accountId' | 'credentialVersion'>;
|
|
5
5
|
export type ProbeStatus = 'attempted' | 'ok' | 'unsupported' | 'unauthorized' | 'unavailable' | 'cancelled';
|
|
6
6
|
export interface ProbeObservation {
|
|
@@ -34,6 +34,9 @@ export interface SubscriptionUsageStore {
|
|
|
34
34
|
claimLocalProbe(key: UsageKey, observedAt: number, minIntervalMs: number): Promise<boolean>;
|
|
35
35
|
claimProbeRequest(requestId: string, expiresAt: number): Promise<boolean>;
|
|
36
36
|
finishProbe(key: UsageKey, observation: ProbeObservation): Promise<void>;
|
|
37
|
+
/** Atomic revalidation and one durable claim immediately before a real spawn. */
|
|
38
|
+
beginVerification(key: AccountIdentity, model: string | undefined, observedAt: number): Promise<UsageWindow[] | undefined>;
|
|
39
|
+
completeVerification(key: UsageKey, attemptAt: number, observedAt: number): Promise<void>;
|
|
37
40
|
}
|
|
38
41
|
export declare function usageStateForSelection(state: StoredAccountUsage, now: number): AccountUsageState;
|
|
39
42
|
export declare class SubscriptionUsageStoreError extends Error {
|
|
@@ -54,4 +57,6 @@ export declare class FileSubscriptionUsageStore implements SubscriptionUsageStor
|
|
|
54
57
|
/** Request claims live in the local data directory, alongside usage, never in a brain. */
|
|
55
58
|
claimProbeRequest(requestId: string, expiresAt: number): Promise<boolean>;
|
|
56
59
|
finishProbe(key: UsageKey, observation: ProbeObservation): Promise<void>;
|
|
60
|
+
beginVerification(key: AccountIdentity, model: string | undefined, observedAt: number): Promise<UsageWindow[] | undefined>;
|
|
61
|
+
completeVerification(key: UsageKey, attemptAt: number, observedAt: number): Promise<void>;
|
|
57
62
|
}
|
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
import { redactStartupText } from './redact-text.js';
|
|
2
|
+
import { logger } from './logger.js';
|
|
2
3
|
import { createHash, randomUUID } from 'node:crypto';
|
|
3
|
-
import { mkdir, readFile, rename, rm, writeFile } from 'node:fs/promises';
|
|
4
|
+
import { mkdir, open, readFile, readdir, rename, rm, stat, writeFile } from 'node:fs/promises';
|
|
4
5
|
import { join, resolve } from 'node:path';
|
|
5
6
|
import { parseClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
6
|
-
import { normalizePassiveUsage, mergePersistedUsageObservations, mergeUsageObservations, record, reactiveEligibleAt, family } from './subscription-selection.js';
|
|
7
|
+
import { normalizePassiveUsage, mergePersistedUsageObservations, mergeUsageObservations, record, reactiveEligibleAt, reactiveUsageWindow, rejectionEvidence, rejectionResetAt, verificationEligibleAt, verificationWindows, selectSubscriptionAccount, family, USAGE_THRESHOLD_PERCENT } from './subscription-selection.js';
|
|
7
8
|
export function usageStateForSelection(state, now) {
|
|
8
9
|
return {
|
|
9
10
|
usage: mergeUsageObservations([
|
|
@@ -80,12 +81,31 @@ function safeUsage(value, source) {
|
|
|
80
81
|
if (observedAt !== undefined)
|
|
81
82
|
validTime(observedAt);
|
|
82
83
|
const source = w['source'] === 'passive' || w['source'] === 'probe' ? w['source'] : undefined;
|
|
83
|
-
return { window, scope, ...((w['kind'] === 'seven_day_overage_included' || w['kind'] === 'legacy-uncertain-seven-day') ? { kind: w['kind'] } : {}), percent, resetAt, ...(status ? { status } : {}), ...(observedAt ? { observedAt } : {}), ...(source ? { source } : {}) };
|
|
84
|
+
return { window, scope, ...((w['kind'] === 'seven_day_overage_included' || w['kind'] === 'legacy-uncertain-seven-day') ? { kind: w['kind'] } : {}), percent, resetAt, ...(status ? { status } : {}), ...(observedAt ? { observedAt } : {}), ...(source ? { source } : {}), ...safeVerificationFields(w) };
|
|
84
85
|
});
|
|
85
86
|
if (usage['windows'].length > 133)
|
|
86
87
|
windows.push({ window: 'seven_day', scope: 'uncertain' });
|
|
87
88
|
return { source, observedAt: usage['observedAt'], windows };
|
|
88
89
|
}
|
|
90
|
+
function safeVerificationFields(value) {
|
|
91
|
+
const result = {};
|
|
92
|
+
const count = value['rejectionCount'];
|
|
93
|
+
if (typeof count === 'number' && Number.isSafeInteger(count) && count >= 1)
|
|
94
|
+
result.rejectionCount = count;
|
|
95
|
+
const attemptAt = value['lastVerificationAttemptAt'];
|
|
96
|
+
if (typeof attemptAt === 'number') {
|
|
97
|
+
validTime(attemptAt);
|
|
98
|
+
result.lastVerificationAttemptAt = attemptAt;
|
|
99
|
+
}
|
|
100
|
+
const evidence = record(value['verificationEvidence']);
|
|
101
|
+
if (typeof evidence['observedAt'] === 'number' && (evidence['source'] === 'probe' || evidence['source'] === 'passive') &&
|
|
102
|
+
(evidence['status'] === 'allowed' || evidence['status'] === 'allowed_warning') &&
|
|
103
|
+
typeof evidence['percent'] === 'number' && Number.isFinite(evidence['percent']) && evidence['percent'] >= 0 && evidence['percent'] <= 100) {
|
|
104
|
+
validTime(evidence['observedAt']);
|
|
105
|
+
result.verificationEvidence = { observedAt: evidence['observedAt'], source: evidence['source'], status: evidence['status'], percent: evidence['percent'] };
|
|
106
|
+
}
|
|
107
|
+
return result;
|
|
108
|
+
}
|
|
89
109
|
function safeProbe(value) {
|
|
90
110
|
const obj = record(value);
|
|
91
111
|
const status = obj['status'];
|
|
@@ -126,7 +146,7 @@ function safeReactive(value) {
|
|
|
126
146
|
if (typeof obj['observedAt'] !== 'number' || !obj['quota'] || typeof obj['quota'] !== 'object')
|
|
127
147
|
throw new SubscriptionUsageStoreError();
|
|
128
148
|
validTime(obj['observedAt']);
|
|
129
|
-
return { observedAt: obj['observedAt'], quota: safeQuota(obj['quota']) };
|
|
149
|
+
return { observedAt: obj['observedAt'], quota: safeQuota(obj['quota']), ...safeVerificationFields(obj) };
|
|
130
150
|
});
|
|
131
151
|
}
|
|
132
152
|
/** One file per version + one reactive file per workspace/account. No credentials or raw provider bodies. */
|
|
@@ -139,6 +159,84 @@ export class FileSubscriptionUsageStore {
|
|
|
139
159
|
throw new SubscriptionUsageStoreError();
|
|
140
160
|
return join(this.#directory, `${createHash('sha256').update(JSON.stringify(parts)).digest('hex')}.json`);
|
|
141
161
|
}
|
|
162
|
+
#claimPath(key, window, fallbackAt) {
|
|
163
|
+
// The epoch identifies the backoff step, so processes with different wall
|
|
164
|
+
// clocks still compete for the same exclusive marker.
|
|
165
|
+
const epoch = verificationEligibleAt(window, window.observedAt ?? fallbackAt);
|
|
166
|
+
return join(this.#directory, 'claims', `${encodeURIComponent(key.accountId)}.${window.scope}.${window.window}.${epoch}.claim`);
|
|
167
|
+
}
|
|
168
|
+
async #claimMarkers(key) {
|
|
169
|
+
const directory = join(this.#directory, 'claims');
|
|
170
|
+
const claims = new Map();
|
|
171
|
+
try {
|
|
172
|
+
const names = await readdir(directory);
|
|
173
|
+
const now = Date.now();
|
|
174
|
+
for (const name of names) {
|
|
175
|
+
if (!name.startsWith(`${encodeURIComponent(key.accountId)}.`) || !name.endsWith('.claim'))
|
|
176
|
+
continue;
|
|
177
|
+
const path = join(directory, name);
|
|
178
|
+
try {
|
|
179
|
+
const raw = await readFile(path, 'utf8');
|
|
180
|
+
let marker;
|
|
181
|
+
try {
|
|
182
|
+
marker = JSON.parse(raw);
|
|
183
|
+
}
|
|
184
|
+
catch {
|
|
185
|
+
// Exclusive creation already owns the step, including the interval
|
|
186
|
+
// before metadata is written or a crash during that write.
|
|
187
|
+
if (now - (await stat(path)).mtimeMs > 24 * 3600_000) {
|
|
188
|
+
await rm(path, { force: true });
|
|
189
|
+
continue;
|
|
190
|
+
}
|
|
191
|
+
const epoch = Number(name.split('.').at(-2));
|
|
192
|
+
validTime(epoch);
|
|
193
|
+
claims.set(path, epoch);
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
validTime(marker.attemptAt);
|
|
197
|
+
validTime(marker.resetAt);
|
|
198
|
+
if (now - marker.attemptAt > 24 * 3600_000 || marker.resetAt <= now) {
|
|
199
|
+
await rm(path, { force: true });
|
|
200
|
+
continue;
|
|
201
|
+
}
|
|
202
|
+
claims.set(path, marker.attemptAt);
|
|
203
|
+
}
|
|
204
|
+
catch (error) {
|
|
205
|
+
if (error.code !== 'ENOENT')
|
|
206
|
+
throw error;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
return claims;
|
|
210
|
+
}
|
|
211
|
+
catch (error) {
|
|
212
|
+
if (error.code === 'ENOENT')
|
|
213
|
+
return claims;
|
|
214
|
+
throw new SubscriptionUsageStoreError();
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
async #claimVerification(key, windows, observedAt) {
|
|
218
|
+
try {
|
|
219
|
+
await mkdir(join(this.#directory, 'claims'), { recursive: true, mode: 0o700 });
|
|
220
|
+
const markers = new Map(windows.map(w => [this.#claimPath(key, w, observedAt), w]));
|
|
221
|
+
for (const [path, window] of [...markers].sort(([a], [b]) => a.localeCompare(b))) {
|
|
222
|
+
const file = await open(path, 'wx', 0o600);
|
|
223
|
+
try {
|
|
224
|
+
await file.writeFile(JSON.stringify({ attemptAt: observedAt, resetAt: rejectionResetAt(window, window.observedAt ?? observedAt) }));
|
|
225
|
+
}
|
|
226
|
+
finally {
|
|
227
|
+
await file.close();
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
return true;
|
|
231
|
+
}
|
|
232
|
+
catch (error) {
|
|
233
|
+
if (error.code === 'EEXIST')
|
|
234
|
+
return false;
|
|
235
|
+
// A marker remains spent even if the JSON write fails. Readers consult
|
|
236
|
+
// markers as well, so restarting cannot launch the same verification.
|
|
237
|
+
throw new SubscriptionUsageStoreError();
|
|
238
|
+
}
|
|
239
|
+
}
|
|
142
240
|
async #load(path, fallback) {
|
|
143
241
|
try {
|
|
144
242
|
const raw = await readFile(path, 'utf8');
|
|
@@ -169,6 +267,49 @@ export class FileSubscriptionUsageStore {
|
|
|
169
267
|
await rm(temporary, { force: true }).catch(() => { });
|
|
170
268
|
}
|
|
171
269
|
}
|
|
270
|
+
#log(message) { logger.info(`[Pool] ${message}`); }
|
|
271
|
+
#logEvidence(previous, window, usage, now, threshold) {
|
|
272
|
+
const evidence = rejectionEvidence(usage, window, now);
|
|
273
|
+
if (evidence && evidence.percent < threshold && evidence.observedAt !== previous?.verificationEvidence?.observedAt &&
|
|
274
|
+
!(window.lastVerificationAttemptAt !== undefined && window.lastVerificationAttemptAt >= (window.observedAt ?? now))) {
|
|
275
|
+
this.#log(`rejection marked verifiable window=${window.window}/${window.scope} count=${window.rejectionCount ?? 1} evidence=${evidence.source}:${evidence.status}:${evidence.percent}% observedAt=${new Date(evidence.observedAt).toISOString()} eligibleAt=${new Date(verificationEligibleAt(window, window.observedAt ?? now)).toISOString()}`);
|
|
276
|
+
}
|
|
277
|
+
}
|
|
278
|
+
async #mergeUsage(key, state, previous, incoming, now) {
|
|
279
|
+
const path = this.#path(key, false);
|
|
280
|
+
const account = await this.#load(path, { reactive: [] });
|
|
281
|
+
account.reactive = safeReactive(account.reactive);
|
|
282
|
+
if (incoming)
|
|
283
|
+
incoming = { ...incoming, windows: incoming.windows.map(w => {
|
|
284
|
+
if (w.status !== 'rejected')
|
|
285
|
+
return w;
|
|
286
|
+
const at = w.observedAt ?? incoming.observedAt;
|
|
287
|
+
const prior = account.reactive.map(reactiveUsageWindow).find(r => r.window === w.window && r.scope === w.scope && r.observedAt <= at && r.resetAt > at);
|
|
288
|
+
return prior ? { ...w, rejectionCount: Math.max(w.rejectionCount ?? 1, (prior.rejectionCount ?? 1) + Number(prior.observedAt < at)) } : w;
|
|
289
|
+
}) };
|
|
290
|
+
state.usage = mergePersistedUsageObservations(previous, incoming, now);
|
|
291
|
+
const threshold = key.useFullQuota ? 100 : USAGE_THRESHOLD_PERCENT;
|
|
292
|
+
if (state.usage)
|
|
293
|
+
state.usage = { ...state.usage, windows: state.usage.windows.map(w => w.status === 'rejected' ? { ...w, rejectionCount: w.rejectionCount ?? 1 } : w) };
|
|
294
|
+
for (const window of state.usage?.windows ?? []) {
|
|
295
|
+
if (window.status !== 'rejected')
|
|
296
|
+
continue;
|
|
297
|
+
const old = previous?.windows.find(w => w.status === 'rejected' && w.window === window.window && w.scope === window.scope);
|
|
298
|
+
this.#logEvidence(old, window, state.usage, now, threshold);
|
|
299
|
+
if (old?.lastVerificationAttemptAt !== undefined && window.observedAt > old.observedAt)
|
|
300
|
+
this.#log(`verification re-rejected window=${window.window}/${window.scope} count=${window.rejectionCount ?? 1} eligibleAt=${new Date(verificationEligibleAt(window, window.observedAt)).toISOString()}`);
|
|
301
|
+
}
|
|
302
|
+
const reactive = safeReactive(account.reactive).map(r => {
|
|
303
|
+
if (!r.quota.window)
|
|
304
|
+
return r;
|
|
305
|
+
const window = reactiveUsageWindow(r);
|
|
306
|
+
const verificationEvidence = rejectionEvidence(state.usage, window, now);
|
|
307
|
+
this.#logEvidence(window, { ...window, verificationEvidence }, state.usage, now, threshold);
|
|
308
|
+
return { ...r, verificationEvidence };
|
|
309
|
+
});
|
|
310
|
+
if (JSON.stringify(reactive) !== JSON.stringify(account.reactive))
|
|
311
|
+
await this.#save(path, { reactive });
|
|
312
|
+
}
|
|
172
313
|
async read(key) {
|
|
173
314
|
const [version, account] = await Promise.all([
|
|
174
315
|
this.#load(this.#path(key, true), {}),
|
|
@@ -177,19 +318,32 @@ export class FileSubscriptionUsageStore {
|
|
|
177
318
|
// Corrupt reactive data must never silently resurrect an exhausted account.
|
|
178
319
|
if (!Array.isArray(account.reactive) || account.reactive.some(r => !r || !Number.isFinite(r.observedAt) || !r.quota))
|
|
179
320
|
throw new SubscriptionUsageStoreError();
|
|
180
|
-
|
|
321
|
+
const state = { ...safeVersion(version), reactive: safeReactive(account.reactive) };
|
|
322
|
+
const claims = await this.#claimMarkers(key);
|
|
323
|
+
const claimedWindow = (w, fallbackAt) => {
|
|
324
|
+
const attemptAt = claims.get(this.#claimPath(key, w, fallbackAt));
|
|
325
|
+
return w.status === 'rejected' && attemptAt !== undefined ? { ...w, lastVerificationAttemptAt: Math.max(w.lastVerificationAttemptAt ?? 0, attemptAt) } : w;
|
|
326
|
+
};
|
|
327
|
+
if (state.usage)
|
|
328
|
+
state.usage = { ...state.usage, windows: state.usage.windows.map(w => claimedWindow(w, state.usage.observedAt)) };
|
|
329
|
+
state.reactive = state.reactive.map(r => ({ ...r, lastVerificationAttemptAt: claimedWindow(reactiveUsageWindow(r), r.observedAt).lastVerificationAttemptAt }));
|
|
330
|
+
return state;
|
|
181
331
|
}
|
|
182
332
|
async recordPassive(key, rateLimitInfo, observedAt, options) {
|
|
183
333
|
validTime(observedAt);
|
|
184
334
|
const path = this.#path(key, true);
|
|
185
|
-
await serial(path, async () => {
|
|
335
|
+
await serial(this.#path(key, false), async () => {
|
|
186
336
|
const state = safeVersion(await this.#load(path, {}));
|
|
187
337
|
if (state.passive && state.passive.observedAt > observedAt && !options?.expireFutureObservations)
|
|
188
338
|
return;
|
|
189
339
|
// Retain Fable observation clocks even when the latest report omits its meter.
|
|
190
340
|
const previous = state.usage ?? usageStateForSelection({ ...state, reactive: [] }, observedAt).usage;
|
|
191
341
|
state.passive = { rateLimitInfo: safeRateLimitInfo(rateLimitInfo), observedAt };
|
|
192
|
-
|
|
342
|
+
const incoming = normalizePassiveUsage(state.passive.rateLimitInfo, observedAt);
|
|
343
|
+
const status = state.passive.rateLimitInfo.status;
|
|
344
|
+
if (status === 'allowed' || status === 'allowed_warning')
|
|
345
|
+
incoming.windows = incoming.windows.map(w => ({ ...w, status: w.status ?? status }));
|
|
346
|
+
await this.#mergeUsage(key, state, previous, incoming, observedAt);
|
|
193
347
|
await this.#save(path, state);
|
|
194
348
|
});
|
|
195
349
|
}
|
|
@@ -199,24 +353,41 @@ export class FileSubscriptionUsageStore {
|
|
|
199
353
|
await serial(path, async () => {
|
|
200
354
|
const state = await this.#load(path, { reactive: [] });
|
|
201
355
|
state.reactive = safeReactive(state.reactive).filter(r => !options?.expireFutureObservations || r.observedAt <= observedAt);
|
|
202
|
-
const next = { observedAt, quota: safeQuota(quota) };
|
|
356
|
+
const next = { observedAt, quota: safeQuota(quota), rejectionCount: 1 };
|
|
203
357
|
const until = reactiveEligibleAt;
|
|
204
358
|
const sameScope = (r) => r.quota.modelScope === next.quota.modelScope && r.quota.window === next.quota.window;
|
|
205
359
|
const previous = state.reactive.find(sameScope);
|
|
360
|
+
const version = safeVersion(await this.#load(this.#path(key, true), {}));
|
|
361
|
+
const window = reactiveUsageWindow(next);
|
|
362
|
+
const passive = version.usage?.windows.find(w => w.status === 'rejected' && w.window === window.window && w.scope === window.scope &&
|
|
363
|
+
(w.observedAt ?? version.usage.observedAt) <= observedAt && rejectionResetAt(w, w.observedAt ?? version.usage.observedAt) > observedAt);
|
|
206
364
|
const reactive = state.reactive.filter(r => !sameScope(r) && until(r) > observedAt);
|
|
207
365
|
const hasExplicitReset = (r) => r.quota.resetAt !== undefined && Date.parse(r.quota.resetAt) > r.observedAt;
|
|
208
366
|
// A recheck is not a reset. New matching quota evidence can replace that
|
|
209
367
|
// provisional deadline with its explicit reset, without clearing a known
|
|
210
368
|
// active reset or accepting an older observation as revalidation.
|
|
211
369
|
const revalidated = previous && next.observedAt >= previous.observedAt && !hasExplicitReset(previous) && hasExplicitReset(next);
|
|
212
|
-
|
|
370
|
+
if (previous && next.observedAt < previous.observedAt)
|
|
371
|
+
reactive.push(previous);
|
|
372
|
+
else if (previous && next.observedAt === previous.observedAt && !revalidated)
|
|
373
|
+
reactive.push(previous);
|
|
374
|
+
else {
|
|
375
|
+
next.rejectionCount = Math.max(previous && until(previous) > observedAt ? (previous.rejectionCount ?? 1) + Number(previous.observedAt < observedAt) : 1, passive ? (passive.rejectionCount ?? 1) + Number((passive.observedAt ?? version.usage.observedAt) < observedAt) : 1);
|
|
376
|
+
if (previous && !next.quota.resetAt && until(previous) > until(next))
|
|
377
|
+
next.quota.resetAt = previous.quota.resetAt;
|
|
378
|
+
if (!next.quota.resetAt && passive?.resetAt)
|
|
379
|
+
next.quota.resetAt = new Date(passive.resetAt).toISOString();
|
|
380
|
+
reactive.push(next);
|
|
381
|
+
if (previous?.lastVerificationAttemptAt !== undefined || passive?.lastVerificationAttemptAt !== undefined)
|
|
382
|
+
this.#log(`verification re-rejected window=${next.quota.window ?? 'seven_day'}/${next.quota.modelScope ?? 'global'} count=${next.rejectionCount} eligibleAt=${new Date(verificationEligibleAt(reactiveUsageWindow(next), observedAt)).toISOString()}`);
|
|
383
|
+
}
|
|
213
384
|
await this.#save(path, { reactive });
|
|
214
385
|
});
|
|
215
386
|
}
|
|
216
387
|
async beginProbe(key, observedAt) {
|
|
217
388
|
validTime(observedAt);
|
|
218
389
|
const path = this.#path(key, true);
|
|
219
|
-
return serial(path, async () => {
|
|
390
|
+
return serial(this.#path(key, false), async () => {
|
|
220
391
|
const state = safeVersion(await this.#load(path, {}));
|
|
221
392
|
if (state.probe)
|
|
222
393
|
return false;
|
|
@@ -229,7 +400,7 @@ export class FileSubscriptionUsageStore {
|
|
|
229
400
|
async claimLocalProbe(key, observedAt, minIntervalMs) {
|
|
230
401
|
validTime(observedAt);
|
|
231
402
|
const path = this.#path(key, true);
|
|
232
|
-
return serial(path, async () => {
|
|
403
|
+
return serial(this.#path(key, false), async () => {
|
|
233
404
|
const state = safeVersion(await this.#load(path, {}));
|
|
234
405
|
if (state.localProbeClaimedAt !== undefined && observedAt >= state.localProbeClaimedAt &&
|
|
235
406
|
observedAt - state.localProbeClaimedAt < minIntervalMs)
|
|
@@ -259,15 +430,77 @@ export class FileSubscriptionUsageStore {
|
|
|
259
430
|
async finishProbe(key, observation) {
|
|
260
431
|
validTime(observation.observedAt);
|
|
261
432
|
const path = this.#path(key, true);
|
|
262
|
-
await serial(path, async () => {
|
|
433
|
+
await serial(this.#path(key, false), async () => {
|
|
263
434
|
const state = safeVersion(await this.#load(path, {}));
|
|
264
435
|
// Network completion order must not discard a newer stored snapshot.
|
|
265
436
|
const mergeAt = Math.max(observation.observedAt, state.usage?.observedAt ?? 0, state.passive?.observedAt ?? 0);
|
|
266
437
|
// Retain Fable observation clocks even when the latest report omits its meter.
|
|
267
438
|
const previous = state.usage ?? usageStateForSelection({ ...state, reactive: [] }, mergeAt).usage;
|
|
268
439
|
state.probe = safeProbe(observation);
|
|
269
|
-
state
|
|
440
|
+
await this.#mergeUsage(key, state, previous, state.probe.usage, mergeAt);
|
|
270
441
|
await this.#save(path, state);
|
|
271
442
|
});
|
|
272
443
|
}
|
|
444
|
+
async beginVerification(key, model, observedAt) {
|
|
445
|
+
validTime(observedAt);
|
|
446
|
+
return serial(this.#path(key, false), async () => {
|
|
447
|
+
const stored = await this.read(key);
|
|
448
|
+
const state = usageStateForSelection(stored, observedAt);
|
|
449
|
+
const selected = selectSubscriptionAccount({ pool: { kind: 'ready', accounts: [key] }, states: { [key.accountId]: state }, model, now: observedAt });
|
|
450
|
+
if (selected.kind !== 'selected')
|
|
451
|
+
return undefined;
|
|
452
|
+
const windows = verificationWindows(key, state, model, observedAt);
|
|
453
|
+
if (!windows.length)
|
|
454
|
+
return [];
|
|
455
|
+
if (!await this.#claimVerification(key, windows, observedAt))
|
|
456
|
+
return undefined;
|
|
457
|
+
const matches = (w) => windows.some(v => v.window === w.window && v.scope === w.scope && v.observedAt === (w.observedAt ?? stored.usage?.observedAt ?? state.usage?.observedAt));
|
|
458
|
+
const version = safeVersion(await this.#load(this.#path(key, true), {}));
|
|
459
|
+
version.usage ??= state.usage;
|
|
460
|
+
if (version.usage)
|
|
461
|
+
version.usage = { ...version.usage, windows: version.usage.windows.map(w => matches(w) ? { ...w, lastVerificationAttemptAt: observedAt } : w) };
|
|
462
|
+
const reactive = stored.reactive.map(r => matches(reactiveUsageWindow(r)) ? { ...r, lastVerificationAttemptAt: observedAt } : r);
|
|
463
|
+
await this.#save(this.#path(key, true), version);
|
|
464
|
+
await this.#save(this.#path(key, false), { reactive });
|
|
465
|
+
for (const w of windows)
|
|
466
|
+
this.#log(`verification attempt started window=${w.window}/${w.scope} count=${w.rejectionCount ?? 1} attemptAt=${new Date(observedAt).toISOString()}`);
|
|
467
|
+
return windows;
|
|
468
|
+
});
|
|
469
|
+
}
|
|
470
|
+
async completeVerification(key, attemptAt, observedAt) {
|
|
471
|
+
validTime(attemptAt);
|
|
472
|
+
validTime(observedAt);
|
|
473
|
+
await serial(this.#path(key, false), async () => {
|
|
474
|
+
const stored = await this.read(key);
|
|
475
|
+
const cleared = [];
|
|
476
|
+
const canClear = (w) => w.status === 'rejected' && w.lastVerificationAttemptAt === attemptAt &&
|
|
477
|
+
(w.observedAt ?? stored.usage?.observedAt ?? observedAt) <= attemptAt && attemptAt <= observedAt;
|
|
478
|
+
const version = safeVersion(await this.#load(this.#path(key, true), {}));
|
|
479
|
+
const windows = version.usage?.windows.filter(w => { if (!canClear(w))
|
|
480
|
+
return true; cleared.push({ ...w, observedAt: w.observedAt ?? version.usage.observedAt }); return false; });
|
|
481
|
+
if (version.usage && windows)
|
|
482
|
+
version.usage = { ...version.usage, windows };
|
|
483
|
+
const reactive = stored.reactive.filter(r => { const w = reactiveUsageWindow(r); if (!canClear(w))
|
|
484
|
+
return true; cleared.push(w); return false; });
|
|
485
|
+
// Raw snapshots are also selection inputs; remove the cleared rejection there.
|
|
486
|
+
const matches = (w, fallback) => cleared.some(c => c.window === w.window && c.scope === w.scope && (w.observedAt ?? fallback) === c.observedAt);
|
|
487
|
+
if (version.passive && normalizePassiveUsage(version.passive.rateLimitInfo, version.passive.observedAt).windows.some(w => w.status === 'rejected' && matches(w, version.passive.observedAt))) {
|
|
488
|
+
// The rejected percentage is not a successful measurement. Keeping it
|
|
489
|
+
// as status-less usage could immediately re-block the cleared account.
|
|
490
|
+
delete version.passive;
|
|
491
|
+
}
|
|
492
|
+
if (version.probe?.usage)
|
|
493
|
+
version.probe.usage = { ...version.probe.usage, windows: version.probe.usage.windows.filter(w => w.status !== 'rejected' || !matches(w, version.probe.usage.observedAt)) };
|
|
494
|
+
await this.#save(this.#path(key, true), version);
|
|
495
|
+
await this.#save(this.#path(key, false), { reactive });
|
|
496
|
+
const markers = await this.#claimMarkers(key);
|
|
497
|
+
await Promise.all([...markers.keys()].filter(path => cleared.some(w => {
|
|
498
|
+
const prefix = join(this.#directory, 'claims', `${encodeURIComponent(key.accountId)}.${w.scope}.${w.window}.`);
|
|
499
|
+
const epoch = Number(path.slice(prefix.length, -'.claim'.length));
|
|
500
|
+
return path.startsWith(prefix) && epoch <= verificationEligibleAt(w, w.observedAt ?? observedAt);
|
|
501
|
+
})).map(path => rm(path, { force: true })));
|
|
502
|
+
for (const w of cleared)
|
|
503
|
+
this.#log(`verification cleared rejection window=${w.window}/${w.scope} count=${w.rejectionCount ?? 1}`);
|
|
504
|
+
});
|
|
505
|
+
}
|
|
273
506
|
}
|
package/dist/types.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type UsageWindow } from './subscription-selection.js';
|
|
1
2
|
/** Owner-facing account policy location. */
|
|
2
3
|
export declare const FULL_QUOTA_SETTINGS_PATH = "Connections (account menu \u2192 Use until exhausted)";
|
|
3
4
|
export interface UsageResetTiming {
|
|
@@ -18,6 +19,8 @@ export interface PoolUsageLimitNotice extends UsageResetTiming {
|
|
|
18
19
|
cause: PoolDeferCause;
|
|
19
20
|
thresholdDeferred?: boolean;
|
|
20
21
|
accounts?: readonly UsageLimitAccount[];
|
|
22
|
+
eligibleAt?: number;
|
|
23
|
+
blockedWindows?: readonly Pick<UsageWindow, 'window' | 'scope'>[];
|
|
21
24
|
}
|
|
22
25
|
export declare function formatUsageReset(input: UsageResetTiming): string | undefined;
|
|
23
26
|
export declare function formatPoolUsageLimitMessage(input: PoolUsageLimitNotice): string;
|
|
@@ -1,6 +1,17 @@
|
|
|
1
1
|
import { USAGE_THRESHOLD_PERCENT } from './subscription-selection.js';
|
|
2
2
|
/** Owner-facing account policy location. */
|
|
3
3
|
export const FULL_QUOTA_SETTINGS_PATH = 'Connections (account menu → Use until exhausted)';
|
|
4
|
+
/** Quota notices include a calendar date and zone across multi-day holds. */
|
|
5
|
+
function formatQuotaTime(at, timeZone) {
|
|
6
|
+
try {
|
|
7
|
+
const parts = new Intl.DateTimeFormat('en-GB', { timeZone: timeZone || 'UTC', weekday: 'short', day: '2-digit', month: 'short', hour: '2-digit', minute: '2-digit', hourCycle: 'h23', timeZoneName: 'short' }).formatToParts(at);
|
|
8
|
+
const part = (type) => parts.find(p => p.type === type)?.value;
|
|
9
|
+
return `${part('weekday')} ${part('day')} ${part('month')} ${part('hour')}:${part('minute')} ${part('timeZoneName')}`;
|
|
10
|
+
}
|
|
11
|
+
catch {
|
|
12
|
+
return formatQuotaTime(at, 'UTC');
|
|
13
|
+
}
|
|
14
|
+
}
|
|
4
15
|
export function formatUsageReset(input) {
|
|
5
16
|
if (input.resetAt === undefined || !Number.isFinite(input.resetAt))
|
|
6
17
|
return undefined;
|
|
@@ -43,6 +54,17 @@ export function formatPoolUsageLimitMessage(input) {
|
|
|
43
54
|
const labels = headroom.filter(a => a.label && a.remaining > 0).map(a => `${a.label} ${Number(a.remaining.toFixed(2))} %`).join(', ');
|
|
44
55
|
return `⏳ All Claude accounts in this workspace are above ${USAGE_THRESHOLD_PERCENT} % of their usage limit, so I'm waiting for the next reset${reset ? ` at ${reset}` : ''}. Your message is saved and will run automatically.\n💡 There is still some capacity left${labels ? ` (${labels})` : ''}. A workspace owner can allow the full quota per account in ${FULL_QUOTA_SETTINGS_PATH}.`;
|
|
45
56
|
}
|
|
57
|
+
if (input.blockedWindows?.length) {
|
|
58
|
+
const windows = [...new Map(input.blockedWindows.map(w => [`${w.window}:${w.scope}`, w])).values()];
|
|
59
|
+
const names = windows.map(w => `${w.scope === 'global' ? 'global' : w.scope === 'uncertain' ? 'Claude' : w.scope[0].toUpperCase() + w.scope.slice(1)} ${w.window === 'seven_day' ? 'weekly' : 'five-hour'} quota`);
|
|
60
|
+
const global = windows.some(w => w.scope === 'global' || w.scope === 'uncertain');
|
|
61
|
+
const resets = input.resetAt !== undefined ? `resets ${formatQuotaTime(input.resetAt, input.timeZone)}` : 'reset time unavailable';
|
|
62
|
+
const retry = input.eligibleAt !== undefined ? `I'll retry automatically at ${formatQuotaTime(input.eligibleAt, input.timeZone)}` : "I'll retry automatically after the next reset";
|
|
63
|
+
const blocked = windows.length === 1 && !global
|
|
64
|
+
? `${names[0]} is exhausted on every account in this workspace (${resets}).`
|
|
65
|
+
: `All Claude accounts in this workspace have hit their usage limit (${names.join(', ')}; ${resets}).`;
|
|
66
|
+
return `⏳ ${blocked} Your message is saved. ${retry}${global ? '.' : '; other models keep working.'}`;
|
|
67
|
+
}
|
|
46
68
|
return `⏳ All Claude accounts in this workspace have hit their usage limit. Your message is saved and will run automatically after ${reset ? `the earliest reset at ${reset}` : 'the next reset'}.`;
|
|
47
69
|
}
|
|
48
70
|
export function formatLegacyUsageLimitMessage(input) {
|
package/dist/usage-probe-job.js
CHANGED
|
@@ -2,7 +2,7 @@ import { spawn } from 'node:child_process';
|
|
|
2
2
|
import { mkdtemp, rm, mkdir } from 'node:fs/promises';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
-
import { selectSubscriptionAccount, normalizePassiveUsage } from './subscription-selection.js';
|
|
5
|
+
import { selectSubscriptionAccount, normalizePassiveUsage, reactiveUsageWindow } from './subscription-selection.js';
|
|
6
6
|
import { safeRateLimitInfo, usageStateForSelection, } from './subscription-usage-store.js';
|
|
7
7
|
import { parseClaudeRateLimitInfo, } from './claude-rate-limit.js';
|
|
8
8
|
import { formatPoolUsageWindows, poolInfo, poolLabel } from './pool-logging.js';
|
|
@@ -210,18 +210,46 @@ export async function runUsageProbe(account, options, context = {}) {
|
|
|
210
210
|
return result;
|
|
211
211
|
const now = Date.now();
|
|
212
212
|
const safe = safeRateLimitInfo(info);
|
|
213
|
+
const normalized = normalizePassiveUsage(safe, now);
|
|
213
214
|
const usage = {
|
|
214
|
-
...
|
|
215
|
+
...normalized,
|
|
215
216
|
source: 'probe',
|
|
217
|
+
windows: normalized.windows.map(w => ({ ...w,
|
|
218
|
+
...(w.status === undefined && w.percent !== undefined && (safe.status === 'allowed' || safe.status === 'allowed_warning') ? { status: safe.status } : {}),
|
|
219
|
+
})),
|
|
216
220
|
};
|
|
217
221
|
measuredWindows = formatPoolUsageWindows(usage.windows);
|
|
222
|
+
const before = usageStateForSelection(await options.store.read(account), now);
|
|
218
223
|
await options.store.finishProbe(account, {
|
|
219
224
|
status: 'ok',
|
|
220
225
|
observedAt: now,
|
|
221
226
|
usage,
|
|
222
227
|
});
|
|
223
228
|
const stored = await options.store.read(account);
|
|
224
|
-
|
|
229
|
+
const after = usageStateForSelection(stored, now);
|
|
230
|
+
// Keep the combined comparison, and compare each rejected scope/window
|
|
231
|
+
// independently so an unrelated, later hold cannot hide an improvement.
|
|
232
|
+
const pairs = new Map();
|
|
233
|
+
for (const state of [before, after])
|
|
234
|
+
for (const w of [...(state.usage?.windows ?? []), ...(state.reactive ?? []).map(reactiveUsageWindow)]) {
|
|
235
|
+
if (w.status === 'rejected')
|
|
236
|
+
pairs.set(`${w.scope}.${w.window}`, w);
|
|
237
|
+
}
|
|
238
|
+
const eligibility = (state, pair) => {
|
|
239
|
+
const matches = (w) => !pair || w.scope === pair.scope && w.window === pair.window;
|
|
240
|
+
const scoped = pair ? {
|
|
241
|
+
usage: state.usage ? { ...state.usage, windows: state.usage.windows.filter(matches) } : undefined,
|
|
242
|
+
reactive: state.reactive?.filter(r => matches(reactiveUsageWindow(r))),
|
|
243
|
+
} : state;
|
|
244
|
+
return selectSubscriptionAccount({ pool: { kind: 'ready', accounts: [account] }, states: { [account.accountId]: scoped }, model: pair?.scope, now });
|
|
245
|
+
};
|
|
246
|
+
const improved = [undefined, ...pairs.values()].some(pair => {
|
|
247
|
+
const previous = eligibility(before, pair);
|
|
248
|
+
const current = eligibility(after, pair);
|
|
249
|
+
return current.kind === 'selected' && (!pair || previous.kind === 'deferred') ||
|
|
250
|
+
current.kind === 'deferred' && previous.kind === 'deferred' && current.eligibleAt < previous.eligibleAt;
|
|
251
|
+
});
|
|
252
|
+
if (improved) {
|
|
225
253
|
try {
|
|
226
254
|
options.onHeadroom?.(account.workspaceId);
|
|
227
255
|
}
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@teamvibe/poller",
|
|
3
|
-
"version": "0.8.
|
|
3
|
+
"version": "0.8.8",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@teamvibe/poller",
|
|
9
|
-
"version": "0.8.
|
|
9
|
+
"version": "0.8.8",
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"@aws-sdk/client-dynamodb": "3.1108.0",
|
|
12
12
|
"@aws-sdk/client-sqs": "3.1108.0",
|