@teamvibe/poller 0.8.7-rc.1 → 0.8.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/claude-spawner.d.ts +2 -0
- package/dist/claude-spawner.js +42 -12
- package/dist/event-queue.js +11 -0
- package/dist/inbox-manager.d.ts +4 -0
- package/dist/inbox-manager.js +29 -0
- package/dist/inbox-watcher.js +16 -3
- package/dist/poller.js +33 -9
- package/dist/pool-defer.d.ts +24 -1
- package/dist/pool-defer.js +101 -5
- package/dist/subscription-runtime.d.ts +2 -0
- package/dist/subscription-runtime.js +8 -2
- package/dist/subscription-selection.d.ts +21 -2
- package/dist/subscription-selection.js +91 -14
- package/dist/subscription-usage-store.d.ts +6 -1
- package/dist/subscription-usage-store.js +246 -13
- package/dist/types.d.ts +2 -0
- package/dist/usage-limit-messages.d.ts +3 -0
- package/dist/usage-limit-messages.js +22 -0
- package/dist/usage-probe-job.d.ts +2 -0
- package/dist/usage-probe-job.js +36 -2
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/claude-spawner.d.ts
CHANGED
|
@@ -28,6 +28,8 @@ export interface SpawnResult {
|
|
|
28
28
|
poolAttempts?: readonly PoolAttempt[];
|
|
29
29
|
quota?: ClaudeQuotaFailure;
|
|
30
30
|
rateLimitInfo?: ClaudeRateLimitInfo;
|
|
31
|
+
/** Rejections anywhere in this attempt survive a later allowed stream event. */
|
|
32
|
+
rateLimitRejections?: readonly ClaudeRateLimitInfo[];
|
|
31
33
|
/** Actual ID passed to this CLI attempt, including a failed first turn. */
|
|
32
34
|
attemptedSessionId?: string;
|
|
33
35
|
success: boolean;
|
package/dist/claude-spawner.js
CHANGED
|
@@ -20,7 +20,7 @@ import { fetchSecretsForSpawn, buildSecretsReminder } from './secrets-client.js'
|
|
|
20
20
|
import { fetchConfigForSpawn } from './config-client.js';
|
|
21
21
|
import { parseClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
22
22
|
import { parseClaudeQuotaFailure } from './claude-quota.js';
|
|
23
|
-
import { abortLocalProbe, accountPin, preparePool, fetchRuntimePool, validateAccount, poolDeferredError, PoolDeferredError } from './subscription-runtime.js';
|
|
23
|
+
import { abortLocalProbe, accountPin, preparePool, fetchRuntimePool, validateAccount, subscriptionUsageStore, poolDeferredError, PoolDeferredError } from './subscription-runtime.js';
|
|
24
24
|
import { selectPoolForResolvedModel } from './subscription-pool.js';
|
|
25
25
|
import { isQuotaPaused, QuotaPausedError } from './quota-pause.js';
|
|
26
26
|
import { redactStartupText } from './startup-auth.js';
|
|
@@ -911,6 +911,14 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
911
911
|
});
|
|
912
912
|
if (runtime?.account)
|
|
913
913
|
await validateAccount(runtime.account, runtime.resolved);
|
|
914
|
+
if (runtime?.account && !runtime.shouldStop()) {
|
|
915
|
+
const store = subscriptionUsageStore();
|
|
916
|
+
const attemptAt = Date.now();
|
|
917
|
+
const windows = await store.beginVerification(runtime.account, runtime.resolved.model, attemptAt);
|
|
918
|
+
if (windows === undefined)
|
|
919
|
+
throw await poolDeferredError('usage', { kind: 'ready', accounts: [runtime.account] }, runtime.resolved, store);
|
|
920
|
+
runtime.verificationAttemptAt = windows.length ? attemptAt : undefined;
|
|
921
|
+
}
|
|
914
922
|
return new Promise((resolve) => {
|
|
915
923
|
// Preparation above awaits remote config/context. Another thread may
|
|
916
924
|
// have exhausted the shared account meanwhile. No awaits before spawn.
|
|
@@ -1083,6 +1091,16 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
1083
1091
|
// Retain CLI error/result evidence, not the full parsed assistant/tool stream.
|
|
1084
1092
|
const failureEvents = [];
|
|
1085
1093
|
let rateLimitInfo;
|
|
1094
|
+
const rateLimitRejections = new Map();
|
|
1095
|
+
const captureRateLimit = (value) => {
|
|
1096
|
+
rateLimitInfo = parseClaudeRateLimitInfo(value);
|
|
1097
|
+
if (!runtime?.account || rateLimitInfo?.status !== 'rejected')
|
|
1098
|
+
return;
|
|
1099
|
+
for (const w of normalizePassiveUsage(rateLimitInfo, Date.now()).windows) {
|
|
1100
|
+
if (w.status === 'rejected')
|
|
1101
|
+
rateLimitRejections.set(`${w.window}:${w.scope}`, rateLimitInfo);
|
|
1102
|
+
}
|
|
1103
|
+
};
|
|
1086
1104
|
let metrics;
|
|
1087
1105
|
let lastAssistant;
|
|
1088
1106
|
// teamvibe.ai#284 (poller-brain#235): end-of-turn guard state — tracked
|
|
@@ -1143,7 +1161,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
1143
1161
|
// (including metadata keys) to logs, telemetry, typing, or Slack.
|
|
1144
1162
|
const event = safeEvent(rawEvent);
|
|
1145
1163
|
if (rawEvent?.type === 'rate_limit_event') {
|
|
1146
|
-
|
|
1164
|
+
captureRateLimit(event.rate_limit_info);
|
|
1147
1165
|
}
|
|
1148
1166
|
if (rawEvent?.type === 'result' || rawEvent?.type === 'error' || (rawEvent?.type === 'assistant' && rawEvent.error)) {
|
|
1149
1167
|
failureEvents.push(rawEvent);
|
|
@@ -1367,7 +1385,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
1367
1385
|
const rawEvent = JSON.parse(stdoutBuffer);
|
|
1368
1386
|
const event = safeEvent(rawEvent);
|
|
1369
1387
|
if (event?.type === 'rate_limit_event') {
|
|
1370
|
-
|
|
1388
|
+
captureRateLimit(event.rate_limit_info);
|
|
1371
1389
|
}
|
|
1372
1390
|
failureEvents.push(rawEvent);
|
|
1373
1391
|
if (event?.type === 'assistant') {
|
|
@@ -1386,6 +1404,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
|
|
|
1386
1404
|
attemptedSessionId: sessionId,
|
|
1387
1405
|
...(quota ? { quota } : {}),
|
|
1388
1406
|
...(rateLimitInfo ? { rateLimitInfo } : {}),
|
|
1407
|
+
...(rateLimitRejections.size ? { rateLimitRejections: [...rateLimitRejections.values()] } : {}),
|
|
1389
1408
|
output: redactOutput(stdout),
|
|
1390
1409
|
exitCode: code,
|
|
1391
1410
|
metrics,
|
|
@@ -1610,7 +1629,7 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1610
1629
|
const modelNormalizedFrom = scoped?.model === resolved.model && scoped?.modelSource === resolved.modelSource
|
|
1611
1630
|
? scoped?.modelNormalizedFrom : undefined;
|
|
1612
1631
|
const runtime = poolRuntime ? {
|
|
1613
|
-
resolved, modelNormalizedFrom, claudeConfigDir, account: prepared?.account,
|
|
1632
|
+
resolved, modelNormalizedFrom, claudeConfigDir, account: prepared?.account, verificationAttemptAt: undefined,
|
|
1614
1633
|
shouldStop: poolRuntime.shouldStop,
|
|
1615
1634
|
resumePending: fetchedPool?.kind !== 'no_accounts_configured' && poolRuntime.pin?.resumePending === true,
|
|
1616
1635
|
onSpawn: () => {
|
|
@@ -1649,10 +1668,12 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1649
1668
|
if (!runtime?.resumePending)
|
|
1650
1669
|
sawSendMessageCall = false;
|
|
1651
1670
|
const spawned = await runClaudeCode(msg, sessionLog, effectiveCwd, directory, id, runtime?.resumePending ? false : first, lastMessageTs, onMessageSent, typingIndicator, runtime, gitDelivery);
|
|
1671
|
+
const rejected = spawned.rateLimitRejections?.find(rejectedForModel);
|
|
1652
1672
|
const result = prepared?.account
|
|
1653
|
-
? { ...spawned, quota: quotaForModel(spawned.quota), success: rejectedForModel(spawned.rateLimitInfo) ? false : spawned.success } : spawned;
|
|
1673
|
+
? { ...spawned, ...(rejected ? { rateLimitInfo: rejected } : {}), quota: quotaForModel(spawned.quota), success: rejected || rejectedForModel(spawned.rateLimitInfo) ? false : spawned.success } : spawned;
|
|
1654
1674
|
sawSendMessageCall ||= result.sawSendMessageCall === true;
|
|
1655
1675
|
if (prepared?.account) {
|
|
1676
|
+
const observedAt = Date.now();
|
|
1656
1677
|
if (poolAttempts.length >= 1000)
|
|
1657
1678
|
poolAttempts.shift();
|
|
1658
1679
|
const selection = prepared.selection;
|
|
@@ -1664,7 +1685,8 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1664
1685
|
accountId: prepared.account.accountId,
|
|
1665
1686
|
credentialVersion: prepared.account.credentialVersion,
|
|
1666
1687
|
accountLabel: prepared.account.label ?? prepared.account.accountId,
|
|
1667
|
-
observedAt
|
|
1688
|
+
observedAt,
|
|
1689
|
+
verificationAttemptAt: runtime?.verificationAttemptAt,
|
|
1668
1690
|
rateLimitInfo: result.rateLimitInfo,
|
|
1669
1691
|
sessionId: id,
|
|
1670
1692
|
model: resolved.model ?? result.metrics?.model,
|
|
@@ -1675,13 +1697,19 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1675
1697
|
metrics: result.metrics,
|
|
1676
1698
|
});
|
|
1677
1699
|
emitEvent('pool.account_attempted', msg.teamvibe.channelId, { workspaceId: msg.teamvibe.workspaceId, threadId: msg.thread_id, ...poolRunTelemetry(poolAttempts, poolRuntime?.quotaDeferEpisodes) });
|
|
1700
|
+
// Retain every rejected scope before completion can clear a verified
|
|
1701
|
+
// hold or return early. A failed write must not skip another scope.
|
|
1702
|
+
const rejectionOutcomes = await Promise.allSettled((result.rateLimitRejections ?? []).map(info => logPoolUsageWrite(sessionLog, prepared.account, info, () => prepared.store.recordPassive(prepared.account, info, observedAt, { expireFutureObservations: true }))));
|
|
1678
1703
|
if (result.success && id && poolRuntime) {
|
|
1679
|
-
// Complete the inbox
|
|
1680
|
-
//
|
|
1704
|
+
// Complete the inbox even if rejection bookkeeping failed. The loop
|
|
1705
|
+
// retains completion independently of the original prompt.
|
|
1681
1706
|
const bookkeepingComplete = await poolRuntime.complete(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: false }, poolAttempts, result.rateLimitInfo);
|
|
1682
1707
|
if (bookkeepingComplete !== false)
|
|
1683
1708
|
await emitCandidateUsage(msg, [prepared.account], prepared.store);
|
|
1684
1709
|
retainWorktree = false;
|
|
1710
|
+
const failed = rejectionOutcomes.find((outcome) => outcome.status === 'rejected');
|
|
1711
|
+
if (failed)
|
|
1712
|
+
throw new PoolDeferredError('infrastructure', { error: failed.reason });
|
|
1685
1713
|
return result;
|
|
1686
1714
|
}
|
|
1687
1715
|
if ((result.quota || rejectedForModel(result.rateLimitInfo)) && !result.killedByPoller) {
|
|
@@ -1692,15 +1720,17 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
|
|
|
1692
1720
|
// retaining the attempt. Selection waits for all writes before switching.
|
|
1693
1721
|
const writes = [];
|
|
1694
1722
|
if (result.quota && !result.killedByPoller)
|
|
1695
|
-
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.quota.window ? { rateLimitType: result.quota.window } : undefined, () => prepared.store.recordQuota(prepared.account, result.quota,
|
|
1696
|
-
if (result.rateLimitInfo)
|
|
1697
|
-
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.rateLimitInfo, () => prepared.store.recordPassive(prepared.account, result.rateLimitInfo,
|
|
1723
|
+
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.quota.window ? { rateLimitType: result.quota.window } : undefined, () => prepared.store.recordQuota(prepared.account, result.quota, observedAt, { expireFutureObservations: true }), 'quota'));
|
|
1724
|
+
if (!result.rateLimitRejections?.length && result.rateLimitInfo)
|
|
1725
|
+
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.rateLimitInfo, () => prepared.store.recordPassive(prepared.account, result.rateLimitInfo, observedAt, { expireFutureObservations: true })));
|
|
1698
1726
|
if (id && poolRuntime)
|
|
1699
1727
|
writes.push(poolRuntime.persist(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: runtime?.resumePending, sawSendMessageCall }, poolAttempts));
|
|
1700
|
-
const outcomes = await Promise.allSettled(writes);
|
|
1728
|
+
const outcomes = [...rejectionOutcomes, ...await Promise.allSettled(writes)];
|
|
1701
1729
|
const failed = outcomes.find((outcome) => outcome.status === 'rejected');
|
|
1702
1730
|
if (failed)
|
|
1703
1731
|
throw new PoolDeferredError('infrastructure', { error: failed.reason });
|
|
1732
|
+
if (runtime?.verificationAttemptAt !== undefined && !result.killedByPoller && result.exitCode !== null)
|
|
1733
|
+
await prepared.store.completeVerification(prepared.account, runtime.verificationAttemptAt, observedAt);
|
|
1704
1734
|
}
|
|
1705
1735
|
return result;
|
|
1706
1736
|
};
|
package/dist/event-queue.js
CHANGED
|
@@ -4,6 +4,8 @@ import { logger } from './logger.js';
|
|
|
4
4
|
const MAX_QUEUE_SIZE = 200;
|
|
5
5
|
const FLUSH_INTERVAL_MS = 30_000;
|
|
6
6
|
const DEDUP_WINDOW_MS = 60_000;
|
|
7
|
+
const rejectedWarnings = new Map();
|
|
8
|
+
const REJECTION_WARN_INTERVAL_MS = 3600_000;
|
|
7
9
|
let queue = [];
|
|
8
10
|
let flushTimer = null;
|
|
9
11
|
let isFlushing = false;
|
|
@@ -60,6 +62,15 @@ async function flushOne(event) {
|
|
|
60
62
|
logger.info(`[Pool] usage probe request=${event.metadata['requestId']} completion downgraded for legacy platform`);
|
|
61
63
|
response = await post();
|
|
62
64
|
}
|
|
65
|
+
if ([400, 404, 422].includes(response.status)) {
|
|
66
|
+
const now = Date.now();
|
|
67
|
+
const previous = rejectedWarnings.get(event.eventType);
|
|
68
|
+
if (previous === undefined || now - previous >= REJECTION_WARN_INTERVAL_MS) {
|
|
69
|
+
rejectedWarnings.set(event.eventType, now);
|
|
70
|
+
logger.warn(`[EventQueue] ${event.eventType} rejected with HTTP ${response.status}; dropping event`);
|
|
71
|
+
}
|
|
72
|
+
return true;
|
|
73
|
+
}
|
|
63
74
|
return response.ok;
|
|
64
75
|
}
|
|
65
76
|
catch {
|
package/dist/inbox-manager.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import type { PoolDefer } from './pool-defer.js';
|
|
1
2
|
import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
2
3
|
import type { PoolAttempt } from './subscription-runtime.js';
|
|
3
4
|
import type { TeamVibeQueueMessage, SubscriptionPin } from './types.js';
|
|
@@ -81,3 +82,6 @@ export declare function discardHandledInboxDrops(brainPath: string, threadId: st
|
|
|
81
82
|
export declare function restartInboxInDirectory(source: string, target: string, threadId: string): void;
|
|
82
83
|
/** Preserve unroutable work for operator review and remove it from replay queues. */
|
|
83
84
|
export declare function quarantineInbox(brainPath: string, threadId: string, includeFailed?: boolean): void;
|
|
85
|
+
/** Only original queue deliveries carry inbound identity; synthetic drops never wake holds. */
|
|
86
|
+
export declare function inboxQueueMessageIds(brainPath: string, threadId: string): string[];
|
|
87
|
+
export declare function hasInboxMessageAfter(brainPath: string, threadId: string, pause: Pick<PoolDefer, 'handledInboundIds'>): boolean;
|
package/dist/inbox-manager.js
CHANGED
|
@@ -305,3 +305,32 @@ export function quarantineInbox(brainPath, threadId, includeFailed = false) {
|
|
|
305
305
|
renameSync(join(root, name), join(quarantine, `${Date.now()}-${name}`));
|
|
306
306
|
}
|
|
307
307
|
}
|
|
308
|
+
/** Only original queue deliveries carry inbound identity; synthetic drops never wake holds. */
|
|
309
|
+
export function inboxQueueMessageIds(brainPath, threadId) {
|
|
310
|
+
const ids = new Set();
|
|
311
|
+
for (const subdir of [PROCESSING_DIR, NEW_DIR]) {
|
|
312
|
+
const directory = join(getInboxPath(brainPath, threadId), subdir);
|
|
313
|
+
let names;
|
|
314
|
+
try {
|
|
315
|
+
names = readdirSync(directory);
|
|
316
|
+
}
|
|
317
|
+
catch {
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
for (const name of names) {
|
|
321
|
+
if (!name.endsWith('.txt'))
|
|
322
|
+
continue;
|
|
323
|
+
try {
|
|
324
|
+
const message = JSON.parse(readFileSync(join(directory, name), 'utf8'));
|
|
325
|
+
if (message.executionOrigin === 'queue' && message.thread_id === threadId && typeof message.id === 'string' && message.id)
|
|
326
|
+
ids.add(message.id);
|
|
327
|
+
}
|
|
328
|
+
catch { /* Plain background drops and concurrent moves have no inbound identity. */ }
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
return [...ids];
|
|
332
|
+
}
|
|
333
|
+
export function hasInboxMessageAfter(brainPath, threadId, pause) {
|
|
334
|
+
const handled = new Set(pause.handledInboundIds ?? []);
|
|
335
|
+
return inboxQueueMessageIds(brainPath, threadId).some(id => !handled.has(id));
|
|
336
|
+
}
|
package/dist/inbox-watcher.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { executionDirectory } from './brain-directory.js';
|
|
2
|
-
import { discardHandledInboxDrops, markProcessed, quarantineInbox, readInboxChannelContext, readInboxContext, readInboxRuntime, restartInboxInDirectory } from './inbox-manager.js';
|
|
2
|
+
import { hasInboxMessageAfter, discardHandledInboxDrops, markProcessed, quarantineInbox, readInboxChannelContext, readInboxContext, readInboxRuntime, restartInboxInDirectory } from './inbox-manager.js';
|
|
3
3
|
import { listThreadRoutings, readThreadRouting, removeThreadRouting, threadRoutingKey } from './thread-routing.js';
|
|
4
|
-
import { isPoolDeferred, readPoolDefer } from './pool-defer.js';
|
|
4
|
+
import { observePoolAccounts, isPoolDeferred, readPoolDefer } from './pool-defer.js';
|
|
5
5
|
import { existsSync, readdirSync, rmSync, watch as fsWatch } from 'fs';
|
|
6
6
|
import { join } from 'path';
|
|
7
7
|
import { config } from './config.js';
|
|
@@ -11,6 +11,7 @@ import { logger } from './logger.js';
|
|
|
11
11
|
import { isAtCapacity } from './claude-spawner.js';
|
|
12
12
|
import { getTrackedBrains } from './maintenance-cron.js';
|
|
13
13
|
import { runInboxLoop, isInboxLoopActive } from './poller.js';
|
|
14
|
+
import { fetchRuntimePool } from './subscription-runtime.js';
|
|
14
15
|
import { isQuotaPaused } from './quota-pause.js';
|
|
15
16
|
import { emitError } from './event-queue.js';
|
|
16
17
|
import { sendSlackMessage } from './slack-client.js';
|
|
@@ -151,6 +152,7 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
|
|
|
151
152
|
const expired = await sweepThreadRouting(deps, brains);
|
|
152
153
|
const started = new Set();
|
|
153
154
|
const recoveries = [];
|
|
155
|
+
const checkedPools = new Set();
|
|
154
156
|
scan: for (const brainPath of brains) {
|
|
155
157
|
if (deps.isAtCapacity())
|
|
156
158
|
break;
|
|
@@ -160,7 +162,7 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
|
|
|
160
162
|
break scan;
|
|
161
163
|
if (expired.has(threadRoutingKey(threadId)))
|
|
162
164
|
continue;
|
|
163
|
-
if (started.has(threadId) || recoveringThreads.has(threadId) || deps.isInboxLoopActive(threadId)
|
|
165
|
+
if (started.has(threadId) || recoveringThreads.has(threadId) || deps.isInboxLoopActive(threadId))
|
|
164
166
|
continue;
|
|
165
167
|
discardHandledInboxDrops(brainPath, threadId);
|
|
166
168
|
const runtime = readInboxRuntime(brainPath, threadId);
|
|
@@ -193,6 +195,17 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
|
|
|
193
195
|
quarantineRetainedThread(deps, brainPath, threadId, durable);
|
|
194
196
|
continue;
|
|
195
197
|
}
|
|
198
|
+
if (isPoolDeferred(threadId)) {
|
|
199
|
+
// Held work still observes pool membership on every watcher tick.
|
|
200
|
+
const key = JSON.stringify([info.workspaceId, info.channelId]);
|
|
201
|
+
if (!checkedPools.has(key)) {
|
|
202
|
+
checkedPools.add(key);
|
|
203
|
+
observePoolAccounts(info.workspaceId, await fetchRuntimePool(message));
|
|
204
|
+
}
|
|
205
|
+
const pause = readPoolDefer(threadId);
|
|
206
|
+
if (isPoolDeferred(threadId) && pause && !hasInboxMessageAfter(brainPath, threadId, pause))
|
|
207
|
+
continue;
|
|
208
|
+
}
|
|
196
209
|
started.add(threadId);
|
|
197
210
|
recoveringThreads.add(threadId);
|
|
198
211
|
recoveries.push((async () => {
|
package/dist/poller.js
CHANGED
|
@@ -14,7 +14,7 @@ import { classifyFailedIteration } from './classify-failed-iteration.js';
|
|
|
14
14
|
import { fetchSecretsForSpawn } from './secrets-client.js';
|
|
15
15
|
import { PoolDeferredError, fetchRuntimePool, subscriptionUsageStore, HELD_PROBE_INTERVAL_MS } from './subscription-runtime.js';
|
|
16
16
|
import { POOL_RECHECK_MS } from './subscription-selection.js';
|
|
17
|
-
import { isPoolDeferred, readPoolDefer, deferPool, setPoolNotice, clearPoolDefer } from './pool-defer.js';
|
|
17
|
+
import { invalidatePoolDefer, invalidateWorkspacePoolDefers, observePoolAccounts, isPoolDeferred, readPoolDefer, deferPool, setPoolNotice, clearPoolDefer } from './pool-defer.js';
|
|
18
18
|
import { isQuotaPaused, pauseForQuota, clearQuotaPauseAfterSuccess, QuotaPausedError } from './quota-pause.js';
|
|
19
19
|
import { acquireSessionLock, releaseSessionLock, updateSessionId } from './session-store.js';
|
|
20
20
|
import { ensureDirectories, ensureBaseBrain, pushBrainChanges } from './brain-manager.js';
|
|
@@ -26,7 +26,7 @@ import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './main
|
|
|
26
26
|
import { bootstrapTrackedBrains } from './brains-bootstrap.js';
|
|
27
27
|
import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
|
|
28
28
|
import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
|
|
29
|
-
import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage, saveInboxContext, saveInboxRuntime, readInboxRuntime, clearInboxRuntime } from './inbox-manager.js';
|
|
29
|
+
import { inboxQueueMessageIds, hasInboxMessageAfter, writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage, saveInboxContext, saveInboxRuntime, readInboxRuntime, clearInboxRuntime } from './inbox-manager.js';
|
|
30
30
|
import { saveThreadRouting } from './thread-routing.js';
|
|
31
31
|
import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
|
|
32
32
|
import { Cron } from 'croner';
|
|
@@ -257,12 +257,17 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
257
257
|
if (isQuotaPaused() && !getPollerId())
|
|
258
258
|
return;
|
|
259
259
|
const initialPool = await fetchRuntimePool(queueMessage);
|
|
260
|
+
observePoolAccounts(queueMessage.teamvibe.workspaceId, initialPool, { log: line => sessionLog.info(line) });
|
|
260
261
|
if (initialPool.kind === 'no_accounts_configured') {
|
|
261
262
|
if (isQuotaPaused())
|
|
262
263
|
return;
|
|
263
264
|
}
|
|
264
|
-
else if (isPoolDeferred(threadId))
|
|
265
|
-
|
|
265
|
+
else if (isPoolDeferred(threadId)) {
|
|
266
|
+
const pause = readPoolDefer(threadId);
|
|
267
|
+
if (!hasInboxMessageAfter(kbPath, threadId, pause))
|
|
268
|
+
return;
|
|
269
|
+
invalidatePoolDefer(threadId, queueMessage.teamvibe.workspaceId, 'new_message', { log: line => sessionLog.info(line) }, inboxQueueMessageIds(kbPath, threadId));
|
|
270
|
+
}
|
|
266
271
|
if (activeLoops.has(threadId)) {
|
|
267
272
|
sessionLog.info(`Loop already running for thread ${threadId}, skipping`);
|
|
268
273
|
return;
|
|
@@ -318,8 +323,10 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
318
323
|
const lastAttempt = state.attempts?.at(-1);
|
|
319
324
|
// Existing telemetry falls back to account IDs. Local pool logs use labels only.
|
|
320
325
|
const accountLabel = lastAttempt?.accountLabel !== state.pin.accountId ? lastAttempt?.accountLabel : undefined;
|
|
326
|
+
// Reuse the spawn's observation clock: the spawner already recorded all
|
|
327
|
+
// rejected scopes, and replaying its final event is the same evidence.
|
|
321
328
|
if (state.completed.rateLimitInfo)
|
|
322
|
-
writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
|
|
329
|
+
writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, lastAttempt?.observedAt ?? state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
|
|
323
330
|
writes.push(persistSessionId(state.sessionId, state.pin));
|
|
324
331
|
const outcomes = await Promise.allSettled(writes);
|
|
325
332
|
if (outcomes.some(outcome => outcome.status === 'rejected' || outcome.value === false)) {
|
|
@@ -332,6 +339,21 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
332
339
|
}
|
|
333
340
|
return false;
|
|
334
341
|
}
|
|
342
|
+
if (lastAttempt?.verificationAttemptAt !== undefined) {
|
|
343
|
+
try {
|
|
344
|
+
await subscriptionUsageStore().completeVerification(state.pin, lastAttempt.verificationAttemptAt, state.completed.observedAt);
|
|
345
|
+
}
|
|
346
|
+
catch {
|
|
347
|
+
sessionLog.warn('[Loop] Verification bookkeeping failed and will retry without running the agent');
|
|
348
|
+
try {
|
|
349
|
+
saveInboxRuntime(kbPath, threadId, { ...state, completed: { ...state.completed, retryAt: Date.now() + POOL_RECHECK_MS } });
|
|
350
|
+
}
|
|
351
|
+
catch {
|
|
352
|
+
sessionLog.warn('[Loop] Could not save bookkeeping retry; completion remains recorded');
|
|
353
|
+
}
|
|
354
|
+
return false;
|
|
355
|
+
}
|
|
356
|
+
}
|
|
335
357
|
currentSessionId = state.sessionId;
|
|
336
358
|
currentPin = state.pin;
|
|
337
359
|
// The spawner publishes live completions only after bookkeeping succeeds.
|
|
@@ -370,6 +392,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
370
392
|
// Dispatch before ANY pool state/context/model preparation. The explicit
|
|
371
393
|
// no-pool outcome runs the original spawn and #408 pause gates.
|
|
372
394
|
const pool = iteration === 0 ? initialPool : await fetchRuntimePool(queueMessage);
|
|
395
|
+
observePoolAccounts(queueMessage.teamvibe.workspaceId, pool, { log: line => sessionLog.info(line) });
|
|
373
396
|
poolIteration = pool.kind !== 'no_accounts_configured';
|
|
374
397
|
if (!poolIteration && isQuotaPaused())
|
|
375
398
|
break;
|
|
@@ -619,9 +642,9 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
619
642
|
quotaPaused = true;
|
|
620
643
|
deferred = true;
|
|
621
644
|
// Revisit held work hourly so background probes can refresh the account.
|
|
622
|
-
// Selection
|
|
645
|
+
// Selection enforces resets or a contradicted rejection's verification backoff.
|
|
623
646
|
const recheckAt = error.cause === 'usage' ? Math.min(error.eligibleAt, Date.now() + HELD_PROBE_INTERVAL_MS) : error.eligibleAt;
|
|
624
|
-
const pause = deferPool(threadId, recheckAt, error.resetAt);
|
|
647
|
+
const pause = deferPool(threadId, recheckAt, error.resetAt, inboxQueueMessageIds(kbPath, threadId));
|
|
625
648
|
const retained = readInboxRuntime(kbPath, threadId);
|
|
626
649
|
// Rechecks of one exhaustion episode do not inflate the episode count.
|
|
627
650
|
quotaDeferEpisodes = Math.max(1, quotaDeferEpisodes);
|
|
@@ -638,7 +661,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
638
661
|
cause: error.cause,
|
|
639
662
|
thresholdDeferred: error.notice?.thresholdDeferred && !error.attempts?.some(a => a.quota),
|
|
640
663
|
resetAt: pause.resetAt ? Date.parse(pause.resetAt) : error.notice?.resetAt,
|
|
641
|
-
timeZone:
|
|
664
|
+
timeZone: deliveryMsg.teamvibe.timeZone ?? error.notice?.timeZone,
|
|
642
665
|
});
|
|
643
666
|
await sendSlackMessage(deliveryMsg, notice, { clientMessageId: pause.episodeId, ...(reconcile ? { reconcile: true } : {}) });
|
|
644
667
|
setPoolNotice(threadId, pause.episodeId, 'delivered');
|
|
@@ -901,7 +924,7 @@ async function processMessage(received) {
|
|
|
901
924
|
const kbPath = directory.path;
|
|
902
925
|
// Write raw message to inbox (spawnClaudeCode handles prompt formatting)
|
|
903
926
|
const { botToken: _inboxToken, ...inboxTeamvibe } = queueMessage.teamvibe;
|
|
904
|
-
writeMessage(kbPath, threadId, JSON.stringify({ ...queueMessage, teamvibe: inboxTeamvibe }));
|
|
927
|
+
writeMessage(kbPath, threadId, JSON.stringify({ ...queueMessage, executionOrigin: receiptHandle ? 'queue' : 'maintenance', teamvibe: inboxTeamvibe }));
|
|
905
928
|
sessionLog.info(`Wrote message to inbox`);
|
|
906
929
|
// Delete SQS message immediately — it's now persisted in the inbox
|
|
907
930
|
if (receiptHandle) {
|
|
@@ -1029,6 +1052,7 @@ export async function pollLoop() {
|
|
|
1029
1052
|
apiUrl: config.TEAMVIBE_API_URL,
|
|
1030
1053
|
pollerToken: config.TEAMVIBE_POLLER_TOKEN || '',
|
|
1031
1054
|
}),
|
|
1055
|
+
onHeadroom: workspaceId => invalidateWorkspacePoolDefers(workspaceId, 'usage_update'),
|
|
1032
1056
|
store: subscriptionUsageStore(),
|
|
1033
1057
|
cliPath: config.CLAUDE_CLI_PATH,
|
|
1034
1058
|
isAtCapacity,
|
package/dist/pool-defer.d.ts
CHANGED
|
@@ -1,4 +1,16 @@
|
|
|
1
|
+
import { emitEvent } from './event-queue.js';
|
|
2
|
+
import type { PoolFetchResult } from './subscription-selection.js';
|
|
1
3
|
export interface PoolDefer {
|
|
4
|
+
/** Additive ownership index; older markers use the existing hashed routing file. */
|
|
5
|
+
scope?: {
|
|
6
|
+
workspaceId: string;
|
|
7
|
+
threadId: string;
|
|
8
|
+
channelId: string;
|
|
9
|
+
};
|
|
10
|
+
/** Enabled capacity known when this hold was last evaluated. */
|
|
11
|
+
poolAccountIds?: string[];
|
|
12
|
+
/** Queue arrivals already considered in this exhaustion episode. */
|
|
13
|
+
handledInboundIds?: string[];
|
|
2
14
|
createdAt: number;
|
|
3
15
|
resumeAt: number;
|
|
4
16
|
resetAt?: string;
|
|
@@ -7,7 +19,18 @@ export interface PoolDefer {
|
|
|
7
19
|
}
|
|
8
20
|
export declare function readPoolDefer(threadId: string): PoolDefer | undefined;
|
|
9
21
|
export declare function isPoolDeferred(threadId: string): boolean;
|
|
10
|
-
export declare function deferPool(threadId: string, eligibleAt: number, resetAt?: string): PoolDefer;
|
|
22
|
+
export declare function deferPool(threadId: string, eligibleAt: number, resetAt?: string, handledInboundIds?: string[]): PoolDefer;
|
|
11
23
|
/** Claim durably before Slack I/O. Ambiguous sends are never blindly replayed. */
|
|
12
24
|
export declare function setPoolNotice(threadId: string, episodeId: string, notice: PoolDefer['notice']): boolean;
|
|
13
25
|
export declare function clearPoolDefer(threadId: string): void;
|
|
26
|
+
export type PoolDeferInvalidationReason = 'usage_update' | 'pool_changed' | 'new_message';
|
|
27
|
+
interface InvalidationObserver {
|
|
28
|
+
emit?: typeof emitEvent;
|
|
29
|
+
log?: (line: string) => void;
|
|
30
|
+
}
|
|
31
|
+
/** Expire atomically, retaining the episode and notice claim until a successful spawn. */
|
|
32
|
+
export declare function invalidatePoolDefer(threadId: string, workspaceId: string, reason: PoolDeferInvalidationReason, observer?: InvalidationObserver, handledInboundIds?: string[]): void;
|
|
33
|
+
export declare function invalidateWorkspacePoolDefers(workspaceId: string, reason: Exclude<PoolDeferInvalidationReason, 'new_message'>, observer?: InvalidationObserver): void;
|
|
34
|
+
/** Only authoritative enabled account sets advance the baseline; never cache credentials. */
|
|
35
|
+
export declare function observePoolAccounts(workspaceId: string, pool: PoolFetchResult, observer?: InvalidationObserver): void;
|
|
36
|
+
export {};
|
package/dist/pool-defer.js
CHANGED
|
@@ -1,19 +1,27 @@
|
|
|
1
1
|
import { createHash, randomUUID } from 'node:crypto';
|
|
2
|
-
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
4
|
import { config } from './config.js';
|
|
5
5
|
import { logger } from './logger.js';
|
|
6
|
+
import { emitEvent } from './event-queue.js';
|
|
7
|
+
import { readThreadRouting, threadRoutingKey } from './thread-routing.js';
|
|
6
8
|
import { POOL_RECHECK_MS } from './subscription-selection.js';
|
|
7
9
|
function pathFor(threadId) {
|
|
8
10
|
return join(config.TEAMVIBE_DATA_DIR, 'pool-defer', `${createHash('sha256').update(threadId).digest('hex')}.json`);
|
|
9
11
|
}
|
|
10
12
|
const warned = new Set();
|
|
11
13
|
export function readPoolDefer(threadId) {
|
|
12
|
-
|
|
14
|
+
return readMarker(pathFor(threadId));
|
|
15
|
+
}
|
|
16
|
+
function readMarker(path) {
|
|
13
17
|
try {
|
|
14
18
|
const state = JSON.parse(readFileSync(path, 'utf8'));
|
|
15
|
-
if (!Number.isFinite(state.createdAt) || !Number.isFinite(state.resumeAt) || state.resumeAt
|
|
19
|
+
if (!Number.isFinite(state.createdAt) || !Number.isFinite(state.resumeAt) || state.resumeAt < state.createdAt || state.resumeAt > 253402300799000 || typeof state.episodeId !== 'string' || !['pending', 'claimed', 'delivered', 'ambiguous'].includes(state.notice))
|
|
16
20
|
throw new Error('Invalid defer');
|
|
21
|
+
for (const ids of [state.poolAccountIds, state.handledInboundIds]) {
|
|
22
|
+
if (ids !== undefined && (!Array.isArray(ids) || ids.some(id => typeof id !== 'string')))
|
|
23
|
+
throw new Error('Invalid defer baseline');
|
|
24
|
+
}
|
|
17
25
|
warned.delete(path);
|
|
18
26
|
return state;
|
|
19
27
|
}
|
|
@@ -56,10 +64,14 @@ function write(threadId, state) {
|
|
|
56
64
|
catch { }
|
|
57
65
|
}
|
|
58
66
|
}
|
|
59
|
-
export function deferPool(threadId, eligibleAt, resetAt) {
|
|
67
|
+
export function deferPool(threadId, eligibleAt, resetAt, handledInboundIds = []) {
|
|
60
68
|
const previous = readPoolDefer(threadId);
|
|
61
69
|
const now = Date.now();
|
|
62
|
-
const
|
|
70
|
+
const routing = readThreadRouting(threadId);
|
|
71
|
+
const scope = routing ? { workspaceId: routing.workspaceId, channelId: routing.channelId, threadId } : previous?.scope;
|
|
72
|
+
const fingerprint = scope ? poolFingerprints.get(poolKey(scope.workspaceId)) : undefined;
|
|
73
|
+
const poolAccountIds = fingerprint === undefined ? previous?.poolAccountIds : JSON.parse(fingerprint);
|
|
74
|
+
const state = { ...(scope ? { scope } : {}), ...(poolAccountIds ? { poolAccountIds } : {}), handledInboundIds: [...new Set([...(previous?.handledInboundIds ?? []), ...handledInboundIds])], createdAt: now, resumeAt: Number.isFinite(eligibleAt) && eligibleAt > now ? eligibleAt : now + POOL_RECHECK_MS, episodeId: previous?.episodeId ?? randomUUID(), notice: previous?.notice ?? 'pending', ...(resetAt ? { resetAt } : {}) };
|
|
63
75
|
write(threadId, state);
|
|
64
76
|
return state;
|
|
65
77
|
}
|
|
@@ -79,3 +91,87 @@ export function clearPoolDefer(threadId) {
|
|
|
79
91
|
logger.warn('[PoolDefer] Completed episode cleanup will be retried');
|
|
80
92
|
}
|
|
81
93
|
}
|
|
94
|
+
/** Expire atomically, retaining the episode and notice claim until a successful spawn. */
|
|
95
|
+
export function invalidatePoolDefer(threadId, workspaceId, reason, observer = {}, handledInboundIds = []) {
|
|
96
|
+
const state = readPoolDefer(threadId);
|
|
97
|
+
if (!state || !isPoolDeferred(threadId))
|
|
98
|
+
return;
|
|
99
|
+
const routing = readThreadRouting(threadId);
|
|
100
|
+
const scope = state.scope ?? (routing ? { ...routing, threadId } : undefined);
|
|
101
|
+
if (!scope || scope.workspaceId !== workspaceId)
|
|
102
|
+
return;
|
|
103
|
+
write(threadId, { ...state, resumeAt: state.createdAt, handledInboundIds: [...new Set([...(state.handledInboundIds ?? []), ...handledInboundIds])] });
|
|
104
|
+
const metadata = { workspaceId, threadId, reason, episodeId: state.episodeId };
|
|
105
|
+
// Observability cannot undo the durable wake-up.
|
|
106
|
+
try {
|
|
107
|
+
(observer.log ?? (line => logger.info(line)))(`[Pool] defer invalidated workspace=${JSON.stringify(workspaceId)} thread=${JSON.stringify(threadId)} reason=${reason} episode=${JSON.stringify(state.episodeId)}`);
|
|
108
|
+
}
|
|
109
|
+
catch { }
|
|
110
|
+
try {
|
|
111
|
+
(observer.emit ?? emitEvent)('pool.defer_invalidated', scope.channelId, metadata);
|
|
112
|
+
}
|
|
113
|
+
catch { }
|
|
114
|
+
}
|
|
115
|
+
/** One directory scan, one bounded routing lookup per legacy marker; no inbox scan. */
|
|
116
|
+
function* workspacePoolDefers(workspaceId) {
|
|
117
|
+
const root = join(config.TEAMVIBE_DATA_DIR, 'pool-defer');
|
|
118
|
+
let names;
|
|
119
|
+
try {
|
|
120
|
+
names = readdirSync(root);
|
|
121
|
+
}
|
|
122
|
+
catch {
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
for (const name of names) {
|
|
126
|
+
if (!/^[a-f0-9]{64}\.json$/.test(name))
|
|
127
|
+
continue;
|
|
128
|
+
try {
|
|
129
|
+
const state = readMarker(join(root, name));
|
|
130
|
+
if (!state)
|
|
131
|
+
continue;
|
|
132
|
+
let scope = state.scope;
|
|
133
|
+
if (!scope) {
|
|
134
|
+
// Pre-#572 markers share their hash with durable thread routing.
|
|
135
|
+
scope = JSON.parse(readFileSync(join(config.TEAMVIBE_DATA_DIR, 'thread-routing', name), 'utf8'));
|
|
136
|
+
}
|
|
137
|
+
if (scope?.workspaceId === workspaceId && typeof scope.threadId === 'string' && threadRoutingKey(scope.threadId) === name)
|
|
138
|
+
yield { state, scope };
|
|
139
|
+
}
|
|
140
|
+
catch {
|
|
141
|
+
logger.warn('[PoolDefer] Workspace invalidation will be retried');
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
export function invalidateWorkspacePoolDefers(workspaceId, reason, observer = {}) {
|
|
146
|
+
for (const { scope } of workspacePoolDefers(workspaceId))
|
|
147
|
+
invalidatePoolDefer(scope.threadId, workspaceId, reason, observer);
|
|
148
|
+
}
|
|
149
|
+
const poolFingerprints = new Map();
|
|
150
|
+
function poolKey(workspaceId) {
|
|
151
|
+
return JSON.stringify([config.TEAMVIBE_DATA_DIR, workspaceId]);
|
|
152
|
+
}
|
|
153
|
+
/** Only authoritative enabled account sets advance the baseline; never cache credentials. */
|
|
154
|
+
export function observePoolAccounts(workspaceId, pool, observer = {}) {
|
|
155
|
+
if (pool.kind !== 'ready' && pool.kind !== 'no_accounts_configured' && !(pool.kind === 'fetch_unavailable' && pool.reason === 'no_usable_accounts'))
|
|
156
|
+
return;
|
|
157
|
+
const poolAccountIds = pool.kind === 'ready' ? [...new Set(pool.accounts.filter(a => a.enabled && a.workspaceId === workspaceId).map(a => a.accountId))].sort() : [];
|
|
158
|
+
const fingerprint = JSON.stringify(poolAccountIds);
|
|
159
|
+
const key = poolKey(workspaceId);
|
|
160
|
+
if (poolFingerprints.get(key) === fingerprint)
|
|
161
|
+
return;
|
|
162
|
+
// The first fetch after boot also compares the durable per-marker baselines.
|
|
163
|
+
for (const { state, scope } of workspacePoolDefers(workspaceId)) {
|
|
164
|
+
const now = Date.now();
|
|
165
|
+
if (now < state.createdAt || now >= state.resumeAt)
|
|
166
|
+
continue;
|
|
167
|
+
if (state.poolAccountIds && poolAccountIds.some(id => !state.poolAccountIds.includes(id))) {
|
|
168
|
+
invalidatePoolDefer(scope.threadId, workspaceId, 'pool_changed', observer);
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
// Reductions never add capacity. Adopt them so a later re-enable wakes work.
|
|
172
|
+
// A legacy marker adopts its first observed baseline without waking work.
|
|
173
|
+
write(scope.threadId, { ...state, poolAccountIds });
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
poolFingerprints.set(key, fingerprint);
|
|
177
|
+
}
|
|
@@ -10,6 +10,8 @@ import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
|
10
10
|
import type { ClaudeQuotaFailure } from './claude-quota.js';
|
|
11
11
|
import type { KilledByPoller } from './classify-failed-iteration.js';
|
|
12
12
|
export interface PoolAttempt extends Partial<HeadroomScore> {
|
|
13
|
+
/** Local completion bookkeeping token; omitted from event transport DTOs. */
|
|
14
|
+
verificationAttemptAt?: number;
|
|
13
15
|
workspaceId: string;
|
|
14
16
|
accountId: string;
|
|
15
17
|
accountLabel?: string;
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { isAtCapacity, isSubscriptionAccountRunning } from './claude-spawner.js';
|
|
2
|
+
import { invalidateWorkspacePoolDefers } from './pool-defer.js';
|
|
2
3
|
import { emitEvent } from './event-queue.js';
|
|
3
4
|
import { logger } from './logger.js';
|
|
4
5
|
import { runUsageProbe } from './usage-probe-job.js';
|
|
@@ -149,6 +150,7 @@ export async function preparePool(msg, resolved, pin, fetched, log, probeOptions
|
|
|
149
150
|
if (pool.kind === 'ready')
|
|
150
151
|
probeUnknownAccounts(pool.accounts, selected.kind === 'selected' ? selected.account.accountId : undefined, resolved, store, {
|
|
151
152
|
cliPath: config.CLAUDE_CLI_PATH, isAtCapacity, emit: emitEvent,
|
|
153
|
+
onHeadroom: workspaceId => invalidateWorkspacePoolDefers(workspaceId, 'usage_update'),
|
|
152
154
|
log: line => (log ?? logger).info(line), ...probeOptions, store,
|
|
153
155
|
}, msg.teamvibe.channelId);
|
|
154
156
|
if (selected.kind === 'selected') {
|
|
@@ -203,6 +205,8 @@ async function poolUsageNotice(pool, resolved, store) {
|
|
|
203
205
|
return {
|
|
204
206
|
label: account.label, useFullQuota: account.useFullQuota, windows: windows.filter(w => binding.some(b => b.window === w.window && b.scope === w.scope)), rejected,
|
|
205
207
|
thresholdBlocked: blockers.length > 0,
|
|
208
|
+
blockedWindows: binding.map(w => ({ window: w.window, scope: w.scope })),
|
|
209
|
+
providerResetAt: binding.length && binding.every(w => w.resetAt !== undefined) ? Math.max(...binding.map(w => w.resetAt)) : undefined,
|
|
206
210
|
resetAt: resets.length && resets.every((r) => r !== undefined) ? Math.max(...resets) : undefined,
|
|
207
211
|
};
|
|
208
212
|
}));
|
|
@@ -210,7 +214,8 @@ async function poolUsageNotice(pool, resolved, store) {
|
|
|
210
214
|
// Snapshot expiry is only a recheck. Use actual window resets for the notice.
|
|
211
215
|
const resets = accounts.map(a => a.resetAt);
|
|
212
216
|
const resetAt = thresholdDeferred && resets.every((r) => r !== undefined) ? Math.min(...resets) : undefined;
|
|
213
|
-
|
|
217
|
+
const providerResets = accounts.map(a => a.providerResetAt);
|
|
218
|
+
return { accounts, thresholdDeferred, resetAt: resetAt ?? (providerResets.length && providerResets.every((r) => r !== undefined) ? Math.min(...providerResets) : undefined), blockedWindows: accounts.flatMap(a => a.blockedWindows) };
|
|
214
219
|
}
|
|
215
220
|
catch {
|
|
216
221
|
return {};
|
|
@@ -224,8 +229,9 @@ export async function poolDeferredError(cause, pool, resolved, store, options =
|
|
|
224
229
|
return new PoolDeferredError(cause, { error: options.error ?? (pool.kind === 'fetch_unavailable' && pool.cause ? { code: pool.cause } : undefined) });
|
|
225
230
|
const notice = await poolUsageNotice(pool, resolved, store);
|
|
226
231
|
const deferredError = (resetAt) => {
|
|
232
|
+
resetAt ??= !notice.thresholdDeferred && notice.resetAt !== undefined ? new Date(notice.resetAt).toISOString() : undefined;
|
|
227
233
|
const error = new PoolDeferredError('usage', { eligibleAt, resetAt });
|
|
228
|
-
error.notice = { ...notice, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
|
|
234
|
+
error.notice = { ...notice, eligibleAt: error.eligibleAt, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
|
|
229
235
|
return error;
|
|
230
236
|
};
|
|
231
237
|
if (eligibleAt === undefined) {
|