@teamvibe/poller 0.8.7-rc.1 → 0.8.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -28,6 +28,8 @@ export interface SpawnResult {
28
28
  poolAttempts?: readonly PoolAttempt[];
29
29
  quota?: ClaudeQuotaFailure;
30
30
  rateLimitInfo?: ClaudeRateLimitInfo;
31
+ /** Rejections anywhere in this attempt survive a later allowed stream event. */
32
+ rateLimitRejections?: readonly ClaudeRateLimitInfo[];
31
33
  /** Actual ID passed to this CLI attempt, including a failed first turn. */
32
34
  attemptedSessionId?: string;
33
35
  success: boolean;
@@ -20,7 +20,7 @@ import { fetchSecretsForSpawn, buildSecretsReminder } from './secrets-client.js'
20
20
  import { fetchConfigForSpawn } from './config-client.js';
21
21
  import { parseClaudeRateLimitInfo } from './claude-rate-limit.js';
22
22
  import { parseClaudeQuotaFailure } from './claude-quota.js';
23
- import { abortLocalProbe, accountPin, preparePool, fetchRuntimePool, validateAccount, poolDeferredError, PoolDeferredError } from './subscription-runtime.js';
23
+ import { abortLocalProbe, accountPin, preparePool, fetchRuntimePool, validateAccount, subscriptionUsageStore, poolDeferredError, PoolDeferredError } from './subscription-runtime.js';
24
24
  import { selectPoolForResolvedModel } from './subscription-pool.js';
25
25
  import { isQuotaPaused, QuotaPausedError } from './quota-pause.js';
26
26
  import { redactStartupText } from './startup-auth.js';
@@ -911,6 +911,14 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
911
911
  });
912
912
  if (runtime?.account)
913
913
  await validateAccount(runtime.account, runtime.resolved);
914
+ if (runtime?.account && !runtime.shouldStop()) {
915
+ const store = subscriptionUsageStore();
916
+ const attemptAt = Date.now();
917
+ const windows = await store.beginVerification(runtime.account, runtime.resolved.model, attemptAt);
918
+ if (windows === undefined)
919
+ throw await poolDeferredError('usage', { kind: 'ready', accounts: [runtime.account] }, runtime.resolved, store);
920
+ runtime.verificationAttemptAt = windows.length ? attemptAt : undefined;
921
+ }
914
922
  return new Promise((resolve) => {
915
923
  // Preparation above awaits remote config/context. Another thread may
916
924
  // have exhausted the shared account meanwhile. No awaits before spawn.
@@ -1083,6 +1091,16 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
1083
1091
  // Retain CLI error/result evidence, not the full parsed assistant/tool stream.
1084
1092
  const failureEvents = [];
1085
1093
  let rateLimitInfo;
1094
+ const rateLimitRejections = new Map();
1095
+ const captureRateLimit = (value) => {
1096
+ rateLimitInfo = parseClaudeRateLimitInfo(value);
1097
+ if (!runtime?.account || rateLimitInfo?.status !== 'rejected')
1098
+ return;
1099
+ for (const w of normalizePassiveUsage(rateLimitInfo, Date.now()).windows) {
1100
+ if (w.status === 'rejected')
1101
+ rateLimitRejections.set(`${w.window}:${w.scope}`, rateLimitInfo);
1102
+ }
1103
+ };
1086
1104
  let metrics;
1087
1105
  let lastAssistant;
1088
1106
  // teamvibe.ai#284 (poller-brain#235): end-of-turn guard state — tracked
@@ -1143,7 +1161,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
1143
1161
  // (including metadata keys) to logs, telemetry, typing, or Slack.
1144
1162
  const event = safeEvent(rawEvent);
1145
1163
  if (rawEvent?.type === 'rate_limit_event') {
1146
- rateLimitInfo = parseClaudeRateLimitInfo(event.rate_limit_info);
1164
+ captureRateLimit(event.rate_limit_info);
1147
1165
  }
1148
1166
  if (rawEvent?.type === 'result' || rawEvent?.type === 'error' || (rawEvent?.type === 'assistant' && rawEvent.error)) {
1149
1167
  failureEvents.push(rawEvent);
@@ -1367,7 +1385,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
1367
1385
  const rawEvent = JSON.parse(stdoutBuffer);
1368
1386
  const event = safeEvent(rawEvent);
1369
1387
  if (event?.type === 'rate_limit_event') {
1370
- rateLimitInfo = parseClaudeRateLimitInfo(event.rate_limit_info);
1388
+ captureRateLimit(event.rate_limit_info);
1371
1389
  }
1372
1390
  failureEvents.push(rawEvent);
1373
1391
  if (event?.type === 'assistant') {
@@ -1386,6 +1404,7 @@ async function runClaudeCode(msg, sessionLog, cwd, directory, sessionId, isFirst
1386
1404
  attemptedSessionId: sessionId,
1387
1405
  ...(quota ? { quota } : {}),
1388
1406
  ...(rateLimitInfo ? { rateLimitInfo } : {}),
1407
+ ...(rateLimitRejections.size ? { rateLimitRejections: [...rateLimitRejections.values()] } : {}),
1389
1408
  output: redactOutput(stdout),
1390
1409
  exitCode: code,
1391
1410
  metrics,
@@ -1610,7 +1629,7 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1610
1629
  const modelNormalizedFrom = scoped?.model === resolved.model && scoped?.modelSource === resolved.modelSource
1611
1630
  ? scoped?.modelNormalizedFrom : undefined;
1612
1631
  const runtime = poolRuntime ? {
1613
- resolved, modelNormalizedFrom, claudeConfigDir, account: prepared?.account,
1632
+ resolved, modelNormalizedFrom, claudeConfigDir, account: prepared?.account, verificationAttemptAt: undefined,
1614
1633
  shouldStop: poolRuntime.shouldStop,
1615
1634
  resumePending: fetchedPool?.kind !== 'no_accounts_configured' && poolRuntime.pin?.resumePending === true,
1616
1635
  onSpawn: () => {
@@ -1649,10 +1668,12 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1649
1668
  if (!runtime?.resumePending)
1650
1669
  sawSendMessageCall = false;
1651
1670
  const spawned = await runClaudeCode(msg, sessionLog, effectiveCwd, directory, id, runtime?.resumePending ? false : first, lastMessageTs, onMessageSent, typingIndicator, runtime, gitDelivery);
1671
+ const rejected = spawned.rateLimitRejections?.find(rejectedForModel);
1652
1672
  const result = prepared?.account
1653
- ? { ...spawned, quota: quotaForModel(spawned.quota), success: rejectedForModel(spawned.rateLimitInfo) ? false : spawned.success } : spawned;
1673
+ ? { ...spawned, ...(rejected ? { rateLimitInfo: rejected } : {}), quota: quotaForModel(spawned.quota), success: rejected || rejectedForModel(spawned.rateLimitInfo) ? false : spawned.success } : spawned;
1654
1674
  sawSendMessageCall ||= result.sawSendMessageCall === true;
1655
1675
  if (prepared?.account) {
1676
+ const observedAt = Date.now();
1656
1677
  if (poolAttempts.length >= 1000)
1657
1678
  poolAttempts.shift();
1658
1679
  const selection = prepared.selection;
@@ -1664,7 +1685,8 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1664
1685
  accountId: prepared.account.accountId,
1665
1686
  credentialVersion: prepared.account.credentialVersion,
1666
1687
  accountLabel: prepared.account.label ?? prepared.account.accountId,
1667
- observedAt: Date.now(),
1688
+ observedAt,
1689
+ verificationAttemptAt: runtime?.verificationAttemptAt,
1668
1690
  rateLimitInfo: result.rateLimitInfo,
1669
1691
  sessionId: id,
1670
1692
  model: resolved.model ?? result.metrics?.model,
@@ -1675,13 +1697,19 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1675
1697
  metrics: result.metrics,
1676
1698
  });
1677
1699
  emitEvent('pool.account_attempted', msg.teamvibe.channelId, { workspaceId: msg.teamvibe.workspaceId, threadId: msg.thread_id, ...poolRunTelemetry(poolAttempts, poolRuntime?.quotaDeferEpisodes) });
1700
+ // Retain every rejected scope before completion can clear a verified
1701
+ // hold or return early. A failed write must not skip another scope.
1702
+ const rejectionOutcomes = await Promise.allSettled((result.rateLimitRejections ?? []).map(info => logPoolUsageWrite(sessionLog, prepared.account, info, () => prepared.store.recordPassive(prepared.account, info, observedAt, { expireFutureObservations: true }))));
1678
1703
  if (result.success && id && poolRuntime) {
1679
- // Complete the inbox before any fallible post-inference store writes.
1680
- // The loop retains bookkeeping independently of the original prompt.
1704
+ // Complete the inbox even if rejection bookkeeping failed. The loop
1705
+ // retains completion independently of the original prompt.
1681
1706
  const bookkeepingComplete = await poolRuntime.complete(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: false }, poolAttempts, result.rateLimitInfo);
1682
1707
  if (bookkeepingComplete !== false)
1683
1708
  await emitCandidateUsage(msg, [prepared.account], prepared.store);
1684
1709
  retainWorktree = false;
1710
+ const failed = rejectionOutcomes.find((outcome) => outcome.status === 'rejected');
1711
+ if (failed)
1712
+ throw new PoolDeferredError('infrastructure', { error: failed.reason });
1685
1713
  return result;
1686
1714
  }
1687
1715
  if ((result.quota || rejectedForModel(result.rateLimitInfo)) && !result.killedByPoller) {
@@ -1692,15 +1720,17 @@ export async function spawnPoolClaudeCode(msg, sessionLog, selectedDirectory, se
1692
1720
  // retaining the attempt. Selection waits for all writes before switching.
1693
1721
  const writes = [];
1694
1722
  if (result.quota && !result.killedByPoller)
1695
- writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.quota.window ? { rateLimitType: result.quota.window } : undefined, () => prepared.store.recordQuota(prepared.account, result.quota, Date.now(), { expireFutureObservations: true }), 'quota'));
1696
- if (result.rateLimitInfo)
1697
- writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.rateLimitInfo, () => prepared.store.recordPassive(prepared.account, result.rateLimitInfo, Date.now(), { expireFutureObservations: true })));
1723
+ writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.quota.window ? { rateLimitType: result.quota.window } : undefined, () => prepared.store.recordQuota(prepared.account, result.quota, observedAt, { expireFutureObservations: true }), 'quota'));
1724
+ if (!result.rateLimitRejections?.length && result.rateLimitInfo)
1725
+ writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.rateLimitInfo, () => prepared.store.recordPassive(prepared.account, result.rateLimitInfo, observedAt, { expireFutureObservations: true })));
1698
1726
  if (id && poolRuntime)
1699
1727
  writes.push(poolRuntime.persist(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: runtime?.resumePending, sawSendMessageCall }, poolAttempts));
1700
- const outcomes = await Promise.allSettled(writes);
1728
+ const outcomes = [...rejectionOutcomes, ...await Promise.allSettled(writes)];
1701
1729
  const failed = outcomes.find((outcome) => outcome.status === 'rejected');
1702
1730
  if (failed)
1703
1731
  throw new PoolDeferredError('infrastructure', { error: failed.reason });
1732
+ if (runtime?.verificationAttemptAt !== undefined && !result.killedByPoller && result.exitCode !== null)
1733
+ await prepared.store.completeVerification(prepared.account, runtime.verificationAttemptAt, observedAt);
1704
1734
  }
1705
1735
  return result;
1706
1736
  };
@@ -4,6 +4,8 @@ import { logger } from './logger.js';
4
4
  const MAX_QUEUE_SIZE = 200;
5
5
  const FLUSH_INTERVAL_MS = 30_000;
6
6
  const DEDUP_WINDOW_MS = 60_000;
7
+ const rejectedWarnings = new Map();
8
+ const REJECTION_WARN_INTERVAL_MS = 3600_000;
7
9
  let queue = [];
8
10
  let flushTimer = null;
9
11
  let isFlushing = false;
@@ -60,6 +62,15 @@ async function flushOne(event) {
60
62
  logger.info(`[Pool] usage probe request=${event.metadata['requestId']} completion downgraded for legacy platform`);
61
63
  response = await post();
62
64
  }
65
+ if ([400, 404, 422].includes(response.status)) {
66
+ const now = Date.now();
67
+ const previous = rejectedWarnings.get(event.eventType);
68
+ if (previous === undefined || now - previous >= REJECTION_WARN_INTERVAL_MS) {
69
+ rejectedWarnings.set(event.eventType, now);
70
+ logger.warn(`[EventQueue] ${event.eventType} rejected with HTTP ${response.status}; dropping event`);
71
+ }
72
+ return true;
73
+ }
63
74
  return response.ok;
64
75
  }
65
76
  catch {
@@ -1,3 +1,4 @@
1
+ import type { PoolDefer } from './pool-defer.js';
1
2
  import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
2
3
  import type { PoolAttempt } from './subscription-runtime.js';
3
4
  import type { TeamVibeQueueMessage, SubscriptionPin } from './types.js';
@@ -81,3 +82,6 @@ export declare function discardHandledInboxDrops(brainPath: string, threadId: st
81
82
  export declare function restartInboxInDirectory(source: string, target: string, threadId: string): void;
82
83
  /** Preserve unroutable work for operator review and remove it from replay queues. */
83
84
  export declare function quarantineInbox(brainPath: string, threadId: string, includeFailed?: boolean): void;
85
+ /** Only original queue deliveries carry inbound identity; synthetic drops never wake holds. */
86
+ export declare function inboxQueueMessageIds(brainPath: string, threadId: string): string[];
87
+ export declare function hasInboxMessageAfter(brainPath: string, threadId: string, pause: Pick<PoolDefer, 'handledInboundIds'>): boolean;
@@ -305,3 +305,32 @@ export function quarantineInbox(brainPath, threadId, includeFailed = false) {
305
305
  renameSync(join(root, name), join(quarantine, `${Date.now()}-${name}`));
306
306
  }
307
307
  }
308
+ /** Only original queue deliveries carry inbound identity; synthetic drops never wake holds. */
309
+ export function inboxQueueMessageIds(brainPath, threadId) {
310
+ const ids = new Set();
311
+ for (const subdir of [PROCESSING_DIR, NEW_DIR]) {
312
+ const directory = join(getInboxPath(brainPath, threadId), subdir);
313
+ let names;
314
+ try {
315
+ names = readdirSync(directory);
316
+ }
317
+ catch {
318
+ continue;
319
+ }
320
+ for (const name of names) {
321
+ if (!name.endsWith('.txt'))
322
+ continue;
323
+ try {
324
+ const message = JSON.parse(readFileSync(join(directory, name), 'utf8'));
325
+ if (message.executionOrigin === 'queue' && message.thread_id === threadId && typeof message.id === 'string' && message.id)
326
+ ids.add(message.id);
327
+ }
328
+ catch { /* Plain background drops and concurrent moves have no inbound identity. */ }
329
+ }
330
+ }
331
+ return [...ids];
332
+ }
333
+ export function hasInboxMessageAfter(brainPath, threadId, pause) {
334
+ const handled = new Set(pause.handledInboundIds ?? []);
335
+ return inboxQueueMessageIds(brainPath, threadId).some(id => !handled.has(id));
336
+ }
@@ -1,7 +1,7 @@
1
1
  import { executionDirectory } from './brain-directory.js';
2
- import { discardHandledInboxDrops, markProcessed, quarantineInbox, readInboxChannelContext, readInboxContext, readInboxRuntime, restartInboxInDirectory } from './inbox-manager.js';
2
+ import { hasInboxMessageAfter, discardHandledInboxDrops, markProcessed, quarantineInbox, readInboxChannelContext, readInboxContext, readInboxRuntime, restartInboxInDirectory } from './inbox-manager.js';
3
3
  import { listThreadRoutings, readThreadRouting, removeThreadRouting, threadRoutingKey } from './thread-routing.js';
4
- import { isPoolDeferred, readPoolDefer } from './pool-defer.js';
4
+ import { observePoolAccounts, isPoolDeferred, readPoolDefer } from './pool-defer.js';
5
5
  import { existsSync, readdirSync, rmSync, watch as fsWatch } from 'fs';
6
6
  import { join } from 'path';
7
7
  import { config } from './config.js';
@@ -11,6 +11,7 @@ import { logger } from './logger.js';
11
11
  import { isAtCapacity } from './claude-spawner.js';
12
12
  import { getTrackedBrains } from './maintenance-cron.js';
13
13
  import { runInboxLoop, isInboxLoopActive } from './poller.js';
14
+ import { fetchRuntimePool } from './subscription-runtime.js';
14
15
  import { isQuotaPaused } from './quota-pause.js';
15
16
  import { emitError } from './event-queue.js';
16
17
  import { sendSlackMessage } from './slack-client.js';
@@ -151,6 +152,7 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
151
152
  const expired = await sweepThreadRouting(deps, brains);
152
153
  const started = new Set();
153
154
  const recoveries = [];
155
+ const checkedPools = new Set();
154
156
  scan: for (const brainPath of brains) {
155
157
  if (deps.isAtCapacity())
156
158
  break;
@@ -160,7 +162,7 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
160
162
  break scan;
161
163
  if (expired.has(threadRoutingKey(threadId)))
162
164
  continue;
163
- if (started.has(threadId) || recoveringThreads.has(threadId) || deps.isInboxLoopActive(threadId) || isPoolDeferred(threadId))
165
+ if (started.has(threadId) || recoveringThreads.has(threadId) || deps.isInboxLoopActive(threadId))
164
166
  continue;
165
167
  discardHandledInboxDrops(brainPath, threadId);
166
168
  const runtime = readInboxRuntime(brainPath, threadId);
@@ -193,6 +195,17 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
193
195
  quarantineRetainedThread(deps, brainPath, threadId, durable);
194
196
  continue;
195
197
  }
198
+ if (isPoolDeferred(threadId)) {
199
+ // Held work still observes pool membership on every watcher tick.
200
+ const key = JSON.stringify([info.workspaceId, info.channelId]);
201
+ if (!checkedPools.has(key)) {
202
+ checkedPools.add(key);
203
+ observePoolAccounts(info.workspaceId, await fetchRuntimePool(message));
204
+ }
205
+ const pause = readPoolDefer(threadId);
206
+ if (isPoolDeferred(threadId) && pause && !hasInboxMessageAfter(brainPath, threadId, pause))
207
+ continue;
208
+ }
196
209
  started.add(threadId);
197
210
  recoveringThreads.add(threadId);
198
211
  recoveries.push((async () => {
package/dist/poller.js CHANGED
@@ -14,7 +14,7 @@ import { classifyFailedIteration } from './classify-failed-iteration.js';
14
14
  import { fetchSecretsForSpawn } from './secrets-client.js';
15
15
  import { PoolDeferredError, fetchRuntimePool, subscriptionUsageStore, HELD_PROBE_INTERVAL_MS } from './subscription-runtime.js';
16
16
  import { POOL_RECHECK_MS } from './subscription-selection.js';
17
- import { isPoolDeferred, readPoolDefer, deferPool, setPoolNotice, clearPoolDefer } from './pool-defer.js';
17
+ import { invalidatePoolDefer, invalidateWorkspacePoolDefers, observePoolAccounts, isPoolDeferred, readPoolDefer, deferPool, setPoolNotice, clearPoolDefer } from './pool-defer.js';
18
18
  import { isQuotaPaused, pauseForQuota, clearQuotaPauseAfterSuccess, QuotaPausedError } from './quota-pause.js';
19
19
  import { acquireSessionLock, releaseSessionLock, updateSessionId } from './session-store.js';
20
20
  import { ensureDirectories, ensureBaseBrain, pushBrainChanges } from './brain-manager.js';
@@ -26,7 +26,7 @@ import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './main
26
26
  import { bootstrapTrackedBrains } from './brains-bootstrap.js';
27
27
  import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
28
28
  import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
29
- import { writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage, saveInboxContext, saveInboxRuntime, readInboxRuntime, clearInboxRuntime } from './inbox-manager.js';
29
+ import { inboxQueueMessageIds, hasInboxMessageAfter, writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage, saveInboxContext, saveInboxRuntime, readInboxRuntime, clearInboxRuntime } from './inbox-manager.js';
30
30
  import { saveThreadRouting } from './thread-routing.js';
31
31
  import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
32
32
  import { Cron } from 'croner';
@@ -257,12 +257,17 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
257
257
  if (isQuotaPaused() && !getPollerId())
258
258
  return;
259
259
  const initialPool = await fetchRuntimePool(queueMessage);
260
+ observePoolAccounts(queueMessage.teamvibe.workspaceId, initialPool, { log: line => sessionLog.info(line) });
260
261
  if (initialPool.kind === 'no_accounts_configured') {
261
262
  if (isQuotaPaused())
262
263
  return;
263
264
  }
264
- else if (isPoolDeferred(threadId))
265
- return;
265
+ else if (isPoolDeferred(threadId)) {
266
+ const pause = readPoolDefer(threadId);
267
+ if (!hasInboxMessageAfter(kbPath, threadId, pause))
268
+ return;
269
+ invalidatePoolDefer(threadId, queueMessage.teamvibe.workspaceId, 'new_message', { log: line => sessionLog.info(line) }, inboxQueueMessageIds(kbPath, threadId));
270
+ }
266
271
  if (activeLoops.has(threadId)) {
267
272
  sessionLog.info(`Loop already running for thread ${threadId}, skipping`);
268
273
  return;
@@ -318,8 +323,10 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
318
323
  const lastAttempt = state.attempts?.at(-1);
319
324
  // Existing telemetry falls back to account IDs. Local pool logs use labels only.
320
325
  const accountLabel = lastAttempt?.accountLabel !== state.pin.accountId ? lastAttempt?.accountLabel : undefined;
326
+ // Reuse the spawn's observation clock: the spawner already recorded all
327
+ // rejected scopes, and replaying its final event is the same evidence.
321
328
  if (state.completed.rateLimitInfo)
322
- writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
329
+ writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, lastAttempt?.observedAt ?? state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
323
330
  writes.push(persistSessionId(state.sessionId, state.pin));
324
331
  const outcomes = await Promise.allSettled(writes);
325
332
  if (outcomes.some(outcome => outcome.status === 'rejected' || outcome.value === false)) {
@@ -332,6 +339,21 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
332
339
  }
333
340
  return false;
334
341
  }
342
+ if (lastAttempt?.verificationAttemptAt !== undefined) {
343
+ try {
344
+ await subscriptionUsageStore().completeVerification(state.pin, lastAttempt.verificationAttemptAt, state.completed.observedAt);
345
+ }
346
+ catch {
347
+ sessionLog.warn('[Loop] Verification bookkeeping failed and will retry without running the agent');
348
+ try {
349
+ saveInboxRuntime(kbPath, threadId, { ...state, completed: { ...state.completed, retryAt: Date.now() + POOL_RECHECK_MS } });
350
+ }
351
+ catch {
352
+ sessionLog.warn('[Loop] Could not save bookkeeping retry; completion remains recorded');
353
+ }
354
+ return false;
355
+ }
356
+ }
335
357
  currentSessionId = state.sessionId;
336
358
  currentPin = state.pin;
337
359
  // The spawner publishes live completions only after bookkeeping succeeds.
@@ -370,6 +392,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
370
392
  // Dispatch before ANY pool state/context/model preparation. The explicit
371
393
  // no-pool outcome runs the original spawn and #408 pause gates.
372
394
  const pool = iteration === 0 ? initialPool : await fetchRuntimePool(queueMessage);
395
+ observePoolAccounts(queueMessage.teamvibe.workspaceId, pool, { log: line => sessionLog.info(line) });
373
396
  poolIteration = pool.kind !== 'no_accounts_configured';
374
397
  if (!poolIteration && isQuotaPaused())
375
398
  break;
@@ -619,9 +642,9 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
619
642
  quotaPaused = true;
620
643
  deferred = true;
621
644
  // Revisit held work hourly so background probes can refresh the account.
622
- // Selection still enforces each window's real reset before inference.
645
+ // Selection enforces resets or a contradicted rejection's verification backoff.
623
646
  const recheckAt = error.cause === 'usage' ? Math.min(error.eligibleAt, Date.now() + HELD_PROBE_INTERVAL_MS) : error.eligibleAt;
624
- const pause = deferPool(threadId, recheckAt, error.resetAt);
647
+ const pause = deferPool(threadId, recheckAt, error.resetAt, inboxQueueMessageIds(kbPath, threadId));
625
648
  const retained = readInboxRuntime(kbPath, threadId);
626
649
  // Rechecks of one exhaustion episode do not inflate the episode count.
627
650
  quotaDeferEpisodes = Math.max(1, quotaDeferEpisodes);
@@ -638,7 +661,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
638
661
  cause: error.cause,
639
662
  thresholdDeferred: error.notice?.thresholdDeferred && !error.attempts?.some(a => a.quota),
640
663
  resetAt: pause.resetAt ? Date.parse(pause.resetAt) : error.notice?.resetAt,
641
- timeZone: config.POLLER_TZ,
664
+ timeZone: deliveryMsg.teamvibe.timeZone ?? error.notice?.timeZone,
642
665
  });
643
666
  await sendSlackMessage(deliveryMsg, notice, { clientMessageId: pause.episodeId, ...(reconcile ? { reconcile: true } : {}) });
644
667
  setPoolNotice(threadId, pause.episodeId, 'delivered');
@@ -901,7 +924,7 @@ async function processMessage(received) {
901
924
  const kbPath = directory.path;
902
925
  // Write raw message to inbox (spawnClaudeCode handles prompt formatting)
903
926
  const { botToken: _inboxToken, ...inboxTeamvibe } = queueMessage.teamvibe;
904
- writeMessage(kbPath, threadId, JSON.stringify({ ...queueMessage, teamvibe: inboxTeamvibe }));
927
+ writeMessage(kbPath, threadId, JSON.stringify({ ...queueMessage, executionOrigin: receiptHandle ? 'queue' : 'maintenance', teamvibe: inboxTeamvibe }));
905
928
  sessionLog.info(`Wrote message to inbox`);
906
929
  // Delete SQS message immediately — it's now persisted in the inbox
907
930
  if (receiptHandle) {
@@ -1029,6 +1052,7 @@ export async function pollLoop() {
1029
1052
  apiUrl: config.TEAMVIBE_API_URL,
1030
1053
  pollerToken: config.TEAMVIBE_POLLER_TOKEN || '',
1031
1054
  }),
1055
+ onHeadroom: workspaceId => invalidateWorkspacePoolDefers(workspaceId, 'usage_update'),
1032
1056
  store: subscriptionUsageStore(),
1033
1057
  cliPath: config.CLAUDE_CLI_PATH,
1034
1058
  isAtCapacity,
@@ -1,4 +1,16 @@
1
+ import { emitEvent } from './event-queue.js';
2
+ import type { PoolFetchResult } from './subscription-selection.js';
1
3
  export interface PoolDefer {
4
+ /** Additive ownership index; older markers use the existing hashed routing file. */
5
+ scope?: {
6
+ workspaceId: string;
7
+ threadId: string;
8
+ channelId: string;
9
+ };
10
+ /** Enabled capacity known when this hold was last evaluated. */
11
+ poolAccountIds?: string[];
12
+ /** Queue arrivals already considered in this exhaustion episode. */
13
+ handledInboundIds?: string[];
2
14
  createdAt: number;
3
15
  resumeAt: number;
4
16
  resetAt?: string;
@@ -7,7 +19,18 @@ export interface PoolDefer {
7
19
  }
8
20
  export declare function readPoolDefer(threadId: string): PoolDefer | undefined;
9
21
  export declare function isPoolDeferred(threadId: string): boolean;
10
- export declare function deferPool(threadId: string, eligibleAt: number, resetAt?: string): PoolDefer;
22
+ export declare function deferPool(threadId: string, eligibleAt: number, resetAt?: string, handledInboundIds?: string[]): PoolDefer;
11
23
  /** Claim durably before Slack I/O. Ambiguous sends are never blindly replayed. */
12
24
  export declare function setPoolNotice(threadId: string, episodeId: string, notice: PoolDefer['notice']): boolean;
13
25
  export declare function clearPoolDefer(threadId: string): void;
26
+ export type PoolDeferInvalidationReason = 'usage_update' | 'pool_changed' | 'new_message';
27
+ interface InvalidationObserver {
28
+ emit?: typeof emitEvent;
29
+ log?: (line: string) => void;
30
+ }
31
+ /** Expire atomically, retaining the episode and notice claim until a successful spawn. */
32
+ export declare function invalidatePoolDefer(threadId: string, workspaceId: string, reason: PoolDeferInvalidationReason, observer?: InvalidationObserver, handledInboundIds?: string[]): void;
33
+ export declare function invalidateWorkspacePoolDefers(workspaceId: string, reason: Exclude<PoolDeferInvalidationReason, 'new_message'>, observer?: InvalidationObserver): void;
34
+ /** Only authoritative enabled account sets advance the baseline; never cache credentials. */
35
+ export declare function observePoolAccounts(workspaceId: string, pool: PoolFetchResult, observer?: InvalidationObserver): void;
36
+ export {};
@@ -1,19 +1,27 @@
1
1
  import { createHash, randomUUID } from 'node:crypto';
2
- import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
2
+ import { mkdirSync, readdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
3
3
  import { join } from 'node:path';
4
4
  import { config } from './config.js';
5
5
  import { logger } from './logger.js';
6
+ import { emitEvent } from './event-queue.js';
7
+ import { readThreadRouting, threadRoutingKey } from './thread-routing.js';
6
8
  import { POOL_RECHECK_MS } from './subscription-selection.js';
7
9
  function pathFor(threadId) {
8
10
  return join(config.TEAMVIBE_DATA_DIR, 'pool-defer', `${createHash('sha256').update(threadId).digest('hex')}.json`);
9
11
  }
10
12
  const warned = new Set();
11
13
  export function readPoolDefer(threadId) {
12
- const path = pathFor(threadId);
14
+ return readMarker(pathFor(threadId));
15
+ }
16
+ function readMarker(path) {
13
17
  try {
14
18
  const state = JSON.parse(readFileSync(path, 'utf8'));
15
- if (!Number.isFinite(state.createdAt) || !Number.isFinite(state.resumeAt) || state.resumeAt <= state.createdAt || state.resumeAt > 253402300799000 || typeof state.episodeId !== 'string' || !['pending', 'claimed', 'delivered', 'ambiguous'].includes(state.notice))
19
+ if (!Number.isFinite(state.createdAt) || !Number.isFinite(state.resumeAt) || state.resumeAt < state.createdAt || state.resumeAt > 253402300799000 || typeof state.episodeId !== 'string' || !['pending', 'claimed', 'delivered', 'ambiguous'].includes(state.notice))
16
20
  throw new Error('Invalid defer');
21
+ for (const ids of [state.poolAccountIds, state.handledInboundIds]) {
22
+ if (ids !== undefined && (!Array.isArray(ids) || ids.some(id => typeof id !== 'string')))
23
+ throw new Error('Invalid defer baseline');
24
+ }
17
25
  warned.delete(path);
18
26
  return state;
19
27
  }
@@ -56,10 +64,14 @@ function write(threadId, state) {
56
64
  catch { }
57
65
  }
58
66
  }
59
- export function deferPool(threadId, eligibleAt, resetAt) {
67
+ export function deferPool(threadId, eligibleAt, resetAt, handledInboundIds = []) {
60
68
  const previous = readPoolDefer(threadId);
61
69
  const now = Date.now();
62
- const state = { createdAt: now, resumeAt: Number.isFinite(eligibleAt) && eligibleAt > now ? eligibleAt : now + POOL_RECHECK_MS, episodeId: previous?.episodeId ?? randomUUID(), notice: previous?.notice ?? 'pending', ...(resetAt ? { resetAt } : {}) };
70
+ const routing = readThreadRouting(threadId);
71
+ const scope = routing ? { workspaceId: routing.workspaceId, channelId: routing.channelId, threadId } : previous?.scope;
72
+ const fingerprint = scope ? poolFingerprints.get(poolKey(scope.workspaceId)) : undefined;
73
+ const poolAccountIds = fingerprint === undefined ? previous?.poolAccountIds : JSON.parse(fingerprint);
74
+ const state = { ...(scope ? { scope } : {}), ...(poolAccountIds ? { poolAccountIds } : {}), handledInboundIds: [...new Set([...(previous?.handledInboundIds ?? []), ...handledInboundIds])], createdAt: now, resumeAt: Number.isFinite(eligibleAt) && eligibleAt > now ? eligibleAt : now + POOL_RECHECK_MS, episodeId: previous?.episodeId ?? randomUUID(), notice: previous?.notice ?? 'pending', ...(resetAt ? { resetAt } : {}) };
63
75
  write(threadId, state);
64
76
  return state;
65
77
  }
@@ -79,3 +91,87 @@ export function clearPoolDefer(threadId) {
79
91
  logger.warn('[PoolDefer] Completed episode cleanup will be retried');
80
92
  }
81
93
  }
94
+ /** Expire atomically, retaining the episode and notice claim until a successful spawn. */
95
+ export function invalidatePoolDefer(threadId, workspaceId, reason, observer = {}, handledInboundIds = []) {
96
+ const state = readPoolDefer(threadId);
97
+ if (!state || !isPoolDeferred(threadId))
98
+ return;
99
+ const routing = readThreadRouting(threadId);
100
+ const scope = state.scope ?? (routing ? { ...routing, threadId } : undefined);
101
+ if (!scope || scope.workspaceId !== workspaceId)
102
+ return;
103
+ write(threadId, { ...state, resumeAt: state.createdAt, handledInboundIds: [...new Set([...(state.handledInboundIds ?? []), ...handledInboundIds])] });
104
+ const metadata = { workspaceId, threadId, reason, episodeId: state.episodeId };
105
+ // Observability cannot undo the durable wake-up.
106
+ try {
107
+ (observer.log ?? (line => logger.info(line)))(`[Pool] defer invalidated workspace=${JSON.stringify(workspaceId)} thread=${JSON.stringify(threadId)} reason=${reason} episode=${JSON.stringify(state.episodeId)}`);
108
+ }
109
+ catch { }
110
+ try {
111
+ (observer.emit ?? emitEvent)('pool.defer_invalidated', scope.channelId, metadata);
112
+ }
113
+ catch { }
114
+ }
115
+ /** One directory scan, one bounded routing lookup per legacy marker; no inbox scan. */
116
+ function* workspacePoolDefers(workspaceId) {
117
+ const root = join(config.TEAMVIBE_DATA_DIR, 'pool-defer');
118
+ let names;
119
+ try {
120
+ names = readdirSync(root);
121
+ }
122
+ catch {
123
+ return;
124
+ }
125
+ for (const name of names) {
126
+ if (!/^[a-f0-9]{64}\.json$/.test(name))
127
+ continue;
128
+ try {
129
+ const state = readMarker(join(root, name));
130
+ if (!state)
131
+ continue;
132
+ let scope = state.scope;
133
+ if (!scope) {
134
+ // Pre-#572 markers share their hash with durable thread routing.
135
+ scope = JSON.parse(readFileSync(join(config.TEAMVIBE_DATA_DIR, 'thread-routing', name), 'utf8'));
136
+ }
137
+ if (scope?.workspaceId === workspaceId && typeof scope.threadId === 'string' && threadRoutingKey(scope.threadId) === name)
138
+ yield { state, scope };
139
+ }
140
+ catch {
141
+ logger.warn('[PoolDefer] Workspace invalidation will be retried');
142
+ }
143
+ }
144
+ }
145
+ export function invalidateWorkspacePoolDefers(workspaceId, reason, observer = {}) {
146
+ for (const { scope } of workspacePoolDefers(workspaceId))
147
+ invalidatePoolDefer(scope.threadId, workspaceId, reason, observer);
148
+ }
149
+ const poolFingerprints = new Map();
150
+ function poolKey(workspaceId) {
151
+ return JSON.stringify([config.TEAMVIBE_DATA_DIR, workspaceId]);
152
+ }
153
+ /** Only authoritative enabled account sets advance the baseline; never cache credentials. */
154
+ export function observePoolAccounts(workspaceId, pool, observer = {}) {
155
+ if (pool.kind !== 'ready' && pool.kind !== 'no_accounts_configured' && !(pool.kind === 'fetch_unavailable' && pool.reason === 'no_usable_accounts'))
156
+ return;
157
+ const poolAccountIds = pool.kind === 'ready' ? [...new Set(pool.accounts.filter(a => a.enabled && a.workspaceId === workspaceId).map(a => a.accountId))].sort() : [];
158
+ const fingerprint = JSON.stringify(poolAccountIds);
159
+ const key = poolKey(workspaceId);
160
+ if (poolFingerprints.get(key) === fingerprint)
161
+ return;
162
+ // The first fetch after boot also compares the durable per-marker baselines.
163
+ for (const { state, scope } of workspacePoolDefers(workspaceId)) {
164
+ const now = Date.now();
165
+ if (now < state.createdAt || now >= state.resumeAt)
166
+ continue;
167
+ if (state.poolAccountIds && poolAccountIds.some(id => !state.poolAccountIds.includes(id))) {
168
+ invalidatePoolDefer(scope.threadId, workspaceId, 'pool_changed', observer);
169
+ }
170
+ else {
171
+ // Reductions never add capacity. Adopt them so a later re-enable wakes work.
172
+ // A legacy marker adopts its first observed baseline without waking work.
173
+ write(scope.threadId, { ...state, poolAccountIds });
174
+ }
175
+ }
176
+ poolFingerprints.set(key, fingerprint);
177
+ }
@@ -10,6 +10,8 @@ import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
10
10
  import type { ClaudeQuotaFailure } from './claude-quota.js';
11
11
  import type { KilledByPoller } from './classify-failed-iteration.js';
12
12
  export interface PoolAttempt extends Partial<HeadroomScore> {
13
+ /** Local completion bookkeeping token; omitted from event transport DTOs. */
14
+ verificationAttemptAt?: number;
13
15
  workspaceId: string;
14
16
  accountId: string;
15
17
  accountLabel?: string;
@@ -1,4 +1,5 @@
1
1
  import { isAtCapacity, isSubscriptionAccountRunning } from './claude-spawner.js';
2
+ import { invalidateWorkspacePoolDefers } from './pool-defer.js';
2
3
  import { emitEvent } from './event-queue.js';
3
4
  import { logger } from './logger.js';
4
5
  import { runUsageProbe } from './usage-probe-job.js';
@@ -149,6 +150,7 @@ export async function preparePool(msg, resolved, pin, fetched, log, probeOptions
149
150
  if (pool.kind === 'ready')
150
151
  probeUnknownAccounts(pool.accounts, selected.kind === 'selected' ? selected.account.accountId : undefined, resolved, store, {
151
152
  cliPath: config.CLAUDE_CLI_PATH, isAtCapacity, emit: emitEvent,
153
+ onHeadroom: workspaceId => invalidateWorkspacePoolDefers(workspaceId, 'usage_update'),
152
154
  log: line => (log ?? logger).info(line), ...probeOptions, store,
153
155
  }, msg.teamvibe.channelId);
154
156
  if (selected.kind === 'selected') {
@@ -203,6 +205,8 @@ async function poolUsageNotice(pool, resolved, store) {
203
205
  return {
204
206
  label: account.label, useFullQuota: account.useFullQuota, windows: windows.filter(w => binding.some(b => b.window === w.window && b.scope === w.scope)), rejected,
205
207
  thresholdBlocked: blockers.length > 0,
208
+ blockedWindows: binding.map(w => ({ window: w.window, scope: w.scope })),
209
+ providerResetAt: binding.length && binding.every(w => w.resetAt !== undefined) ? Math.max(...binding.map(w => w.resetAt)) : undefined,
206
210
  resetAt: resets.length && resets.every((r) => r !== undefined) ? Math.max(...resets) : undefined,
207
211
  };
208
212
  }));
@@ -210,7 +214,8 @@ async function poolUsageNotice(pool, resolved, store) {
210
214
  // Snapshot expiry is only a recheck. Use actual window resets for the notice.
211
215
  const resets = accounts.map(a => a.resetAt);
212
216
  const resetAt = thresholdDeferred && resets.every((r) => r !== undefined) ? Math.min(...resets) : undefined;
213
- return { accounts, thresholdDeferred, resetAt };
217
+ const providerResets = accounts.map(a => a.providerResetAt);
218
+ return { accounts, thresholdDeferred, resetAt: resetAt ?? (providerResets.length && providerResets.every((r) => r !== undefined) ? Math.min(...providerResets) : undefined), blockedWindows: accounts.flatMap(a => a.blockedWindows) };
214
219
  }
215
220
  catch {
216
221
  return {};
@@ -224,8 +229,9 @@ export async function poolDeferredError(cause, pool, resolved, store, options =
224
229
  return new PoolDeferredError(cause, { error: options.error ?? (pool.kind === 'fetch_unavailable' && pool.cause ? { code: pool.cause } : undefined) });
225
230
  const notice = await poolUsageNotice(pool, resolved, store);
226
231
  const deferredError = (resetAt) => {
232
+ resetAt ??= !notice.thresholdDeferred && notice.resetAt !== undefined ? new Date(notice.resetAt).toISOString() : undefined;
227
233
  const error = new PoolDeferredError('usage', { eligibleAt, resetAt });
228
- error.notice = { ...notice, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
234
+ error.notice = { ...notice, eligibleAt: error.eligibleAt, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
229
235
  return error;
230
236
  };
231
237
  if (eligibleAt === undefined) {