@teamvibe/poller 0.8.7 → 0.8.9-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/README.md +5 -0
  2. package/bin/tv-git-credential.mjs +11 -6
  3. package/dist/brain-manager.js +4 -4
  4. package/dist/claude-spawner.d.ts +4 -2
  5. package/dist/claude-spawner.js +50 -16
  6. package/dist/config.d.ts +4 -0
  7. package/dist/config.js +1 -0
  8. package/dist/connections-client.d.ts +16 -4
  9. package/dist/connections-client.js +42 -11
  10. package/dist/end-of-turn-sentinel.d.ts +2 -0
  11. package/dist/end-of-turn-sentinel.js +2 -0
  12. package/dist/git-credential-bindings.d.ts +2 -0
  13. package/dist/git-credential-bindings.js +4 -0
  14. package/dist/git-credential-env.d.ts +2 -2
  15. package/dist/git-credential-env.js +22 -9
  16. package/dist/git-credential-session.d.ts +6 -6
  17. package/dist/git-credential-session.js +24 -4
  18. package/dist/inbox-watcher.d.ts +2 -2
  19. package/dist/inbox-watcher.js +20 -13
  20. package/dist/poller.d.ts +4 -3
  21. package/dist/poller.js +63 -11
  22. package/dist/redact-text.js +1 -1
  23. package/dist/slack-client.d.ts +3 -1
  24. package/dist/slack-client.js +84 -12
  25. package/dist/subscription-runtime.d.ts +2 -0
  26. package/dist/subscription-runtime.js +6 -2
  27. package/dist/subscription-selection.d.ts +21 -2
  28. package/dist/subscription-selection.js +91 -14
  29. package/dist/subscription-usage-store.d.ts +6 -1
  30. package/dist/subscription-usage-store.js +246 -13
  31. package/dist/thread-routing.d.ts +7 -0
  32. package/dist/thread-routing.js +7 -0
  33. package/dist/types.d.ts +2 -0
  34. package/dist/usage-limit-messages.d.ts +3 -0
  35. package/dist/usage-limit-messages.js +22 -0
  36. package/dist/usage-probe-job.js +31 -3
  37. package/npm-shrinkwrap.json +2 -2
  38. package/package.json +1 -1
@@ -38,8 +38,8 @@ function defaultDeps() {
38
38
  * threads (see modules-teamvibe/hooks/slack-events.ts). Synthetic threadIds
39
39
  * used elsewhere (`maintenance_<brainId>_<ts>`, and
40
40
  * `scheduled:<scheduleId>:<ts>` from
41
- * modules-teamvibe/src/functions/scheduler.ts:86) never sit idle waiting on
42
- * a background-task drop, so they're skipped rather than guessed at.
41
+ * modules-teamvibe/src/functions/scheduler.ts) carry no Slack destination;
42
+ * scheduler recovery needs the poller-owned routing record exclusively.
43
43
  * `scheduled:*:*` has the same 3-colon-segment shape as a real Slack
44
44
  * threadId, so it's rejected explicitly rather than by segment count alone
45
45
  * — otherwise a leftover unprocessed file in a stale scheduled-session inbox
@@ -65,20 +65,16 @@ function platformRouting(info) {
65
65
  brain: { brainId: info.brainId, gitRepoUrl: info.gitRepoUrl, branch: info.branch },
66
66
  };
67
67
  }
68
- function buildSyntheticMessage(info, threadId, parsed) {
68
+ function buildSyntheticMessage(info, threadId, slack, source = 'slack') {
69
69
  return {
70
70
  id: `inbox_watch_${threadId}`,
71
- source: 'slack',
71
+ source,
72
72
  thread_id: threadId,
73
73
  sender: { id: 'system', name: 'Background Task' },
74
74
  text: '',
75
75
  attachments: [],
76
76
  response_context: {
77
- slack: {
78
- channel: parsed.channel,
79
- thread_ts: parsed.threadTs,
80
- message_ts: parsed.threadTs,
81
- },
77
+ slack,
82
78
  },
83
79
  type: 'message',
84
80
  teamvibe: platformRouting(info),
@@ -174,9 +170,13 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
174
170
  continue;
175
171
  const durable = readInboxContext(brainPath, threadId);
176
172
  const parsed = parseSlackThreadId(threadId);
173
+ const scheduled = threadId.startsWith('scheduled:');
177
174
  // Context or poller-owned per-thread routing selects a channel in the
178
175
  // full map. Never infer ownership from a directory's current channels.
179
- const storedRoute = readInboxChannelContext(brainPath, threadId) ?? readThreadRouting(threadId);
176
+ const routingRecord = readThreadRouting(threadId);
177
+ // Scheduler routing is trusted poller state. Brain-writable envelopes and
178
+ // sidecars cannot select either its platform channel or Slack destination.
179
+ const storedRoute = scheduled ? routingRecord : readInboxChannelContext(brainPath, threadId) ?? routingRecord;
180
180
  const route = storedRoute
181
181
  ? routes.find(({ info }) => info.workspaceId === storedRoute.workspaceId && info.channelId === storedRoute.channelId)
182
182
  : undefined;
@@ -186,9 +186,16 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
186
186
  }
187
187
  const info = route.info;
188
188
  const directory = route.directory;
189
- const current = parsed ? buildSyntheticMessage(info, threadId, parsed) : undefined;
189
+ // Synthetic IDs need the original Slack destination captured at spawn.
190
+ // Match its platform scope before using it alongside the current route.
191
+ const slack = parsed
192
+ ? { channel: parsed.channel, thread_ts: parsed.threadTs, message_ts: parsed.threadTs }
193
+ : routingRecord?.workspaceId === info.workspaceId && routingRecord.channelId === info.channelId
194
+ ? routingRecord.responseContext?.slack
195
+ : undefined;
196
+ const current = slack ? buildSyntheticMessage(info, threadId, slack, scheduled ? routingRecord?.source ?? 'cron' : 'slack') : undefined;
190
197
  // Stored context supplies content, never the selected directory identity.
191
- const message = durable
198
+ const message = scheduled ? current : durable
192
199
  ? { ...durable, teamvibe: platformRouting(info) }
193
200
  : current;
194
201
  if (!message) {
@@ -324,7 +331,7 @@ function quarantineRetainedThread(deps, brainPath, threadId, durable, expired =
324
331
  const info = [...deps.getTrackedBrains().values()].find(info => info.botId === threadId.split(':')[0] && (!durable || info.workspaceId === durable.teamvibe.workspaceId));
325
332
  if (!info)
326
333
  return true;
327
- void deps.sendSlackMessage(buildSyntheticMessage(info, threadId, parsed), 'A retained background result could not be routed to a current channel. It has been saved for operator review; please contact your poller operator.')
334
+ void deps.sendSlackMessage(buildSyntheticMessage(info, threadId, { channel: parsed.channel, thread_ts: parsed.threadTs, message_ts: parsed.threadTs }), 'A retained background result could not be routed to a current channel. It has been saved for operator review; please contact your poller operator.')
328
335
  .catch(() => logger.error('[InboxWatcher] Could not send unroutable inbox notice'));
329
336
  return true;
330
337
  }
package/dist/poller.d.ts CHANGED
@@ -1,5 +1,6 @@
1
1
  import { type BrainDirectory } from './brain-directory.js';
2
2
  import { logger } from './logger.js';
3
+ import { type ReceivedMessage } from './sqs-poller.js';
3
4
  import { type SpawnResult } from './claude-spawner.js';
4
5
  import { type KilledByPoller } from './classify-failed-iteration.js';
5
6
  import type { TeamVibeQueueMessage } from './types.js';
@@ -31,11 +32,11 @@ export declare function isPollerInitiatedFailure(result: {
31
32
  killedByPoller?: KilledByPoller;
32
33
  }): boolean;
33
34
  export declare function describeFailureReason(killedByPoller: KilledByPoller | undefined, exitCode: number | null): string;
34
- export declare const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = "NO_FURTHER_RESPONSE_NEEDED";
35
+ export { END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL } from './end-of-turn-sentinel.js';
35
36
  export declare function endOfTurnFallbackText(hasSlackContext: boolean, result: Pick<SpawnResult, 'success' | 'sawSendMessageCall' | 'lastPlainTextContent'>): string | undefined;
36
- export declare function runInboxLoop(threadId: string, selectedDirectory: BrainDirectory | string, queueMessage: TeamVibeQueueMessage, sessionLog: ReturnType<typeof logger.createSession>): Promise<void>;
37
+ export declare function runInboxLoop(threadId: string, selectedDirectory: BrainDirectory | string, queueMessage: TeamVibeQueueMessage, sessionLog: ReturnType<typeof logger.createSession>, executionOrigin?: 'queue' | 'recovery'): Promise<void>;
38
+ export declare function processMessage(received: ReceivedMessage): Promise<void>;
37
39
  export declare function pollLoop(): Promise<void>;
38
40
  export declare function __setSleepForTest(impl: (ms: number) => Promise<void>): void;
39
41
  export declare function __setShuttingDownForTest(value: boolean): void;
40
42
  export declare function startPoller(): Promise<void>;
41
- export {};
package/dist/poller.js CHANGED
@@ -1,3 +1,4 @@
1
+ import { END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL } from './end-of-turn-sentinel.js';
1
2
  import { startGitCredentialJanitor, registeredGitCredentialTasks } from './git-credential-session.js';
2
3
  import { isUsageProbe, handleUsageProbe } from './usage-probe-job.js';
3
4
  import { fetchSubscriptionPool } from './subscription-pool-client.js';
@@ -26,8 +27,8 @@ import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './main
26
27
  import { bootstrapTrackedBrains } from './brains-bootstrap.js';
27
28
  import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
28
29
  import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
29
- import { inboxQueueMessageIds, hasInboxMessageAfter, writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage, saveInboxContext, saveInboxRuntime, readInboxRuntime, clearInboxRuntime } from './inbox-manager.js';
30
- import { saveThreadRouting } from './thread-routing.js';
30
+ import { inboxQueueMessageIds, hasInboxMessageAfter, writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage, saveInboxContext, saveInboxRuntime, readInboxRuntime, clearInboxRuntime, quarantineInbox } from './inbox-manager.js';
31
+ import { readThreadRouting, saveThreadRouting } from './thread-routing.js';
31
32
  import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
32
33
  import { Cron } from 'croner';
33
34
  import { isControlMessage, handleControlMessage } from './control-handler.js';
@@ -232,9 +233,9 @@ export function describeFailureReason(killedByPoller, exitCode) {
232
233
  // string match, not a content heuristic — anything else (including real
233
234
  // content an agent forgot to send via send_message) still delivers as
234
235
  // before, so this can't silently drop a forgotten reply.
235
- export const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = 'NO_FURTHER_RESPONSE_NEEDED';
236
+ export { END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL } from './end-of-turn-sentinel.js';
236
237
  // tv#358 (revised): the sentinel agents emit and the guard matches is
237
- // unchanged — still the exact ALL_CAPS literal above, still an exact-match
238
+ // unchanged — still the exact shared ALL_CAPS literal, still an exact-match
238
239
  // suppression. What changes is purely cosmetic: poller-brain#428's leak
239
240
  // failure mode (a sentence written before the sentinel breaks the exact
240
241
  // match, so the whole thing delivers as-is) showed the raw literal reads as
@@ -251,9 +252,23 @@ export function endOfTurnFallbackText(hasSlackContext, result) {
251
252
  }
252
253
  return text.split(END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL).join(END_OF_TURN_SENTINEL_DISPLAY);
253
254
  }
254
- export async function runInboxLoop(threadId, selectedDirectory, queueMessage, sessionLog) {
255
+ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, sessionLog, executionOrigin = 'recovery') {
255
256
  const directory = executionDirectory(selectedDirectory);
256
257
  const kbPath = directory.path;
258
+ const scheduled = threadId.startsWith('scheduled:');
259
+ // A trusted queue job may legitimately have no destination. Only retained
260
+ // inbox recovery requires a captured destination before executing its drop.
261
+ if (scheduled && executionOrigin === 'recovery') {
262
+ const routing = readThreadRouting(threadId);
263
+ if (!routing?.responseContext?.slack) {
264
+ quarantineInbox(kbPath, threadId);
265
+ const detail = 'Retained scheduler inbox has no trusted Slack destination; saved in unroutable/ for operator review';
266
+ sessionLog.error(`[Loop] ${detail}: ${threadId}`);
267
+ emitError('session', detail, { context: { threadId, reason: 'unroutable_inbox' } });
268
+ return;
269
+ }
270
+ queueMessage = { ...queueMessage, source: routing.source ?? 'cron', response_context: routing.responseContext };
271
+ }
257
272
  if (isQuotaPaused() && !getPollerId())
258
273
  return;
259
274
  const initialPool = await fetchRuntimePool(queueMessage);
@@ -275,6 +290,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
275
290
  activeLoops.add(threadId);
276
291
  let iteration = 0;
277
292
  const failureCounts = new Map();
293
+ const loggedRoutingDrops = new Set();
278
294
  try {
279
295
  let lockResult = await acquireSessionLock(threadId, kbPath);
280
296
  if (!lockResult.success) {
@@ -323,8 +339,10 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
323
339
  const lastAttempt = state.attempts?.at(-1);
324
340
  // Existing telemetry falls back to account IDs. Local pool logs use labels only.
325
341
  const accountLabel = lastAttempt?.accountLabel !== state.pin.accountId ? lastAttempt?.accountLabel : undefined;
342
+ // Reuse the spawn's observation clock: the spawner already recorded all
343
+ // rejected scopes, and replaying its final event is the same evidence.
326
344
  if (state.completed.rateLimitInfo)
327
- writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
345
+ writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, lastAttempt?.observedAt ?? state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
328
346
  writes.push(persistSessionId(state.sessionId, state.pin));
329
347
  const outcomes = await Promise.allSettled(writes);
330
348
  if (outcomes.some(outcome => outcome.status === 'rejected' || outcome.value === false)) {
@@ -337,6 +355,21 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
337
355
  }
338
356
  return false;
339
357
  }
358
+ if (lastAttempt?.verificationAttemptAt !== undefined) {
359
+ try {
360
+ await subscriptionUsageStore().completeVerification(state.pin, lastAttempt.verificationAttemptAt, state.completed.observedAt);
361
+ }
362
+ catch {
363
+ sessionLog.warn('[Loop] Verification bookkeeping failed and will retry without running the agent');
364
+ try {
365
+ saveInboxRuntime(kbPath, threadId, { ...state, completed: { ...state.completed, retryAt: Date.now() + POOL_RECHECK_MS } });
366
+ }
367
+ catch {
368
+ sessionLog.warn('[Loop] Could not save bookkeeping retry; completion remains recorded');
369
+ }
370
+ return false;
371
+ }
372
+ }
340
373
  currentSessionId = state.sessionId;
341
374
  currentPin = state.pin;
342
375
  // The spawner publishes live completions only after bookkeeping succeeds.
@@ -413,6 +446,10 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
413
446
  try {
414
447
  // Reconstruct queue message from stored inbox data
415
448
  const storedData = JSON.parse(message.content);
449
+ if (scheduled && 'response_context' in storedData && !loggedRoutingDrops.has(message.filename)) {
450
+ logger.debug(`[Loop] Ignoring inbox response_context for scheduler routing: ${threadId} (${message.filename})`);
451
+ loggedRoutingDrops.add(message.filename);
452
+ }
416
453
  const storedScope = storedData.teamvibe;
417
454
  const scopeMismatch = storedScope && (storedScope.workspaceId !== queueMessage.teamvibe.workspaceId ||
418
455
  storedScope.channelId !== queueMessage.teamvibe.channelId ||
@@ -427,7 +464,8 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
427
464
  // inbox data. This guard applies to both direct and pool execution.
428
465
  thread_id: threadId,
429
466
  teamvibe: { ...storedScope, ...queueMessage.teamvibe, brain: queueMessage.teamvibe.brain },
430
- ...(scopeMismatch ? { response_context: queueMessage.response_context } : {}),
467
+ ...(scopeMismatch || scheduled ? { response_context: queueMessage.response_context } : {}),
468
+ ...(scheduled ? { source: queueMessage.source } : {}),
431
469
  };
432
470
  // Legacy delivery uses the original routing, just as on main.
433
471
  let result;
@@ -625,7 +663,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
625
663
  quotaPaused = true;
626
664
  deferred = true;
627
665
  // Revisit held work hourly so background probes can refresh the account.
628
- // Selection still enforces each window's real reset before inference.
666
+ // Selection enforces resets or a contradicted rejection's verification backoff.
629
667
  const recheckAt = error.cause === 'usage' ? Math.min(error.eligibleAt, Date.now() + HELD_PROBE_INTERVAL_MS) : error.eligibleAt;
630
668
  const pause = deferPool(threadId, recheckAt, error.resetAt, inboxQueueMessageIds(kbPath, threadId));
631
669
  const retained = readInboxRuntime(kbPath, threadId);
@@ -644,7 +682,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
644
682
  cause: error.cause,
645
683
  thresholdDeferred: error.notice?.thresholdDeferred && !error.attempts?.some(a => a.quota),
646
684
  resetAt: pause.resetAt ? Date.parse(pause.resetAt) : error.notice?.resetAt,
647
- timeZone: config.POLLER_TZ,
685
+ timeZone: deliveryMsg.teamvibe.timeZone ?? error.notice?.timeZone,
648
686
  });
649
687
  await sendSlackMessage(deliveryMsg, notice, { clientMessageId: pause.episodeId, ...(reconcile ? { reconcile: true } : {}) });
650
688
  setPoolNotice(threadId, pause.episodeId, 'delivered');
@@ -860,7 +898,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
860
898
  activeLocks.delete(threadId);
861
899
  }
862
900
  }
863
- async function processMessage(received) {
901
+ export async function processMessage(received) {
864
902
  const { queueMessage, receiptHandle, messageId } = received;
865
903
  const logSessionId = messageId.slice(0, 8);
866
904
  const threadId = queueMessage.thread_id;
@@ -905,6 +943,20 @@ async function processMessage(received) {
905
943
  // Register brain so the maintenance cron knows about it.
906
944
  registerBrain(queueMessage, directory);
907
945
  const kbPath = directory.path;
946
+ // This is the trusted queue boundary, before creating any brain-writable
947
+ // inbox copy. Recovery may update assignment/bookkeeping, never destination
948
+ // or source. Synthetic scheduler IDs retain their independent lock identity.
949
+ if (threadId.startsWith('scheduled:')) {
950
+ saveThreadRouting(threadId, {
951
+ workspaceId: queueMessage.teamvibe.workspaceId,
952
+ channelId: queueMessage.teamvibe.channelId,
953
+ brainId: queueMessage.teamvibe.brain?.brainId,
954
+ directory: kbPath,
955
+ source: queueMessage.source,
956
+ responseContext: queueMessage.response_context.slack?.channel
957
+ ? { slack: { ...queueMessage.response_context.slack } } : undefined,
958
+ });
959
+ }
908
960
  // Write raw message to inbox (spawnClaudeCode handles prompt formatting)
909
961
  const { botToken: _inboxToken, ...inboxTeamvibe } = queueMessage.teamvibe;
910
962
  writeMessage(kbPath, threadId, JSON.stringify({ ...queueMessage, executionOrigin: receiptHandle ? 'queue' : 'maintenance', teamvibe: inboxTeamvibe }));
@@ -916,7 +968,7 @@ async function processMessage(received) {
916
968
  }
917
969
  processingMessages.add(messageId);
918
970
  try {
919
- await runInboxLoop(threadId, directory, queueMessage, sessionLog);
971
+ await runInboxLoop(threadId, directory, queueMessage, sessionLog, 'queue');
920
972
  }
921
973
  catch (error) {
922
974
  sessionLog.error(`Error in inbox loop: ${error instanceof Error ? error.message : error}`);
@@ -19,7 +19,7 @@ export function redactStartupText(text, configuredTokens = []) {
19
19
  let safe = text
20
20
  .replace(/https?:\/\/[^\s/@]+@/gi, 'https://[REDACTED]@')
21
21
  .replace(/\b(?:Authorization\s*:\s*)?Bearer\s+[^\s,;]+/gi, 'Bearer [REDACTED]')
22
- .replace(/\bx-access-token\s*:\s*[^\s/@,;]+/gi, 'x-access-token:[REDACTED]')
22
+ .replace(/\b(x-access-token|oauth2)\s*:\s*[^\s/@,;]+/gi, '$1:[REDACTED]')
23
23
  // Cover both Claude's token name and the opt-in subprocess alias in
24
24
  // shell assignments and JSON env diagnostics, even without a token list.
25
25
  .replace(/(\b(?:CLAUDE_CODE_OAUTH_TOKEN|TEAMVIBE_CLAUDE_OAUTH_TOKEN)["']?\s*[:=]\s*)(?:"(?:\\.|[^"\\])*"|'[^']*'|[^\s,;]+)/gi, '$1[REDACTED]');
@@ -17,6 +17,8 @@ export interface TypingIndicator {
17
17
  /** Stop the typing indicator and clear the status */
18
18
  stop: () => void;
19
19
  }
20
- export declare function startTypingIndicator(msg: TeamVibeQueueMessage, statusText?: string): TypingIndicator;
20
+ export declare function startTypingIndicator(msg: TeamVibeQueueMessage, statusText?: string): TypingIndicator & {
21
+ __flushForTest: () => Promise<void>;
22
+ };
21
23
  export declare function removeReaction(msg: TeamVibeQueueMessage, emoji: string): Promise<void>;
22
24
  export {};
@@ -1,4 +1,4 @@
1
- import { WebClient } from '@slack/web-api';
1
+ import { LogLevel, WebClient } from '@slack/web-api';
2
2
  import { logger } from './logger.js';
3
3
  import { emitError } from './event-queue.js';
4
4
  import { config } from './config.js';
@@ -6,6 +6,20 @@ import { config } from './config.js';
6
6
  const userCache = new Map();
7
7
  // Per-token Slack client cache
8
8
  const slackClients = new Map();
9
+ const slackStatusClients = new Map();
10
+ // tv#589 review: the SDK's own logger prints raw `response_metadata.messages`,
11
+ // which can echo the status text (agent narration) on an API error. The
12
+ // status client therefore gets a silent logger; application code owns the
13
+ // redacted warnings in requestThreadStatus below.
14
+ const silentSlackLogger = {
15
+ debug: () => { },
16
+ info: () => { },
17
+ warn: () => { },
18
+ error: () => { },
19
+ setLevel: () => { },
20
+ getLevel: () => LogLevel.ERROR,
21
+ setName: () => { },
22
+ };
9
23
  // Slack: "must be less than 51 characters [json-pointer:/loading_messages/0]"
10
24
  const SLACK_LOADING_MESSAGE_MAX = 50;
11
25
  function getSlackClient(botToken) {
@@ -16,6 +30,21 @@ function getSlackClient(botToken) {
16
30
  }
17
31
  return client;
18
32
  }
33
+ function getSlackStatusClient(botToken) {
34
+ let client = slackStatusClients.get(botToken);
35
+ if (!client) {
36
+ client = new WebClient(botToken, {
37
+ // The SDK passes timeout to Axios, which aborts the underlying HTTP request.
38
+ timeout: config.TYPING_STATUS_REQUEST_TIMEOUT_MS,
39
+ // Let the indicator recover immediately and own the single clear retry.
40
+ retryConfig: { retries: 0 },
41
+ rejectRateLimitedCalls: true,
42
+ logger: silentSlackLogger, // Never let the SDK echo status text; warnings are redacted below.
43
+ });
44
+ slackStatusClients.set(botToken, client);
45
+ }
46
+ return client;
47
+ }
19
48
  export async function getUserInfo(botToken, userId) {
20
49
  const cached = userCache.get(userId);
21
50
  if (cached)
@@ -111,16 +140,29 @@ export async function addReaction(msg, emoji) {
111
140
  }
112
141
  }
113
142
  export async function setThreadStatus(msg, status) {
143
+ // Standalone callers retain best-effort delivery; indicators handle retries.
144
+ await requestThreadStatus(msg, status).catch(() => { });
145
+ }
146
+ async function requestThreadStatus(msg, status, onTimeout = () => logger.warn('Slack thread status request timed out')) {
114
147
  if (!msg.response_context.slack)
115
148
  return;
116
149
  const { channel, thread_ts } = msg.response_context.slack;
117
- const slack = getSlackClient(msg.teamvibe.botToken);
150
+ const slack = getSlackStatusClient(msg.teamvibe.botToken);
118
151
  const safeStatus = neutralizeSlackMarkup(status);
119
152
  const payload = {
120
153
  channel_id: channel,
121
154
  thread_ts,
122
155
  status: safeStatus,
123
156
  };
157
+ const reportFailure = (error) => {
158
+ // WebClient wraps Axios errors in a WebAPIRequestError with the original attached.
159
+ const code = error?.original?.code ?? error?.code;
160
+ if (code === 'ECONNABORTED' || code === 'ETIMEDOUT')
161
+ onTimeout();
162
+ else
163
+ logger.warn('Failed to set thread status');
164
+ emitError('slack', 'Failed to set thread status', { severity: 'warn' });
165
+ };
124
166
  try {
125
167
  await slack.apiCall('assistant.threads.setStatus', {
126
168
  ...payload,
@@ -137,14 +179,14 @@ export async function setThreadStatus(msg, status) {
137
179
  await slack.apiCall('assistant.threads.setStatus', payload);
138
180
  return;
139
181
  }
140
- catch {
141
- emitError('slack', 'Failed to set thread status', { severity: 'warn' });
142
- return;
182
+ catch (retryError) {
183
+ reportFailure(retryError);
184
+ throw retryError;
143
185
  }
144
186
  }
145
187
  // API errors may echo request fields; status can contain private narration.
146
- logger.warn('Failed to set thread status');
147
- emitError('slack', `Failed to set thread status`, { severity: 'warn' });
188
+ reportFailure(error);
189
+ throw error;
148
190
  }
149
191
  }
150
192
  function truncateLoadingMessage(text) {
@@ -176,9 +218,36 @@ export function startTypingIndicator(msg, statusText = 'Thinking...') {
176
218
  else
177
219
  logger.info(`[Typing] status -> ${source} (${status.length} chars)`);
178
220
  };
221
+ // Serialize all writes so an older request cannot land after a newer status.
222
+ let statusChain = Promise.resolve();
223
+ let pendingRequests = 0;
224
+ let timeoutWarned = false;
225
+ const warnTimeout = () => {
226
+ if (timeoutWarned)
227
+ return;
228
+ timeoutWarned = true;
229
+ logger.warn('Slack thread status request timed out');
230
+ };
231
+ const enqueueStatus = (text, retryClear = false) => {
232
+ pendingRequests++;
233
+ const request = statusChain.then(async () => {
234
+ try {
235
+ await requestThreadStatus(msg, text, warnTimeout);
236
+ }
237
+ catch (error) {
238
+ if (!retryClear)
239
+ throw error;
240
+ await new Promise(resolve => setTimeout(resolve, 500));
241
+ await requestThreadStatus(msg, text, warnTimeout);
242
+ }
243
+ });
244
+ // Recover the chain so a failed write never blocks later writes or the clear.
245
+ statusChain = request.catch(() => { }).finally(() => { pendingRequests--; });
246
+ return request;
247
+ };
179
248
  // Set initial status
180
249
  logStatus(currentStatus, currentSource);
181
- setThreadStatus(msg, currentStatus);
250
+ void enqueueStatus(currentStatus).catch(() => { });
182
251
  const cancelHold = () => {
183
252
  if (holdTimer !== undefined)
184
253
  clearTimeout(holdTimer);
@@ -190,17 +259,19 @@ export function startTypingIndicator(msg, statusText = 'Thinking...') {
190
259
  currentStatus = text;
191
260
  currentSource = text ? source : 'default';
192
261
  if (text && source === 'comment')
193
- commentShownAt = Date.now();
262
+ commentShownAt = Date.now(); // Pre-existing enqueue-time hold, before successful display (#589 review P2-2).
194
263
  if (changed) {
195
264
  logStatus(text, source);
196
- setThreadStatus(msg, text);
265
+ void enqueueStatus(text).catch(() => { });
197
266
  }
198
267
  };
199
268
  // Keepalive every 3 seconds (Slack clears after ~2 min timeout)
200
269
  let failures = 0;
201
270
  const interval = setInterval(async () => {
271
+ if (stopped || pendingRequests > 0)
272
+ return;
202
273
  try {
203
- await setThreadStatus(msg, currentStatus);
274
+ await enqueueStatus(currentStatus);
204
275
  failures = 0;
205
276
  }
206
277
  catch {
@@ -211,6 +282,7 @@ export function startTypingIndicator(msg, statusText = 'Thinking...') {
211
282
  }
212
283
  }, 3000);
213
284
  return {
285
+ __flushForTest: () => statusChain,
214
286
  updateStatus: (newStatus, source = 'comment') => {
215
287
  if (stopped)
216
288
  return;
@@ -244,7 +316,7 @@ export function startTypingIndicator(msg, statusText = 'Thinking...') {
244
316
  clearInterval(interval);
245
317
  if (currentStatus)
246
318
  logStatus('', 'default');
247
- setThreadStatus(msg, '');
319
+ void enqueueStatus('', true).catch(() => { });
248
320
  },
249
321
  };
250
322
  }
@@ -10,6 +10,8 @@ import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
10
10
  import type { ClaudeQuotaFailure } from './claude-quota.js';
11
11
  import type { KilledByPoller } from './classify-failed-iteration.js';
12
12
  export interface PoolAttempt extends Partial<HeadroomScore> {
13
+ /** Local completion bookkeeping token; omitted from event transport DTOs. */
14
+ verificationAttemptAt?: number;
13
15
  workspaceId: string;
14
16
  accountId: string;
15
17
  accountLabel?: string;
@@ -205,6 +205,8 @@ async function poolUsageNotice(pool, resolved, store) {
205
205
  return {
206
206
  label: account.label, useFullQuota: account.useFullQuota, windows: windows.filter(w => binding.some(b => b.window === w.window && b.scope === w.scope)), rejected,
207
207
  thresholdBlocked: blockers.length > 0,
208
+ blockedWindows: binding.map(w => ({ window: w.window, scope: w.scope })),
209
+ providerResetAt: binding.length && binding.every(w => w.resetAt !== undefined) ? Math.max(...binding.map(w => w.resetAt)) : undefined,
208
210
  resetAt: resets.length && resets.every((r) => r !== undefined) ? Math.max(...resets) : undefined,
209
211
  };
210
212
  }));
@@ -212,7 +214,8 @@ async function poolUsageNotice(pool, resolved, store) {
212
214
  // Snapshot expiry is only a recheck. Use actual window resets for the notice.
213
215
  const resets = accounts.map(a => a.resetAt);
214
216
  const resetAt = thresholdDeferred && resets.every((r) => r !== undefined) ? Math.min(...resets) : undefined;
215
- return { accounts, thresholdDeferred, resetAt };
217
+ const providerResets = accounts.map(a => a.providerResetAt);
218
+ return { accounts, thresholdDeferred, resetAt: resetAt ?? (providerResets.length && providerResets.every((r) => r !== undefined) ? Math.min(...providerResets) : undefined), blockedWindows: accounts.flatMap(a => a.blockedWindows) };
216
219
  }
217
220
  catch {
218
221
  return {};
@@ -226,8 +229,9 @@ export async function poolDeferredError(cause, pool, resolved, store, options =
226
229
  return new PoolDeferredError(cause, { error: options.error ?? (pool.kind === 'fetch_unavailable' && pool.cause ? { code: pool.cause } : undefined) });
227
230
  const notice = await poolUsageNotice(pool, resolved, store);
228
231
  const deferredError = (resetAt) => {
232
+ resetAt ??= !notice.thresholdDeferred && notice.resetAt !== undefined ? new Date(notice.resetAt).toISOString() : undefined;
229
233
  const error = new PoolDeferredError('usage', { eligibleAt, resetAt });
230
- error.notice = { ...notice, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
234
+ error.notice = { ...notice, eligibleAt: error.eligibleAt, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
231
235
  return error;
232
236
  };
233
237
  if (eligibleAt === undefined) {
@@ -10,6 +10,13 @@ export declare const PROBE_USAGE_TTL_MS = 60000;
10
10
  export declare const POOL_RECHECK_MS: number;
11
11
  /** Safety hold from the rejection observation when the provider gives no reset. */
12
12
  export declare const REJECTED_BLOCK_MS: number;
13
+ export declare const VERIFICATION_MAX_BACKOFF_MS: number;
14
+ export interface VerificationEvidence {
15
+ observedAt: number;
16
+ source: 'passive' | 'probe';
17
+ status: 'allowed' | 'allowed_warning';
18
+ percent: number;
19
+ }
13
20
  export interface AccountIdentity {
14
21
  workspaceId: string;
15
22
  accountId: string;
@@ -47,6 +54,11 @@ export interface UsageWindow {
47
54
  source?: 'passive' | 'probe';
48
55
  percent?: number;
49
56
  resetAt?: number;
57
+ /** Missing on legacy rejections means the first consecutive rejection. */
58
+ rejectionCount?: number;
59
+ lastVerificationAttemptAt?: number;
60
+ /** A genuine same-window measurement; never inferred from an omitted meter. */
61
+ verificationEvidence?: VerificationEvidence;
50
62
  }
51
63
  export interface UsageObservation {
52
64
  pollerVersion?: string;
@@ -67,11 +79,11 @@ export declare function projectLegacyFableUsage<T extends UsageObservation | und
67
79
  * Deduplicate uncertain windows before splitting measurements from rejections.
68
80
  */
69
81
  export declare function supersedeLegacyFableProjections(observations: readonly (UsageObservation | undefined)[], now: number): (UsageObservation | undefined)[];
70
- /** Latest measurements plus durable genuine rejections; omission or success never clears a genuine hold. */
82
+ /** Latest measurements plus durable rejections; only a claimed real spawn can clear a hold. */
71
83
  export declare function mergeUsageObservations(observations: readonly (UsageObservation | undefined)[], now: number): UsageObservation | undefined;
72
84
  /** Preserve last-known meters, projecting legacy identity before its global companion is replaced. */
73
85
  export declare function mergePersistedUsageObservations(previous: UsageObservation | undefined, incoming: UsageObservation | undefined, now: number): UsageObservation | undefined;
74
- export interface ReactiveExhaustion {
86
+ export interface ReactiveExhaustion extends Pick<UsageWindow, 'rejectionCount' | 'lastVerificationAttemptAt' | 'verificationEvidence'> {
75
87
  observedAt: number;
76
88
  quota: ClaudeQuotaFailure;
77
89
  }
@@ -106,6 +118,13 @@ export declare function record(value: unknown): Record<string, unknown>;
106
118
  export declare function normalizePassiveUsage(value: unknown, observedAt: number): UsageObservation;
107
119
  export declare function normalizeProbeUsage(value: unknown, observedAt: number): UsageObservation;
108
120
  export declare function family(model: string | undefined): 'sonnet' | 'opus' | 'haiku' | 'fable' | undefined;
121
+ export declare function rejectionResetAt(window: UsageWindow, observedAt: number): number;
122
+ export declare function verificationEligibleAt(window: UsageWindow, observedAt: number): number;
123
+ export declare function rejectionEvidence(usage: UsageObservation | undefined, window: UsageWindow, now: number): VerificationEvidence | undefined;
124
+ export declare function rejectionEligibleAt(usage: UsageObservation | undefined, window: UsageWindow, account: AccountIdentity, now: number): number;
125
+ export declare function reactiveUsageWindow(reactive: ReactiveExhaustion): UsageWindow;
126
+ /** Due windows are claimed durably at the real spawn boundary, never by pure selection. */
127
+ export declare function verificationWindows(account: AccountIdentity, state: AccountUsageState, model: string | undefined, now: number): UsageWindow[];
109
128
  export declare function reactiveEligibleAt(reactive: ReactiveExhaustion): number;
110
129
  /** Pure policy. Unknown is eligible after every known/reactive blocker is ruled out. */
111
130
  export declare function selectSubscriptionAccount<A extends AccountIdentity>(input: {