@teamvibe/poller 0.8.7 → 0.8.9-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -0
- package/bin/tv-git-credential.mjs +11 -6
- package/dist/brain-manager.js +4 -4
- package/dist/claude-spawner.d.ts +4 -2
- package/dist/claude-spawner.js +50 -16
- package/dist/config.d.ts +4 -0
- package/dist/config.js +1 -0
- package/dist/connections-client.d.ts +16 -4
- package/dist/connections-client.js +42 -11
- package/dist/end-of-turn-sentinel.d.ts +2 -0
- package/dist/end-of-turn-sentinel.js +2 -0
- package/dist/git-credential-bindings.d.ts +2 -0
- package/dist/git-credential-bindings.js +4 -0
- package/dist/git-credential-env.d.ts +2 -2
- package/dist/git-credential-env.js +22 -9
- package/dist/git-credential-session.d.ts +6 -6
- package/dist/git-credential-session.js +24 -4
- package/dist/inbox-watcher.d.ts +2 -2
- package/dist/inbox-watcher.js +20 -13
- package/dist/poller.d.ts +4 -3
- package/dist/poller.js +63 -11
- package/dist/redact-text.js +1 -1
- package/dist/slack-client.d.ts +3 -1
- package/dist/slack-client.js +84 -12
- package/dist/subscription-runtime.d.ts +2 -0
- package/dist/subscription-runtime.js +6 -2
- package/dist/subscription-selection.d.ts +21 -2
- package/dist/subscription-selection.js +91 -14
- package/dist/subscription-usage-store.d.ts +6 -1
- package/dist/subscription-usage-store.js +246 -13
- package/dist/thread-routing.d.ts +7 -0
- package/dist/thread-routing.js +7 -0
- package/dist/types.d.ts +2 -0
- package/dist/usage-limit-messages.d.ts +3 -0
- package/dist/usage-limit-messages.js +22 -0
- package/dist/usage-probe-job.js +31 -3
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
package/dist/inbox-watcher.js
CHANGED
|
@@ -38,8 +38,8 @@ function defaultDeps() {
|
|
|
38
38
|
* threads (see modules-teamvibe/hooks/slack-events.ts). Synthetic threadIds
|
|
39
39
|
* used elsewhere (`maintenance_<brainId>_<ts>`, and
|
|
40
40
|
* `scheduled:<scheduleId>:<ts>` from
|
|
41
|
-
* modules-teamvibe/src/functions/scheduler.ts
|
|
42
|
-
*
|
|
41
|
+
* modules-teamvibe/src/functions/scheduler.ts) carry no Slack destination;
|
|
42
|
+
* scheduler recovery needs the poller-owned routing record exclusively.
|
|
43
43
|
* `scheduled:*:*` has the same 3-colon-segment shape as a real Slack
|
|
44
44
|
* threadId, so it's rejected explicitly rather than by segment count alone
|
|
45
45
|
* — otherwise a leftover unprocessed file in a stale scheduled-session inbox
|
|
@@ -65,20 +65,16 @@ function platformRouting(info) {
|
|
|
65
65
|
brain: { brainId: info.brainId, gitRepoUrl: info.gitRepoUrl, branch: info.branch },
|
|
66
66
|
};
|
|
67
67
|
}
|
|
68
|
-
function buildSyntheticMessage(info, threadId,
|
|
68
|
+
function buildSyntheticMessage(info, threadId, slack, source = 'slack') {
|
|
69
69
|
return {
|
|
70
70
|
id: `inbox_watch_${threadId}`,
|
|
71
|
-
source
|
|
71
|
+
source,
|
|
72
72
|
thread_id: threadId,
|
|
73
73
|
sender: { id: 'system', name: 'Background Task' },
|
|
74
74
|
text: '',
|
|
75
75
|
attachments: [],
|
|
76
76
|
response_context: {
|
|
77
|
-
slack
|
|
78
|
-
channel: parsed.channel,
|
|
79
|
-
thread_ts: parsed.threadTs,
|
|
80
|
-
message_ts: parsed.threadTs,
|
|
81
|
-
},
|
|
77
|
+
slack,
|
|
82
78
|
},
|
|
83
79
|
type: 'message',
|
|
84
80
|
teamvibe: platformRouting(info),
|
|
@@ -174,9 +170,13 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
|
|
|
174
170
|
continue;
|
|
175
171
|
const durable = readInboxContext(brainPath, threadId);
|
|
176
172
|
const parsed = parseSlackThreadId(threadId);
|
|
173
|
+
const scheduled = threadId.startsWith('scheduled:');
|
|
177
174
|
// Context or poller-owned per-thread routing selects a channel in the
|
|
178
175
|
// full map. Never infer ownership from a directory's current channels.
|
|
179
|
-
const
|
|
176
|
+
const routingRecord = readThreadRouting(threadId);
|
|
177
|
+
// Scheduler routing is trusted poller state. Brain-writable envelopes and
|
|
178
|
+
// sidecars cannot select either its platform channel or Slack destination.
|
|
179
|
+
const storedRoute = scheduled ? routingRecord : readInboxChannelContext(brainPath, threadId) ?? routingRecord;
|
|
180
180
|
const route = storedRoute
|
|
181
181
|
? routes.find(({ info }) => info.workspaceId === storedRoute.workspaceId && info.channelId === storedRoute.channelId)
|
|
182
182
|
: undefined;
|
|
@@ -186,9 +186,16 @@ export async function checkInboxForNewSessions(deps = defaultDeps()) {
|
|
|
186
186
|
}
|
|
187
187
|
const info = route.info;
|
|
188
188
|
const directory = route.directory;
|
|
189
|
-
|
|
189
|
+
// Synthetic IDs need the original Slack destination captured at spawn.
|
|
190
|
+
// Match its platform scope before using it alongside the current route.
|
|
191
|
+
const slack = parsed
|
|
192
|
+
? { channel: parsed.channel, thread_ts: parsed.threadTs, message_ts: parsed.threadTs }
|
|
193
|
+
: routingRecord?.workspaceId === info.workspaceId && routingRecord.channelId === info.channelId
|
|
194
|
+
? routingRecord.responseContext?.slack
|
|
195
|
+
: undefined;
|
|
196
|
+
const current = slack ? buildSyntheticMessage(info, threadId, slack, scheduled ? routingRecord?.source ?? 'cron' : 'slack') : undefined;
|
|
190
197
|
// Stored context supplies content, never the selected directory identity.
|
|
191
|
-
const message = durable
|
|
198
|
+
const message = scheduled ? current : durable
|
|
192
199
|
? { ...durable, teamvibe: platformRouting(info) }
|
|
193
200
|
: current;
|
|
194
201
|
if (!message) {
|
|
@@ -324,7 +331,7 @@ function quarantineRetainedThread(deps, brainPath, threadId, durable, expired =
|
|
|
324
331
|
const info = [...deps.getTrackedBrains().values()].find(info => info.botId === threadId.split(':')[0] && (!durable || info.workspaceId === durable.teamvibe.workspaceId));
|
|
325
332
|
if (!info)
|
|
326
333
|
return true;
|
|
327
|
-
void deps.sendSlackMessage(buildSyntheticMessage(info, threadId, parsed), 'A retained background result could not be routed to a current channel. It has been saved for operator review; please contact your poller operator.')
|
|
334
|
+
void deps.sendSlackMessage(buildSyntheticMessage(info, threadId, { channel: parsed.channel, thread_ts: parsed.threadTs, message_ts: parsed.threadTs }), 'A retained background result could not be routed to a current channel. It has been saved for operator review; please contact your poller operator.')
|
|
328
335
|
.catch(() => logger.error('[InboxWatcher] Could not send unroutable inbox notice'));
|
|
329
336
|
return true;
|
|
330
337
|
}
|
package/dist/poller.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { type BrainDirectory } from './brain-directory.js';
|
|
2
2
|
import { logger } from './logger.js';
|
|
3
|
+
import { type ReceivedMessage } from './sqs-poller.js';
|
|
3
4
|
import { type SpawnResult } from './claude-spawner.js';
|
|
4
5
|
import { type KilledByPoller } from './classify-failed-iteration.js';
|
|
5
6
|
import type { TeamVibeQueueMessage } from './types.js';
|
|
@@ -31,11 +32,11 @@ export declare function isPollerInitiatedFailure(result: {
|
|
|
31
32
|
killedByPoller?: KilledByPoller;
|
|
32
33
|
}): boolean;
|
|
33
34
|
export declare function describeFailureReason(killedByPoller: KilledByPoller | undefined, exitCode: number | null): string;
|
|
34
|
-
export
|
|
35
|
+
export { END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL } from './end-of-turn-sentinel.js';
|
|
35
36
|
export declare function endOfTurnFallbackText(hasSlackContext: boolean, result: Pick<SpawnResult, 'success' | 'sawSendMessageCall' | 'lastPlainTextContent'>): string | undefined;
|
|
36
|
-
export declare function runInboxLoop(threadId: string, selectedDirectory: BrainDirectory | string, queueMessage: TeamVibeQueueMessage, sessionLog: ReturnType<typeof logger.createSession
|
|
37
|
+
export declare function runInboxLoop(threadId: string, selectedDirectory: BrainDirectory | string, queueMessage: TeamVibeQueueMessage, sessionLog: ReturnType<typeof logger.createSession>, executionOrigin?: 'queue' | 'recovery'): Promise<void>;
|
|
38
|
+
export declare function processMessage(received: ReceivedMessage): Promise<void>;
|
|
37
39
|
export declare function pollLoop(): Promise<void>;
|
|
38
40
|
export declare function __setSleepForTest(impl: (ms: number) => Promise<void>): void;
|
|
39
41
|
export declare function __setShuttingDownForTest(value: boolean): void;
|
|
40
42
|
export declare function startPoller(): Promise<void>;
|
|
41
|
-
export {};
|
package/dist/poller.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL } from './end-of-turn-sentinel.js';
|
|
1
2
|
import { startGitCredentialJanitor, registeredGitCredentialTasks } from './git-credential-session.js';
|
|
2
3
|
import { isUsageProbe, handleUsageProbe } from './usage-probe-job.js';
|
|
3
4
|
import { fetchSubscriptionPool } from './subscription-pool-client.js';
|
|
@@ -26,8 +27,8 @@ import { registerBrain, startMaintenanceLoop, stopMaintenanceLoop } from './main
|
|
|
26
27
|
import { bootstrapTrackedBrains } from './brains-bootstrap.js';
|
|
27
28
|
import { startEventQueue, stopEventQueue, flush as flushEventQueue, emitEvent, emitError } from './event-queue.js';
|
|
28
29
|
import { startMetricsCron, stopMetricsCron } from './metrics/scheduler.js';
|
|
29
|
-
import { inboxQueueMessageIds, hasInboxMessageAfter, writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage, saveInboxContext, saveInboxRuntime, readInboxRuntime, clearInboxRuntime } from './inbox-manager.js';
|
|
30
|
-
import { saveThreadRouting } from './thread-routing.js';
|
|
30
|
+
import { inboxQueueMessageIds, hasInboxMessageAfter, writeMessage, pickNextMessage, markProcessed, markFailed, hasUnprocessed, requeueMessage, saveInboxContext, saveInboxRuntime, readInboxRuntime, clearInboxRuntime, quarantineInbox } from './inbox-manager.js';
|
|
31
|
+
import { readThreadRouting, saveThreadRouting } from './thread-routing.js';
|
|
31
32
|
import { startInboxWatcher, stopInboxWatcher } from './inbox-watcher.js';
|
|
32
33
|
import { Cron } from 'croner';
|
|
33
34
|
import { isControlMessage, handleControlMessage } from './control-handler.js';
|
|
@@ -232,9 +233,9 @@ export function describeFailureReason(killedByPoller, exitCode) {
|
|
|
232
233
|
// string match, not a content heuristic — anything else (including real
|
|
233
234
|
// content an agent forgot to send via send_message) still delivers as
|
|
234
235
|
// before, so this can't silently drop a forgotten reply.
|
|
235
|
-
export
|
|
236
|
+
export { END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL } from './end-of-turn-sentinel.js';
|
|
236
237
|
// tv#358 (revised): the sentinel agents emit and the guard matches is
|
|
237
|
-
// unchanged — still the exact ALL_CAPS literal
|
|
238
|
+
// unchanged — still the exact shared ALL_CAPS literal, still an exact-match
|
|
238
239
|
// suppression. What changes is purely cosmetic: poller-brain#428's leak
|
|
239
240
|
// failure mode (a sentence written before the sentinel breaks the exact
|
|
240
241
|
// match, so the whole thing delivers as-is) showed the raw literal reads as
|
|
@@ -251,9 +252,23 @@ export function endOfTurnFallbackText(hasSlackContext, result) {
|
|
|
251
252
|
}
|
|
252
253
|
return text.split(END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL).join(END_OF_TURN_SENTINEL_DISPLAY);
|
|
253
254
|
}
|
|
254
|
-
export async function runInboxLoop(threadId, selectedDirectory, queueMessage, sessionLog) {
|
|
255
|
+
export async function runInboxLoop(threadId, selectedDirectory, queueMessage, sessionLog, executionOrigin = 'recovery') {
|
|
255
256
|
const directory = executionDirectory(selectedDirectory);
|
|
256
257
|
const kbPath = directory.path;
|
|
258
|
+
const scheduled = threadId.startsWith('scheduled:');
|
|
259
|
+
// A trusted queue job may legitimately have no destination. Only retained
|
|
260
|
+
// inbox recovery requires a captured destination before executing its drop.
|
|
261
|
+
if (scheduled && executionOrigin === 'recovery') {
|
|
262
|
+
const routing = readThreadRouting(threadId);
|
|
263
|
+
if (!routing?.responseContext?.slack) {
|
|
264
|
+
quarantineInbox(kbPath, threadId);
|
|
265
|
+
const detail = 'Retained scheduler inbox has no trusted Slack destination; saved in unroutable/ for operator review';
|
|
266
|
+
sessionLog.error(`[Loop] ${detail}: ${threadId}`);
|
|
267
|
+
emitError('session', detail, { context: { threadId, reason: 'unroutable_inbox' } });
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
queueMessage = { ...queueMessage, source: routing.source ?? 'cron', response_context: routing.responseContext };
|
|
271
|
+
}
|
|
257
272
|
if (isQuotaPaused() && !getPollerId())
|
|
258
273
|
return;
|
|
259
274
|
const initialPool = await fetchRuntimePool(queueMessage);
|
|
@@ -275,6 +290,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
275
290
|
activeLoops.add(threadId);
|
|
276
291
|
let iteration = 0;
|
|
277
292
|
const failureCounts = new Map();
|
|
293
|
+
const loggedRoutingDrops = new Set();
|
|
278
294
|
try {
|
|
279
295
|
let lockResult = await acquireSessionLock(threadId, kbPath);
|
|
280
296
|
if (!lockResult.success) {
|
|
@@ -323,8 +339,10 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
323
339
|
const lastAttempt = state.attempts?.at(-1);
|
|
324
340
|
// Existing telemetry falls back to account IDs. Local pool logs use labels only.
|
|
325
341
|
const accountLabel = lastAttempt?.accountLabel !== state.pin.accountId ? lastAttempt?.accountLabel : undefined;
|
|
342
|
+
// Reuse the spawn's observation clock: the spawner already recorded all
|
|
343
|
+
// rejected scopes, and replaying its final event is the same evidence.
|
|
326
344
|
if (state.completed.rateLimitInfo)
|
|
327
|
-
writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
|
|
345
|
+
writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, lastAttempt?.observedAt ?? state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
|
|
328
346
|
writes.push(persistSessionId(state.sessionId, state.pin));
|
|
329
347
|
const outcomes = await Promise.allSettled(writes);
|
|
330
348
|
if (outcomes.some(outcome => outcome.status === 'rejected' || outcome.value === false)) {
|
|
@@ -337,6 +355,21 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
337
355
|
}
|
|
338
356
|
return false;
|
|
339
357
|
}
|
|
358
|
+
if (lastAttempt?.verificationAttemptAt !== undefined) {
|
|
359
|
+
try {
|
|
360
|
+
await subscriptionUsageStore().completeVerification(state.pin, lastAttempt.verificationAttemptAt, state.completed.observedAt);
|
|
361
|
+
}
|
|
362
|
+
catch {
|
|
363
|
+
sessionLog.warn('[Loop] Verification bookkeeping failed and will retry without running the agent');
|
|
364
|
+
try {
|
|
365
|
+
saveInboxRuntime(kbPath, threadId, { ...state, completed: { ...state.completed, retryAt: Date.now() + POOL_RECHECK_MS } });
|
|
366
|
+
}
|
|
367
|
+
catch {
|
|
368
|
+
sessionLog.warn('[Loop] Could not save bookkeeping retry; completion remains recorded');
|
|
369
|
+
}
|
|
370
|
+
return false;
|
|
371
|
+
}
|
|
372
|
+
}
|
|
340
373
|
currentSessionId = state.sessionId;
|
|
341
374
|
currentPin = state.pin;
|
|
342
375
|
// The spawner publishes live completions only after bookkeeping succeeds.
|
|
@@ -413,6 +446,10 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
413
446
|
try {
|
|
414
447
|
// Reconstruct queue message from stored inbox data
|
|
415
448
|
const storedData = JSON.parse(message.content);
|
|
449
|
+
if (scheduled && 'response_context' in storedData && !loggedRoutingDrops.has(message.filename)) {
|
|
450
|
+
logger.debug(`[Loop] Ignoring inbox response_context for scheduler routing: ${threadId} (${message.filename})`);
|
|
451
|
+
loggedRoutingDrops.add(message.filename);
|
|
452
|
+
}
|
|
416
453
|
const storedScope = storedData.teamvibe;
|
|
417
454
|
const scopeMismatch = storedScope && (storedScope.workspaceId !== queueMessage.teamvibe.workspaceId ||
|
|
418
455
|
storedScope.channelId !== queueMessage.teamvibe.channelId ||
|
|
@@ -427,7 +464,8 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
427
464
|
// inbox data. This guard applies to both direct and pool execution.
|
|
428
465
|
thread_id: threadId,
|
|
429
466
|
teamvibe: { ...storedScope, ...queueMessage.teamvibe, brain: queueMessage.teamvibe.brain },
|
|
430
|
-
...(scopeMismatch ? { response_context: queueMessage.response_context } : {}),
|
|
467
|
+
...(scopeMismatch || scheduled ? { response_context: queueMessage.response_context } : {}),
|
|
468
|
+
...(scheduled ? { source: queueMessage.source } : {}),
|
|
431
469
|
};
|
|
432
470
|
// Legacy delivery uses the original routing, just as on main.
|
|
433
471
|
let result;
|
|
@@ -625,7 +663,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
625
663
|
quotaPaused = true;
|
|
626
664
|
deferred = true;
|
|
627
665
|
// Revisit held work hourly so background probes can refresh the account.
|
|
628
|
-
// Selection
|
|
666
|
+
// Selection enforces resets or a contradicted rejection's verification backoff.
|
|
629
667
|
const recheckAt = error.cause === 'usage' ? Math.min(error.eligibleAt, Date.now() + HELD_PROBE_INTERVAL_MS) : error.eligibleAt;
|
|
630
668
|
const pause = deferPool(threadId, recheckAt, error.resetAt, inboxQueueMessageIds(kbPath, threadId));
|
|
631
669
|
const retained = readInboxRuntime(kbPath, threadId);
|
|
@@ -644,7 +682,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
644
682
|
cause: error.cause,
|
|
645
683
|
thresholdDeferred: error.notice?.thresholdDeferred && !error.attempts?.some(a => a.quota),
|
|
646
684
|
resetAt: pause.resetAt ? Date.parse(pause.resetAt) : error.notice?.resetAt,
|
|
647
|
-
timeZone:
|
|
685
|
+
timeZone: deliveryMsg.teamvibe.timeZone ?? error.notice?.timeZone,
|
|
648
686
|
});
|
|
649
687
|
await sendSlackMessage(deliveryMsg, notice, { clientMessageId: pause.episodeId, ...(reconcile ? { reconcile: true } : {}) });
|
|
650
688
|
setPoolNotice(threadId, pause.episodeId, 'delivered');
|
|
@@ -860,7 +898,7 @@ export async function runInboxLoop(threadId, selectedDirectory, queueMessage, se
|
|
|
860
898
|
activeLocks.delete(threadId);
|
|
861
899
|
}
|
|
862
900
|
}
|
|
863
|
-
async function processMessage(received) {
|
|
901
|
+
export async function processMessage(received) {
|
|
864
902
|
const { queueMessage, receiptHandle, messageId } = received;
|
|
865
903
|
const logSessionId = messageId.slice(0, 8);
|
|
866
904
|
const threadId = queueMessage.thread_id;
|
|
@@ -905,6 +943,20 @@ async function processMessage(received) {
|
|
|
905
943
|
// Register brain so the maintenance cron knows about it.
|
|
906
944
|
registerBrain(queueMessage, directory);
|
|
907
945
|
const kbPath = directory.path;
|
|
946
|
+
// This is the trusted queue boundary, before creating any brain-writable
|
|
947
|
+
// inbox copy. Recovery may update assignment/bookkeeping, never destination
|
|
948
|
+
// or source. Synthetic scheduler IDs retain their independent lock identity.
|
|
949
|
+
if (threadId.startsWith('scheduled:')) {
|
|
950
|
+
saveThreadRouting(threadId, {
|
|
951
|
+
workspaceId: queueMessage.teamvibe.workspaceId,
|
|
952
|
+
channelId: queueMessage.teamvibe.channelId,
|
|
953
|
+
brainId: queueMessage.teamvibe.brain?.brainId,
|
|
954
|
+
directory: kbPath,
|
|
955
|
+
source: queueMessage.source,
|
|
956
|
+
responseContext: queueMessage.response_context.slack?.channel
|
|
957
|
+
? { slack: { ...queueMessage.response_context.slack } } : undefined,
|
|
958
|
+
});
|
|
959
|
+
}
|
|
908
960
|
// Write raw message to inbox (spawnClaudeCode handles prompt formatting)
|
|
909
961
|
const { botToken: _inboxToken, ...inboxTeamvibe } = queueMessage.teamvibe;
|
|
910
962
|
writeMessage(kbPath, threadId, JSON.stringify({ ...queueMessage, executionOrigin: receiptHandle ? 'queue' : 'maintenance', teamvibe: inboxTeamvibe }));
|
|
@@ -916,7 +968,7 @@ async function processMessage(received) {
|
|
|
916
968
|
}
|
|
917
969
|
processingMessages.add(messageId);
|
|
918
970
|
try {
|
|
919
|
-
await runInboxLoop(threadId, directory, queueMessage, sessionLog);
|
|
971
|
+
await runInboxLoop(threadId, directory, queueMessage, sessionLog, 'queue');
|
|
920
972
|
}
|
|
921
973
|
catch (error) {
|
|
922
974
|
sessionLog.error(`Error in inbox loop: ${error instanceof Error ? error.message : error}`);
|
package/dist/redact-text.js
CHANGED
|
@@ -19,7 +19,7 @@ export function redactStartupText(text, configuredTokens = []) {
|
|
|
19
19
|
let safe = text
|
|
20
20
|
.replace(/https?:\/\/[^\s/@]+@/gi, 'https://[REDACTED]@')
|
|
21
21
|
.replace(/\b(?:Authorization\s*:\s*)?Bearer\s+[^\s,;]+/gi, 'Bearer [REDACTED]')
|
|
22
|
-
.replace(/\
|
|
22
|
+
.replace(/\b(x-access-token|oauth2)\s*:\s*[^\s/@,;]+/gi, '$1:[REDACTED]')
|
|
23
23
|
// Cover both Claude's token name and the opt-in subprocess alias in
|
|
24
24
|
// shell assignments and JSON env diagnostics, even without a token list.
|
|
25
25
|
.replace(/(\b(?:CLAUDE_CODE_OAUTH_TOKEN|TEAMVIBE_CLAUDE_OAUTH_TOKEN)["']?\s*[:=]\s*)(?:"(?:\\.|[^"\\])*"|'[^']*'|[^\s,;]+)/gi, '$1[REDACTED]');
|
package/dist/slack-client.d.ts
CHANGED
|
@@ -17,6 +17,8 @@ export interface TypingIndicator {
|
|
|
17
17
|
/** Stop the typing indicator and clear the status */
|
|
18
18
|
stop: () => void;
|
|
19
19
|
}
|
|
20
|
-
export declare function startTypingIndicator(msg: TeamVibeQueueMessage, statusText?: string): TypingIndicator
|
|
20
|
+
export declare function startTypingIndicator(msg: TeamVibeQueueMessage, statusText?: string): TypingIndicator & {
|
|
21
|
+
__flushForTest: () => Promise<void>;
|
|
22
|
+
};
|
|
21
23
|
export declare function removeReaction(msg: TeamVibeQueueMessage, emoji: string): Promise<void>;
|
|
22
24
|
export {};
|
package/dist/slack-client.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { WebClient } from '@slack/web-api';
|
|
1
|
+
import { LogLevel, WebClient } from '@slack/web-api';
|
|
2
2
|
import { logger } from './logger.js';
|
|
3
3
|
import { emitError } from './event-queue.js';
|
|
4
4
|
import { config } from './config.js';
|
|
@@ -6,6 +6,20 @@ import { config } from './config.js';
|
|
|
6
6
|
const userCache = new Map();
|
|
7
7
|
// Per-token Slack client cache
|
|
8
8
|
const slackClients = new Map();
|
|
9
|
+
const slackStatusClients = new Map();
|
|
10
|
+
// tv#589 review: the SDK's own logger prints raw `response_metadata.messages`,
|
|
11
|
+
// which can echo the status text (agent narration) on an API error. The
|
|
12
|
+
// status client therefore gets a silent logger; application code owns the
|
|
13
|
+
// redacted warnings in requestThreadStatus below.
|
|
14
|
+
const silentSlackLogger = {
|
|
15
|
+
debug: () => { },
|
|
16
|
+
info: () => { },
|
|
17
|
+
warn: () => { },
|
|
18
|
+
error: () => { },
|
|
19
|
+
setLevel: () => { },
|
|
20
|
+
getLevel: () => LogLevel.ERROR,
|
|
21
|
+
setName: () => { },
|
|
22
|
+
};
|
|
9
23
|
// Slack: "must be less than 51 characters [json-pointer:/loading_messages/0]"
|
|
10
24
|
const SLACK_LOADING_MESSAGE_MAX = 50;
|
|
11
25
|
function getSlackClient(botToken) {
|
|
@@ -16,6 +30,21 @@ function getSlackClient(botToken) {
|
|
|
16
30
|
}
|
|
17
31
|
return client;
|
|
18
32
|
}
|
|
33
|
+
function getSlackStatusClient(botToken) {
|
|
34
|
+
let client = slackStatusClients.get(botToken);
|
|
35
|
+
if (!client) {
|
|
36
|
+
client = new WebClient(botToken, {
|
|
37
|
+
// The SDK passes timeout to Axios, which aborts the underlying HTTP request.
|
|
38
|
+
timeout: config.TYPING_STATUS_REQUEST_TIMEOUT_MS,
|
|
39
|
+
// Let the indicator recover immediately and own the single clear retry.
|
|
40
|
+
retryConfig: { retries: 0 },
|
|
41
|
+
rejectRateLimitedCalls: true,
|
|
42
|
+
logger: silentSlackLogger, // Never let the SDK echo status text; warnings are redacted below.
|
|
43
|
+
});
|
|
44
|
+
slackStatusClients.set(botToken, client);
|
|
45
|
+
}
|
|
46
|
+
return client;
|
|
47
|
+
}
|
|
19
48
|
export async function getUserInfo(botToken, userId) {
|
|
20
49
|
const cached = userCache.get(userId);
|
|
21
50
|
if (cached)
|
|
@@ -111,16 +140,29 @@ export async function addReaction(msg, emoji) {
|
|
|
111
140
|
}
|
|
112
141
|
}
|
|
113
142
|
export async function setThreadStatus(msg, status) {
|
|
143
|
+
// Standalone callers retain best-effort delivery; indicators handle retries.
|
|
144
|
+
await requestThreadStatus(msg, status).catch(() => { });
|
|
145
|
+
}
|
|
146
|
+
async function requestThreadStatus(msg, status, onTimeout = () => logger.warn('Slack thread status request timed out')) {
|
|
114
147
|
if (!msg.response_context.slack)
|
|
115
148
|
return;
|
|
116
149
|
const { channel, thread_ts } = msg.response_context.slack;
|
|
117
|
-
const slack =
|
|
150
|
+
const slack = getSlackStatusClient(msg.teamvibe.botToken);
|
|
118
151
|
const safeStatus = neutralizeSlackMarkup(status);
|
|
119
152
|
const payload = {
|
|
120
153
|
channel_id: channel,
|
|
121
154
|
thread_ts,
|
|
122
155
|
status: safeStatus,
|
|
123
156
|
};
|
|
157
|
+
const reportFailure = (error) => {
|
|
158
|
+
// WebClient wraps Axios errors in a WebAPIRequestError with the original attached.
|
|
159
|
+
const code = error?.original?.code ?? error?.code;
|
|
160
|
+
if (code === 'ECONNABORTED' || code === 'ETIMEDOUT')
|
|
161
|
+
onTimeout();
|
|
162
|
+
else
|
|
163
|
+
logger.warn('Failed to set thread status');
|
|
164
|
+
emitError('slack', 'Failed to set thread status', { severity: 'warn' });
|
|
165
|
+
};
|
|
124
166
|
try {
|
|
125
167
|
await slack.apiCall('assistant.threads.setStatus', {
|
|
126
168
|
...payload,
|
|
@@ -137,14 +179,14 @@ export async function setThreadStatus(msg, status) {
|
|
|
137
179
|
await slack.apiCall('assistant.threads.setStatus', payload);
|
|
138
180
|
return;
|
|
139
181
|
}
|
|
140
|
-
catch {
|
|
141
|
-
|
|
142
|
-
|
|
182
|
+
catch (retryError) {
|
|
183
|
+
reportFailure(retryError);
|
|
184
|
+
throw retryError;
|
|
143
185
|
}
|
|
144
186
|
}
|
|
145
187
|
// API errors may echo request fields; status can contain private narration.
|
|
146
|
-
|
|
147
|
-
|
|
188
|
+
reportFailure(error);
|
|
189
|
+
throw error;
|
|
148
190
|
}
|
|
149
191
|
}
|
|
150
192
|
function truncateLoadingMessage(text) {
|
|
@@ -176,9 +218,36 @@ export function startTypingIndicator(msg, statusText = 'Thinking...') {
|
|
|
176
218
|
else
|
|
177
219
|
logger.info(`[Typing] status -> ${source} (${status.length} chars)`);
|
|
178
220
|
};
|
|
221
|
+
// Serialize all writes so an older request cannot land after a newer status.
|
|
222
|
+
let statusChain = Promise.resolve();
|
|
223
|
+
let pendingRequests = 0;
|
|
224
|
+
let timeoutWarned = false;
|
|
225
|
+
const warnTimeout = () => {
|
|
226
|
+
if (timeoutWarned)
|
|
227
|
+
return;
|
|
228
|
+
timeoutWarned = true;
|
|
229
|
+
logger.warn('Slack thread status request timed out');
|
|
230
|
+
};
|
|
231
|
+
const enqueueStatus = (text, retryClear = false) => {
|
|
232
|
+
pendingRequests++;
|
|
233
|
+
const request = statusChain.then(async () => {
|
|
234
|
+
try {
|
|
235
|
+
await requestThreadStatus(msg, text, warnTimeout);
|
|
236
|
+
}
|
|
237
|
+
catch (error) {
|
|
238
|
+
if (!retryClear)
|
|
239
|
+
throw error;
|
|
240
|
+
await new Promise(resolve => setTimeout(resolve, 500));
|
|
241
|
+
await requestThreadStatus(msg, text, warnTimeout);
|
|
242
|
+
}
|
|
243
|
+
});
|
|
244
|
+
// Recover the chain so a failed write never blocks later writes or the clear.
|
|
245
|
+
statusChain = request.catch(() => { }).finally(() => { pendingRequests--; });
|
|
246
|
+
return request;
|
|
247
|
+
};
|
|
179
248
|
// Set initial status
|
|
180
249
|
logStatus(currentStatus, currentSource);
|
|
181
|
-
|
|
250
|
+
void enqueueStatus(currentStatus).catch(() => { });
|
|
182
251
|
const cancelHold = () => {
|
|
183
252
|
if (holdTimer !== undefined)
|
|
184
253
|
clearTimeout(holdTimer);
|
|
@@ -190,17 +259,19 @@ export function startTypingIndicator(msg, statusText = 'Thinking...') {
|
|
|
190
259
|
currentStatus = text;
|
|
191
260
|
currentSource = text ? source : 'default';
|
|
192
261
|
if (text && source === 'comment')
|
|
193
|
-
commentShownAt = Date.now();
|
|
262
|
+
commentShownAt = Date.now(); // Pre-existing enqueue-time hold, before successful display (#589 review P2-2).
|
|
194
263
|
if (changed) {
|
|
195
264
|
logStatus(text, source);
|
|
196
|
-
|
|
265
|
+
void enqueueStatus(text).catch(() => { });
|
|
197
266
|
}
|
|
198
267
|
};
|
|
199
268
|
// Keepalive every 3 seconds (Slack clears after ~2 min timeout)
|
|
200
269
|
let failures = 0;
|
|
201
270
|
const interval = setInterval(async () => {
|
|
271
|
+
if (stopped || pendingRequests > 0)
|
|
272
|
+
return;
|
|
202
273
|
try {
|
|
203
|
-
await
|
|
274
|
+
await enqueueStatus(currentStatus);
|
|
204
275
|
failures = 0;
|
|
205
276
|
}
|
|
206
277
|
catch {
|
|
@@ -211,6 +282,7 @@ export function startTypingIndicator(msg, statusText = 'Thinking...') {
|
|
|
211
282
|
}
|
|
212
283
|
}, 3000);
|
|
213
284
|
return {
|
|
285
|
+
__flushForTest: () => statusChain,
|
|
214
286
|
updateStatus: (newStatus, source = 'comment') => {
|
|
215
287
|
if (stopped)
|
|
216
288
|
return;
|
|
@@ -244,7 +316,7 @@ export function startTypingIndicator(msg, statusText = 'Thinking...') {
|
|
|
244
316
|
clearInterval(interval);
|
|
245
317
|
if (currentStatus)
|
|
246
318
|
logStatus('', 'default');
|
|
247
|
-
|
|
319
|
+
void enqueueStatus('', true).catch(() => { });
|
|
248
320
|
},
|
|
249
321
|
};
|
|
250
322
|
}
|
|
@@ -10,6 +10,8 @@ import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
|
10
10
|
import type { ClaudeQuotaFailure } from './claude-quota.js';
|
|
11
11
|
import type { KilledByPoller } from './classify-failed-iteration.js';
|
|
12
12
|
export interface PoolAttempt extends Partial<HeadroomScore> {
|
|
13
|
+
/** Local completion bookkeeping token; omitted from event transport DTOs. */
|
|
14
|
+
verificationAttemptAt?: number;
|
|
13
15
|
workspaceId: string;
|
|
14
16
|
accountId: string;
|
|
15
17
|
accountLabel?: string;
|
|
@@ -205,6 +205,8 @@ async function poolUsageNotice(pool, resolved, store) {
|
|
|
205
205
|
return {
|
|
206
206
|
label: account.label, useFullQuota: account.useFullQuota, windows: windows.filter(w => binding.some(b => b.window === w.window && b.scope === w.scope)), rejected,
|
|
207
207
|
thresholdBlocked: blockers.length > 0,
|
|
208
|
+
blockedWindows: binding.map(w => ({ window: w.window, scope: w.scope })),
|
|
209
|
+
providerResetAt: binding.length && binding.every(w => w.resetAt !== undefined) ? Math.max(...binding.map(w => w.resetAt)) : undefined,
|
|
208
210
|
resetAt: resets.length && resets.every((r) => r !== undefined) ? Math.max(...resets) : undefined,
|
|
209
211
|
};
|
|
210
212
|
}));
|
|
@@ -212,7 +214,8 @@ async function poolUsageNotice(pool, resolved, store) {
|
|
|
212
214
|
// Snapshot expiry is only a recheck. Use actual window resets for the notice.
|
|
213
215
|
const resets = accounts.map(a => a.resetAt);
|
|
214
216
|
const resetAt = thresholdDeferred && resets.every((r) => r !== undefined) ? Math.min(...resets) : undefined;
|
|
215
|
-
|
|
217
|
+
const providerResets = accounts.map(a => a.providerResetAt);
|
|
218
|
+
return { accounts, thresholdDeferred, resetAt: resetAt ?? (providerResets.length && providerResets.every((r) => r !== undefined) ? Math.min(...providerResets) : undefined), blockedWindows: accounts.flatMap(a => a.blockedWindows) };
|
|
216
219
|
}
|
|
217
220
|
catch {
|
|
218
221
|
return {};
|
|
@@ -226,8 +229,9 @@ export async function poolDeferredError(cause, pool, resolved, store, options =
|
|
|
226
229
|
return new PoolDeferredError(cause, { error: options.error ?? (pool.kind === 'fetch_unavailable' && pool.cause ? { code: pool.cause } : undefined) });
|
|
227
230
|
const notice = await poolUsageNotice(pool, resolved, store);
|
|
228
231
|
const deferredError = (resetAt) => {
|
|
232
|
+
resetAt ??= !notice.thresholdDeferred && notice.resetAt !== undefined ? new Date(notice.resetAt).toISOString() : undefined;
|
|
229
233
|
const error = new PoolDeferredError('usage', { eligibleAt, resetAt });
|
|
230
|
-
error.notice = { ...notice, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
|
|
234
|
+
error.notice = { ...notice, eligibleAt: error.eligibleAt, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
|
|
231
235
|
return error;
|
|
232
236
|
};
|
|
233
237
|
if (eligibleAt === undefined) {
|
|
@@ -10,6 +10,13 @@ export declare const PROBE_USAGE_TTL_MS = 60000;
|
|
|
10
10
|
export declare const POOL_RECHECK_MS: number;
|
|
11
11
|
/** Safety hold from the rejection observation when the provider gives no reset. */
|
|
12
12
|
export declare const REJECTED_BLOCK_MS: number;
|
|
13
|
+
export declare const VERIFICATION_MAX_BACKOFF_MS: number;
|
|
14
|
+
export interface VerificationEvidence {
|
|
15
|
+
observedAt: number;
|
|
16
|
+
source: 'passive' | 'probe';
|
|
17
|
+
status: 'allowed' | 'allowed_warning';
|
|
18
|
+
percent: number;
|
|
19
|
+
}
|
|
13
20
|
export interface AccountIdentity {
|
|
14
21
|
workspaceId: string;
|
|
15
22
|
accountId: string;
|
|
@@ -47,6 +54,11 @@ export interface UsageWindow {
|
|
|
47
54
|
source?: 'passive' | 'probe';
|
|
48
55
|
percent?: number;
|
|
49
56
|
resetAt?: number;
|
|
57
|
+
/** Missing on legacy rejections means the first consecutive rejection. */
|
|
58
|
+
rejectionCount?: number;
|
|
59
|
+
lastVerificationAttemptAt?: number;
|
|
60
|
+
/** A genuine same-window measurement; never inferred from an omitted meter. */
|
|
61
|
+
verificationEvidence?: VerificationEvidence;
|
|
50
62
|
}
|
|
51
63
|
export interface UsageObservation {
|
|
52
64
|
pollerVersion?: string;
|
|
@@ -67,11 +79,11 @@ export declare function projectLegacyFableUsage<T extends UsageObservation | und
|
|
|
67
79
|
* Deduplicate uncertain windows before splitting measurements from rejections.
|
|
68
80
|
*/
|
|
69
81
|
export declare function supersedeLegacyFableProjections(observations: readonly (UsageObservation | undefined)[], now: number): (UsageObservation | undefined)[];
|
|
70
|
-
/** Latest measurements plus durable
|
|
82
|
+
/** Latest measurements plus durable rejections; only a claimed real spawn can clear a hold. */
|
|
71
83
|
export declare function mergeUsageObservations(observations: readonly (UsageObservation | undefined)[], now: number): UsageObservation | undefined;
|
|
72
84
|
/** Preserve last-known meters, projecting legacy identity before its global companion is replaced. */
|
|
73
85
|
export declare function mergePersistedUsageObservations(previous: UsageObservation | undefined, incoming: UsageObservation | undefined, now: number): UsageObservation | undefined;
|
|
74
|
-
export interface ReactiveExhaustion {
|
|
86
|
+
export interface ReactiveExhaustion extends Pick<UsageWindow, 'rejectionCount' | 'lastVerificationAttemptAt' | 'verificationEvidence'> {
|
|
75
87
|
observedAt: number;
|
|
76
88
|
quota: ClaudeQuotaFailure;
|
|
77
89
|
}
|
|
@@ -106,6 +118,13 @@ export declare function record(value: unknown): Record<string, unknown>;
|
|
|
106
118
|
export declare function normalizePassiveUsage(value: unknown, observedAt: number): UsageObservation;
|
|
107
119
|
export declare function normalizeProbeUsage(value: unknown, observedAt: number): UsageObservation;
|
|
108
120
|
export declare function family(model: string | undefined): 'sonnet' | 'opus' | 'haiku' | 'fable' | undefined;
|
|
121
|
+
export declare function rejectionResetAt(window: UsageWindow, observedAt: number): number;
|
|
122
|
+
export declare function verificationEligibleAt(window: UsageWindow, observedAt: number): number;
|
|
123
|
+
export declare function rejectionEvidence(usage: UsageObservation | undefined, window: UsageWindow, now: number): VerificationEvidence | undefined;
|
|
124
|
+
export declare function rejectionEligibleAt(usage: UsageObservation | undefined, window: UsageWindow, account: AccountIdentity, now: number): number;
|
|
125
|
+
export declare function reactiveUsageWindow(reactive: ReactiveExhaustion): UsageWindow;
|
|
126
|
+
/** Due windows are claimed durably at the real spawn boundary, never by pure selection. */
|
|
127
|
+
export declare function verificationWindows(account: AccountIdentity, state: AccountUsageState, model: string | undefined, now: number): UsageWindow[];
|
|
109
128
|
export declare function reactiveEligibleAt(reactive: ReactiveExhaustion): number;
|
|
110
129
|
/** Pure policy. Unknown is eligible after every known/reactive blocker is ruled out. */
|
|
111
130
|
export declare function selectSubscriptionAccount<A extends AccountIdentity>(input: {
|