@teamvibe/poller 0.8.3-rc.2 → 0.8.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/classify-failed-iteration.js +6 -2
- package/dist/claude-spawner.js +24 -7
- package/dist/event-queue.js +2 -2
- package/dist/poller.d.ts +1 -0
- package/dist/poller.js +60 -9
- package/dist/pool-logging.d.ts +18 -0
- package/dist/pool-logging.js +79 -0
- package/dist/subscription-pool-client.d.ts +4 -1
- package/dist/subscription-pool-client.js +7 -3
- package/dist/subscription-pool.d.ts +3 -0
- package/dist/subscription-pool.js +7 -1
- package/dist/subscription-runtime.d.ts +7 -3
- package/dist/subscription-runtime.js +64 -12
- package/dist/subscription-selection.d.ts +3 -1
- package/dist/subscription-selection.js +19 -4
- package/dist/subscription-usage-store.d.ts +3 -0
- package/dist/subscription-usage-store.js +20 -3
- package/dist/types.d.ts +10 -1
- package/dist/usage-limit-messages.d.ts +22 -0
- package/dist/usage-limit-messages.js +44 -0
- package/dist/usage-probe-job.d.ts +17 -0
- package/dist/usage-probe-job.js +186 -0
- package/npm-shrinkwrap.json +2 -2
- package/package.json +1 -1
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
// Exit codes are fragile — the Claude binary can legitimately return 143
|
|
14
14
|
// for unrelated reasons; the marker decides on poller intent, not process
|
|
15
15
|
// output.
|
|
16
|
+
import { config } from './config.js';
|
|
17
|
+
import { formatLegacyUsageLimitMessage } from './usage-limit-messages.js';
|
|
16
18
|
export function classifyFailedIteration(result) {
|
|
17
19
|
switch (result.killedByPoller) {
|
|
18
20
|
case 'idle':
|
|
@@ -68,10 +70,12 @@ export function classifyFailedIteration(result) {
|
|
|
68
70
|
case 'timeout':
|
|
69
71
|
case undefined: {
|
|
70
72
|
if (!result.killedByPoller && result.quota) {
|
|
71
|
-
const reset = result.quota.resetAt ? `Resets at ${result.quota.resetAt}.` : 'Reset time unavailable.';
|
|
72
73
|
return {
|
|
73
74
|
kind: 'quota', quota: result.quota,
|
|
74
|
-
text:
|
|
75
|
+
text: formatLegacyUsageLimitMessage({
|
|
76
|
+
resetAt: result.quota.resetAt ? Date.parse(result.quota.resetAt) : undefined,
|
|
77
|
+
timeZone: config.POLLER_TZ,
|
|
78
|
+
}),
|
|
75
79
|
reaction: 'hourglass',
|
|
76
80
|
};
|
|
77
81
|
}
|
package/dist/claude-spawner.js
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { logPoolFetch, logPoolUsageWrite, poolInfo, poolLabel } from './pool-logging.js';
|
|
1
2
|
import { accountUsageTelemetry, emitCandidateUsage, poolRunTelemetry } from './pool-telemetry.js';
|
|
2
3
|
import { spawn } from 'child_process';
|
|
3
4
|
import { join } from 'path';
|
|
@@ -938,6 +939,8 @@ async function runClaudeCode(msg, sessionLog, cwd, brainPath, sessionId, isFirst
|
|
|
938
939
|
const envPath = config.PERSISTENT_STORAGE_PATH
|
|
939
940
|
? `${config.PERSISTENT_STORAGE_PATH}/bin:${process.env['PATH'] || ''}`
|
|
940
941
|
: process.env['PATH'];
|
|
942
|
+
if (runtime?.account)
|
|
943
|
+
poolInfo(sessionLog, `token target account=${poolLabel(runtime.account)} session=${JSON.stringify(sessionId ?? 'new')} credential=selected account token`);
|
|
941
944
|
const proc = spawn(config.CLAUDE_CLI_PATH, args, {
|
|
942
945
|
cwd,
|
|
943
946
|
env: buildSpawnEnv({
|
|
@@ -1418,6 +1421,7 @@ export async function spawnPoolClaudeCode(msg, sessionLog, cwd, sessionId, isFir
|
|
|
1418
1421
|
// the main working dir can't trample in-flight writes or pre-push commits.
|
|
1419
1422
|
// See teamvibe.ai#120. cwd remains the main brain path on fallback / failure.
|
|
1420
1423
|
let effectiveCwd = cwd;
|
|
1424
|
+
let loggingPool = false;
|
|
1421
1425
|
let retainWorktree = poolRuntime?.pin?.resumePending === true;
|
|
1422
1426
|
const poolAttempts = poolRuntime?.attempts?.slice(-1000) ?? [];
|
|
1423
1427
|
let worktreeOwner;
|
|
@@ -1462,12 +1466,15 @@ export async function spawnPoolClaudeCode(msg, sessionLog, cwd, sessionId, isFir
|
|
|
1462
1466
|
effectiveSessionId = randomUUID();
|
|
1463
1467
|
}
|
|
1464
1468
|
const fetchedPool = poolRuntime ? poolRuntime.pool ?? await fetchRuntimePool(msg) : undefined;
|
|
1469
|
+
loggingPool = fetchedPool !== undefined && fetchedPool.kind !== 'no_accounts_configured';
|
|
1470
|
+
if (fetchedPool)
|
|
1471
|
+
logPoolFetch(sessionLog, fetchedPool);
|
|
1465
1472
|
const scoped = poolRuntime ? await fetchConfigForSpawn({ pollerId: getPollerId() || '', workspaceId: msg.teamvibe.workspaceId, channelId: msg.teamvibe.channelId }) : undefined;
|
|
1466
1473
|
const settings = readBrainSettings(effectiveCwd);
|
|
1467
1474
|
const resolved = fetchedPool?.kind !== 'no_accounts_configured' && poolRuntime?.pin?.workspaceId === msg.teamvibe.workspaceId
|
|
1468
1475
|
? { model: poolRuntime.pin.model, modelSource: poolRuntime.pin.modelSource }
|
|
1469
1476
|
: resolveModel(typeof settings['model'] === 'string' ? settings['model'] : undefined, scoped ?? {}, config.CLAUDE_MODEL);
|
|
1470
|
-
const prepared = poolRuntime ? await preparePool(msg, resolved, poolRuntime.pin, fetchedPool) : undefined;
|
|
1477
|
+
const prepared = poolRuntime ? await preparePool(msg, resolved, poolRuntime.pin, fetchedPool, sessionLog) : undefined;
|
|
1471
1478
|
const claudeConfigDir = poolRuntime?.pin?.resumePending ? poolRuntime.pin.claudeConfigDir ?? getBaseBrainPath() : getBaseBrainPath();
|
|
1472
1479
|
const modelNormalizedFrom = scoped?.model === resolved.model && scoped?.modelSource === resolved.modelSource
|
|
1473
1480
|
? scoped?.modelNormalizedFrom : undefined;
|
|
@@ -1496,8 +1503,9 @@ export async function spawnPoolClaudeCode(msg, sessionLog, cwd, sessionId, isFir
|
|
|
1496
1503
|
if (result.success && id && poolRuntime) {
|
|
1497
1504
|
// Complete the inbox before any fallible post-inference store writes.
|
|
1498
1505
|
// The loop retains bookkeeping independently of the original prompt.
|
|
1499
|
-
await poolRuntime.complete(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: false }, poolAttempts, result.rateLimitInfo);
|
|
1500
|
-
|
|
1506
|
+
const bookkeepingComplete = await poolRuntime.complete(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: false }, poolAttempts, result.rateLimitInfo);
|
|
1507
|
+
if (bookkeepingComplete !== false)
|
|
1508
|
+
await emitCandidateUsage(msg, [prepared.account], prepared.store);
|
|
1501
1509
|
retainWorktree = false;
|
|
1502
1510
|
return result;
|
|
1503
1511
|
}
|
|
@@ -1509,9 +1517,9 @@ export async function spawnPoolClaudeCode(msg, sessionLog, cwd, sessionId, isFir
|
|
|
1509
1517
|
// retaining the attempt. Selection waits for all writes before switching.
|
|
1510
1518
|
const writes = [];
|
|
1511
1519
|
if (result.quota && !result.killedByPoller)
|
|
1512
|
-
writes.push(prepared.store.recordQuota(prepared.account, result.quota, Date.now(), { expireFutureObservations: true }));
|
|
1520
|
+
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.quota.window ? { rateLimitType: result.quota.window } : undefined, () => prepared.store.recordQuota(prepared.account, result.quota, Date.now(), { expireFutureObservations: true }), 'quota'));
|
|
1513
1521
|
if (result.rateLimitInfo)
|
|
1514
|
-
writes.push(prepared.store.recordPassive(prepared.account, result.rateLimitInfo, Date.now(), { expireFutureObservations: true }));
|
|
1522
|
+
writes.push(logPoolUsageWrite(sessionLog, prepared.account, result.rateLimitInfo, () => prepared.store.recordPassive(prepared.account, result.rateLimitInfo, Date.now(), { expireFutureObservations: true })));
|
|
1515
1523
|
if (id && poolRuntime)
|
|
1516
1524
|
writes.push(poolRuntime.persist(id, { ...accountPin(prepared.account, resolved), effectiveCwd, claudeConfigDir, resumePending: runtime?.resumePending, sawSendMessageCall }, poolAttempts));
|
|
1517
1525
|
const outcomes = await Promise.allSettled(writes);
|
|
@@ -1529,17 +1537,24 @@ export async function spawnPoolClaudeCode(msg, sessionLog, cwd, sessionId, isFir
|
|
|
1529
1537
|
if (poolRuntime.shouldStop() || attemptedAccounts.size >= 1000)
|
|
1530
1538
|
throw new PoolDeferredError();
|
|
1531
1539
|
prepared.pool = await fetchRuntimePool(msg);
|
|
1540
|
+
logPoolFetch(sessionLog, prepared.pool);
|
|
1532
1541
|
if (prepared.pool.kind === 'ready')
|
|
1533
1542
|
await emitCandidateUsage(msg, prepared.pool.accounts, prepared.store);
|
|
1534
1543
|
// Reactive evidence was persisted by attempt() before choosing another account.
|
|
1544
|
+
if (prepared.pool.kind === 'ready')
|
|
1545
|
+
for (const account of prepared.pool.accounts) {
|
|
1546
|
+
if (attemptedAccounts.has(account.accountId))
|
|
1547
|
+
poolInfo(sessionLog, `skipped account=${poolLabel(account)} reason=already exhausted in this session`);
|
|
1548
|
+
}
|
|
1535
1549
|
const remaining = prepared.pool.kind === 'ready' ? prepared.pool.accounts.filter(a => !attemptedAccounts.has(a.accountId)) : [];
|
|
1536
|
-
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: msg.teamvibe.workspaceId, pool: remaining.length
|
|
1550
|
+
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: msg.teamvibe.workspaceId, pool: remaining.length && prepared.pool.kind === 'ready' ? { ...prepared.pool, accounts: remaining } : prepared.pool, store: prepared.store, now: Date.now(), log: sessionLog, logAllExhausted: remaining.length === 0 });
|
|
1537
1551
|
if (selected.kind !== 'selected' || attemptedAccounts.has(selected.account.accountId))
|
|
1538
|
-
throw await poolDeferredError(prepared.pool, resolved, prepared.store);
|
|
1552
|
+
throw await poolDeferredError(prepared.pool, resolved, prepared.store, undefined, remaining.length ? sessionLog : undefined);
|
|
1539
1553
|
effectiveSessionId = result.attemptedSessionId ?? effectiveSessionId;
|
|
1540
1554
|
if (!effectiveSessionId)
|
|
1541
1555
|
throw new PoolDeferredError();
|
|
1542
1556
|
const fromAccount = prepared.account;
|
|
1557
|
+
poolInfo(sessionLog, `switch from=${poolLabel(fromAccount)} to=${poolLabel(selected.account)} session=${JSON.stringify(effectiveSessionId)}`);
|
|
1543
1558
|
prepared.account = selected.account;
|
|
1544
1559
|
runtime.account = selected.account;
|
|
1545
1560
|
runtime.resumePending = true;
|
|
@@ -1574,6 +1589,8 @@ export async function spawnPoolClaudeCode(msg, sessionLog, cwd, sessionId, isFir
|
|
|
1574
1589
|
}
|
|
1575
1590
|
catch (error) {
|
|
1576
1591
|
if (error instanceof PoolDeferredError) {
|
|
1592
|
+
if (loggingPool)
|
|
1593
|
+
poolInfo(sessionLog, `work deferred eligibleAt=${new Date(error.eligibleAt).toISOString()} providerResetAt=${error.resetAt ?? 'unknown'}`);
|
|
1577
1594
|
retainWorktree = true;
|
|
1578
1595
|
error.attempts = poolAttempts;
|
|
1579
1596
|
}
|
package/dist/event-queue.js
CHANGED
|
@@ -69,7 +69,7 @@ export async function flush() {
|
|
|
69
69
|
return;
|
|
70
70
|
isFlushing = true;
|
|
71
71
|
try {
|
|
72
|
-
const batch =
|
|
72
|
+
const batch = queue.splice(0);
|
|
73
73
|
const failed = [];
|
|
74
74
|
for (const event of batch) {
|
|
75
75
|
const ok = await flushOne(event);
|
|
@@ -77,7 +77,7 @@ export async function flush() {
|
|
|
77
77
|
failed.push(event);
|
|
78
78
|
}
|
|
79
79
|
}
|
|
80
|
-
queue = failed;
|
|
80
|
+
queue = [...failed, ...queue].slice(-MAX_QUEUE_SIZE);
|
|
81
81
|
if (failed.length > 0) {
|
|
82
82
|
logger.debug(`[EventQueue] ${failed.length} event(s) failed to flush, will retry`);
|
|
83
83
|
}
|
package/dist/poller.d.ts
CHANGED
|
@@ -33,6 +33,7 @@ export declare function describeFailureReason(killedByPoller: KilledByPoller | u
|
|
|
33
33
|
export declare const END_OF_TURN_NO_FURTHER_RESPONSE_SENTINEL = "NO_FURTHER_RESPONSE_NEEDED";
|
|
34
34
|
export declare function endOfTurnFallbackText(hasSlackContext: boolean, result: Pick<SpawnResult, 'success' | 'sawSendMessageCall' | 'lastPlainTextContent'>): string | undefined;
|
|
35
35
|
export declare function runInboxLoop(threadId: string, kbPath: string, queueMessage: TeamVibeQueueMessage, sessionLog: ReturnType<typeof logger.createSession>): Promise<void>;
|
|
36
|
+
export declare function pollLoop(): Promise<void>;
|
|
36
37
|
export declare function __setSleepForTest(impl: (ms: number) => Promise<void>): void;
|
|
37
38
|
export declare function __setShuttingDownForTest(value: boolean): void;
|
|
38
39
|
export declare function startPoller(): Promise<void>;
|
package/dist/poller.js
CHANGED
|
@@ -1,4 +1,8 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { isUsageProbe, handleUsageProbe } from './usage-probe-job.js';
|
|
2
|
+
import { fetchSubscriptionPool } from './subscription-pool-client.js';
|
|
3
|
+
import { formatPoolUsageLimitMessage } from './usage-limit-messages.js';
|
|
4
|
+
import { logPoolUsageWrite } from './pool-logging.js';
|
|
5
|
+
import { emitCandidateUsage, poolRunTelemetry } from './pool-telemetry.js';
|
|
2
6
|
import { config, POLLER_VERSION } from './config.js';
|
|
3
7
|
import { logger, setLogContext } from './logger.js';
|
|
4
8
|
import { pollMessages, deleteMessage, } from './sqs-poller.js';
|
|
@@ -284,14 +288,17 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
284
288
|
let currentPin = session.subscription_pin;
|
|
285
289
|
let deferred = false;
|
|
286
290
|
let poolIteration = false;
|
|
287
|
-
const retryCompletedBookkeeping = async (state, liveCompletion = false) => {
|
|
291
|
+
const retryCompletedBookkeeping = async (state, pool, liveCompletion = false) => {
|
|
288
292
|
if (!state.completed)
|
|
289
293
|
return true;
|
|
290
294
|
const writes = [];
|
|
291
295
|
// A live run may expire evidence after clock rollback; replaying an old
|
|
292
296
|
// completion must preserve observations produced by newer concurrent runs.
|
|
297
|
+
const lastAttempt = state.attempts?.at(-1);
|
|
298
|
+
// Existing telemetry falls back to account IDs. Local pool logs use labels only.
|
|
299
|
+
const accountLabel = lastAttempt?.accountLabel !== state.pin.accountId ? lastAttempt?.accountLabel : undefined;
|
|
293
300
|
if (state.completed.rateLimitInfo)
|
|
294
|
-
writes.push(subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined));
|
|
301
|
+
writes.push(logPoolUsageWrite(sessionLog, { label: accountLabel }, state.completed.rateLimitInfo, () => subscriptionUsageStore().recordPassive(state.pin, state.completed.rateLimitInfo, state.completed.observedAt, liveCompletion ? { expireFutureObservations: true } : undefined)));
|
|
295
302
|
writes.push(lockToken ? updateSessionId(threadId, lockToken, state.sessionId, state.pin) : Promise.reject(new Error('Missing lock')));
|
|
296
303
|
const outcomes = await Promise.allSettled(writes);
|
|
297
304
|
if (outcomes.some(outcome => outcome.status === 'rejected' || outcome.value === false)) {
|
|
@@ -306,6 +313,17 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
306
313
|
}
|
|
307
314
|
currentSessionId = state.sessionId;
|
|
308
315
|
currentPin = state.pin;
|
|
316
|
+
// The spawner publishes live completions only after bookkeeping succeeds.
|
|
317
|
+
// Recovery uses a validated current account, without attributing an old
|
|
318
|
+
// credential's observation to its replacement.
|
|
319
|
+
if (!liveCompletion && pool.kind === 'ready') {
|
|
320
|
+
const account = pool.accounts.find(account => account.workspaceId === state.pin.workspaceId &&
|
|
321
|
+
account.workspaceId === queueMessage.teamvibe.workspaceId &&
|
|
322
|
+
account.accountId === state.pin.accountId &&
|
|
323
|
+
account.credentialVersion === state.pin.credentialVersion);
|
|
324
|
+
if (account)
|
|
325
|
+
await emitCandidateUsage(queueMessage, [account], subscriptionUsageStore());
|
|
326
|
+
}
|
|
309
327
|
clearInboxRuntime(kbPath, threadId);
|
|
310
328
|
return true;
|
|
311
329
|
};
|
|
@@ -340,7 +358,7 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
340
358
|
// Completed prompts never enter pickNextMessage/spawn again, even
|
|
341
359
|
// when the usage store or the session pin needs a later repair.
|
|
342
360
|
markProcessed(kbPath, threadId, completion.filename);
|
|
343
|
-
if (Date.now() >= completion.completed.observedAt && Date.now() < completion.completed.retryAt || !await retryCompletedBookkeeping(completion)) {
|
|
361
|
+
if (Date.now() >= completion.completed.observedAt && Date.now() < completion.completed.retryAt || !await retryCompletedBookkeeping(completion, pool)) {
|
|
344
362
|
deferred = hasUnprocessed(kbPath, threadId);
|
|
345
363
|
break;
|
|
346
364
|
}
|
|
@@ -428,7 +446,7 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
428
446
|
currentPin = pin;
|
|
429
447
|
succeeded = true;
|
|
430
448
|
clearPoolDefer(threadId);
|
|
431
|
-
|
|
449
|
+
return retryCompletedBookkeeping(state, pool, true);
|
|
432
450
|
},
|
|
433
451
|
persist: async (id, pin, attempts) => {
|
|
434
452
|
pin = { ...pin, inboxFilename: message.filename };
|
|
@@ -579,7 +597,13 @@ export async function runInboxLoop(threadId, kbPath, queueMessage, sessionLog) {
|
|
|
579
597
|
if (!reconcile && !setPoolNotice(threadId, pause.episodeId, 'claimed'))
|
|
580
598
|
throw new PoolDeferredError(error.eligibleAt);
|
|
581
599
|
try {
|
|
582
|
-
|
|
600
|
+
const notice = formatPoolUsageLimitMessage({
|
|
601
|
+
...error.notice,
|
|
602
|
+
thresholdDeferred: error.notice?.thresholdDeferred && !error.attempts?.some(a => a.quota),
|
|
603
|
+
resetAt: pause.resetAt ? Date.parse(pause.resetAt) : error.notice?.resetAt,
|
|
604
|
+
timeZone: config.POLLER_TZ,
|
|
605
|
+
});
|
|
606
|
+
await sendSlackMessage(deliveryMsg, notice, { clientMessageId: pause.episodeId, ...(reconcile ? { reconcile: true } : {}) });
|
|
583
607
|
setPoolNotice(threadId, pause.episodeId, 'delivered');
|
|
584
608
|
}
|
|
585
609
|
catch (deliveryError) {
|
|
@@ -910,7 +934,7 @@ function isAuthError(error) {
|
|
|
910
934
|
message.includes('security token') ||
|
|
911
935
|
message.includes('Auth failed'));
|
|
912
936
|
}
|
|
913
|
-
async function pollLoop() {
|
|
937
|
+
export async function pollLoop() {
|
|
914
938
|
logger.info('Poll loop started');
|
|
915
939
|
while (!shuttingDown) {
|
|
916
940
|
try {
|
|
@@ -951,8 +975,35 @@ async function pollLoop() {
|
|
|
951
975
|
continue;
|
|
952
976
|
}
|
|
953
977
|
logger.info(`Received ${messages.length} message(s) from SQS`);
|
|
954
|
-
//
|
|
955
|
-
const
|
|
978
|
+
// Workspace jobs have no thread and bypass every channel/session path.
|
|
979
|
+
const channelMessages = [];
|
|
980
|
+
for (const msg of messages) {
|
|
981
|
+
if (isUsageProbe(msg.queueMessage)) {
|
|
982
|
+
const job = msg.queueMessage;
|
|
983
|
+
await handleUsageProbe(job, {
|
|
984
|
+
fetchPool: () => fetchSubscriptionPool({
|
|
985
|
+
pollerId: getPollerId() || '',
|
|
986
|
+
workspaceId: job.workspaceId,
|
|
987
|
+
accountId: job.accountId,
|
|
988
|
+
credentialVersion: job.credentialVersion,
|
|
989
|
+
requestId: job.requestId,
|
|
990
|
+
}, {
|
|
991
|
+
apiUrl: config.TEAMVIBE_API_URL,
|
|
992
|
+
pollerToken: config.TEAMVIBE_POLLER_TOKEN || '',
|
|
993
|
+
}),
|
|
994
|
+
store: subscriptionUsageStore(),
|
|
995
|
+
cliPath: config.CLAUDE_CLI_PATH,
|
|
996
|
+
isAtCapacity,
|
|
997
|
+
emit: emitEvent,
|
|
998
|
+
log: line => logger.info(line),
|
|
999
|
+
});
|
|
1000
|
+
await deleteMessage(msg.receiptHandle);
|
|
1001
|
+
continue;
|
|
1002
|
+
}
|
|
1003
|
+
channelMessages.push(msg);
|
|
1004
|
+
}
|
|
1005
|
+
// Deduplicate real messages for the same thread, keeping the latest.
|
|
1006
|
+
const { unique, duplicates } = deduplicateMessages(channelMessages);
|
|
956
1007
|
for (const dup of duplicates) {
|
|
957
1008
|
logger.info(`Deleting duplicate message for thread ${dup.queueMessage.thread_id} (messageId: ${dup.messageId})`);
|
|
958
1009
|
await deleteMessage(dup.receiptHandle);
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { SessionLogger } from './logger.js';
|
|
2
|
+
import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
3
|
+
import { type AccountIdentity, type AccountUsageState, type PoolFetchResult, type SelectionOutcome } from './subscription-selection.js';
|
|
4
|
+
export type PoolInfoLogger = Pick<SessionLogger, 'info'>;
|
|
5
|
+
type LabeledAccount = AccountIdentity & {
|
|
6
|
+
label?: string;
|
|
7
|
+
};
|
|
8
|
+
export declare function poolInfo(log: PoolInfoLogger, message: string): void;
|
|
9
|
+
export declare function poolLabel(account: {
|
|
10
|
+
label?: string;
|
|
11
|
+
}): string;
|
|
12
|
+
export declare function logPoolFetch(log: PoolInfoLogger, pool: PoolFetchResult): void;
|
|
13
|
+
export declare function logPoolSelection<A extends LabeledAccount>(log: PoolInfoLogger, pool: PoolFetchResult<A>, states: Readonly<Record<string, AccountUsageState>>, model: string | undefined, now: number, selected: SelectionOutcome<A>, logAllExhausted?: boolean): void;
|
|
14
|
+
/** Log the real store promise without changing its rejection or retry behavior. */
|
|
15
|
+
export declare function logPoolUsageWrite(log: PoolInfoLogger, account: {
|
|
16
|
+
label?: string;
|
|
17
|
+
}, info: ClaudeRateLimitInfo | undefined, write: () => Promise<void>, source?: 'passive' | 'quota'): Promise<void>;
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { redactStartupText } from './startup-auth.js';
|
|
2
|
+
import { normalizePassiveUsage, selectSubscriptionAccount } from './subscription-selection.js';
|
|
3
|
+
import { safeRateLimitInfo } from './subscription-usage-store.js';
|
|
4
|
+
// Never serialize accounts, credentials, provider strings or caught errors.
|
|
5
|
+
// Labels have already been checked by the pool client; JSON quoting keeps
|
|
6
|
+
// arbitrary model/label whitespace on one physical log line.
|
|
7
|
+
export function poolInfo(log, message) {
|
|
8
|
+
log.info(redactStartupText(`[Pool] ${message}`).replace(/[\r\n]/g, ' '));
|
|
9
|
+
}
|
|
10
|
+
export function poolLabel(account) {
|
|
11
|
+
const label = account.label;
|
|
12
|
+
return JSON.stringify(label && label.length <= 128 && !/sk-ant-|Bearer\s|[\r\n]/i.test(label) ? label : 'unlabeled');
|
|
13
|
+
}
|
|
14
|
+
export function logPoolFetch(log, pool) {
|
|
15
|
+
if (pool.kind === 'no_accounts_configured')
|
|
16
|
+
return;
|
|
17
|
+
// The API exposes usable members only; hidden disabled labels are unavailable.
|
|
18
|
+
const enabled = pool.kind === 'ready' ? pool.accounts.filter(a => a.enabled).length : pool.kind === 'fetch_unavailable' && pool.reason === 'no_usable_accounts' ? 0 : 'unknown';
|
|
19
|
+
const configured = pool.kind === 'ready' || pool.kind === 'fetch_unavailable' && pool.reason === 'no_usable_accounts' ? 'true' : 'unknown';
|
|
20
|
+
poolInfo(log, `fetch poolConfigured=${configured} enabled=${enabled} usable=${enabled} result=${pool.kind}${pool.kind === 'fetch_unavailable' ? ` reason=${pool.reason}` : ''}`);
|
|
21
|
+
}
|
|
22
|
+
export function logPoolSelection(log, pool, states, model, now, selected, logAllExhausted = true) {
|
|
23
|
+
if (pool.kind !== 'ready')
|
|
24
|
+
return;
|
|
25
|
+
for (const account of pool.accounts) {
|
|
26
|
+
if (!account.enabled) {
|
|
27
|
+
poolInfo(log, `skipped account=${poolLabel(account)} reason=disabled`);
|
|
28
|
+
continue;
|
|
29
|
+
}
|
|
30
|
+
const candidate = selectSubscriptionAccount({ pool: { ...pool, accounts: [account] }, states, model, now });
|
|
31
|
+
if (candidate.kind === 'deferred') {
|
|
32
|
+
const usageOnly = selectSubscriptionAccount({ pool: { ...pool, accounts: [account] }, states: { [account.accountId]: { usage: states[account.accountId]?.usage } }, model, now });
|
|
33
|
+
poolInfo(log, `skipped account=${poolLabel(account)} reason=${usageOnly.kind === 'deferred' ? `usage >=${account.useFullQuota ? 100 : 90}%` : 'reactive quota'} eligibleAt=${new Date(candidate.eligibleAt).toISOString()}`);
|
|
34
|
+
}
|
|
35
|
+
else if (candidate.kind === 'selected' && selected.kind === 'selected' && account.accountId !== selected.account.accountId) {
|
|
36
|
+
const reason = candidate.provenance === 'unknown' && selected.provenance === 'known' ? 'known observation preferred' : candidate.provenance === 'known' && selected.provenance === 'known' && candidate.score > selected.score ? 'higher usage score' : 'deterministic account order';
|
|
37
|
+
poolInfo(log, `skipped account=${poolLabel(account)} reason=${reason}`);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
if (selected.kind === 'deferred' && logAllExhausted) {
|
|
41
|
+
// Ask the same policy about each observation so model scope, freshness and
|
|
42
|
+
// reset expiry stay identical to selection. An account must clear ALL its
|
|
43
|
+
// blockers; the pool can resume when its first account clears them.
|
|
44
|
+
const resets = pool.accounts.filter(a => a.enabled).map(account => {
|
|
45
|
+
const state = states[account.accountId];
|
|
46
|
+
const deadlines = [];
|
|
47
|
+
for (const window of state?.usage?.windows ?? []) {
|
|
48
|
+
const candidate = selectSubscriptionAccount({ pool: { ...pool, accounts: [account] }, states: { [account.accountId]: { usage: { ...state.usage, windows: [window] } } }, model, now });
|
|
49
|
+
if (candidate.kind === 'deferred')
|
|
50
|
+
deadlines.push(window.resetAt);
|
|
51
|
+
}
|
|
52
|
+
for (const reactive of state?.reactive ?? []) {
|
|
53
|
+
const candidate = selectSubscriptionAccount({ pool: { ...pool, accounts: [account] }, states: { [account.accountId]: { reactive: [reactive] } }, model, now });
|
|
54
|
+
if (candidate.kind === 'deferred')
|
|
55
|
+
deadlines.push(reactive.quota.resetAt ? Date.parse(reactive.quota.resetAt) : undefined);
|
|
56
|
+
}
|
|
57
|
+
return deadlines.length && deadlines.every(d => d !== undefined && Number.isFinite(d)) ? Math.max(...deadlines) : undefined;
|
|
58
|
+
});
|
|
59
|
+
const earliestResetAt = resets.length && resets.every(r => r !== undefined) ? new Date(Math.min(...resets)).toISOString() : 'unknown';
|
|
60
|
+
poolInfo(log, `deferred reason=all accounts exhausted earliestResetAt=${earliestResetAt} eligibleAt=${new Date(selected.eligibleAt).toISOString()}`);
|
|
61
|
+
}
|
|
62
|
+
if (selected.kind === 'selected') {
|
|
63
|
+
const score = selected.provenance === 'known' ? `${selected.score}% headroom=${100 - selected.score}%` : 'unknown (no observation)';
|
|
64
|
+
poolInfo(log, `selected account=${poolLabel(selected.account)} model=${JSON.stringify(model ?? 'CLI default')} score=${score}`);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
/** Log the real store promise without changing its rejection or retry behavior. */
|
|
68
|
+
export async function logPoolUsageWrite(log, account, info, write, source = 'passive') {
|
|
69
|
+
const windows = info ? normalizePassiveUsage(safeRateLimitInfo(info), Date.now()).windows.map(w => `${w.window}${source === 'passive' ? `/${w.scope}` : ''}:${w.percent === undefined ? 'unknown' : `${Number(w.percent.toFixed(2))}%`}`).join(',') : '';
|
|
70
|
+
try {
|
|
71
|
+
await write();
|
|
72
|
+
poolInfo(log, `usage write account=${poolLabel(account)} result=ok windows=${windows || 'none'} source=${source}`);
|
|
73
|
+
}
|
|
74
|
+
catch (error) {
|
|
75
|
+
const name = error instanceof Error && ['Error', 'TypeError', 'RangeError', 'AbortError', 'TimeoutError', 'SubscriptionUsageStoreError'].includes(error.name) ? error.name : 'UnknownError';
|
|
76
|
+
poolInfo(log, `usage write account=${poolLabel(account)} result=failed errorClass=${name} windows=${windows || 'none'} source=${source}`);
|
|
77
|
+
throw error;
|
|
78
|
+
}
|
|
79
|
+
}
|
|
@@ -14,8 +14,11 @@ export interface SubscriptionAccount extends AccountIdentity {
|
|
|
14
14
|
}
|
|
15
15
|
export interface PoolFetchInput {
|
|
16
16
|
workspaceId: string;
|
|
17
|
-
channelId
|
|
17
|
+
channelId?: string;
|
|
18
18
|
pollerId: string;
|
|
19
|
+
accountId?: string;
|
|
20
|
+
requestId?: string;
|
|
21
|
+
credentialVersion?: string;
|
|
19
22
|
}
|
|
20
23
|
export interface PoolClientOptions {
|
|
21
24
|
apiUrl: string;
|
|
@@ -71,14 +71,18 @@ export async function readSubscriptionJson(response) {
|
|
|
71
71
|
export async function fetchSubscriptionPool(input, options) {
|
|
72
72
|
let url;
|
|
73
73
|
try {
|
|
74
|
-
if (!input.pollerId || !input.workspaceId || !input.channelId || !options.pollerToken)
|
|
74
|
+
if (!input.pollerId || !input.workspaceId || (!input.channelId && !input.requestId) || !options.pollerToken)
|
|
75
75
|
throw new Error('Missing configuration');
|
|
76
76
|
url = new URL('/subscription-accounts/spawn', options.apiUrl);
|
|
77
77
|
if (!['https:', 'http:'].includes(url.protocol) || url.username || url.password)
|
|
78
78
|
throw new Error('Invalid URL');
|
|
79
79
|
url.searchParams.set('pollerId', input.pollerId);
|
|
80
80
|
url.searchParams.set('workspaceId', input.workspaceId);
|
|
81
|
-
|
|
81
|
+
if (input.channelId)
|
|
82
|
+
url.searchParams.set('channelId', input.channelId);
|
|
83
|
+
for (const name of ['accountId', 'requestId', 'credentialVersion'])
|
|
84
|
+
if (input[name])
|
|
85
|
+
url.searchParams.set(name, input[name]);
|
|
82
86
|
}
|
|
83
87
|
catch {
|
|
84
88
|
return { kind: 'fetch_unavailable', reason: 'configuration' };
|
|
@@ -121,7 +125,7 @@ export async function fetchSubscriptionPool(input, options) {
|
|
|
121
125
|
return { kind: 'fetch_unavailable', reason: 'invalid_response' };
|
|
122
126
|
}
|
|
123
127
|
ids.add(item['accountId']);
|
|
124
|
-
accounts.push({ accountId: item['accountId'], workspaceId: input.workspaceId, credentialVersion: item['credentialVersion'], enabled: true, harness: 'CLAUDE', ...(typeof item['label'] === 'string' && item['label'].length <= 128 && !/sk-ant-|Bearer\s|[\r\n]/i.test(item['label']) ? { label: item['label'] } : {}), credential: new SubscriptionCredential(credential['token']) });
|
|
128
|
+
accounts.push({ accountId: item['accountId'], workspaceId: input.workspaceId, credentialVersion: item['credentialVersion'], enabled: true, useFullQuota: item['useFullQuota'] === true, harness: 'CLAUDE', ...(typeof item['label'] === 'string' && item['label'].length <= 128 && !/sk-ant-|Bearer\s|[\r\n]/i.test(item['label']) ? { label: item['label'] } : {}), credential: new SubscriptionCredential(credential['token']) });
|
|
125
129
|
}
|
|
126
130
|
if (accounts.length)
|
|
127
131
|
return { kind: 'ready', accounts };
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type PoolInfoLogger } from './pool-logging.js';
|
|
1
2
|
import type { resolveModel } from './claude-spawner.js';
|
|
2
3
|
import { type AccountIdentity, type PoolFetchResult, type SelectionOutcome } from './subscription-selection.js';
|
|
3
4
|
import { type SubscriptionUsageStore } from './subscription-usage-store.js';
|
|
@@ -15,4 +16,6 @@ export declare function selectPoolForResolvedModel<A extends AccountIdentity>(re
|
|
|
15
16
|
pool: PoolFetchResult<A>;
|
|
16
17
|
store: SubscriptionUsageStore;
|
|
17
18
|
now: number;
|
|
19
|
+
log?: PoolInfoLogger;
|
|
20
|
+
logAllExhausted?: boolean;
|
|
18
21
|
}): Promise<ResolvedPoolSelection<A>>;
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { logPoolSelection } from './pool-logging.js';
|
|
1
2
|
import { selectSubscriptionAccount } from './subscription-selection.js';
|
|
2
3
|
import { usageStateForSelection } from './subscription-usage-store.js';
|
|
3
4
|
/**
|
|
@@ -9,6 +10,8 @@ export async function selectPoolForResolvedModel(resolvedModel, input) {
|
|
|
9
10
|
if (input.pool.kind !== 'ready')
|
|
10
11
|
return { ...input.pool, ...resolvedModel };
|
|
11
12
|
if (!input.pool.accounts.length || input.pool.accounts.some(a => a.workspaceId !== input.workspaceId || !a.enabled)) {
|
|
13
|
+
if (input.log)
|
|
14
|
+
logPoolSelection(input.log, input.pool, {}, resolvedModel.model, input.now, { kind: 'fetch_unavailable', reason: 'invalid_response' });
|
|
12
15
|
return { kind: 'fetch_unavailable', reason: 'invalid_response', ...resolvedModel };
|
|
13
16
|
}
|
|
14
17
|
const states = Object.create(null);
|
|
@@ -20,5 +23,8 @@ export async function selectPoolForResolvedModel(resolvedModel, input) {
|
|
|
20
23
|
catch {
|
|
21
24
|
return { kind: 'usage_unavailable', ...resolvedModel };
|
|
22
25
|
}
|
|
23
|
-
|
|
26
|
+
const selected = selectSubscriptionAccount({ pool: input.pool, states, model: resolvedModel.model, now: input.now });
|
|
27
|
+
if (input.log)
|
|
28
|
+
logPoolSelection(input.log, input.pool, states, resolvedModel.model, input.now, selected, input.logAllExhausted);
|
|
29
|
+
return { ...selected, ...resolvedModel };
|
|
24
30
|
}
|
|
@@ -1,8 +1,10 @@
|
|
|
1
|
+
import { type PoolInfoLogger } from './pool-logging.js';
|
|
1
2
|
import { type SubscriptionAccount } from './subscription-pool-client.js';
|
|
2
3
|
import { type ResolvedPoolModel } from './subscription-pool.js';
|
|
3
4
|
import { FileSubscriptionUsageStore, type SubscriptionUsageStore } from './subscription-usage-store.js';
|
|
4
5
|
import { type PoolFetchResult } from './subscription-selection.js';
|
|
5
6
|
import type { SubscriptionPin, TeamVibeQueueMessage, SessionMetrics } from './types.js';
|
|
7
|
+
import type { PoolUsageLimitNotice } from './usage-limit-messages.js';
|
|
6
8
|
import type { ClaudeRateLimitInfo } from './claude-rate-limit.js';
|
|
7
9
|
import type { ClaudeQuotaFailure } from './claude-quota.js';
|
|
8
10
|
import type { KilledByPoller } from './classify-failed-iteration.js';
|
|
@@ -25,6 +27,7 @@ export declare class PoolDeferredError extends Error {
|
|
|
25
27
|
readonly eligibleAt: number;
|
|
26
28
|
readonly resetAt?: string | undefined;
|
|
27
29
|
attempts?: readonly PoolAttempt[];
|
|
30
|
+
notice?: PoolUsageLimitNotice;
|
|
28
31
|
constructor(eligibleAt?: number, resetAt?: string | undefined);
|
|
29
32
|
}
|
|
30
33
|
export interface PoolRuntimeOptions {
|
|
@@ -33,12 +36,13 @@ export interface PoolRuntimeOptions {
|
|
|
33
36
|
attempts?: readonly PoolAttempt[];
|
|
34
37
|
quotaDeferEpisodes?: number;
|
|
35
38
|
persist: (sessionId: string, pin: SubscriptionPin, attempts?: readonly PoolAttempt[]) => Promise<void>;
|
|
36
|
-
|
|
39
|
+
/** False retains bookkeeping for recovery, which owns the usage emission. */
|
|
40
|
+
complete: (sessionId: string, pin: SubscriptionPin, attempts: readonly PoolAttempt[], rateLimitInfo?: ClaudeRateLimitInfo) => Promise<boolean | void>;
|
|
37
41
|
shouldStop: () => boolean;
|
|
38
42
|
}
|
|
39
43
|
export declare function subscriptionUsageStore(): FileSubscriptionUsageStore;
|
|
40
44
|
export declare function fetchRuntimePool(msg: TeamVibeQueueMessage): Promise<PoolFetchResult<SubscriptionAccount>>;
|
|
41
|
-
export declare function preparePool(msg: TeamVibeQueueMessage, resolved: ResolvedPoolModel, pin?: SubscriptionPin, fetched?: PoolFetchResult<SubscriptionAccount
|
|
45
|
+
export declare function preparePool(msg: TeamVibeQueueMessage, resolved: ResolvedPoolModel, pin?: SubscriptionPin, fetched?: PoolFetchResult<SubscriptionAccount>, log?: PoolInfoLogger): Promise<{
|
|
42
46
|
account: SubscriptionAccount;
|
|
43
47
|
store: FileSubscriptionUsageStore;
|
|
44
48
|
pool: PoolFetchResult<SubscriptionAccount>;
|
|
@@ -50,4 +54,4 @@ export declare function preparePool(msg: TeamVibeQueueMessage, resolved: Resolve
|
|
|
50
54
|
export declare function accountPin(account: SubscriptionAccount, resolved: ResolvedPoolModel): SubscriptionPin;
|
|
51
55
|
export declare function validateAccount(account: SubscriptionAccount, resolved: ResolvedPoolModel): Promise<void>;
|
|
52
56
|
/** A snapshot TTL/recheck is not a provider reset. Report a reset only when reactive evidence establishes it. */
|
|
53
|
-
export declare function poolDeferredError(pool: PoolFetchResult<SubscriptionAccount>, resolved: ResolvedPoolModel, store: SubscriptionUsageStore, eligibleAt?: number): Promise<PoolDeferredError>;
|
|
57
|
+
export declare function poolDeferredError(pool: PoolFetchResult<SubscriptionAccount>, resolved: ResolvedPoolModel, store: SubscriptionUsageStore, eligibleAt?: number, log?: PoolInfoLogger): Promise<PoolDeferredError>;
|
|
@@ -1,15 +1,17 @@
|
|
|
1
|
+
import { poolInfo, poolLabel } from './pool-logging.js';
|
|
1
2
|
import { emitCandidateUsage } from './pool-telemetry.js';
|
|
2
3
|
import { join } from 'node:path';
|
|
3
4
|
import { config } from './config.js';
|
|
4
5
|
import { getPollerId } from './auth-provider.js';
|
|
5
6
|
import { fetchSubscriptionPool } from './subscription-pool-client.js';
|
|
6
7
|
import { selectPoolForResolvedModel } from './subscription-pool.js';
|
|
7
|
-
import { FileSubscriptionUsageStore } from './subscription-usage-store.js';
|
|
8
|
-
import { POOL_RECHECK_MS } from './subscription-selection.js';
|
|
8
|
+
import { FileSubscriptionUsageStore, usageStateForSelection } from './subscription-usage-store.js';
|
|
9
|
+
import { PASSIVE_USAGE_TTL_MS, PROBE_USAGE_TTL_MS, reactiveEligibleAt, USAGE_THRESHOLD_PERCENT, POOL_RECHECK_MS } from './subscription-selection.js';
|
|
9
10
|
export class PoolDeferredError extends Error {
|
|
10
11
|
eligibleAt;
|
|
11
12
|
resetAt;
|
|
12
13
|
attempts;
|
|
14
|
+
notice;
|
|
13
15
|
constructor(eligibleAt = Date.now() + POOL_RECHECK_MS, resetAt) {
|
|
14
16
|
super('Subscription pool unavailable; work retained');
|
|
15
17
|
this.eligibleAt = eligibleAt;
|
|
@@ -22,7 +24,7 @@ export function subscriptionUsageStore() {
|
|
|
22
24
|
export async function fetchRuntimePool(msg) {
|
|
23
25
|
return fetchSubscriptionPool({ pollerId: getPollerId() || '', workspaceId: msg.teamvibe.workspaceId, channelId: msg.teamvibe.channelId }, { apiUrl: config.TEAMVIBE_API_URL, pollerToken: config.TEAMVIBE_POLLER_TOKEN || '' });
|
|
24
26
|
}
|
|
25
|
-
export async function preparePool(msg, resolved, pin, fetched) {
|
|
27
|
+
export async function preparePool(msg, resolved, pin, fetched, log) {
|
|
26
28
|
const pool = fetched ?? await fetchRuntimePool(msg);
|
|
27
29
|
const store = subscriptionUsageStore();
|
|
28
30
|
if (pool.kind === 'ready')
|
|
@@ -30,14 +32,25 @@ export async function preparePool(msg, resolved, pin, fetched) {
|
|
|
30
32
|
if (pool.kind === 'ready' && pin?.workspaceId === msg.teamvibe.workspaceId) {
|
|
31
33
|
const account = pool.accounts.find(a => a.accountId === pin.accountId);
|
|
32
34
|
if (account) {
|
|
33
|
-
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: msg.teamvibe.workspaceId, pool: {
|
|
34
|
-
if (selected.kind === 'selected')
|
|
35
|
+
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: msg.teamvibe.workspaceId, pool: { ...pool, accounts: [account] }, store, now: Date.now(), log, logAllExhausted: false });
|
|
36
|
+
if (selected.kind === 'selected') {
|
|
37
|
+
if (log)
|
|
38
|
+
for (const other of pool.accounts) {
|
|
39
|
+
if (other.accountId !== account.accountId)
|
|
40
|
+
poolInfo(log, `skipped account=${poolLabel(other)} reason=session pinned elsewhere`);
|
|
41
|
+
}
|
|
35
42
|
return { account, store, pool };
|
|
43
|
+
}
|
|
36
44
|
}
|
|
37
45
|
}
|
|
38
|
-
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: msg.teamvibe.workspaceId, pool, store, now: Date.now() });
|
|
39
|
-
if (selected.kind === 'selected')
|
|
46
|
+
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: msg.teamvibe.workspaceId, pool, store, now: Date.now(), log });
|
|
47
|
+
if (selected.kind === 'selected') {
|
|
48
|
+
if (log && pool.kind === 'ready' && pin?.workspaceId === msg.teamvibe.workspaceId && pin.accountId !== selected.account.accountId) {
|
|
49
|
+
const previous = pool.accounts.find(a => a.accountId === pin.accountId);
|
|
50
|
+
poolInfo(log, `switch from=${poolLabel(previous ?? {})} to=${poolLabel(selected.account)} reason=pinned account unavailable`);
|
|
51
|
+
}
|
|
40
52
|
return { account: selected.account, store, pool };
|
|
53
|
+
}
|
|
41
54
|
if (selected.kind === 'no_accounts_configured')
|
|
42
55
|
return { account: undefined, store, pool };
|
|
43
56
|
throw await poolDeferredError(pool, resolved, store, selected.kind === 'deferred' ? selected.eligibleAt : undefined);
|
|
@@ -50,25 +63,64 @@ export async function validateAccount(account, resolved) {
|
|
|
50
63
|
if (selected.kind !== 'selected')
|
|
51
64
|
throw new PoolDeferredError(selected.kind === 'deferred' ? selected.eligibleAt : undefined);
|
|
52
65
|
}
|
|
66
|
+
/** Message-only evidence; never changes selection, eligibility or durable pause state. */
|
|
67
|
+
async function poolUsageNotice(pool, store) {
|
|
68
|
+
if (pool.kind !== 'ready')
|
|
69
|
+
return {};
|
|
70
|
+
const now = Date.now();
|
|
71
|
+
try {
|
|
72
|
+
const accounts = await Promise.all(pool.accounts.map(async (account) => {
|
|
73
|
+
const state = usageStateForSelection(await store.read(account), now);
|
|
74
|
+
const usage = state.usage;
|
|
75
|
+
const ttl = usage?.source === 'passive' ? PASSIVE_USAGE_TTL_MS : PROBE_USAGE_TTL_MS;
|
|
76
|
+
const windows = usage && usage.observedAt <= now && now < usage.observedAt + ttl
|
|
77
|
+
? usage.windows.filter(w => w.resetAt === undefined || w.resetAt > now) : [];
|
|
78
|
+
const rejected = state.reactive?.some(r => r.observedAt <= now && reactiveEligibleAt(r) > now) ?? false;
|
|
79
|
+
const blockers = windows.filter(w => w.percent !== undefined && w.percent >= (account.useFullQuota ? 100 : USAGE_THRESHOLD_PERCENT));
|
|
80
|
+
const resets = blockers.map(w => w.resetAt);
|
|
81
|
+
return {
|
|
82
|
+
label: account.label, useFullQuota: account.useFullQuota, windows, rejected,
|
|
83
|
+
thresholdBlocked: blockers.length > 0,
|
|
84
|
+
resetAt: resets.length && resets.every((r) => r !== undefined) ? Math.max(...resets) : undefined,
|
|
85
|
+
};
|
|
86
|
+
}));
|
|
87
|
+
const thresholdDeferred = accounts.length > 0 && accounts.every(a => a.thresholdBlocked) && !accounts.some(a => a.rejected);
|
|
88
|
+
// Snapshot expiry is only a recheck. Use actual window resets for the notice.
|
|
89
|
+
const resets = accounts.map(a => a.resetAt);
|
|
90
|
+
const resetAt = thresholdDeferred && resets.every((r) => r !== undefined) ? Math.min(...resets) : undefined;
|
|
91
|
+
return { accounts, thresholdDeferred, resetAt };
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
return {};
|
|
95
|
+
}
|
|
96
|
+
}
|
|
53
97
|
/** A snapshot TTL/recheck is not a provider reset. Report a reset only when reactive evidence establishes it. */
|
|
54
|
-
export async function poolDeferredError(pool, resolved, store, eligibleAt) {
|
|
98
|
+
export async function poolDeferredError(pool, resolved, store, eligibleAt, log) {
|
|
99
|
+
const notice = await poolUsageNotice(pool, store);
|
|
100
|
+
let usageDeferred = eligibleAt !== undefined;
|
|
101
|
+
const deferredError = (resetAt) => {
|
|
102
|
+
const error = new PoolDeferredError(eligibleAt, resetAt);
|
|
103
|
+
error.notice = { ...notice, thresholdDeferred: usageDeferred && notice.thresholdDeferred, resetAt: resetAt ? Date.parse(resetAt) : notice.resetAt };
|
|
104
|
+
return error;
|
|
105
|
+
};
|
|
55
106
|
if (eligibleAt === undefined) {
|
|
56
|
-
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: pool.kind === 'ready' ? pool.accounts[0]?.workspaceId ?? '' : '', pool, store, now: Date.now() });
|
|
107
|
+
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: pool.kind === 'ready' ? pool.accounts[0]?.workspaceId ?? '' : '', pool, store, now: Date.now(), log });
|
|
108
|
+
usageDeferred = selected.kind === 'deferred';
|
|
57
109
|
eligibleAt = selected.kind === 'deferred' ? selected.eligibleAt : Date.now() + POOL_RECHECK_MS;
|
|
58
110
|
}
|
|
59
111
|
if (pool.kind === 'ready') {
|
|
60
112
|
try {
|
|
61
113
|
for (const account of pool.accounts) {
|
|
62
|
-
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: account.workspaceId, pool: {
|
|
114
|
+
const selected = await selectPoolForResolvedModel(resolved, { workspaceId: account.workspaceId, pool: { ...pool, accounts: [account] }, store, now: Date.now() });
|
|
63
115
|
if (selected.kind !== 'deferred' || selected.eligibleAt !== eligibleAt)
|
|
64
116
|
continue;
|
|
65
117
|
const state = await store.read(account);
|
|
66
118
|
const live = state.reactive.filter(r => !r.quota.resetAt || Date.parse(r.quota.resetAt) > Date.now());
|
|
67
119
|
if (live.length && live.every(r => r.quota.resetAt && Number.isFinite(Date.parse(r.quota.resetAt))) && live.some(r => Date.parse(r.quota.resetAt) === eligibleAt))
|
|
68
|
-
return
|
|
120
|
+
return deferredError(new Date(eligibleAt).toISOString());
|
|
69
121
|
}
|
|
70
122
|
}
|
|
71
123
|
catch { }
|
|
72
124
|
}
|
|
73
|
-
return
|
|
125
|
+
return deferredError();
|
|
74
126
|
}
|
|
@@ -8,6 +8,7 @@ export interface AccountIdentity {
|
|
|
8
8
|
workspaceId: string;
|
|
9
9
|
accountId: string;
|
|
10
10
|
credentialVersion: string;
|
|
11
|
+
useFullQuota?: boolean;
|
|
11
12
|
enabled: boolean;
|
|
12
13
|
}
|
|
13
14
|
export type PoolFetchResult<A extends AccountIdentity = AccountIdentity> = {
|
|
@@ -27,7 +28,8 @@ export type PoolFetchResult<A extends AccountIdentity = AccountIdentity> = {
|
|
|
27
28
|
};
|
|
28
29
|
export interface UsageWindow {
|
|
29
30
|
window: 'five_hour' | 'seven_day';
|
|
30
|
-
scope: 'global' | 'sonnet' | 'opus' | 'haiku' | 'uncertain';
|
|
31
|
+
scope: 'global' | 'sonnet' | 'opus' | 'haiku' | 'fable' | 'uncertain';
|
|
32
|
+
status?: 'allowed' | 'allowed_warning' | 'rejected';
|
|
31
33
|
percent?: number;
|
|
32
34
|
resetAt?: number;
|
|
33
35
|
}
|
|
@@ -35,12 +35,27 @@ export function normalizePassiveUsage(value, observedAt) {
|
|
|
35
35
|
}
|
|
36
36
|
const type = info['rateLimitType'];
|
|
37
37
|
if (typeof type === 'string') {
|
|
38
|
-
const legacy = type.match(/^(five_hour|seven_day)(?:_(sonnet|opus|haiku))?$/);
|
|
39
|
-
|
|
38
|
+
const legacy = type.match(/^(five_hour|seven_day)(?:_(sonnet|opus|haiku|fable))?$/);
|
|
39
|
+
const candidate = {
|
|
40
40
|
window: legacy?.[1] === 'five_hour' ? 'five_hour' : 'seven_day',
|
|
41
41
|
scope: legacy ? (legacy[2] ?? 'global') : 'uncertain',
|
|
42
42
|
percent: percent(info['utilization'], 1), resetAt: epoch(info['resetsAt']),
|
|
43
|
-
|
|
43
|
+
...(info['status'] === 'allowed' || info['status'] === 'allowed_warning' || info['status'] === 'rejected'
|
|
44
|
+
? { status: info['status'] } : {}),
|
|
45
|
+
};
|
|
46
|
+
const existing = windows.find(w => w.window === candidate.window && w.scope === candidate.scope);
|
|
47
|
+
if (existing) {
|
|
48
|
+
if (candidate.status !== undefined)
|
|
49
|
+
existing.status = candidate.status;
|
|
50
|
+
// Collapse the same quota while retaining conservative conflict evidence.
|
|
51
|
+
if (candidate.percent !== undefined)
|
|
52
|
+
existing.percent = Math.max(existing.percent ?? candidate.percent, candidate.percent);
|
|
53
|
+
if (candidate.resetAt !== undefined)
|
|
54
|
+
existing.resetAt = Math.max(existing.resetAt ?? candidate.resetAt, candidate.resetAt);
|
|
55
|
+
}
|
|
56
|
+
else {
|
|
57
|
+
windows.push(candidate);
|
|
58
|
+
}
|
|
44
59
|
}
|
|
45
60
|
return { source: 'passive', observedAt, windows };
|
|
46
61
|
}
|
|
@@ -118,7 +133,7 @@ export function selectSubscriptionAccount(input) {
|
|
|
118
133
|
const windows = fresh ? usage.windows.filter(w => applicable(w.scope, input.model) && (w.resetAt === undefined || w.resetAt > input.now)) : [];
|
|
119
134
|
const uncertain = windows.some(w => w.scope === 'uncertain' || (w.scope !== 'global' && !family(input.model)));
|
|
120
135
|
for (const w of windows) {
|
|
121
|
-
if (w.percent !== undefined && w.percent >= USAGE_THRESHOLD_PERCENT)
|
|
136
|
+
if (w.percent !== undefined && w.percent >= (account.useFullQuota ? 100 : USAGE_THRESHOLD_PERCENT))
|
|
122
137
|
blockers.push(Math.min(w.resetAt ?? expiry, expiry));
|
|
123
138
|
}
|
|
124
139
|
if (blockers.length) {
|
|
@@ -27,6 +27,7 @@ export interface SubscriptionUsageStore {
|
|
|
27
27
|
recordQuota(key: UsageKey, quota: ClaudeQuotaFailure, observedAt: number, options?: UsageWriteOptions): Promise<void>;
|
|
28
28
|
/** Persist BEFORE network I/O; one attempt per credential version, including restart. */
|
|
29
29
|
beginProbe(key: UsageKey, observedAt: number): Promise<boolean>;
|
|
30
|
+
claimProbeRequest(requestId: string, expiresAt: number): Promise<boolean>;
|
|
30
31
|
finishProbe(key: UsageKey, observation: ProbeObservation): Promise<void>;
|
|
31
32
|
}
|
|
32
33
|
export declare function usageStateForSelection(state: StoredAccountUsage, now: number): AccountUsageState;
|
|
@@ -43,5 +44,7 @@ export declare class FileSubscriptionUsageStore implements SubscriptionUsageStor
|
|
|
43
44
|
recordPassive(key: UsageKey, rateLimitInfo: ClaudeRateLimitInfo, observedAt: number, options?: UsageWriteOptions): Promise<void>;
|
|
44
45
|
recordQuota(key: UsageKey, quota: ClaudeQuotaFailure, observedAt: number, options?: UsageWriteOptions): Promise<void>;
|
|
45
46
|
beginProbe(key: UsageKey, observedAt: number): Promise<boolean>;
|
|
47
|
+
/** Request claims live in the local data directory, alongside usage, never in a brain. */
|
|
48
|
+
claimProbeRequest(requestId: string, expiresAt: number): Promise<boolean>;
|
|
46
49
|
finishProbe(key: UsageKey, observation: ProbeObservation): Promise<void>;
|
|
47
50
|
}
|
|
@@ -33,9 +33,9 @@ function validTime(now) {
|
|
|
33
33
|
export function safeRateLimitInfo(info) {
|
|
34
34
|
const parsed = parseClaudeRateLimitInfo(info) ?? {};
|
|
35
35
|
// Never persist arbitrary status strings or incidental stream fields.
|
|
36
|
-
if (parsed.status !== 'allowed' && parsed.status !== 'allowed_warning')
|
|
36
|
+
if (parsed.status !== 'allowed' && parsed.status !== 'allowed_warning' && parsed.status !== 'rejected')
|
|
37
37
|
delete parsed.status;
|
|
38
|
-
if (parsed.rateLimitType && !/^(five_hour|seven_day)(?:_(sonnet|opus|haiku))?$/.test(parsed.rateLimitType))
|
|
38
|
+
if (parsed.rateLimitType && !/^(five_hour|seven_day)(?:_(sonnet|opus|haiku|fable))?$/.test(parsed.rateLimitType))
|
|
39
39
|
parsed.rateLimitType = 'uncertain';
|
|
40
40
|
delete parsed.overageStatus;
|
|
41
41
|
delete parsed.overageDisabledReason;
|
|
@@ -69,7 +69,7 @@ function safeProbe(value) {
|
|
|
69
69
|
const windows = usage['windows'].slice(0, 133).map(raw => {
|
|
70
70
|
const w = record(raw);
|
|
71
71
|
const window = w['window'] === 'five_hour' ? 'five_hour' : 'seven_day';
|
|
72
|
-
const scope = typeof w['scope'] === 'string' && ['global', 'sonnet', 'opus', 'haiku'].includes(w['scope']) ? w['scope'] : 'uncertain';
|
|
72
|
+
const scope = typeof w['scope'] === 'string' && ['global', 'sonnet', 'opus', 'haiku', 'fable'].includes(w['scope']) ? w['scope'] : 'uncertain';
|
|
73
73
|
const percent = typeof w['percent'] === 'number' && Number.isFinite(w['percent']) && w['percent'] >= 0 && w['percent'] <= 100 ? w['percent'] : undefined;
|
|
74
74
|
const resetAt = typeof w['resetAt'] === 'number' && Number.isFinite(w['resetAt']) && w['resetAt'] > 0 && w['resetAt'] <= 253402300799000 ? w['resetAt'] : undefined;
|
|
75
75
|
return { window, scope, percent, resetAt };
|
|
@@ -194,6 +194,23 @@ export class FileSubscriptionUsageStore {
|
|
|
194
194
|
return true;
|
|
195
195
|
});
|
|
196
196
|
}
|
|
197
|
+
/** Request claims live in the local data directory, alongside usage, never in a brain. */
|
|
198
|
+
async claimProbeRequest(requestId, expiresAt) {
|
|
199
|
+
validTime(expiresAt);
|
|
200
|
+
const path = join(this.#directory, 'processed-probe-requests.json');
|
|
201
|
+
return serial(path, async () => {
|
|
202
|
+
const now = Date.now();
|
|
203
|
+
const stored = await this.#load(path, {});
|
|
204
|
+
if (Object.values(stored).some(expiry => !Number.isFinite(expiry)))
|
|
205
|
+
throw new SubscriptionUsageStoreError();
|
|
206
|
+
const claims = Object.fromEntries(Object.entries(stored).filter(([, expiry]) => expiry > now));
|
|
207
|
+
if (Object.hasOwn(claims, requestId))
|
|
208
|
+
return false;
|
|
209
|
+
claims[requestId] = Math.max(expiresAt, now + 30 * 60_000);
|
|
210
|
+
await this.#save(path, claims);
|
|
211
|
+
return true;
|
|
212
|
+
});
|
|
213
|
+
}
|
|
197
214
|
async finishProbe(key, observation) {
|
|
198
215
|
validTime(observation.observedAt);
|
|
199
216
|
const path = this.#path(key, true);
|
package/dist/types.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ export interface ControlMessage {
|
|
|
4
4
|
action: 'restart' | 'update';
|
|
5
5
|
targetVersion?: string;
|
|
6
6
|
}
|
|
7
|
-
export type QueueMessage = TeamVibeQueueMessage | ControlMessage;
|
|
7
|
+
export type QueueMessage = TeamVibeQueueMessage | ControlMessage | UsageProbeMessage;
|
|
8
8
|
export interface TeamVibeQueueMessage {
|
|
9
9
|
id: string;
|
|
10
10
|
source: 'slack' | 'email' | 'cron' | 'maintenance' | 'webhook';
|
|
@@ -107,3 +107,12 @@ export interface SessionMetrics {
|
|
|
107
107
|
cache_creation_input_tokens?: number;
|
|
108
108
|
model?: string;
|
|
109
109
|
}
|
|
110
|
+
export interface UsageProbeMessage {
|
|
111
|
+
kind: 'usage_probe';
|
|
112
|
+
id: string;
|
|
113
|
+
workspaceId: string;
|
|
114
|
+
accountId: string;
|
|
115
|
+
credentialVersion: string;
|
|
116
|
+
requestId: string;
|
|
117
|
+
expiresAt: number;
|
|
118
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/** Owner-facing account policy location. */
|
|
2
|
+
export declare const FULL_QUOTA_SETTINGS_PATH = "Settings \u2192 Connections (account menu \u2192 Use until exhausted)";
|
|
3
|
+
export interface UsageResetTiming {
|
|
4
|
+
resetAt?: number;
|
|
5
|
+
now?: number;
|
|
6
|
+
timeZone?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface UsageLimitAccount {
|
|
9
|
+
useFullQuota?: boolean;
|
|
10
|
+
label?: string;
|
|
11
|
+
windows: readonly {
|
|
12
|
+
percent?: number;
|
|
13
|
+
}[];
|
|
14
|
+
rejected?: boolean;
|
|
15
|
+
}
|
|
16
|
+
export interface PoolUsageLimitNotice extends UsageResetTiming {
|
|
17
|
+
thresholdDeferred?: boolean;
|
|
18
|
+
accounts?: readonly UsageLimitAccount[];
|
|
19
|
+
}
|
|
20
|
+
export declare function formatUsageReset(input: UsageResetTiming): string | undefined;
|
|
21
|
+
export declare function formatPoolUsageLimitMessage(input: PoolUsageLimitNotice): string;
|
|
22
|
+
export declare function formatLegacyUsageLimitMessage(input: UsageResetTiming): string;
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/** Owner-facing account policy location. */
|
|
2
|
+
export const FULL_QUOTA_SETTINGS_PATH = 'Settings → Connections (account menu → Use until exhausted)';
|
|
3
|
+
export function formatUsageReset(input) {
|
|
4
|
+
if (input.resetAt === undefined || !Number.isFinite(input.resetAt))
|
|
5
|
+
return undefined;
|
|
6
|
+
const now = input.now ?? Date.now();
|
|
7
|
+
let time;
|
|
8
|
+
try {
|
|
9
|
+
time = new Intl.DateTimeFormat('en-GB', {
|
|
10
|
+
timeZone: input.timeZone || 'UTC', hour: '2-digit', minute: '2-digit', hourCycle: 'h23',
|
|
11
|
+
}).format(input.resetAt) + (input.timeZone ? '' : ' UTC');
|
|
12
|
+
}
|
|
13
|
+
catch {
|
|
14
|
+
time = new Intl.DateTimeFormat('en-GB', {
|
|
15
|
+
timeZone: 'UTC', hour: '2-digit', minute: '2-digit', hourCycle: 'h23',
|
|
16
|
+
}).format(input.resetAt) + ' UTC';
|
|
17
|
+
}
|
|
18
|
+
// Round up so a reset less than a minute away is not advertised as immediate.
|
|
19
|
+
const minutes = Math.max(0, Math.ceil((input.resetAt - now) / 60_000));
|
|
20
|
+
const days = Math.floor(minutes / 1440);
|
|
21
|
+
const hours = Math.floor(minutes % 1440 / 60);
|
|
22
|
+
const remainder = minutes % 60;
|
|
23
|
+
const relative = [days ? `${days} d` : '', hours ? `${hours} h` : '', remainder || !minutes ? `${remainder} min` : ''].filter(Boolean).join(' ');
|
|
24
|
+
return `${time} (in ${relative})`;
|
|
25
|
+
}
|
|
26
|
+
export function formatPoolUsageLimitMessage(input) {
|
|
27
|
+
const accounts = input.accounts ?? [];
|
|
28
|
+
const headroom = accounts.filter(account => !account.useFullQuota && !account.rejected).map(account => {
|
|
29
|
+
const percentages = account.windows.map(w => w.percent).filter((p) => p !== undefined && Number.isFinite(p) && p >= 0 && p <= 100);
|
|
30
|
+
return { label: account.label, remaining: percentages.length ? 100 - Math.max(...percentages) : 0 };
|
|
31
|
+
});
|
|
32
|
+
const threshold = input.thresholdDeferred === true &&
|
|
33
|
+
!accounts.some(a => a.rejected) && headroom.reduce((sum, a) => sum + a.remaining, 0) > 0;
|
|
34
|
+
const reset = formatUsageReset(input);
|
|
35
|
+
if (threshold) {
|
|
36
|
+
const labels = headroom.filter(a => a.label && a.remaining > 0).map(a => `${a.label} ${Number(a.remaining.toFixed(2))} %`).join(', ');
|
|
37
|
+
return `⏳ All Claude accounts in this workspace are above 90 % of their usage limit, so I'm waiting for the next reset${reset ? ` at ${reset}` : ''}. Your message is saved and will run automatically.\n💡 There is still some capacity left${labels ? ` (${labels})` : ''}. A workspace owner can allow the full quota per account in ${FULL_QUOTA_SETTINGS_PATH}.`;
|
|
38
|
+
}
|
|
39
|
+
return `⏳ All Claude accounts in this workspace have hit their usage limit. Your message is saved and will run automatically after ${reset ? `the earliest reset at ${reset}` : 'the next reset'}.`;
|
|
40
|
+
}
|
|
41
|
+
export function formatLegacyUsageLimitMessage(input) {
|
|
42
|
+
const reset = formatUsageReset(input);
|
|
43
|
+
return `⏳ This poller's Claude login has hit its usage limit. Your message is saved and will run automatically after ${reset ? `the reset at ${reset}` : 'the next reset'}.`;
|
|
44
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import type { UsageProbeMessage } from './types.js';
|
|
2
|
+
import type { PoolFetchResult } from './subscription-selection.js';
|
|
3
|
+
import type { SubscriptionAccount } from './subscription-pool-client.js';
|
|
4
|
+
import { type SubscriptionUsageStore } from './subscription-usage-store.js';
|
|
5
|
+
interface Options {
|
|
6
|
+
fetchPool: () => Promise<PoolFetchResult<SubscriptionAccount>>;
|
|
7
|
+
store: SubscriptionUsageStore;
|
|
8
|
+
cliPath: string;
|
|
9
|
+
timeoutMs?: number;
|
|
10
|
+
isAtCapacity: () => boolean;
|
|
11
|
+
emit: (type: string, channelId: string, metadata: Record<string, unknown>) => void;
|
|
12
|
+
log: (line: string) => void;
|
|
13
|
+
}
|
|
14
|
+
export declare function isUsageProbe(value: unknown): value is UsageProbeMessage;
|
|
15
|
+
/** This run deliberately has no message/session/brain context or normal spawn bookkeeping. */
|
|
16
|
+
export declare function handleUsageProbe(job: UsageProbeMessage, options: Options): Promise<void>;
|
|
17
|
+
export {};
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { mkdtemp, rm, mkdir } from 'node:fs/promises';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { normalizePassiveUsage } from './subscription-selection.js';
|
|
6
|
+
import { safeRateLimitInfo, } from './subscription-usage-store.js';
|
|
7
|
+
import { parseClaudeRateLimitInfo, } from './claude-rate-limit.js';
|
|
8
|
+
export function isUsageProbe(value) {
|
|
9
|
+
return (!!value &&
|
|
10
|
+
typeof value === 'object' &&
|
|
11
|
+
value.kind === 'usage_probe');
|
|
12
|
+
}
|
|
13
|
+
/** This run deliberately has no message/session/brain context or normal spawn bookkeeping. */
|
|
14
|
+
export async function handleUsageProbe(job, options) {
|
|
15
|
+
let result = 'unknown';
|
|
16
|
+
let duplicate = false;
|
|
17
|
+
let cwd;
|
|
18
|
+
if (!['workspaceId', 'accountId', 'credentialVersion', 'requestId'].every((k) => {
|
|
19
|
+
const value = job[k];
|
|
20
|
+
return (typeof value === 'string' &&
|
|
21
|
+
/^[A-Za-z0-9_-]{1,128}$/.test(value) &&
|
|
22
|
+
!/sk-ant-/i.test(value));
|
|
23
|
+
}) ||
|
|
24
|
+
!Number.isFinite(job.expiresAt) ||
|
|
25
|
+
job.expiresAt <= Date.now())
|
|
26
|
+
return;
|
|
27
|
+
const metadata = {
|
|
28
|
+
workspaceId: job.workspaceId,
|
|
29
|
+
accountId: job.accountId,
|
|
30
|
+
credentialVersion: job.credentialVersion,
|
|
31
|
+
requestId: job.requestId,
|
|
32
|
+
source: 'probe',
|
|
33
|
+
};
|
|
34
|
+
try {
|
|
35
|
+
if (!(await options.store.claimProbeRequest(job.requestId, job.expiresAt))) {
|
|
36
|
+
duplicate = true;
|
|
37
|
+
options.log(`[Pool] usage probe request=${job.requestId} result=duplicate_ignored`);
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
const pool = await options.fetchPool();
|
|
41
|
+
const account = pool.kind === 'ready'
|
|
42
|
+
? pool.accounts.find((a) => a.workspaceId === job.workspaceId &&
|
|
43
|
+
a.accountId === job.accountId &&
|
|
44
|
+
a.credentialVersion === job.credentialVersion &&
|
|
45
|
+
a.enabled)
|
|
46
|
+
: undefined;
|
|
47
|
+
if (!account)
|
|
48
|
+
return;
|
|
49
|
+
if (options.isAtCapacity()) {
|
|
50
|
+
result = 'cli_at_capacity';
|
|
51
|
+
return;
|
|
52
|
+
}
|
|
53
|
+
cwd = await mkdtemp(join(tmpdir(), 'teamvibe-usage-'));
|
|
54
|
+
const configDir = join(cwd, 'config');
|
|
55
|
+
await mkdir(configDir);
|
|
56
|
+
// Allow only OS execution essentials. No ambient credentials, Slack, API,
|
|
57
|
+
// brain, hooks, or user's Claude settings reach this isolated child.
|
|
58
|
+
const env = {
|
|
59
|
+
PATH: process.env['PATH'],
|
|
60
|
+
HOME: cwd,
|
|
61
|
+
TMPDIR: cwd,
|
|
62
|
+
CLAUDE_CONFIG_DIR: configDir,
|
|
63
|
+
CLAUDE_CODE_OAUTH_TOKEN: account.credential.unwrap(),
|
|
64
|
+
CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC: '1',
|
|
65
|
+
};
|
|
66
|
+
let info;
|
|
67
|
+
const outcome = await new Promise((resolve) => {
|
|
68
|
+
let done = false, buffer = '', errorText = '';
|
|
69
|
+
const child = spawn(options.cliPath, [
|
|
70
|
+
'-p',
|
|
71
|
+
'Reply OK.',
|
|
72
|
+
'--output-format',
|
|
73
|
+
'stream-json',
|
|
74
|
+
'--verbose',
|
|
75
|
+
'--max-turns',
|
|
76
|
+
'1',
|
|
77
|
+
'--tools',
|
|
78
|
+
'',
|
|
79
|
+
'--strict-mcp-config',
|
|
80
|
+
'--mcp-config',
|
|
81
|
+
'{"mcpServers":{}}',
|
|
82
|
+
'--no-session-persistence',
|
|
83
|
+
'--setting-sources',
|
|
84
|
+
'',
|
|
85
|
+
], { cwd, env, stdio: ['ignore', 'pipe', 'pipe'], detached: true });
|
|
86
|
+
const kill = () => {
|
|
87
|
+
if (child.pid) {
|
|
88
|
+
try {
|
|
89
|
+
process.kill(-child.pid, 'SIGKILL');
|
|
90
|
+
}
|
|
91
|
+
catch {
|
|
92
|
+
child.kill('SIGKILL');
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
};
|
|
96
|
+
const finish = (reason) => {
|
|
97
|
+
if (done)
|
|
98
|
+
return;
|
|
99
|
+
done = true;
|
|
100
|
+
clearTimeout(timer);
|
|
101
|
+
resolve(reason);
|
|
102
|
+
};
|
|
103
|
+
const timer = setTimeout(() => {
|
|
104
|
+
kill();
|
|
105
|
+
finish('timeout');
|
|
106
|
+
}, Math.min(options.timeoutMs ?? 60_000, Math.max(1, job.expiresAt - Date.now())));
|
|
107
|
+
const line = (text) => {
|
|
108
|
+
try {
|
|
109
|
+
const event = JSON.parse(text);
|
|
110
|
+
if (event?.type === 'rate_limit_event')
|
|
111
|
+
info = parseClaudeRateLimitInfo(event.rate_limit_info);
|
|
112
|
+
if (event?.type === 'result' && event.is_error)
|
|
113
|
+
errorText = (errorText + String(event.result ?? '').slice(0, 4096)).slice(-8192);
|
|
114
|
+
}
|
|
115
|
+
catch { }
|
|
116
|
+
};
|
|
117
|
+
child.stdout.on('data', (chunk) => {
|
|
118
|
+
buffer += chunk.toString();
|
|
119
|
+
if (buffer.length > 1024 * 1024) {
|
|
120
|
+
kill();
|
|
121
|
+
finish('unknown');
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
124
|
+
let end;
|
|
125
|
+
while ((end = buffer.indexOf('\n')) >= 0) {
|
|
126
|
+
line(buffer.slice(0, end));
|
|
127
|
+
buffer = buffer.slice(end + 1);
|
|
128
|
+
}
|
|
129
|
+
});
|
|
130
|
+
child.stderr.on('data', (chunk) => {
|
|
131
|
+
errorText = (errorText + chunk.toString()).slice(-8192);
|
|
132
|
+
});
|
|
133
|
+
child.once('error', () => finish('unknown'));
|
|
134
|
+
// Quota rejection is still a successful observation: it reveals reset
|
|
135
|
+
// time without requiring the account to complete an assistant turn.
|
|
136
|
+
child.once('close', (code) => {
|
|
137
|
+
if (buffer)
|
|
138
|
+
line(buffer);
|
|
139
|
+
finish(/401|unauthorized|invalid.*token|invalid api key|authentication.*(?:fail|error)/i.test(errorText)
|
|
140
|
+
? 'token_rejected'
|
|
141
|
+
: /capacity|overloaded/i.test(errorText)
|
|
142
|
+
? 'cli_at_capacity'
|
|
143
|
+
: info && (code === 0 || info.status === 'rejected')
|
|
144
|
+
? 'ok'
|
|
145
|
+
: 'unknown');
|
|
146
|
+
});
|
|
147
|
+
});
|
|
148
|
+
result = outcome;
|
|
149
|
+
if (result !== 'ok' || !info)
|
|
150
|
+
return;
|
|
151
|
+
const now = Date.now();
|
|
152
|
+
const safe = safeRateLimitInfo(info);
|
|
153
|
+
const usage = {
|
|
154
|
+
...normalizePassiveUsage(safe, now),
|
|
155
|
+
source: 'probe',
|
|
156
|
+
};
|
|
157
|
+
await options.store.finishProbe(account, {
|
|
158
|
+
status: 'ok',
|
|
159
|
+
observedAt: now,
|
|
160
|
+
usage,
|
|
161
|
+
});
|
|
162
|
+
options.emit('pool.usage_observed', 'usage_probe', {
|
|
163
|
+
...metadata,
|
|
164
|
+
accountLabel: account.label ?? account.accountId,
|
|
165
|
+
reportedAt: now,
|
|
166
|
+
usage,
|
|
167
|
+
lastRateLimitInfo: safe,
|
|
168
|
+
probeStatus: 'ok',
|
|
169
|
+
probeObservedAt: now,
|
|
170
|
+
});
|
|
171
|
+
}
|
|
172
|
+
catch {
|
|
173
|
+
result = 'unknown';
|
|
174
|
+
}
|
|
175
|
+
finally {
|
|
176
|
+
if (cwd)
|
|
177
|
+
await rm(cwd, { recursive: true, force: true }).catch(() => { });
|
|
178
|
+
if (!duplicate && result !== 'ok')
|
|
179
|
+
options.emit('pool.probe_failed', 'usage_probe', {
|
|
180
|
+
...metadata,
|
|
181
|
+
reason: result,
|
|
182
|
+
});
|
|
183
|
+
if (!duplicate)
|
|
184
|
+
options.log(`[Pool] usage probe account=${job.accountId} result=${result}`);
|
|
185
|
+
}
|
|
186
|
+
}
|
package/npm-shrinkwrap.json
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@teamvibe/poller",
|
|
3
|
-
"version": "0.8.3
|
|
3
|
+
"version": "0.8.3",
|
|
4
4
|
"lockfileVersion": 3,
|
|
5
5
|
"requires": true,
|
|
6
6
|
"packages": {
|
|
7
7
|
"": {
|
|
8
8
|
"name": "@teamvibe/poller",
|
|
9
|
-
"version": "0.8.3
|
|
9
|
+
"version": "0.8.3",
|
|
10
10
|
"dependencies": {
|
|
11
11
|
"@aws-sdk/client-dynamodb": "3.1108.0",
|
|
12
12
|
"@aws-sdk/client-sqs": "3.1108.0",
|