@gobing-ai/ts-ai-runner 0.4.55 → 0.4.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ai-runner.d.ts +9 -0
- package/dist/ai-runner.d.ts.map +1 -1
- package/dist/ai-runner.js +15 -0
- package/dist/events.d.ts +5 -0
- package/dist/events.d.ts.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/model-health-probe.d.ts +23 -1
- package/dist/model-health-probe.d.ts.map +1 -1
- package/dist/model-health-probe.js +23 -0
- package/dist/quota.d.ts +98 -0
- package/dist/quota.d.ts.map +1 -0
- package/dist/quota.js +123 -0
- package/dist/team-agent-process.d.ts +16 -1
- package/dist/team-agent-process.d.ts.map +1 -1
- package/dist/team-agent-process.js +38 -4
- package/package.json +4 -4
- package/src/ai-runner.ts +28 -0
- package/src/events.ts +5 -0
- package/src/index.ts +1 -0
- package/src/model-health-probe.ts +46 -1
- package/src/quota.ts +188 -0
- package/src/team-agent-process.ts +52 -5
package/dist/ai-runner.d.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { type BusLifecycleEvents, EventBus, type Logger } from '@gobing-ai/ts-in
|
|
|
2
2
|
import { type ProcessExecutor, type ProcessOutputChunk, type TracerPort } from '@gobing-ai/ts-runtime';
|
|
3
3
|
import { type AgentName, type PromptOptions, type ShimCommand } from './agents/shims';
|
|
4
4
|
import type { AgentEvents, AiRunnerProcessEvents } from './events';
|
|
5
|
+
import { type QuotaAttribution } from './quota';
|
|
5
6
|
/** Result returned by every AI runner dispatch method. */
|
|
6
7
|
export interface AgentRunResult {
|
|
7
8
|
/** Process exit code; null indicates signal or timeout termination. */
|
|
@@ -27,6 +28,12 @@ export interface AgentRunOptions {
|
|
|
27
28
|
onOutput?: (output: ProcessOutputChunk) => void;
|
|
28
29
|
/** Optional application-owned correlation propagated unchanged to agent lifecycle events. */
|
|
29
30
|
correlation?: AgentRunCorrelation;
|
|
31
|
+
/**
|
|
32
|
+
* Optional exact attribution for quota observations (Spur task 0798). Purely
|
|
33
|
+
* additive metadata: existing callers without it keep byte-identical results,
|
|
34
|
+
* and missing fields stay absent on emitted observations — never inferred.
|
|
35
|
+
*/
|
|
36
|
+
quotaContext?: QuotaAttribution;
|
|
30
37
|
}
|
|
31
38
|
/** Application-owned execution identity carried without coupling the runner to a workflow model. */
|
|
32
39
|
export interface AgentRunCorrelation {
|
|
@@ -92,6 +99,8 @@ export declare class AiRunner {
|
|
|
92
99
|
private readonly defaultTimeout;
|
|
93
100
|
private readonly logger;
|
|
94
101
|
private readonly events;
|
|
102
|
+
/** Single producer path for quota events; no-ops when no agent event bus is configured. */
|
|
103
|
+
private readonly quotaProducer;
|
|
95
104
|
/** Internal process-level observability bus, parented to `lifecycleBus` when auto-constructed. Exposed for introspection/testing. */
|
|
96
105
|
readonly processEvents: EventBus<AiRunnerProcessEvents> | undefined;
|
|
97
106
|
constructor(options?: AiRunnerOptions);
|
package/dist/ai-runner.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ai-runner.d.ts","sourceRoot":"","sources":["../src/ai-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAE,QAAQ,EAAa,KAAK,MAAM,EAAc,MAAM,qBAAqB,CAAC;AAC5G,OAAO,EAGH,KAAK,eAAe,EACpB,KAAK,kBAAkB,EAEvB,KAAK,UAAU,EAClB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,KAAK,SAAS,EAAgB,KAAK,aAAa,EAAE,KAAK,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACpG,OAAO,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;
|
|
1
|
+
{"version":3,"file":"ai-runner.d.ts","sourceRoot":"","sources":["../src/ai-runner.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,kBAAkB,EAAE,QAAQ,EAAa,KAAK,MAAM,EAAc,MAAM,qBAAqB,CAAC;AAC5G,OAAO,EAGH,KAAK,eAAe,EACpB,KAAK,kBAAkB,EAEvB,KAAK,UAAU,EAClB,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,KAAK,SAAS,EAAgB,KAAK,aAAa,EAAE,KAAK,WAAW,EAAE,MAAM,gBAAgB,CAAC;AACpG,OAAO,KAAK,EAAE,WAAW,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAEnE,OAAO,EAGH,KAAK,gBAAgB,EAExB,MAAM,SAAS,CAAC;AAGjB,0DAA0D;AAC1D,MAAM,WAAW,cAAc;IAC3B,uEAAuE;IACvE,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,uBAAuB;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,oEAAoE;IACpE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,2CAA2C;IAC3C,UAAU,EAAE,MAAM,CAAC;CACtB;AAED,sCAAsC;AACtC,MAAM,WAAW,eAAe;IAC5B,4CAA4C;IAC5C,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,+BAA+B;IAC/B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,yFAAyF;IACzF,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,gFAAgF;IAChF,QAAQ,CAAC,EAAE,CAAC,MAAM,EAAE,kBAAkB,KAAK,IAAI,CAAC;IAChD,6FAA6F;IAC7F,WAAW,CAAC,EAAE,mBAAmB,CAAC;IAClC;;;;OAIG;IACH,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACnC;AAED,oGAAoG;AACpG,MAAM,WAAW,mBAAmB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;CAC9B;AACD;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,eAAO,MAAM,gBAAgB,gBAAgB,CAAC;AAC9C,kFAAkF;AAClF,eAAO,MAAM,sBAAsB,sBAAsB,CAAC;AAC1D,wFAAwF;AACxF,eAAO,MAAM,mBAAmB,mBAAmB,CAAC;AAiBpD,wCAAwC;AACxC,MAAM,WAAW,eAAe;IAC5B,4DAA4D;IAC5D,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,iDAAiD;IACjD,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,uCAAuC;IACvC,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,+EAA+E;IAC/E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iFAAiF;IACjF,aAAa,CAAC,EAAE,QAAQ,CAAC,qBAAqB,CAAC,CAAC;IAChD,gEAAgE;IAChE,MAAM,CAAC,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B;;;;OAIG;IACH,YAAY,CAAC,EAAE,QAAQ,CAAC,kBAAkB,CAAC,CAAC;IAC5C,kFAAkF;IAClF,MAAM,CAAC,EAAE,UAAU,CAAC;CACvB;AAED,uEAAuE;AACvE,qBAAa,QAAQ;IACjB,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqB;IAChD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAqB;IACpD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAoC;IAC3D,2FAA2F;IAC3F,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA2B;IACzD,qIAAqI;IACrI,QAAQ,CAAC,aAAa,EAAE,QAAQ,CAAC,qBAAqB,CAAC,GAAG,SAAS,CAAC;gBAExD,OAAO,GAAE,eAAoB;IAiCzC,iCAAiC;IACjC,cAAc,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC;IAIxF,oCAAoC;IACpC,iBAAiB,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC;IAI3F,mCAAmC;IACnC,gBAAgB,CACZ,KAAK,EAAE,SAAS,EAChB,aAAa,EAAE,aAAa,EAC5B,OAAO,GAAE,eAAoB,GAC9B,OAAO,CAAC,cAAc,CAAC;IAI1B,6EAA6E;IAC7E,eAAe,CACX,KAAK,EAAE,SAAS,EAChB,KAAK,EAAE,MAAM,EACb,aAAa,EAAE,aAAa,EAC5B,OAAO,GAAE,eAAoB,GAC9B,OAAO,CAAC,cAAc,CAAC;IAI1B,0DAA0D;IAC1D,kBAAkB,CAAC,KAAK,EAAE,SAAS,EAAE,aAAa,EAAE,aAAa,EAAE,OAAO,GAAE,eAAoB,GAAG,WAAW;IAa9G,4EAA4E;IAC5E,cAAc,CAAC,KAAK,EAAE,SAAS,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,GAAG,IAAI;YAKjF,MAAM;CAmEvB;AAWD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAC7B,KAAK,EAAE,SAAS,EAChB,aAAa,EAAE,aAAa,EAC5B,OAAO,EAAE;IAAE,SAAS,EAAE,MAAM,CAAA;CAAE,GAC/B,WAAW,CAKb"}
|
package/dist/ai-runner.js
CHANGED
|
@@ -2,6 +2,7 @@ import { EventBus, getLogger, traceAsync } from '@gobing-ai/ts-infra';
|
|
|
2
2
|
import { getProcessCwd, nodeBunFactory, } from '@gobing-ai/ts-runtime';
|
|
3
3
|
import { getAgentShim } from './agents/shims.js';
|
|
4
4
|
import { buildIdentityPreamble } from './identity.js';
|
|
5
|
+
import { buildQuotaObservation, classifyQuotaErrorRecord, QuotaObservationProducer, } from './quota.js';
|
|
5
6
|
import { translateSlashCommand } from './slash-command.js';
|
|
6
7
|
/**
|
|
7
8
|
* Public environment-variable contract for agent run correlation.
|
|
@@ -52,6 +53,8 @@ export class AiRunner {
|
|
|
52
53
|
defaultTimeout;
|
|
53
54
|
logger;
|
|
54
55
|
events;
|
|
56
|
+
/** Single producer path for quota events; no-ops when no agent event bus is configured. */
|
|
57
|
+
quotaProducer;
|
|
55
58
|
/** Internal process-level observability bus, parented to `lifecycleBus` when auto-constructed. Exposed for introspection/testing. */
|
|
56
59
|
processEvents;
|
|
57
60
|
constructor(options = {}) {
|
|
@@ -81,6 +84,7 @@ export class AiRunner {
|
|
|
81
84
|
this.defaultTimeout = options.defaultTimeout;
|
|
82
85
|
this.logger = options.logger ?? getLogger('ai-runner');
|
|
83
86
|
this.events = events;
|
|
87
|
+
this.quotaProducer = new QuotaObservationProducer(events);
|
|
84
88
|
this.processEvents = processEvents;
|
|
85
89
|
}
|
|
86
90
|
/** Run an agent help command. */
|
|
@@ -153,6 +157,17 @@ export class AiRunner {
|
|
|
153
157
|
...(options.correlation !== undefined ? { correlation: options.correlation } : {}),
|
|
154
158
|
severity: result.exitCode === 0 || result.exitCode === null ? 'info' : 'error',
|
|
155
159
|
});
|
|
160
|
+
if (result.exitCode !== null && result.exitCode !== 0) {
|
|
161
|
+
const classification = classifyQuotaErrorRecord(result.stderr);
|
|
162
|
+
if (classification.quota) {
|
|
163
|
+
this.quotaProducer.produce(buildQuotaObservation({
|
|
164
|
+
source: 'buffered-error',
|
|
165
|
+
reason: classification.reason,
|
|
166
|
+
attribution: { agent, ...options.quotaContext },
|
|
167
|
+
...(options.correlation !== undefined ? { correlation: options.correlation } : {}),
|
|
168
|
+
}));
|
|
169
|
+
}
|
|
170
|
+
}
|
|
156
171
|
return {
|
|
157
172
|
exitCode: result.exitCode,
|
|
158
173
|
stdout: result.stdout,
|
package/dist/events.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { EventSeverity } from '@gobing-ai/ts-infra';
|
|
2
2
|
import type { ProcessEvents } from '@gobing-ai/ts-runtime';
|
|
3
3
|
import type { AgentRunCorrelation } from './ai-runner';
|
|
4
|
+
import type { AgentQuotaObservation, AgentQuotaRecovery } from './quota';
|
|
4
5
|
/** Typed event map for agent-runner observability. All events prefixed `agent.`. */
|
|
5
6
|
export type AgentEvents = {
|
|
6
7
|
/** Emitted immediately before an agent CLI invocation starts. */
|
|
@@ -41,6 +42,10 @@ export type AgentEvents = {
|
|
|
41
42
|
ok: boolean;
|
|
42
43
|
severity: EventSeverity;
|
|
43
44
|
}) => void;
|
|
45
|
+
/** Emitted once per verified quota-exhaustion observation (confirmed usage-allowance or credit exhaustion). */
|
|
46
|
+
'agent.quota.exhausted': (data: AgentQuotaObservation) => void;
|
|
47
|
+
/** Explicit-recovery signal; type contract only — no automatic producer exists upstream. */
|
|
48
|
+
'agent.quota.recovered': (data: AgentQuotaRecovery) => void;
|
|
44
49
|
};
|
|
45
50
|
/** Event map for process-level observability emitted by AiRunner-owned executors. */
|
|
46
51
|
export type AiRunnerProcessEvents = ProcessEvents;
|
package/dist/events.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../src/events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"events.d.ts","sourceRoot":"","sources":["../src/events.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,uBAAuB,CAAC;AAC3D,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,qBAAqB,EAAE,kBAAkB,EAAE,MAAM,SAAS,CAAC;AAEzE,oFAAoF;AACpF,MAAM,MAAM,WAAW,GAAG;IACtB,iEAAiE;IACjE,oBAAoB,EAAE,CAAC,IAAI,EAAE;QACzB,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,WAAW,CAAC,EAAE,mBAAmB,CAAC;QAClC,QAAQ,EAAE,aAAa,CAAC;KAC3B,KAAK,IAAI,CAAC;IACX,mDAAmD;IACnD,mBAAmB,EAAE,CAAC,IAAI,EAAE;QACxB,KAAK,EAAE,MAAM,CAAC;QACd,SAAS,EAAE,MAAM,CAAC;QAClB,KAAK,EAAE,MAAM,CAAC;QACd,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QACxB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,UAAU,EAAE,MAAM,CAAC;QACnB,WAAW,CAAC,EAAE,mBAAmB,CAAC;QAClC,QAAQ,EAAE,aAAa,CAAC;KAC3B,KAAK,IAAI,CAAC;IACX,6DAA6D;IAC7D,eAAe,EAAE,CAAC,IAAI,EAAE;QACpB,OAAO,EAAE,MAAM,CAAC;QAChB,SAAS,EAAE,MAAM,CAAC;QAClB,GAAG,EAAE,MAAM,GAAG,IAAI,CAAC;QACnB,QAAQ,EAAE,aAAa,CAAC;KAC3B,KAAK,IAAI,CAAC;IACX,4DAA4D;IAC5D,eAAe,EAAE,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAA;KAAE,KAAK,IAAI,CAAC;IACvG,8DAA8D;IAC9D,oBAAoB,EAAE,CAAC,IAAI,EAAE;QAAE,OAAO,EAAE,MAAM,CAAC;QAAC,EAAE,EAAE,OAAO,CAAC;QAAC,QAAQ,EAAE,aAAa,CAAA;KAAE,KAAK,IAAI,CAAC;IAChG,+GAA+G;IAC/G,uBAAuB,EAAE,CAAC,IAAI,EAAE,qBAAqB,KAAK,IAAI,CAAC;IAC/D,4FAA4F;IAC5F,uBAAuB,EAAE,CAAC,IAAI,EAAE,kBAAkB,KAAK,IAAI,CAAC;CAC/D,CAAC;AAEF,qFAAqF;AACrF,MAAM,MAAM,qBAAqB,GAAG,aAAa,CAAC"}
|
package/dist/index.d.ts
CHANGED
|
@@ -9,6 +9,7 @@ export * from './identity';
|
|
|
9
9
|
export * from './message-store';
|
|
10
10
|
export * from './messages';
|
|
11
11
|
export * from './model-health-probe';
|
|
12
|
+
export * from './quota';
|
|
12
13
|
export * from './slash-command';
|
|
13
14
|
export * from './team-agent-process';
|
|
14
15
|
export * from './team-orchestrator';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,iBAAiB,CAAC;AAChC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,kBAAkB,CAAC;AACjC,cAAc,cAAc,CAAC;AAC7B,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,UAAU,CAAC;AACzB,cAAc,YAAY,CAAC;AAC3B,cAAc,iBAAiB,CAAC;AAChC,cAAc,YAAY,CAAC;AAC3B,cAAc,sBAAsB,CAAC;AACrC,cAAc,SAAS,CAAC;AACxB,cAAc,iBAAiB,CAAC;AAChC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -9,6 +9,7 @@ export * from './identity.js';
|
|
|
9
9
|
export * from './message-store.js';
|
|
10
10
|
export * from './messages.js';
|
|
11
11
|
export * from './model-health-probe.js';
|
|
12
|
+
export * from './quota.js';
|
|
12
13
|
export * from './slash-command.js';
|
|
13
14
|
export * from './team-agent-process.js';
|
|
14
15
|
export * from './team-orchestrator.js';
|
|
@@ -10,7 +10,10 @@
|
|
|
10
10
|
* issuing a minimal 1-token completion request and interpreting the HTTP
|
|
11
11
|
* response (R3).
|
|
12
12
|
*/
|
|
13
|
-
import { type APIClientConfig } from '@gobing-ai/ts-infra';
|
|
13
|
+
import { type APIClientConfig, type EventBus } from '@gobing-ai/ts-infra';
|
|
14
|
+
import type { AgentRunCorrelation } from './ai-runner';
|
|
15
|
+
import type { AgentEvents } from './events';
|
|
16
|
+
import { type AgentQuotaObservation, type QuotaAttribution, QuotaObservationProducer } from './quota';
|
|
14
17
|
/** Health status for a single model endpoint. */
|
|
15
18
|
export type ModelHealthStatus = 'available' | 'quota_exhausted' | 'rate_limited' | 'unavailable' | 'unknown';
|
|
16
19
|
/** Result of probing one model's health (R2). */
|
|
@@ -91,4 +94,23 @@ export declare class OmpModelProbe implements ModelHealthProbe {
|
|
|
91
94
|
}
|
|
92
95
|
/** Default probe timeout in milliseconds (R8). */
|
|
93
96
|
export { DEFAULT_PROBE_TIMEOUT_MS };
|
|
97
|
+
/** Options for the explicitly opted-in quota health observation. */
|
|
98
|
+
export interface QuotaHealthObservationOptions {
|
|
99
|
+
/** Agent event bus receiving `agent.quota.exhausted`. */
|
|
100
|
+
events: EventBus<AgentEvents>;
|
|
101
|
+
/** Optional exact attribution; absent fields stay absent — never inferred. */
|
|
102
|
+
attribution?: QuotaAttribution;
|
|
103
|
+
/** Optional run correlation carried unchanged onto the observation. */
|
|
104
|
+
correlation?: AgentRunCorrelation;
|
|
105
|
+
/** Optional shared producer so multiple observations dedupe across one observation session. */
|
|
106
|
+
producer?: QuotaObservationProducer;
|
|
107
|
+
}
|
|
108
|
+
/**
|
|
109
|
+
* Explicitly opted-in health observation → quota event. The ONLY producer seam
|
|
110
|
+
* on the health path: ordinary doctor runs never call this, so read-only
|
|
111
|
+
* behavior is preserved. Returns the emitted observation, or null when the
|
|
112
|
+
* result is not a confirmed quota exhaustion (rate_limited / unavailable /
|
|
113
|
+
* unknown never produce events).
|
|
114
|
+
*/
|
|
115
|
+
export declare function observeQuotaHealthResult(result: ModelHealthResult, options: QuotaHealthObservationOptions): AgentQuotaObservation | null;
|
|
94
116
|
//# sourceMappingURL=model-health-probe.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"model-health-probe.d.ts","sourceRoot":"","sources":["../src/model-health-probe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAa,KAAK,eAAe,
|
|
1
|
+
{"version":3,"file":"model-health-probe.d.ts","sourceRoot":"","sources":["../src/model-health-probe.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AACH,OAAO,EAAa,KAAK,eAAe,EAAY,KAAK,QAAQ,EAAwB,MAAM,qBAAqB,CAAC;AACrH,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EACH,KAAK,qBAAqB,EAE1B,KAAK,gBAAgB,EACrB,wBAAwB,EAC3B,MAAM,SAAS,CAAC;AAEjB,iDAAiD;AACjD,MAAM,MAAM,iBAAiB,GAAG,WAAW,GAAG,iBAAiB,GAAG,cAAc,GAAG,aAAa,GAAG,SAAS,CAAC;AAE7G,iDAAiD;AACjD,MAAM,WAAW,iBAAiB;IAC9B,MAAM,EAAE,iBAAiB,CAAC;IAC1B,6EAA6E;IAC7E,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sDAAsD;IACtD,SAAS,EAAE,MAAM,CAAC;CACrB;AAED,8EAA8E;AAC9E,MAAM,WAAW,WAAW;IACxB,gFAAgF;IAChF,MAAM,EAAE,MAAM,CAAC;IACf,2FAA2F;IAC3F,SAAS,EAAE,MAAM,CAAC;IAClB,sEAAsE;IACtE,QAAQ,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,8CAA8C;AAC9C,MAAM,WAAW,gBAAgB;IAC7B,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAAC;CAC3F;AAED;;;;GAIG;AACH,qBAAa,wBAAwB;IACjC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAuC;IAE9D,mEAAmE;IACnE,QAAQ,CAAC,cAAc,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,GAAG,IAAI;IAI/D,iFAAiF;IACjF,OAAO,CAAC,KAAK,EAAE,MAAM,GAAG,gBAAgB,GAAG,IAAI;CAKlD;AAED,kEAAkE;AAClE,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGrD;AAED,6DAA6D;AAC7D,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,CAGtD;AAgBD,QAAA,MAAM,wBAAwB,QAAS,CAAC;AAExC,yDAAyD;AACzD,MAAM,WAAW,oBAAoB;IACjC;;;;;OAKG;IACH,eAAe,CAAC,EAAE,IAAI,CAAC,eAAe,EAAE,SAAS,CAAC,CAAC;CACtD;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,qBAAa,aAAc,YAAW,gBAAgB;IAClD,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAC,CAAmC;gBAExD,OAAO,CAAC,EAAE,oBAAoB;IAIpC,KAAK,CAAC,QAAQ,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC;IA6B7F,kEAAkE;YACpD,YAAY;IAiD1B,oEAAoE;IACpE,OAAO,CAAC,iBAAiB;IAwBzB,4DAA4D;IAC5D,OAAO,CAAC,SAAS;CAOpB;AAED,kDAAkD;AAClD,OAAO,EAAE,wBAAwB,EAAE,CAAC;AAEpC,oEAAoE;AACpE,MAAM,WAAW,6BAA6B;IAC1C,yDAAyD;IACzD,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC9B,8EAA8E;IAC9E,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,uEAAuE;IACvE,WAAW,CAAC,EAAE,mBAAmB,CAAC;IAClC,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,wBAAwB,CAAC;CACvC;AAED;;;;;;GAMG;AACH,wBAAgB,wBAAwB,CACpC,MAAM,EAAE,iBAAiB,EACzB,OAAO,EAAE,6BAA6B,GACvC,qBAAqB,GAAG,IAAI,CAa9B"}
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
* response (R3).
|
|
12
12
|
*/
|
|
13
13
|
import { APIClient, APIError } from '@gobing-ai/ts-infra';
|
|
14
|
+
import { buildQuotaObservation, QuotaObservationProducer, } from './quota.js';
|
|
14
15
|
/**
|
|
15
16
|
* Maps provider prefixes to probe implementations (R9).
|
|
16
17
|
* The provider prefix is extracted from the model string's first path segment
|
|
@@ -151,3 +152,25 @@ export class OmpModelProbe {
|
|
|
151
152
|
}
|
|
152
153
|
/** Default probe timeout in milliseconds (R8). */
|
|
153
154
|
export { DEFAULT_PROBE_TIMEOUT_MS };
|
|
155
|
+
/**
|
|
156
|
+
* Explicitly opted-in health observation → quota event. The ONLY producer seam
|
|
157
|
+
* on the health path: ordinary doctor runs never call this, so read-only
|
|
158
|
+
* behavior is preserved. Returns the emitted observation, or null when the
|
|
159
|
+
* result is not a confirmed quota exhaustion (rate_limited / unavailable /
|
|
160
|
+
* unknown never produce events).
|
|
161
|
+
*/
|
|
162
|
+
export function observeQuotaHealthResult(result, options) {
|
|
163
|
+
if (result.status !== 'quota_exhausted')
|
|
164
|
+
return null;
|
|
165
|
+
const producer = options.producer ?? new QuotaObservationProducer(options.events);
|
|
166
|
+
const observation = buildQuotaObservation({
|
|
167
|
+
source: 'health-probe',
|
|
168
|
+
reason: 'provider_quota_exhausted',
|
|
169
|
+
observedAt: new Date(result.checkedAt),
|
|
170
|
+
...(result.detail !== undefined ? { detail: result.detail } : {}),
|
|
171
|
+
attribution: options.attribution,
|
|
172
|
+
...(options.correlation !== undefined ? { correlation: options.correlation } : {}),
|
|
173
|
+
});
|
|
174
|
+
producer.produce(observation);
|
|
175
|
+
return observation;
|
|
176
|
+
}
|
package/dist/quota.d.ts
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
import type { EventBus } from '@gobing-ai/ts-infra';
|
|
2
|
+
import type { AgentRunCorrelation } from './ai-runner';
|
|
3
|
+
import type { AgentEvents } from './events';
|
|
4
|
+
/**
|
|
5
|
+
* Canonical quota-exhaustion codes accepted in structured error-record positions
|
|
6
|
+
* (`error.type` / `error.code`). The set is an exact-match allowlist on purpose:
|
|
7
|
+
* free text never confirms quota (R2 — Spur task 0798). Generic HTTP 429 without
|
|
8
|
+
* one of these codes is rate limiting, not quota exhaustion.
|
|
9
|
+
*/
|
|
10
|
+
export type QuotaExhaustionReason = 'insufficient_quota' | 'insufficient_credit_balance' | 'quota_exceeded' | 'usage_limit_reached' | 'credits_exhausted' | 'billing_hard_limit_reached'
|
|
11
|
+
/** Verified by the shipped health probe's quota-aware 429 interpretation (`OmpModelProbe`). */
|
|
12
|
+
| 'provider_quota_exhausted';
|
|
13
|
+
/** Where a quota observation was verified. Streaming evidence is a bounded trailing window, never a transcript. */
|
|
14
|
+
export type QuotaEvidenceSource = 'buffered-error' | 'streaming-error' | 'health-probe';
|
|
15
|
+
/**
|
|
16
|
+
* Exact attribution carried on a quota observation. Every field is optional and
|
|
17
|
+
* only populated from caller-supplied context — the runner never infers project
|
|
18
|
+
* or executor identity (missing attribution stays observable, not guessed).
|
|
19
|
+
*/
|
|
20
|
+
export interface QuotaAttribution {
|
|
21
|
+
projectId?: string;
|
|
22
|
+
executor?: string;
|
|
23
|
+
agent?: string;
|
|
24
|
+
model?: string;
|
|
25
|
+
}
|
|
26
|
+
/** A verified quota-exhaustion observation. `observationId` is deterministic, so redelivery keeps identity. */
|
|
27
|
+
export interface AgentQuotaObservation {
|
|
28
|
+
/** Stable across redelivery: digest over evidence source, reason, and attribution identity — never timestamps. */
|
|
29
|
+
observationId: string;
|
|
30
|
+
/** UTC ISO-8601 with millisecond precision. */
|
|
31
|
+
observedAt: string;
|
|
32
|
+
evidenceSource: QuotaEvidenceSource;
|
|
33
|
+
reason: QuotaExhaustionReason;
|
|
34
|
+
/** Sanitized provider detail. Only carried when the source guarantees sanitization (health probe). */
|
|
35
|
+
detail?: string;
|
|
36
|
+
/** Exact attribution when supplied by the caller; absent fields mean unknown. */
|
|
37
|
+
attribution?: QuotaAttribution;
|
|
38
|
+
correlation?: AgentRunCorrelation;
|
|
39
|
+
}
|
|
40
|
+
/** Explicit-recovery payload for `agent.quota.recovered`. Type contract only — no automatic producer, timer, or polling. */
|
|
41
|
+
export interface AgentQuotaRecovery {
|
|
42
|
+
/** The observationId whose quota state recovered. */
|
|
43
|
+
observationId: string;
|
|
44
|
+
/** UTC ISO-8601 with millisecond precision. */
|
|
45
|
+
recoveredAt: string;
|
|
46
|
+
attribution?: QuotaAttribution;
|
|
47
|
+
correlation?: AgentRunCorrelation;
|
|
48
|
+
}
|
|
49
|
+
/** Result of classifying one bounded error record. */
|
|
50
|
+
export type QuotaClassification = {
|
|
51
|
+
quota: true;
|
|
52
|
+
reason: QuotaExhaustionReason;
|
|
53
|
+
} | {
|
|
54
|
+
quota: false;
|
|
55
|
+
};
|
|
56
|
+
/** Upper bound for error evidence considered by the classifier (bytes, UTF-8). Keeps streaming memory bounded. */
|
|
57
|
+
export declare const MAX_QUOTA_EVIDENCE_BYTES = 8192;
|
|
58
|
+
/**
|
|
59
|
+
* Classify one error record as quota exhaustion. Accepts only structured
|
|
60
|
+
* provider error envelopes — a JSON object with an `error` object whose
|
|
61
|
+
* `type` or `code` exactly matches a canonical quota code. Free text, quoted
|
|
62
|
+
* prompt content inside `error.message`, plain HTTP 429 rate-limit codes,
|
|
63
|
+
* throttling/overload, authentication, context/output limits, and timeouts
|
|
64
|
+
* all classify as negative (R2). Input is bounded to
|
|
65
|
+
* {@link MAX_QUOTA_EVIDENCE_BYTES} trailing bytes.
|
|
66
|
+
*/
|
|
67
|
+
export declare function classifyQuotaErrorRecord(record: string): QuotaClassification;
|
|
68
|
+
/** Normalize an observation timestamp to UTC ISO-8601 with millisecond precision. */
|
|
69
|
+
export declare function normalizeObservedAt(at: Date): string;
|
|
70
|
+
/**
|
|
71
|
+
* Build a quota observation. `observationId` is a SHA-256 digest over the
|
|
72
|
+
* evidence source, reason, and attribution identity — stable across
|
|
73
|
+
* redelivery because it excludes timestamps, correlation, and free text.
|
|
74
|
+
*/
|
|
75
|
+
export declare function buildQuotaObservation(params: {
|
|
76
|
+
source: QuotaEvidenceSource;
|
|
77
|
+
reason: QuotaExhaustionReason;
|
|
78
|
+
observedAt?: Date;
|
|
79
|
+
detail?: string;
|
|
80
|
+
attribution?: QuotaAttribution;
|
|
81
|
+
correlation?: AgentRunCorrelation;
|
|
82
|
+
}): AgentQuotaObservation;
|
|
83
|
+
/**
|
|
84
|
+
* Single producer path for quota events. Emits `agent.quota.exhausted` at most
|
|
85
|
+
* once per observationId per producer instance (redelivery of the same quota
|
|
86
|
+
* fact is suppressed). Recovery delivery is explicit-only — callers own
|
|
87
|
+
* recovery detection; this class never schedules, polls, or times anything.
|
|
88
|
+
*/
|
|
89
|
+
export declare class QuotaObservationProducer {
|
|
90
|
+
private readonly events;
|
|
91
|
+
private readonly emitted;
|
|
92
|
+
constructor(events: EventBus<AgentEvents> | undefined);
|
|
93
|
+
/** Emit one `agent.quota.exhausted`. Returns true when emitted, false when deduped or bus-less. */
|
|
94
|
+
produce(observation: AgentQuotaObservation): boolean;
|
|
95
|
+
/** Deliver an explicit `agent.quota.recovered`. Returns true when emitted, false when bus-less. */
|
|
96
|
+
produceRecovery(recovery: AgentQuotaRecovery): boolean;
|
|
97
|
+
}
|
|
98
|
+
//# sourceMappingURL=quota.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"quota.d.ts","sourceRoot":"","sources":["../src/quota.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AACpD,OAAO,KAAK,EAAE,mBAAmB,EAAE,MAAM,aAAa,CAAC;AACvD,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAE5C;;;;;GAKG;AACH,MAAM,MAAM,qBAAqB,GAC3B,oBAAoB,GACpB,6BAA6B,GAC7B,gBAAgB,GAChB,qBAAqB,GACrB,mBAAmB,GACnB,4BAA4B;AAC9B,+FAA+F;GAC7F,0BAA0B,CAAC;AAEjC,mHAAmH;AACnH,MAAM,MAAM,mBAAmB,GAAG,gBAAgB,GAAG,iBAAiB,GAAG,cAAc,CAAC;AAExF;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,+GAA+G;AAC/G,MAAM,WAAW,qBAAqB;IAClC,kHAAkH;IAClH,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,UAAU,EAAE,MAAM,CAAC;IACnB,cAAc,EAAE,mBAAmB,CAAC;IACpC,MAAM,EAAE,qBAAqB,CAAC;IAC9B,sGAAsG;IACtG,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iFAAiF;IACjF,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,WAAW,CAAC,EAAE,mBAAmB,CAAC;CACrC;AAED,4HAA4H;AAC5H,MAAM,WAAW,kBAAkB;IAC/B,qDAAqD;IACrD,aAAa,EAAE,MAAM,CAAC;IACtB,+CAA+C;IAC/C,WAAW,EAAE,MAAM,CAAC;IACpB,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,WAAW,CAAC,EAAE,mBAAmB,CAAC;CACrC;AAED,sDAAsD;AACtD,MAAM,MAAM,mBAAmB,GAAG;IAAE,KAAK,EAAE,IAAI,CAAC;IAAC,MAAM,EAAE,qBAAqB,CAAA;CAAE,GAAG;IAAE,KAAK,EAAE,KAAK,CAAA;CAAE,CAAC;AAEpG,kHAAkH;AAClH,eAAO,MAAM,wBAAwB,OAAO,CAAC;AAW7C;;;;;;;;GAQG;AACH,wBAAgB,wBAAwB,CAAC,MAAM,EAAE,MAAM,GAAG,mBAAmB,CAU5E;AAkBD,qFAAqF;AACrF,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,IAAI,GAAG,MAAM,CAEpD;AAYD;;;;GAIG;AACH,wBAAgB,qBAAqB,CAAC,MAAM,EAAE;IAC1C,MAAM,EAAE,mBAAmB,CAAC;IAC5B,MAAM,EAAE,qBAAqB,CAAC;IAC9B,UAAU,CAAC,EAAE,IAAI,CAAC;IAClB,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,WAAW,CAAC,EAAE,gBAAgB,CAAC;IAC/B,WAAW,CAAC,EAAE,mBAAmB,CAAC;CACrC,GAAG,qBAAqB,CAoBxB;AAED;;;;;GAKG;AACH,qBAAa,wBAAwB;IAGrB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAFnC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAqB;gBAEhB,MAAM,EAAE,QAAQ,CAAC,WAAW,CAAC,GAAG,SAAS;IAEtE,mGAAmG;IACnG,OAAO,CAAC,WAAW,EAAE,qBAAqB,GAAG,OAAO;IAOpD,mGAAmG;IACnG,eAAe,CAAC,QAAQ,EAAE,kBAAkB,GAAG,OAAO;CAKzD"}
|
package/dist/quota.js
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
/** Upper bound for error evidence considered by the classifier (bytes, UTF-8). Keeps streaming memory bounded. */
|
|
3
|
+
export const MAX_QUOTA_EVIDENCE_BYTES = 8192;
|
|
4
|
+
const QUOTA_CODES = new Set([
|
|
5
|
+
'insufficient_quota',
|
|
6
|
+
'insufficient_credit_balance',
|
|
7
|
+
'quota_exceeded',
|
|
8
|
+
'usage_limit_reached',
|
|
9
|
+
'credits_exhausted',
|
|
10
|
+
'billing_hard_limit_reached',
|
|
11
|
+
]);
|
|
12
|
+
/**
|
|
13
|
+
* Classify one error record as quota exhaustion. Accepts only structured
|
|
14
|
+
* provider error envelopes — a JSON object with an `error` object whose
|
|
15
|
+
* `type` or `code` exactly matches a canonical quota code. Free text, quoted
|
|
16
|
+
* prompt content inside `error.message`, plain HTTP 429 rate-limit codes,
|
|
17
|
+
* throttling/overload, authentication, context/output limits, and timeouts
|
|
18
|
+
* all classify as negative (R2). Input is bounded to
|
|
19
|
+
* {@link MAX_QUOTA_EVIDENCE_BYTES} trailing bytes.
|
|
20
|
+
*/
|
|
21
|
+
export function classifyQuotaErrorRecord(record) {
|
|
22
|
+
const bounded = record.length > MAX_QUOTA_EVIDENCE_BYTES ? record.slice(-MAX_QUOTA_EVIDENCE_BYTES) : record;
|
|
23
|
+
const envelope = tryParseErrorEnvelope(bounded);
|
|
24
|
+
if (envelope === null)
|
|
25
|
+
return { quota: false };
|
|
26
|
+
const type = typeof envelope.type === 'string' ? envelope.type.toLowerCase() : '';
|
|
27
|
+
const code = typeof envelope.code === 'string' ? envelope.code.toLowerCase() : '';
|
|
28
|
+
for (const candidate of [type, code]) {
|
|
29
|
+
if (QUOTA_CODES.has(candidate))
|
|
30
|
+
return { quota: true, reason: candidate };
|
|
31
|
+
}
|
|
32
|
+
return { quota: false };
|
|
33
|
+
}
|
|
34
|
+
/** Best-effort parse of a JSON provider error envelope (`{ error: { type?, code?, message? } }`). */
|
|
35
|
+
function tryParseErrorEnvelope(record) {
|
|
36
|
+
const start = record.indexOf('{');
|
|
37
|
+
const end = record.lastIndexOf('}');
|
|
38
|
+
if (start < 0 || end <= start)
|
|
39
|
+
return null;
|
|
40
|
+
try {
|
|
41
|
+
const parsed = JSON.parse(record.slice(start, end + 1));
|
|
42
|
+
if (typeof parsed !== 'object' || parsed === null)
|
|
43
|
+
return null;
|
|
44
|
+
const error = parsed.error;
|
|
45
|
+
if (typeof error !== 'object' || error === null)
|
|
46
|
+
return null;
|
|
47
|
+
return error;
|
|
48
|
+
}
|
|
49
|
+
catch {
|
|
50
|
+
return null;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
/** Normalize an observation timestamp to UTC ISO-8601 with millisecond precision. */
|
|
54
|
+
export function normalizeObservedAt(at) {
|
|
55
|
+
return at.toISOString();
|
|
56
|
+
}
|
|
57
|
+
/** Keep only defined, non-empty attribution fields; return undefined when nothing is known. */
|
|
58
|
+
function cleanAttribution(attribution) {
|
|
59
|
+
const cleaned = {};
|
|
60
|
+
if (attribution.projectId)
|
|
61
|
+
cleaned.projectId = attribution.projectId;
|
|
62
|
+
if (attribution.executor)
|
|
63
|
+
cleaned.executor = attribution.executor;
|
|
64
|
+
if (attribution.agent)
|
|
65
|
+
cleaned.agent = attribution.agent;
|
|
66
|
+
if (attribution.model)
|
|
67
|
+
cleaned.model = attribution.model;
|
|
68
|
+
return Object.keys(cleaned).length > 0 ? cleaned : undefined;
|
|
69
|
+
}
|
|
70
|
+
/**
|
|
71
|
+
* Build a quota observation. `observationId` is a SHA-256 digest over the
|
|
72
|
+
* evidence source, reason, and attribution identity — stable across
|
|
73
|
+
* redelivery because it excludes timestamps, correlation, and free text.
|
|
74
|
+
*/
|
|
75
|
+
export function buildQuotaObservation(params) {
|
|
76
|
+
const attribution = params.attribution === undefined ? undefined : cleanAttribution(params.attribution);
|
|
77
|
+
const identity = [
|
|
78
|
+
params.source,
|
|
79
|
+
params.reason,
|
|
80
|
+
attribution?.projectId ?? '',
|
|
81
|
+
attribution?.executor ?? '',
|
|
82
|
+
attribution?.agent ?? '',
|
|
83
|
+
attribution?.model ?? '',
|
|
84
|
+
].join('\0');
|
|
85
|
+
const observationId = `qo-${createHash('sha256').update(identity).digest('hex').slice(0, 24)}`;
|
|
86
|
+
return {
|
|
87
|
+
observationId,
|
|
88
|
+
observedAt: normalizeObservedAt(params.observedAt ?? new Date()),
|
|
89
|
+
evidenceSource: params.source,
|
|
90
|
+
reason: params.reason,
|
|
91
|
+
...(params.detail !== undefined && params.detail !== '' ? { detail: params.detail } : {}),
|
|
92
|
+
...(attribution !== undefined ? { attribution } : {}),
|
|
93
|
+
...(params.correlation !== undefined ? { correlation: params.correlation } : {}),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
/**
|
|
97
|
+
* Single producer path for quota events. Emits `agent.quota.exhausted` at most
|
|
98
|
+
* once per observationId per producer instance (redelivery of the same quota
|
|
99
|
+
* fact is suppressed). Recovery delivery is explicit-only — callers own
|
|
100
|
+
* recovery detection; this class never schedules, polls, or times anything.
|
|
101
|
+
*/
|
|
102
|
+
export class QuotaObservationProducer {
|
|
103
|
+
events;
|
|
104
|
+
emitted = new Set();
|
|
105
|
+
constructor(events) {
|
|
106
|
+
this.events = events;
|
|
107
|
+
}
|
|
108
|
+
/** Emit one `agent.quota.exhausted`. Returns true when emitted, false when deduped or bus-less. */
|
|
109
|
+
produce(observation) {
|
|
110
|
+
if (this.events === undefined || this.emitted.has(observation.observationId))
|
|
111
|
+
return false;
|
|
112
|
+
this.emitted.add(observation.observationId);
|
|
113
|
+
void this.events.emit('agent.quota.exhausted', observation);
|
|
114
|
+
return true;
|
|
115
|
+
}
|
|
116
|
+
/** Deliver an explicit `agent.quota.recovered`. Returns true when emitted, false when bus-less. */
|
|
117
|
+
produceRecovery(recovery) {
|
|
118
|
+
if (this.events === undefined)
|
|
119
|
+
return false;
|
|
120
|
+
void this.events.emit('agent.quota.recovered', recovery);
|
|
121
|
+
return true;
|
|
122
|
+
}
|
|
123
|
+
}
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
|
-
import { type Logger } from '@gobing-ai/ts-infra';
|
|
2
|
+
import { type EventBus, type Logger } from '@gobing-ai/ts-infra';
|
|
3
3
|
import { type ProcessExecutor } from '@gobing-ai/ts-runtime';
|
|
4
4
|
import type { AgentSpec } from './agent-spec';
|
|
5
|
+
import type { AgentEvents } from './events';
|
|
6
|
+
import { type QuotaAttribution } from './quota';
|
|
5
7
|
/** Options for spawning a team agent subprocess. */
|
|
6
8
|
export interface AgentProcessOptions {
|
|
7
9
|
spec: AgentSpec;
|
|
@@ -10,6 +12,10 @@ export interface AgentProcessOptions {
|
|
|
10
12
|
cwd?: string;
|
|
11
13
|
processExecutor?: ProcessExecutor;
|
|
12
14
|
logger?: Logger;
|
|
15
|
+
/** Optional agent event bus; when present, confirmed quota failures emit `agent.quota.exhausted` (Spur task 0798). */
|
|
16
|
+
events?: EventBus<AgentEvents>;
|
|
17
|
+
/** Optional exact quota attribution; absent fields stay absent on observations — never inferred. */
|
|
18
|
+
quotaContext?: QuotaAttribution;
|
|
13
19
|
}
|
|
14
20
|
type ProcessStatus = 'running' | 'stopped' | 'errored';
|
|
15
21
|
/**
|
|
@@ -27,6 +33,11 @@ export declare class TeamAgentProcess {
|
|
|
27
33
|
private status;
|
|
28
34
|
private exitCode;
|
|
29
35
|
private readonly subscribers;
|
|
36
|
+
/** Bounded trailing stderr window retained for quota classification — never a full transcript. */
|
|
37
|
+
private stderrTail;
|
|
38
|
+
private readonly quotaProducer;
|
|
39
|
+
private readonly spec;
|
|
40
|
+
private readonly quotaContext;
|
|
30
41
|
constructor(options: AgentProcessOptions);
|
|
31
42
|
start(): Promise<void>;
|
|
32
43
|
stop(): Promise<void>;
|
|
@@ -38,6 +49,10 @@ export declare class TeamAgentProcess {
|
|
|
38
49
|
getPid(): number | null;
|
|
39
50
|
getExitCode(): number | null;
|
|
40
51
|
private pipe;
|
|
52
|
+
/** Keep only the trailing {@link MAX_QUOTA_EVIDENCE_BYTES} of stderr; subscribers still see every chunk unchanged. */
|
|
53
|
+
private retainStderrTail;
|
|
54
|
+
/** Classify the retained stderr window and emit one attributed quota observation on confirmation. */
|
|
55
|
+
private classifyQuota;
|
|
41
56
|
private warn;
|
|
42
57
|
}
|
|
43
58
|
export {};
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"team-agent-process.d.ts","sourceRoot":"","sources":["../src/team-agent-process.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAa,KAAK,MAAM,EAAE,MAAM,qBAAqB,CAAC;
|
|
1
|
+
{"version":3,"file":"team-agent-process.d.ts","sourceRoot":"","sources":["../src/team-agent-process.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,KAAK,QAAQ,EAAa,KAAK,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAC5E,OAAO,EAAoC,KAAK,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAC/F,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AAC9C,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,UAAU,CAAC;AAC5C,OAAO,EAIH,KAAK,gBAAgB,EAExB,MAAM,SAAS,CAAC;AAEjB,oDAAoD;AACpD,MAAM,WAAW,mBAAmB;IAChC,IAAI,EAAE,SAAS,CAAC;IAChB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,eAAe,CAAC,EAAE,eAAe,CAAC;IAClC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,sHAAsH;IACtH,MAAM,CAAC,EAAE,QAAQ,CAAC,WAAW,CAAC,CAAC;IAC/B,oGAAoG;IACpG,YAAY,CAAC,EAAE,gBAAgB,CAAC;CACnC;AAED,KAAK,aAAa,GAAG,SAAS,GAAG,SAAS,GAAG,SAAS,CAAC;AAEvD;;;GAGG;AACH,qBAAa,gBAAgB;IACzB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAW;IACnC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAqC;IACzD,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAqB;IACzC,OAAO,CAAC,QAAQ,CAAC,eAAe,CAAkB;IAClD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,UAAU,CAA4B;IAC9C,OAAO,CAAC,MAAM,CAA4B;IAC1C,OAAO,CAAC,QAAQ,CAAuB;IACvC,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAqC;IACjE,kGAAkG;IAClG,OAAO,CAAC,UAAU,CAAM;IACxB,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA2B;IACzD,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAY;IACjC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAA+B;gBAEhD,OAAO,EAAE,mBAAmB;IAYlC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAyBtB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IA0BrB,IAAI,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC;QAAE,EAAE,EAAE,OAAO,CAAA;KAAE,CAAC;IAerD,SAAS,CAAC,QAAQ,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,IAAI,GAAG,MAAM,IAAI;IAOvD,SAAS,IAAI,aAAa;IAI1B,MAAM,IAAI,MAAM,GAAG,IAAI;IAIvB,WAAW,IAAI,MAAM,GAAG,IAAI;YAId,IAAI;IAkBlB,sHAAsH;IACtH,OAAO,CAAC,gBAAgB;IAIxB,qGAAqG;IACrG,OAAO,CAAC,aAAa;IAgBrB,OAAO,CAAC,IAAI;CAOf"}
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
2
|
import { getLogger } from '@gobing-ai/ts-infra';
|
|
3
3
|
import { nodeBunFactory } from '@gobing-ai/ts-runtime';
|
|
4
|
+
import { buildQuotaObservation, classifyQuotaErrorRecord, MAX_QUOTA_EVIDENCE_BYTES, QuotaObservationProducer, } from './quota.js';
|
|
4
5
|
/**
|
|
5
6
|
* Manages the lifecycle of a single agent subprocess — start, stop, message send, and stdout/stderr subscription.
|
|
6
7
|
* The identity preamble is built by `TeamOrchestrator` and baked into `command` before the process is constructed.
|
|
@@ -16,6 +17,11 @@ export class TeamAgentProcess {
|
|
|
16
17
|
status = 'stopped';
|
|
17
18
|
exitCode = null;
|
|
18
19
|
subscribers = new Set();
|
|
20
|
+
/** Bounded trailing stderr window retained for quota classification — never a full transcript. */
|
|
21
|
+
stderrTail = '';
|
|
22
|
+
quotaProducer;
|
|
23
|
+
spec;
|
|
24
|
+
quotaContext;
|
|
19
25
|
constructor(options) {
|
|
20
26
|
this.agentId = options.spec.id;
|
|
21
27
|
this.command = options.command;
|
|
@@ -23,6 +29,9 @@ export class TeamAgentProcess {
|
|
|
23
29
|
this.cwd = options.cwd ?? options.spec.workspace;
|
|
24
30
|
this.processExecutor = options.processExecutor ?? nodeBunFactory.createProcessExecutor();
|
|
25
31
|
this.logger = options.logger ?? getLogger('team-agent');
|
|
32
|
+
this.quotaProducer = new QuotaObservationProducer(options.events);
|
|
33
|
+
this.spec = options.spec;
|
|
34
|
+
this.quotaContext = options.quotaContext;
|
|
26
35
|
}
|
|
27
36
|
async start() {
|
|
28
37
|
if (this.status === 'running')
|
|
@@ -39,14 +48,18 @@ export class TeamAgentProcess {
|
|
|
39
48
|
});
|
|
40
49
|
this.status = 'running';
|
|
41
50
|
this.exitCode = null;
|
|
51
|
+
this.stderrTail = '';
|
|
42
52
|
if (this.subprocess.stdout !== null)
|
|
43
|
-
this.pipe(this.subprocess.stdout);
|
|
53
|
+
this.pipe(this.subprocess.stdout, false);
|
|
44
54
|
if (this.subprocess.stderr !== null)
|
|
45
|
-
this.pipe(this.subprocess.stderr);
|
|
55
|
+
this.pipe(this.subprocess.stderr, true);
|
|
46
56
|
void this.subprocess.exited.then((code) => {
|
|
47
57
|
this.exitCode = code;
|
|
48
|
-
if (this.status === 'running')
|
|
58
|
+
if (this.status === 'running') {
|
|
49
59
|
this.status = code === 0 ? 'stopped' : 'errored';
|
|
60
|
+
if (this.status === 'errored')
|
|
61
|
+
this.classifyQuota();
|
|
62
|
+
}
|
|
50
63
|
});
|
|
51
64
|
}
|
|
52
65
|
async stop() {
|
|
@@ -106,7 +119,7 @@ export class TeamAgentProcess {
|
|
|
106
119
|
getExitCode() {
|
|
107
120
|
return this.exitCode;
|
|
108
121
|
}
|
|
109
|
-
async pipe(stream) {
|
|
122
|
+
async pipe(stream, isStderr) {
|
|
110
123
|
const reader = stream.getReader();
|
|
111
124
|
try {
|
|
112
125
|
while (true) {
|
|
@@ -114,6 +127,8 @@ export class TeamAgentProcess {
|
|
|
114
127
|
if (chunk.done)
|
|
115
128
|
break;
|
|
116
129
|
const buffer = Buffer.from(chunk.value);
|
|
130
|
+
if (isStderr)
|
|
131
|
+
this.retainStderrTail(buffer.toString('utf8'));
|
|
117
132
|
for (const subscriber of this.subscribers)
|
|
118
133
|
subscriber(buffer);
|
|
119
134
|
}
|
|
@@ -127,6 +142,25 @@ export class TeamAgentProcess {
|
|
|
127
142
|
reader.releaseLock();
|
|
128
143
|
}
|
|
129
144
|
}
|
|
145
|
+
/** Keep only the trailing {@link MAX_QUOTA_EVIDENCE_BYTES} of stderr; subscribers still see every chunk unchanged. */
|
|
146
|
+
retainStderrTail(text) {
|
|
147
|
+
this.stderrTail = (this.stderrTail + text).slice(-MAX_QUOTA_EVIDENCE_BYTES);
|
|
148
|
+
}
|
|
149
|
+
/** Classify the retained stderr window and emit one attributed quota observation on confirmation. */
|
|
150
|
+
classifyQuota() {
|
|
151
|
+
const classification = classifyQuotaErrorRecord(this.stderrTail);
|
|
152
|
+
if (!classification.quota)
|
|
153
|
+
return;
|
|
154
|
+
this.quotaProducer.produce(buildQuotaObservation({
|
|
155
|
+
source: 'streaming-error',
|
|
156
|
+
reason: classification.reason,
|
|
157
|
+
attribution: {
|
|
158
|
+
...this.quotaContext,
|
|
159
|
+
agent: this.quotaContext?.agent ?? this.spec.id,
|
|
160
|
+
executor: this.quotaContext?.executor ?? this.spec.executor ?? this.spec.type,
|
|
161
|
+
},
|
|
162
|
+
}));
|
|
163
|
+
}
|
|
130
164
|
warn(message, op, error) {
|
|
131
165
|
this.logger.warn(message, {
|
|
132
166
|
agentId: this.agentId,
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@gobing-ai/ts-ai-runner",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.57",
|
|
4
4
|
"description": "@gobing-ai/ts-ai-runner — Coding-agent shims, detection, doctor checks, and prompt execution.",
|
|
5
5
|
"keywords": [
|
|
6
6
|
"typescript",
|
|
@@ -47,12 +47,12 @@
|
|
|
47
47
|
"release": "echo 'Manual publish is disabled. Releases go through GitHub Actions via Trusted Publishing — push a tag: git tag @gobing-ai/ts-ai-runner-v<version> && git push --tags' && exit 1"
|
|
48
48
|
},
|
|
49
49
|
"dependencies": {
|
|
50
|
-
"@gobing-ai/ts-infra": "^0.4.
|
|
51
|
-
"@gobing-ai/ts-runtime": "^0.4.
|
|
50
|
+
"@gobing-ai/ts-infra": "^0.4.57",
|
|
51
|
+
"@gobing-ai/ts-runtime": "^0.4.57"
|
|
52
52
|
},
|
|
53
53
|
"devDependencies": {
|
|
54
54
|
"@types/bun": "1.3.14",
|
|
55
|
-
"@gobing-ai/ts-db": "^0.4.
|
|
55
|
+
"@gobing-ai/ts-db": "^0.4.57"
|
|
56
56
|
},
|
|
57
57
|
"publishConfig": {
|
|
58
58
|
"access": "public"
|
package/src/ai-runner.ts
CHANGED
|
@@ -10,6 +10,12 @@ import {
|
|
|
10
10
|
import { type AgentName, getAgentShim, type PromptOptions, type ShimCommand } from './agents/shims';
|
|
11
11
|
import type { AgentEvents, AiRunnerProcessEvents } from './events';
|
|
12
12
|
import { buildIdentityPreamble } from './identity';
|
|
13
|
+
import {
|
|
14
|
+
buildQuotaObservation,
|
|
15
|
+
classifyQuotaErrorRecord,
|
|
16
|
+
type QuotaAttribution,
|
|
17
|
+
QuotaObservationProducer,
|
|
18
|
+
} from './quota';
|
|
13
19
|
import { translateSlashCommand } from './slash-command';
|
|
14
20
|
|
|
15
21
|
/** Result returned by every AI runner dispatch method. */
|
|
@@ -38,6 +44,12 @@ export interface AgentRunOptions {
|
|
|
38
44
|
onOutput?: (output: ProcessOutputChunk) => void;
|
|
39
45
|
/** Optional application-owned correlation propagated unchanged to agent lifecycle events. */
|
|
40
46
|
correlation?: AgentRunCorrelation;
|
|
47
|
+
/**
|
|
48
|
+
* Optional exact attribution for quota observations (Spur task 0798). Purely
|
|
49
|
+
* additive metadata: existing callers without it keep byte-identical results,
|
|
50
|
+
* and missing fields stay absent on emitted observations — never inferred.
|
|
51
|
+
*/
|
|
52
|
+
quotaContext?: QuotaAttribution;
|
|
41
53
|
}
|
|
42
54
|
|
|
43
55
|
/** Application-owned execution identity carried without coupling the runner to a workflow model. */
|
|
@@ -121,6 +133,8 @@ export class AiRunner {
|
|
|
121
133
|
private readonly defaultTimeout: number | undefined;
|
|
122
134
|
private readonly logger: Logger;
|
|
123
135
|
private readonly events: EventBus<AgentEvents> | undefined;
|
|
136
|
+
/** Single producer path for quota events; no-ops when no agent event bus is configured. */
|
|
137
|
+
private readonly quotaProducer: QuotaObservationProducer;
|
|
124
138
|
/** Internal process-level observability bus, parented to `lifecycleBus` when auto-constructed. Exposed for introspection/testing. */
|
|
125
139
|
readonly processEvents: EventBus<AiRunnerProcessEvents> | undefined;
|
|
126
140
|
|
|
@@ -153,6 +167,7 @@ export class AiRunner {
|
|
|
153
167
|
this.defaultTimeout = options.defaultTimeout;
|
|
154
168
|
this.logger = options.logger ?? getLogger('ai-runner');
|
|
155
169
|
this.events = events;
|
|
170
|
+
this.quotaProducer = new QuotaObservationProducer(events);
|
|
156
171
|
this.processEvents = processEvents;
|
|
157
172
|
}
|
|
158
173
|
|
|
@@ -251,6 +266,19 @@ export class AiRunner {
|
|
|
251
266
|
...(options.correlation !== undefined ? { correlation: options.correlation } : {}),
|
|
252
267
|
severity: result.exitCode === 0 || result.exitCode === null ? 'info' : 'error',
|
|
253
268
|
});
|
|
269
|
+
if (result.exitCode !== null && result.exitCode !== 0) {
|
|
270
|
+
const classification = classifyQuotaErrorRecord(result.stderr);
|
|
271
|
+
if (classification.quota) {
|
|
272
|
+
this.quotaProducer.produce(
|
|
273
|
+
buildQuotaObservation({
|
|
274
|
+
source: 'buffered-error',
|
|
275
|
+
reason: classification.reason,
|
|
276
|
+
attribution: { agent, ...options.quotaContext },
|
|
277
|
+
...(options.correlation !== undefined ? { correlation: options.correlation } : {}),
|
|
278
|
+
}),
|
|
279
|
+
);
|
|
280
|
+
}
|
|
281
|
+
}
|
|
254
282
|
return {
|
|
255
283
|
exitCode: result.exitCode,
|
|
256
284
|
stdout: result.stdout,
|
package/src/events.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import type { EventSeverity } from '@gobing-ai/ts-infra';
|
|
2
2
|
import type { ProcessEvents } from '@gobing-ai/ts-runtime';
|
|
3
3
|
import type { AgentRunCorrelation } from './ai-runner';
|
|
4
|
+
import type { AgentQuotaObservation, AgentQuotaRecovery } from './quota';
|
|
4
5
|
|
|
5
6
|
/** Typed event map for agent-runner observability. All events prefixed `agent.`. */
|
|
6
7
|
export type AgentEvents = {
|
|
@@ -34,6 +35,10 @@ export type AgentEvents = {
|
|
|
34
35
|
'agent.stopped': (data: { agentId: string; exitCode: number | null; severity: EventSeverity }) => void;
|
|
35
36
|
/** Emitted when a message is sent to a team agent process. */
|
|
36
37
|
'agent.message.sent': (data: { agentId: string; ok: boolean; severity: EventSeverity }) => void;
|
|
38
|
+
/** Emitted once per verified quota-exhaustion observation (confirmed usage-allowance or credit exhaustion). */
|
|
39
|
+
'agent.quota.exhausted': (data: AgentQuotaObservation) => void;
|
|
40
|
+
/** Explicit-recovery signal; type contract only — no automatic producer exists upstream. */
|
|
41
|
+
'agent.quota.recovered': (data: AgentQuotaRecovery) => void;
|
|
37
42
|
};
|
|
38
43
|
|
|
39
44
|
/** Event map for process-level observability emitted by AiRunner-owned executors. */
|
package/src/index.ts
CHANGED
|
@@ -9,6 +9,7 @@ export * from './identity';
|
|
|
9
9
|
export * from './message-store';
|
|
10
10
|
export * from './messages';
|
|
11
11
|
export * from './model-health-probe';
|
|
12
|
+
export * from './quota';
|
|
12
13
|
export * from './slash-command';
|
|
13
14
|
export * from './team-agent-process';
|
|
14
15
|
export * from './team-orchestrator';
|
|
@@ -10,7 +10,15 @@
|
|
|
10
10
|
* issuing a minimal 1-token completion request and interpreting the HTTP
|
|
11
11
|
* response (R3).
|
|
12
12
|
*/
|
|
13
|
-
import { APIClient, type APIClientConfig, APIError, type RawHttpResponse } from '@gobing-ai/ts-infra';
|
|
13
|
+
import { APIClient, type APIClientConfig, APIError, type EventBus, type RawHttpResponse } from '@gobing-ai/ts-infra';
|
|
14
|
+
import type { AgentRunCorrelation } from './ai-runner';
|
|
15
|
+
import type { AgentEvents } from './events';
|
|
16
|
+
import {
|
|
17
|
+
type AgentQuotaObservation,
|
|
18
|
+
buildQuotaObservation,
|
|
19
|
+
type QuotaAttribution,
|
|
20
|
+
QuotaObservationProducer,
|
|
21
|
+
} from './quota';
|
|
14
22
|
|
|
15
23
|
/** Health status for a single model endpoint. */
|
|
16
24
|
export type ModelHealthStatus = 'available' | 'quota_exhausted' | 'rate_limited' | 'unavailable' | 'unknown';
|
|
@@ -239,3 +247,40 @@ export class OmpModelProbe implements ModelHealthProbe {
|
|
|
239
247
|
|
|
240
248
|
/** Default probe timeout in milliseconds (R8). */
|
|
241
249
|
export { DEFAULT_PROBE_TIMEOUT_MS };
|
|
250
|
+
|
|
251
|
+
/** Options for the explicitly opted-in quota health observation. */
|
|
252
|
+
export interface QuotaHealthObservationOptions {
|
|
253
|
+
/** Agent event bus receiving `agent.quota.exhausted`. */
|
|
254
|
+
events: EventBus<AgentEvents>;
|
|
255
|
+
/** Optional exact attribution; absent fields stay absent — never inferred. */
|
|
256
|
+
attribution?: QuotaAttribution;
|
|
257
|
+
/** Optional run correlation carried unchanged onto the observation. */
|
|
258
|
+
correlation?: AgentRunCorrelation;
|
|
259
|
+
/** Optional shared producer so multiple observations dedupe across one observation session. */
|
|
260
|
+
producer?: QuotaObservationProducer;
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Explicitly opted-in health observation → quota event. The ONLY producer seam
|
|
265
|
+
* on the health path: ordinary doctor runs never call this, so read-only
|
|
266
|
+
* behavior is preserved. Returns the emitted observation, or null when the
|
|
267
|
+
* result is not a confirmed quota exhaustion (rate_limited / unavailable /
|
|
268
|
+
* unknown never produce events).
|
|
269
|
+
*/
|
|
270
|
+
export function observeQuotaHealthResult(
|
|
271
|
+
result: ModelHealthResult,
|
|
272
|
+
options: QuotaHealthObservationOptions,
|
|
273
|
+
): AgentQuotaObservation | null {
|
|
274
|
+
if (result.status !== 'quota_exhausted') return null;
|
|
275
|
+
const producer = options.producer ?? new QuotaObservationProducer(options.events);
|
|
276
|
+
const observation = buildQuotaObservation({
|
|
277
|
+
source: 'health-probe',
|
|
278
|
+
reason: 'provider_quota_exhausted',
|
|
279
|
+
observedAt: new Date(result.checkedAt),
|
|
280
|
+
...(result.detail !== undefined ? { detail: result.detail } : {}),
|
|
281
|
+
attribution: options.attribution,
|
|
282
|
+
...(options.correlation !== undefined ? { correlation: options.correlation } : {}),
|
|
283
|
+
});
|
|
284
|
+
producer.produce(observation);
|
|
285
|
+
return observation;
|
|
286
|
+
}
|
package/src/quota.ts
ADDED
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import type { EventBus } from '@gobing-ai/ts-infra';
|
|
3
|
+
import type { AgentRunCorrelation } from './ai-runner';
|
|
4
|
+
import type { AgentEvents } from './events';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Canonical quota-exhaustion codes accepted in structured error-record positions
|
|
8
|
+
* (`error.type` / `error.code`). The set is an exact-match allowlist on purpose:
|
|
9
|
+
* free text never confirms quota (R2 — Spur task 0798). Generic HTTP 429 without
|
|
10
|
+
* one of these codes is rate limiting, not quota exhaustion.
|
|
11
|
+
*/
|
|
12
|
+
export type QuotaExhaustionReason =
|
|
13
|
+
| 'insufficient_quota'
|
|
14
|
+
| 'insufficient_credit_balance'
|
|
15
|
+
| 'quota_exceeded'
|
|
16
|
+
| 'usage_limit_reached'
|
|
17
|
+
| 'credits_exhausted'
|
|
18
|
+
| 'billing_hard_limit_reached'
|
|
19
|
+
/** Verified by the shipped health probe's quota-aware 429 interpretation (`OmpModelProbe`). */
|
|
20
|
+
| 'provider_quota_exhausted';
|
|
21
|
+
|
|
22
|
+
/** Where a quota observation was verified. Streaming evidence is a bounded trailing window, never a transcript. */
|
|
23
|
+
export type QuotaEvidenceSource = 'buffered-error' | 'streaming-error' | 'health-probe';
|
|
24
|
+
|
|
25
|
+
/**
|
|
26
|
+
* Exact attribution carried on a quota observation. Every field is optional and
|
|
27
|
+
* only populated from caller-supplied context — the runner never infers project
|
|
28
|
+
* or executor identity (missing attribution stays observable, not guessed).
|
|
29
|
+
*/
|
|
30
|
+
export interface QuotaAttribution {
|
|
31
|
+
projectId?: string;
|
|
32
|
+
executor?: string;
|
|
33
|
+
agent?: string;
|
|
34
|
+
model?: string;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
/** A verified quota-exhaustion observation. `observationId` is deterministic, so redelivery keeps identity. */
|
|
38
|
+
export interface AgentQuotaObservation {
|
|
39
|
+
/** Stable across redelivery: digest over evidence source, reason, and attribution identity — never timestamps. */
|
|
40
|
+
observationId: string;
|
|
41
|
+
/** UTC ISO-8601 with millisecond precision. */
|
|
42
|
+
observedAt: string;
|
|
43
|
+
evidenceSource: QuotaEvidenceSource;
|
|
44
|
+
reason: QuotaExhaustionReason;
|
|
45
|
+
/** Sanitized provider detail. Only carried when the source guarantees sanitization (health probe). */
|
|
46
|
+
detail?: string;
|
|
47
|
+
/** Exact attribution when supplied by the caller; absent fields mean unknown. */
|
|
48
|
+
attribution?: QuotaAttribution;
|
|
49
|
+
correlation?: AgentRunCorrelation;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
/** Explicit-recovery payload for `agent.quota.recovered`. Type contract only — no automatic producer, timer, or polling. */
|
|
53
|
+
export interface AgentQuotaRecovery {
|
|
54
|
+
/** The observationId whose quota state recovered. */
|
|
55
|
+
observationId: string;
|
|
56
|
+
/** UTC ISO-8601 with millisecond precision. */
|
|
57
|
+
recoveredAt: string;
|
|
58
|
+
attribution?: QuotaAttribution;
|
|
59
|
+
correlation?: AgentRunCorrelation;
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** Result of classifying one bounded error record. */
|
|
63
|
+
export type QuotaClassification = { quota: true; reason: QuotaExhaustionReason } | { quota: false };
|
|
64
|
+
|
|
65
|
+
/** Upper bound for error evidence considered by the classifier (bytes, UTF-8). Keeps streaming memory bounded. */
|
|
66
|
+
export const MAX_QUOTA_EVIDENCE_BYTES = 8192;
|
|
67
|
+
|
|
68
|
+
const QUOTA_CODES: ReadonlySet<string> = new Set([
|
|
69
|
+
'insufficient_quota',
|
|
70
|
+
'insufficient_credit_balance',
|
|
71
|
+
'quota_exceeded',
|
|
72
|
+
'usage_limit_reached',
|
|
73
|
+
'credits_exhausted',
|
|
74
|
+
'billing_hard_limit_reached',
|
|
75
|
+
]);
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Classify one error record as quota exhaustion. Accepts only structured
|
|
79
|
+
* provider error envelopes — a JSON object with an `error` object whose
|
|
80
|
+
* `type` or `code` exactly matches a canonical quota code. Free text, quoted
|
|
81
|
+
* prompt content inside `error.message`, plain HTTP 429 rate-limit codes,
|
|
82
|
+
* throttling/overload, authentication, context/output limits, and timeouts
|
|
83
|
+
* all classify as negative (R2). Input is bounded to
|
|
84
|
+
* {@link MAX_QUOTA_EVIDENCE_BYTES} trailing bytes.
|
|
85
|
+
*/
|
|
86
|
+
export function classifyQuotaErrorRecord(record: string): QuotaClassification {
|
|
87
|
+
const bounded = record.length > MAX_QUOTA_EVIDENCE_BYTES ? record.slice(-MAX_QUOTA_EVIDENCE_BYTES) : record;
|
|
88
|
+
const envelope = tryParseErrorEnvelope(bounded);
|
|
89
|
+
if (envelope === null) return { quota: false };
|
|
90
|
+
const type = typeof envelope.type === 'string' ? envelope.type.toLowerCase() : '';
|
|
91
|
+
const code = typeof envelope.code === 'string' ? envelope.code.toLowerCase() : '';
|
|
92
|
+
for (const candidate of [type, code]) {
|
|
93
|
+
if (QUOTA_CODES.has(candidate)) return { quota: true, reason: candidate as QuotaExhaustionReason };
|
|
94
|
+
}
|
|
95
|
+
return { quota: false };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** Best-effort parse of a JSON provider error envelope (`{ error: { type?, code?, message? } }`). */
|
|
99
|
+
function tryParseErrorEnvelope(record: string): { type?: unknown; code?: unknown } | null {
|
|
100
|
+
const start = record.indexOf('{');
|
|
101
|
+
const end = record.lastIndexOf('}');
|
|
102
|
+
if (start < 0 || end <= start) return null;
|
|
103
|
+
try {
|
|
104
|
+
const parsed: unknown = JSON.parse(record.slice(start, end + 1));
|
|
105
|
+
if (typeof parsed !== 'object' || parsed === null) return null;
|
|
106
|
+
const error = (parsed as { error?: unknown }).error;
|
|
107
|
+
if (typeof error !== 'object' || error === null) return null;
|
|
108
|
+
return error as { type?: unknown; code?: unknown };
|
|
109
|
+
} catch {
|
|
110
|
+
return null;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Normalize an observation timestamp to UTC ISO-8601 with millisecond precision. */
|
|
115
|
+
export function normalizeObservedAt(at: Date): string {
|
|
116
|
+
return at.toISOString();
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
/** Keep only defined, non-empty attribution fields; return undefined when nothing is known. */
|
|
120
|
+
function cleanAttribution(attribution: QuotaAttribution): QuotaAttribution | undefined {
|
|
121
|
+
const cleaned: QuotaAttribution = {};
|
|
122
|
+
if (attribution.projectId) cleaned.projectId = attribution.projectId;
|
|
123
|
+
if (attribution.executor) cleaned.executor = attribution.executor;
|
|
124
|
+
if (attribution.agent) cleaned.agent = attribution.agent;
|
|
125
|
+
if (attribution.model) cleaned.model = attribution.model;
|
|
126
|
+
return Object.keys(cleaned).length > 0 ? cleaned : undefined;
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Build a quota observation. `observationId` is a SHA-256 digest over the
|
|
131
|
+
* evidence source, reason, and attribution identity — stable across
|
|
132
|
+
* redelivery because it excludes timestamps, correlation, and free text.
|
|
133
|
+
*/
|
|
134
|
+
export function buildQuotaObservation(params: {
|
|
135
|
+
source: QuotaEvidenceSource;
|
|
136
|
+
reason: QuotaExhaustionReason;
|
|
137
|
+
observedAt?: Date;
|
|
138
|
+
detail?: string;
|
|
139
|
+
attribution?: QuotaAttribution;
|
|
140
|
+
correlation?: AgentRunCorrelation;
|
|
141
|
+
}): AgentQuotaObservation {
|
|
142
|
+
const attribution = params.attribution === undefined ? undefined : cleanAttribution(params.attribution);
|
|
143
|
+
const identity = [
|
|
144
|
+
params.source,
|
|
145
|
+
params.reason,
|
|
146
|
+
attribution?.projectId ?? '',
|
|
147
|
+
attribution?.executor ?? '',
|
|
148
|
+
attribution?.agent ?? '',
|
|
149
|
+
attribution?.model ?? '',
|
|
150
|
+
].join('\0');
|
|
151
|
+
const observationId = `qo-${createHash('sha256').update(identity).digest('hex').slice(0, 24)}`;
|
|
152
|
+
return {
|
|
153
|
+
observationId,
|
|
154
|
+
observedAt: normalizeObservedAt(params.observedAt ?? new Date()),
|
|
155
|
+
evidenceSource: params.source,
|
|
156
|
+
reason: params.reason,
|
|
157
|
+
...(params.detail !== undefined && params.detail !== '' ? { detail: params.detail } : {}),
|
|
158
|
+
...(attribution !== undefined ? { attribution } : {}),
|
|
159
|
+
...(params.correlation !== undefined ? { correlation: params.correlation } : {}),
|
|
160
|
+
};
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* Single producer path for quota events. Emits `agent.quota.exhausted` at most
|
|
165
|
+
* once per observationId per producer instance (redelivery of the same quota
|
|
166
|
+
* fact is suppressed). Recovery delivery is explicit-only — callers own
|
|
167
|
+
* recovery detection; this class never schedules, polls, or times anything.
|
|
168
|
+
*/
|
|
169
|
+
export class QuotaObservationProducer {
|
|
170
|
+
private readonly emitted = new Set<string>();
|
|
171
|
+
|
|
172
|
+
constructor(private readonly events: EventBus<AgentEvents> | undefined) {}
|
|
173
|
+
|
|
174
|
+
/** Emit one `agent.quota.exhausted`. Returns true when emitted, false when deduped or bus-less. */
|
|
175
|
+
produce(observation: AgentQuotaObservation): boolean {
|
|
176
|
+
if (this.events === undefined || this.emitted.has(observation.observationId)) return false;
|
|
177
|
+
this.emitted.add(observation.observationId);
|
|
178
|
+
void this.events.emit('agent.quota.exhausted', observation);
|
|
179
|
+
return true;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** Deliver an explicit `agent.quota.recovered`. Returns true when emitted, false when bus-less. */
|
|
183
|
+
produceRecovery(recovery: AgentQuotaRecovery): boolean {
|
|
184
|
+
if (this.events === undefined) return false;
|
|
185
|
+
void this.events.emit('agent.quota.recovered', recovery);
|
|
186
|
+
return true;
|
|
187
|
+
}
|
|
188
|
+
}
|
|
@@ -1,7 +1,15 @@
|
|
|
1
1
|
import { Buffer } from 'node:buffer';
|
|
2
|
-
import { getLogger, type Logger } from '@gobing-ai/ts-infra';
|
|
2
|
+
import { type EventBus, getLogger, type Logger } from '@gobing-ai/ts-infra';
|
|
3
3
|
import { nodeBunFactory, type PipeProcess, type ProcessExecutor } from '@gobing-ai/ts-runtime';
|
|
4
4
|
import type { AgentSpec } from './agent-spec';
|
|
5
|
+
import type { AgentEvents } from './events';
|
|
6
|
+
import {
|
|
7
|
+
buildQuotaObservation,
|
|
8
|
+
classifyQuotaErrorRecord,
|
|
9
|
+
MAX_QUOTA_EVIDENCE_BYTES,
|
|
10
|
+
type QuotaAttribution,
|
|
11
|
+
QuotaObservationProducer,
|
|
12
|
+
} from './quota';
|
|
5
13
|
|
|
6
14
|
/** Options for spawning a team agent subprocess. */
|
|
7
15
|
export interface AgentProcessOptions {
|
|
@@ -11,6 +19,10 @@ export interface AgentProcessOptions {
|
|
|
11
19
|
cwd?: string;
|
|
12
20
|
processExecutor?: ProcessExecutor;
|
|
13
21
|
logger?: Logger;
|
|
22
|
+
/** Optional agent event bus; when present, confirmed quota failures emit `agent.quota.exhausted` (Spur task 0798). */
|
|
23
|
+
events?: EventBus<AgentEvents>;
|
|
24
|
+
/** Optional exact quota attribution; absent fields stay absent on observations — never inferred. */
|
|
25
|
+
quotaContext?: QuotaAttribution;
|
|
14
26
|
}
|
|
15
27
|
|
|
16
28
|
type ProcessStatus = 'running' | 'stopped' | 'errored';
|
|
@@ -30,6 +42,11 @@ export class TeamAgentProcess {
|
|
|
30
42
|
private status: ProcessStatus = 'stopped';
|
|
31
43
|
private exitCode: number | null = null;
|
|
32
44
|
private readonly subscribers = new Set<(data: Buffer) => void>();
|
|
45
|
+
/** Bounded trailing stderr window retained for quota classification — never a full transcript. */
|
|
46
|
+
private stderrTail = '';
|
|
47
|
+
private readonly quotaProducer: QuotaObservationProducer;
|
|
48
|
+
private readonly spec: AgentSpec;
|
|
49
|
+
private readonly quotaContext: QuotaAttribution | undefined;
|
|
33
50
|
|
|
34
51
|
constructor(options: AgentProcessOptions) {
|
|
35
52
|
this.agentId = options.spec.id;
|
|
@@ -38,6 +55,9 @@ export class TeamAgentProcess {
|
|
|
38
55
|
this.cwd = options.cwd ?? options.spec.workspace;
|
|
39
56
|
this.processExecutor = options.processExecutor ?? nodeBunFactory.createProcessExecutor();
|
|
40
57
|
this.logger = options.logger ?? getLogger('team-agent');
|
|
58
|
+
this.quotaProducer = new QuotaObservationProducer(options.events);
|
|
59
|
+
this.spec = options.spec;
|
|
60
|
+
this.quotaContext = options.quotaContext;
|
|
41
61
|
}
|
|
42
62
|
|
|
43
63
|
async start(): Promise<void> {
|
|
@@ -53,11 +73,15 @@ export class TeamAgentProcess {
|
|
|
53
73
|
});
|
|
54
74
|
this.status = 'running';
|
|
55
75
|
this.exitCode = null;
|
|
56
|
-
|
|
57
|
-
if (this.subprocess.
|
|
76
|
+
this.stderrTail = '';
|
|
77
|
+
if (this.subprocess.stdout !== null) this.pipe(this.subprocess.stdout, false);
|
|
78
|
+
if (this.subprocess.stderr !== null) this.pipe(this.subprocess.stderr, true);
|
|
58
79
|
void this.subprocess.exited.then((code) => {
|
|
59
80
|
this.exitCode = code;
|
|
60
|
-
if (this.status === 'running')
|
|
81
|
+
if (this.status === 'running') {
|
|
82
|
+
this.status = code === 0 ? 'stopped' : 'errored';
|
|
83
|
+
if (this.status === 'errored') this.classifyQuota();
|
|
84
|
+
}
|
|
61
85
|
});
|
|
62
86
|
}
|
|
63
87
|
|
|
@@ -121,13 +145,14 @@ export class TeamAgentProcess {
|
|
|
121
145
|
return this.exitCode;
|
|
122
146
|
}
|
|
123
147
|
|
|
124
|
-
private async pipe(stream: ReadableStream<Uint8Array
|
|
148
|
+
private async pipe(stream: ReadableStream<Uint8Array>, isStderr: boolean): Promise<void> {
|
|
125
149
|
const reader = stream.getReader();
|
|
126
150
|
try {
|
|
127
151
|
while (true) {
|
|
128
152
|
const chunk = await reader.read();
|
|
129
153
|
if (chunk.done) break;
|
|
130
154
|
const buffer = Buffer.from(chunk.value);
|
|
155
|
+
if (isStderr) this.retainStderrTail(buffer.toString('utf8'));
|
|
131
156
|
for (const subscriber of this.subscribers) subscriber(buffer);
|
|
132
157
|
}
|
|
133
158
|
} catch (error) {
|
|
@@ -138,6 +163,28 @@ export class TeamAgentProcess {
|
|
|
138
163
|
}
|
|
139
164
|
}
|
|
140
165
|
|
|
166
|
+
/** Keep only the trailing {@link MAX_QUOTA_EVIDENCE_BYTES} of stderr; subscribers still see every chunk unchanged. */
|
|
167
|
+
private retainStderrTail(text: string): void {
|
|
168
|
+
this.stderrTail = (this.stderrTail + text).slice(-MAX_QUOTA_EVIDENCE_BYTES);
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
/** Classify the retained stderr window and emit one attributed quota observation on confirmation. */
|
|
172
|
+
private classifyQuota(): void {
|
|
173
|
+
const classification = classifyQuotaErrorRecord(this.stderrTail);
|
|
174
|
+
if (!classification.quota) return;
|
|
175
|
+
this.quotaProducer.produce(
|
|
176
|
+
buildQuotaObservation({
|
|
177
|
+
source: 'streaming-error',
|
|
178
|
+
reason: classification.reason,
|
|
179
|
+
attribution: {
|
|
180
|
+
...this.quotaContext,
|
|
181
|
+
agent: this.quotaContext?.agent ?? this.spec.id,
|
|
182
|
+
executor: this.quotaContext?.executor ?? this.spec.executor ?? this.spec.type,
|
|
183
|
+
},
|
|
184
|
+
}),
|
|
185
|
+
);
|
|
186
|
+
}
|
|
187
|
+
|
|
141
188
|
private warn(message: string, op: string, error?: unknown): void {
|
|
142
189
|
this.logger.warn(message, {
|
|
143
190
|
agentId: this.agentId,
|