@dogpile/sdk 0.4.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +92 -0
- package/dist/browser/index.js +4156 -4611
- package/dist/browser/index.js.map +1 -1
- package/dist/index.d.ts +3 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/providers/openai-compatible.d.ts.map +1 -1
- package/dist/providers/openai-compatible.js +6 -1
- package/dist/providers/openai-compatible.js.map +1 -1
- package/dist/runtime/audit.d.ts +42 -0
- package/dist/runtime/audit.d.ts.map +1 -0
- package/dist/runtime/audit.js +73 -0
- package/dist/runtime/audit.js.map +1 -0
- package/dist/runtime/broadcast.d.ts +1 -0
- package/dist/runtime/broadcast.d.ts.map +1 -1
- package/dist/runtime/broadcast.js +171 -105
- package/dist/runtime/broadcast.js.map +1 -1
- package/dist/runtime/coordinator.d.ts +9 -2
- package/dist/runtime/coordinator.d.ts.map +1 -1
- package/dist/runtime/coordinator.js +164 -78
- package/dist/runtime/coordinator.js.map +1 -1
- package/dist/runtime/defaults.d.ts.map +1 -1
- package/dist/runtime/defaults.js +14 -5
- package/dist/runtime/defaults.js.map +1 -1
- package/dist/runtime/engine.d.ts +17 -4
- package/dist/runtime/engine.d.ts.map +1 -1
- package/dist/runtime/engine.js +577 -52
- package/dist/runtime/engine.js.map +1 -1
- package/dist/runtime/health.d.ts +51 -0
- package/dist/runtime/health.d.ts.map +1 -0
- package/dist/runtime/health.js +85 -0
- package/dist/runtime/health.js.map +1 -0
- package/dist/runtime/introspection.d.ts +96 -0
- package/dist/runtime/introspection.d.ts.map +1 -0
- package/dist/runtime/introspection.js +31 -0
- package/dist/runtime/introspection.js.map +1 -0
- package/dist/runtime/metrics.d.ts +44 -0
- package/dist/runtime/metrics.d.ts.map +1 -0
- package/dist/runtime/metrics.js +12 -0
- package/dist/runtime/metrics.js.map +1 -0
- package/dist/runtime/model.d.ts.map +1 -1
- package/dist/runtime/model.js +40 -10
- package/dist/runtime/model.js.map +1 -1
- package/dist/runtime/provenance.d.ts +25 -0
- package/dist/runtime/provenance.d.ts.map +1 -0
- package/dist/runtime/provenance.js +13 -0
- package/dist/runtime/provenance.js.map +1 -0
- package/dist/runtime/redaction.d.ts +13 -0
- package/dist/runtime/redaction.d.ts.map +1 -0
- package/dist/runtime/redaction.js +278 -0
- package/dist/runtime/redaction.js.map +1 -0
- package/dist/runtime/sanitization.d.ts +4 -0
- package/dist/runtime/sanitization.d.ts.map +1 -0
- package/dist/runtime/sanitization.js +63 -0
- package/dist/runtime/sanitization.js.map +1 -0
- package/dist/runtime/sequential.d.ts.map +1 -1
- package/dist/runtime/sequential.js +39 -36
- package/dist/runtime/sequential.js.map +1 -1
- package/dist/runtime/shared.d.ts +1 -0
- package/dist/runtime/shared.d.ts.map +1 -1
- package/dist/runtime/shared.js +167 -101
- package/dist/runtime/shared.js.map +1 -1
- package/dist/runtime/tools/built-in.d.ts +2 -0
- package/dist/runtime/tools/built-in.d.ts.map +1 -1
- package/dist/runtime/tools/built-in.js +153 -15
- package/dist/runtime/tools/built-in.js.map +1 -1
- package/dist/runtime/tools.d.ts.map +1 -1
- package/dist/runtime/tools.js +29 -7
- package/dist/runtime/tools.js.map +1 -1
- package/dist/runtime/tracing.d.ts +31 -0
- package/dist/runtime/tracing.d.ts.map +1 -0
- package/dist/runtime/tracing.js +18 -0
- package/dist/runtime/tracing.js.map +1 -0
- package/dist/runtime/validation.d.ts.map +1 -1
- package/dist/runtime/validation.js +3 -0
- package/dist/runtime/validation.js.map +1 -1
- package/dist/types/events.d.ts +13 -7
- package/dist/types/events.d.ts.map +1 -1
- package/dist/types/replay.d.ts +5 -1
- package/dist/types/replay.d.ts.map +1 -1
- package/dist/types.d.ts +144 -1
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/package.json +46 -1
- package/src/index.ts +5 -0
- package/src/providers/openai-compatible.ts +6 -1
- package/src/runtime/audit.ts +121 -0
- package/src/runtime/broadcast.ts +195 -108
- package/src/runtime/coordinator.ts +197 -86
- package/src/runtime/defaults.ts +15 -5
- package/src/runtime/engine.ts +725 -58
- package/src/runtime/health.ts +136 -0
- package/src/runtime/introspection.ts +122 -0
- package/src/runtime/metrics.ts +45 -0
- package/src/runtime/model.ts +44 -9
- package/src/runtime/provenance.ts +43 -0
- package/src/runtime/redaction.ts +355 -0
- package/src/runtime/sanitization.ts +81 -0
- package/src/runtime/sequential.ts +40 -37
- package/src/runtime/shared.ts +191 -104
- package/src/runtime/tools/built-in.ts +168 -15
- package/src/runtime/tools.ts +39 -8
- package/src/runtime/tracing.ts +35 -0
- package/src/runtime/validation.ts +3 -0
- package/src/types/events.ts +13 -7
- package/src/types/replay.ts +5 -1
- package/src/types.ts +152 -1
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAyBA,MAAM,iBAAiB,GAAG;IACxB,uBAAuB;IACvB,SAAS;IACT,SAAS;IACT,yBAAyB;IACzB,0BAA0B;IAC1B,2BAA2B;IAC3B,oBAAoB;IACpB,uBAAuB;IACvB,kBAAkB;IAClB,sBAAsB;IACtB,sBAAsB;IACtB,gBAAgB;IAChB,SAAS;CACD,CAAC;AAqGX,MAAM,gBAAiB,SAAQ,KAAK;IACzB,IAAI,GAAG,cAAuB,CAAC;IACxC,0CAA0C;IACjC,IAAI,CAAmB;IAChC,uEAAuE;IAC9D,SAAS,CAAW;IAC7B,0EAA0E;IACjE,UAAU,CAAU;IAC7B,+CAA+C;IACtC,MAAM,CAAc;IAC7B,uEAAuE;IAC9D,KAAK,CAAW;IAEzB,YAAY,OAA4B;QACtC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;QACvB,IAAI,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC;QAEzB,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;YACpC,IAAI,CAAC,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;QACrC,CAAC;QACD,IAAI,OAAO,CAAC,UAAU,KAAK,SAAS,EAAE,CAAC;YACrC,IAAI,CAAC,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC;QACvC,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACjC,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;QAC/B,CAAC;QACD,IAAI,OAAO,CAAC,KAAK,KAAK,SAAS,EAAE,CAAC;YAChC,IAAI,CAAC,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;QAC7B,CAAC;QAED,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;IACpD,CAAC;IAED;;OAEG;IACH,MAAM,CAAC,UAAU,CAAC,KAAc;QAC9B,IAAI,KAAK,YAAY,gBAAgB,EAAE,CAAC;YACtC,OAAO,IAAI,CAAC;QACd,CAAC;QAED,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;YACrB,OAAO,KAAK,CAAC;QACf,CAAC;QAED,OAAO,KAAK,CAAC,IAAI,KAAK,cAAc,IAAI,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,OAAO,KAAK,CAAC,OAAO,KAAK,QAAQ,CAAC;IAC9G,CAAC;IAED;;OAEG;IACH,MAAM;QACJ,OAAO;YACL,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,GAAG,CAAC,IAAI,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtE,GAAG,CAAC,IAAI,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9D,CAAC;IACJ,CAAC;CACF;AAED;;GAEG;AACH,MAAM,CAAC,MAAM,YAAY,GAAG,gBAA2C,CAAC;AAExE,SAAS,kBAAkB,CAAC,KAAc;IACxC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,iBAAiB,CAAC,QAAQ,CAAC,KAAyB,CAAC,CAAC;AAC5F,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@dogpile/sdk",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.6.0",
|
|
4
4
|
"description": "TypeScript SDK for running multi-agent LLM coordination protocols.",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"type": "module",
|
|
@@ -45,6 +45,11 @@
|
|
|
45
45
|
"import": "./dist/types.js",
|
|
46
46
|
"default": "./dist/types.js"
|
|
47
47
|
},
|
|
48
|
+
"./runtime/audit": {
|
|
49
|
+
"types": "./dist/runtime/audit.d.ts",
|
|
50
|
+
"import": "./dist/runtime/audit.js",
|
|
51
|
+
"default": "./dist/runtime/audit.js"
|
|
52
|
+
},
|
|
48
53
|
"./runtime/broadcast": {
|
|
49
54
|
"types": "./dist/runtime/broadcast.d.ts",
|
|
50
55
|
"import": "./dist/runtime/broadcast.js",
|
|
@@ -65,11 +70,41 @@
|
|
|
65
70
|
"import": "./dist/runtime/engine.js",
|
|
66
71
|
"default": "./dist/runtime/engine.js"
|
|
67
72
|
},
|
|
73
|
+
"./runtime/health": {
|
|
74
|
+
"types": "./dist/runtime/health.d.ts",
|
|
75
|
+
"import": "./dist/runtime/health.js",
|
|
76
|
+
"default": "./dist/runtime/health.js"
|
|
77
|
+
},
|
|
78
|
+
"./runtime/introspection": {
|
|
79
|
+
"types": "./dist/runtime/introspection.d.ts",
|
|
80
|
+
"import": "./dist/runtime/introspection.js",
|
|
81
|
+
"default": "./dist/runtime/introspection.js"
|
|
82
|
+
},
|
|
68
83
|
"./runtime/model": {
|
|
69
84
|
"types": "./dist/runtime/model.d.ts",
|
|
70
85
|
"import": "./dist/runtime/model.js",
|
|
71
86
|
"default": "./dist/runtime/model.js"
|
|
72
87
|
},
|
|
88
|
+
"./runtime/provenance": {
|
|
89
|
+
"types": "./dist/runtime/provenance.d.ts",
|
|
90
|
+
"import": "./dist/runtime/provenance.js",
|
|
91
|
+
"default": "./dist/runtime/provenance.js"
|
|
92
|
+
},
|
|
93
|
+
"./runtime/tracing": {
|
|
94
|
+
"types": "./dist/runtime/tracing.d.ts",
|
|
95
|
+
"import": "./dist/runtime/tracing.js",
|
|
96
|
+
"default": "./dist/runtime/tracing.js"
|
|
97
|
+
},
|
|
98
|
+
"./runtime/metrics": {
|
|
99
|
+
"types": "./dist/runtime/metrics.d.ts",
|
|
100
|
+
"import": "./dist/runtime/metrics.js",
|
|
101
|
+
"default": "./dist/runtime/metrics.js"
|
|
102
|
+
},
|
|
103
|
+
"./runtime/redaction": {
|
|
104
|
+
"types": "./dist/runtime/redaction.d.ts",
|
|
105
|
+
"import": "./dist/runtime/redaction.js",
|
|
106
|
+
"default": "./dist/runtime/redaction.js"
|
|
107
|
+
},
|
|
73
108
|
"./providers/openai-compatible": {
|
|
74
109
|
"types": "./dist/providers/openai-compatible.d.ts",
|
|
75
110
|
"import": "./dist/providers/openai-compatible.js",
|
|
@@ -142,22 +177,30 @@
|
|
|
142
177
|
"src/types/replay.ts",
|
|
143
178
|
"src/browser/index.ts",
|
|
144
179
|
"src/providers/openai-compatible.ts",
|
|
180
|
+
"src/runtime/audit.ts",
|
|
145
181
|
"src/runtime/broadcast.ts",
|
|
146
182
|
"src/runtime/cancellation.ts",
|
|
147
183
|
"src/runtime/coordinator.ts",
|
|
148
184
|
"src/runtime/decisions.ts",
|
|
149
185
|
"src/runtime/defaults.ts",
|
|
150
186
|
"src/runtime/engine.ts",
|
|
187
|
+
"src/runtime/health.ts",
|
|
151
188
|
"src/runtime/ids.ts",
|
|
189
|
+
"src/runtime/introspection.ts",
|
|
152
190
|
"src/runtime/logger.ts",
|
|
191
|
+
"src/runtime/metrics.ts",
|
|
153
192
|
"src/runtime/model.ts",
|
|
193
|
+
"src/runtime/provenance.ts",
|
|
194
|
+
"src/runtime/redaction.ts",
|
|
154
195
|
"src/runtime/retry.ts",
|
|
196
|
+
"src/runtime/sanitization.ts",
|
|
155
197
|
"src/runtime/sequential.ts",
|
|
156
198
|
"src/runtime/shared.ts",
|
|
157
199
|
"src/runtime/termination.ts",
|
|
158
200
|
"src/runtime/tools.ts",
|
|
159
201
|
"src/runtime/tools/built-in.ts",
|
|
160
202
|
"src/runtime/tools/vercel-ai.ts",
|
|
203
|
+
"src/runtime/tracing.ts",
|
|
161
204
|
"src/runtime/wrap-up.ts",
|
|
162
205
|
"src/runtime/validation.ts",
|
|
163
206
|
"README.md",
|
|
@@ -184,6 +227,8 @@
|
|
|
184
227
|
"verify": "pnpm run package:identity && pnpm run build && pnpm run package:artifacts && pnpm run quickstart:smoke -- --skip-build && pnpm run typecheck && pnpm run test"
|
|
185
228
|
},
|
|
186
229
|
"devDependencies": {
|
|
230
|
+
"@opentelemetry/api": "1.9.1",
|
|
231
|
+
"@opentelemetry/sdk-trace-base": "2.7.1",
|
|
187
232
|
"@types/node": "^25.6.0",
|
|
188
233
|
"ai": "^6.0.168",
|
|
189
234
|
"typescript": "^6.0.3",
|
package/src/index.ts
CHANGED
|
@@ -12,6 +12,8 @@ export { consoleLogger, loggerFromEvents, noopLogger } from "./runtime/logger.js
|
|
|
12
12
|
export type { Logger, LoggerFromEventsOptions, LogLevel } from "./runtime/logger.js";
|
|
13
13
|
export { DEFAULT_RETRYABLE_DOGPILE_CODES, withRetry } from "./runtime/retry.js";
|
|
14
14
|
export type { RetryAttemptInfo, RetryJitterMode, RetryPolicy } from "./runtime/retry.js";
|
|
15
|
+
export { DOGPILE_SPAN_NAMES } from "./runtime/tracing.js";
|
|
16
|
+
export type { DogpileSpan, DogpileSpanOptions, DogpileTracer } from "./runtime/tracing.js";
|
|
15
17
|
export { DogpileError } from "./types.js";
|
|
16
18
|
export type {
|
|
17
19
|
OpenAICompatibleChatCompletionChoice,
|
|
@@ -79,6 +81,7 @@ export type {
|
|
|
79
81
|
AgentSpec,
|
|
80
82
|
AgentDecision,
|
|
81
83
|
AgentParticipation,
|
|
84
|
+
AnomalyCode,
|
|
82
85
|
BroadcastContribution,
|
|
83
86
|
BroadcastEvent,
|
|
84
87
|
BroadcastProtocolConfig,
|
|
@@ -102,6 +105,7 @@ export type {
|
|
|
102
105
|
EngineOptions,
|
|
103
106
|
RunCallOptions,
|
|
104
107
|
FinalEvent,
|
|
108
|
+
HealthAnomaly,
|
|
105
109
|
FirstOfTerminationCondition,
|
|
106
110
|
FirstOfTerminationConditions,
|
|
107
111
|
FirstOfTerminationInput,
|
|
@@ -150,6 +154,7 @@ export type {
|
|
|
150
154
|
RunEvaluator,
|
|
151
155
|
RunEventLog,
|
|
152
156
|
RunEvent,
|
|
157
|
+
RunHealthSummary,
|
|
153
158
|
RunMetadata,
|
|
154
159
|
RunResult,
|
|
155
160
|
RunUsage,
|
|
@@ -9,6 +9,7 @@ import type {
|
|
|
9
9
|
ModelRequest,
|
|
10
10
|
ModelResponse
|
|
11
11
|
} from "../types.js";
|
|
12
|
+
import { sanitizeProviderJsonValue, sanitizeProviderResponseHeaders } from "../runtime/sanitization.js";
|
|
12
13
|
|
|
13
14
|
const defaultBaseURL = "https://api.openai.com/v1";
|
|
14
15
|
const defaultPath = "/chat/completions";
|
|
@@ -91,6 +92,7 @@ export function createOpenAICompatibleProvider(options: OpenAICompatibleProvider
|
|
|
91
92
|
|
|
92
93
|
return {
|
|
93
94
|
id: providerId,
|
|
95
|
+
modelId: options.model,
|
|
94
96
|
metadata: { locality: resolvedLocality },
|
|
95
97
|
async generate(request: ModelRequest): Promise<ModelResponse> {
|
|
96
98
|
let response: Response;
|
|
@@ -419,6 +421,8 @@ function responseMetadata(response: OpenAICompatibleChatCompletionResponse): Jso
|
|
|
419
421
|
function createProviderError(response: Response, payload: unknown, providerId: string): DogpileError {
|
|
420
422
|
const code = codeForStatus(response.status);
|
|
421
423
|
const timeoutSource = code === "provider-timeout" ? { source: "provider" as const } : {};
|
|
424
|
+
const responseHeaders = sanitizeProviderResponseHeaders(response.headers);
|
|
425
|
+
const sanitizedPayload = sanitizeProviderJsonValue(payload);
|
|
422
426
|
return new DogpileError({
|
|
423
427
|
code,
|
|
424
428
|
message: providerResponseErrorMessage(response, payload),
|
|
@@ -428,7 +432,8 @@ function createProviderError(response: Response, payload: unknown, providerId: s
|
|
|
428
432
|
statusCode: response.status,
|
|
429
433
|
statusText: response.statusText,
|
|
430
434
|
...timeoutSource,
|
|
431
|
-
|
|
435
|
+
responseHeaders,
|
|
436
|
+
response: sanitizedPayload
|
|
432
437
|
})
|
|
433
438
|
});
|
|
434
439
|
}
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
import type { Protocol, Tier, Trace } from "../types.js";
|
|
2
|
+
import type { BudgetStopEvent, FinalEvent, SubRunCompletedEvent, TurnEvent } from "../types/events.js";
|
|
3
|
+
|
|
4
|
+
export type AuditOutcomeStatus = "completed" | "budget-stopped" | "aborted";
|
|
5
|
+
|
|
6
|
+
export interface AuditOutcome {
|
|
7
|
+
readonly status: AuditOutcomeStatus;
|
|
8
|
+
readonly terminationCode?: string;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export interface AuditCost {
|
|
12
|
+
readonly usd: number;
|
|
13
|
+
readonly inputTokens: number;
|
|
14
|
+
readonly outputTokens: number;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
export interface AuditAgentRecord {
|
|
18
|
+
readonly id: string;
|
|
19
|
+
readonly role: string;
|
|
20
|
+
readonly turnCount: number;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
export interface AuditRecord {
|
|
24
|
+
readonly auditSchemaVersion: "1";
|
|
25
|
+
readonly runId: string;
|
|
26
|
+
readonly intent: string;
|
|
27
|
+
readonly startedAt: string;
|
|
28
|
+
readonly completedAt: string;
|
|
29
|
+
readonly protocol: Protocol;
|
|
30
|
+
readonly tier: Tier;
|
|
31
|
+
readonly modelProviderId: string;
|
|
32
|
+
readonly agentCount: number;
|
|
33
|
+
readonly turnCount: number;
|
|
34
|
+
readonly outcome: AuditOutcome;
|
|
35
|
+
readonly cost: AuditCost;
|
|
36
|
+
readonly agents: readonly AuditAgentRecord[];
|
|
37
|
+
readonly childRunIds?: readonly string[];
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Derive a versioned, schema-stable audit record from a completed run trace.
|
|
42
|
+
*
|
|
43
|
+
* Pure function - no side effects, no I/O, no storage access. Deterministic:
|
|
44
|
+
* given the same trace, always produces the same AuditRecord.
|
|
45
|
+
*
|
|
46
|
+
* @param trace - Completed run trace (from RunResult.trace or a stored/replayed trace).
|
|
47
|
+
*/
|
|
48
|
+
export function createAuditRecord(trace: Trace): AuditRecord {
|
|
49
|
+
const finalEvent = trace.events.find((event): event is FinalEvent => event.type === "final");
|
|
50
|
+
const budgetStopEvent = trace.events.find((event): event is BudgetStopEvent => event.type === "budget-stop");
|
|
51
|
+
|
|
52
|
+
const outcome: AuditOutcome = budgetStopEvent
|
|
53
|
+
? { status: "budget-stopped", terminationCode: budgetStopEvent.reason }
|
|
54
|
+
: finalEvent
|
|
55
|
+
? { status: "completed" }
|
|
56
|
+
: { status: "aborted" };
|
|
57
|
+
|
|
58
|
+
const lastTurnCost = [...trace.events]
|
|
59
|
+
.reverse()
|
|
60
|
+
.find((event): event is TurnEvent => event.type === "agent-turn")?.cost;
|
|
61
|
+
const costSource = finalEvent?.cost ?? budgetStopEvent?.cost ?? lastTurnCost;
|
|
62
|
+
const cost: AuditCost = {
|
|
63
|
+
usd: costSource?.usd ?? 0,
|
|
64
|
+
inputTokens: costSource?.inputTokens ?? 0,
|
|
65
|
+
outputTokens: costSource?.outputTokens ?? 0
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
const turnEvents = trace.events.filter((event): event is TurnEvent => event.type === "agent-turn");
|
|
69
|
+
const agentTurnMap = new Map<string, { role: string; count: number }>();
|
|
70
|
+
for (const event of turnEvents) {
|
|
71
|
+
const existing = agentTurnMap.get(event.agentId);
|
|
72
|
+
if (existing !== undefined) {
|
|
73
|
+
existing.count++;
|
|
74
|
+
} else {
|
|
75
|
+
agentTurnMap.set(event.agentId, { role: event.role, count: 1 });
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const agents: AuditAgentRecord[] = [...agentTurnMap.entries()]
|
|
80
|
+
.map(([id, { role, count }]) => ({ id, role, turnCount: count }))
|
|
81
|
+
.sort((a, b) => a.id.localeCompare(b.id));
|
|
82
|
+
|
|
83
|
+
const childRunIds = trace.events
|
|
84
|
+
.filter((event): event is SubRunCompletedEvent => event.type === "sub-run-completed")
|
|
85
|
+
.map((event) => event.childRunId);
|
|
86
|
+
|
|
87
|
+
const startedAt = eventStartedAt(trace.events[0]);
|
|
88
|
+
|
|
89
|
+
return {
|
|
90
|
+
auditSchemaVersion: "1",
|
|
91
|
+
runId: trace.runId,
|
|
92
|
+
intent: trace.inputs.intent,
|
|
93
|
+
startedAt,
|
|
94
|
+
completedAt: trace.finalOutput.completedAt,
|
|
95
|
+
protocol: trace.protocol,
|
|
96
|
+
tier: trace.tier,
|
|
97
|
+
modelProviderId: trace.modelProviderId,
|
|
98
|
+
agentCount: agentTurnMap.size,
|
|
99
|
+
turnCount: turnEvents.length,
|
|
100
|
+
outcome,
|
|
101
|
+
cost,
|
|
102
|
+
agents,
|
|
103
|
+
...(childRunIds.length > 0 ? { childRunIds } : {})
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function eventStartedAt(event: Trace["events"][number] | undefined): string {
|
|
108
|
+
if (event === undefined) {
|
|
109
|
+
return "";
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
if ("at" in event) {
|
|
113
|
+
return event.at;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if ("startedAt" in event) {
|
|
117
|
+
return event.startedAt;
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
return "";
|
|
121
|
+
}
|
package/src/runtime/broadcast.ts
CHANGED
|
@@ -17,6 +17,7 @@ import type {
|
|
|
17
17
|
TerminationCondition,
|
|
18
18
|
TerminationStopRecord,
|
|
19
19
|
Tier,
|
|
20
|
+
Trace,
|
|
20
21
|
TranscriptEntry
|
|
21
22
|
} from "../types.js";
|
|
22
23
|
import { createRunId, elapsedMs, nowMs, providerCallIdFor } from "./ids.js";
|
|
@@ -35,6 +36,7 @@ import {
|
|
|
35
36
|
createTranscriptLink,
|
|
36
37
|
emptyCost
|
|
37
38
|
} from "./defaults.js";
|
|
39
|
+
import { computeHealth, DEFAULT_HEALTH_THRESHOLDS } from "./health.js";
|
|
38
40
|
import { throwIfAborted } from "./cancellation.js";
|
|
39
41
|
import { parseAgentDecision } from "./decisions.js";
|
|
40
42
|
import { generateModelTurn } from "./model.js";
|
|
@@ -55,9 +57,12 @@ interface BroadcastRunOptions {
|
|
|
55
57
|
readonly signal?: AbortSignal;
|
|
56
58
|
readonly terminate?: TerminationCondition;
|
|
57
59
|
readonly wrapUpHint?: DogpileOptions["wrapUpHint"];
|
|
60
|
+
readonly maxConcurrentAgentTurns?: number;
|
|
58
61
|
readonly emit?: (event: RunEvent) => void;
|
|
59
62
|
}
|
|
60
63
|
|
|
64
|
+
const DEFAULT_MAX_CONCURRENT_AGENT_TURNS = 4;
|
|
65
|
+
|
|
61
66
|
export async function runBroadcast(options: BroadcastRunOptions): Promise<RunResult> {
|
|
62
67
|
const runId = createRunId();
|
|
63
68
|
const events: RunEvent[] = [];
|
|
@@ -129,81 +134,92 @@ export async function runBroadcast(options: BroadcastRunOptions): Promise<RunRes
|
|
|
129
134
|
}
|
|
130
135
|
|
|
131
136
|
const providerCallSlots: ReplayTraceProviderCall[] = [];
|
|
132
|
-
const
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
137
|
+
const fanout = createFanoutAbortController(options.signal);
|
|
138
|
+
const turnResults = await (async () => {
|
|
139
|
+
try {
|
|
140
|
+
return await mapWithConcurrency(
|
|
141
|
+
options.agents,
|
|
142
|
+
options.maxConcurrentAgentTurns ?? DEFAULT_MAX_CONCURRENT_AGENT_TURNS,
|
|
143
|
+
fanout,
|
|
144
|
+
async (agent, agentIndex) => {
|
|
145
|
+
throwIfAborted(fanout.signal, options.model.id);
|
|
146
|
+
const turn = transcript.length + agentIndex + 1;
|
|
147
|
+
const input = buildBroadcastInput(options.intent, round, maxRounds, firstRoundContributions);
|
|
148
|
+
const request: ModelRequest = {
|
|
149
|
+
temperature: options.temperature,
|
|
150
|
+
signal: fanout.signal,
|
|
151
|
+
metadata: {
|
|
152
|
+
runId,
|
|
153
|
+
protocol: "broadcast",
|
|
154
|
+
agentId: agent.id,
|
|
155
|
+
role: agent.role,
|
|
156
|
+
tier: options.tier,
|
|
157
|
+
round,
|
|
158
|
+
...toolAvailability
|
|
153
159
|
},
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
+
messages: wrapUpHint.inject(
|
|
161
|
+
[
|
|
162
|
+
{
|
|
163
|
+
role: "system",
|
|
164
|
+
content: buildSystemPrompt(agent)
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
role: "user",
|
|
168
|
+
content: input
|
|
169
|
+
}
|
|
170
|
+
],
|
|
171
|
+
{
|
|
172
|
+
runId,
|
|
173
|
+
protocol: "broadcast",
|
|
174
|
+
cost: totalCost,
|
|
175
|
+
events,
|
|
176
|
+
transcript,
|
|
177
|
+
iteration: transcript.length,
|
|
178
|
+
elapsedMs: elapsedMs(startedAtMs)
|
|
179
|
+
}
|
|
180
|
+
)
|
|
181
|
+
};
|
|
182
|
+
const response = await generateModelTurn({
|
|
183
|
+
model: options.model,
|
|
184
|
+
request,
|
|
160
185
|
runId,
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
186
|
+
agent,
|
|
187
|
+
input,
|
|
188
|
+
emit,
|
|
189
|
+
callId: providerCallIdFor(runId, providerCalls.length + agentIndex + 1),
|
|
190
|
+
onProviderCall(call): void {
|
|
191
|
+
providerCallSlots[agentIndex] = call;
|
|
192
|
+
}
|
|
193
|
+
});
|
|
194
|
+
const decision = parseAgentDecision(response.text);
|
|
195
|
+
const toolCalls = await executeModelResponseToolRequests({
|
|
196
|
+
response,
|
|
197
|
+
executor: toolExecutor,
|
|
198
|
+
agentId: agent.id,
|
|
199
|
+
role: agent.role,
|
|
200
|
+
turn,
|
|
201
|
+
metadata: {
|
|
202
|
+
round
|
|
203
|
+
}
|
|
204
|
+
});
|
|
205
|
+
throwIfAborted(fanout.signal, options.model.id);
|
|
206
|
+
|
|
207
|
+
return {
|
|
208
|
+
agent,
|
|
209
|
+
agentIndex,
|
|
210
|
+
turn,
|
|
211
|
+
input,
|
|
212
|
+
response,
|
|
213
|
+
decision,
|
|
214
|
+
toolCalls,
|
|
215
|
+
turnCost: responseCost(response)
|
|
216
|
+
};
|
|
191
217
|
}
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
agentIndex,
|
|
198
|
-
turn,
|
|
199
|
-
input,
|
|
200
|
-
response,
|
|
201
|
-
decision,
|
|
202
|
-
toolCalls,
|
|
203
|
-
turnCost: responseCost(response)
|
|
204
|
-
};
|
|
205
|
-
})
|
|
206
|
-
);
|
|
218
|
+
);
|
|
219
|
+
} finally {
|
|
220
|
+
fanout.cleanup();
|
|
221
|
+
}
|
|
222
|
+
})();
|
|
207
223
|
providerCalls.push(...providerCallSlots.filter((call): call is ReplayTraceProviderCall => call !== undefined));
|
|
208
224
|
|
|
209
225
|
const contributions: BroadcastContribution[] = [];
|
|
@@ -289,45 +305,46 @@ export async function runBroadcast(options: BroadcastRunOptions): Promise<RunRes
|
|
|
289
305
|
transcriptEntryCount: transcript.length
|
|
290
306
|
});
|
|
291
307
|
const finalEvent = events.at(-1);
|
|
308
|
+
const trace: Trace = {
|
|
309
|
+
schemaVersion: "1.0",
|
|
310
|
+
runId,
|
|
311
|
+
protocol: "broadcast",
|
|
312
|
+
tier: options.tier,
|
|
313
|
+
modelProviderId: options.model.id,
|
|
314
|
+
agentsUsed: options.agents,
|
|
315
|
+
inputs: createReplayTraceRunInputs({
|
|
316
|
+
intent: options.intent,
|
|
317
|
+
protocol: options.protocol,
|
|
318
|
+
tier: options.tier,
|
|
319
|
+
modelProviderId: options.model.id,
|
|
320
|
+
agents: options.agents,
|
|
321
|
+
temperature: options.temperature
|
|
322
|
+
}),
|
|
323
|
+
budget: createReplayTraceBudget({
|
|
324
|
+
tier: options.tier,
|
|
325
|
+
...(options.budget ? { caps: options.budget } : {}),
|
|
326
|
+
...(options.terminate ? { termination: options.terminate } : {})
|
|
327
|
+
}),
|
|
328
|
+
budgetStateChanges: createReplayTraceBudgetStateChanges(events),
|
|
329
|
+
seed: createReplayTraceSeed(options.seed),
|
|
330
|
+
protocolDecisions,
|
|
331
|
+
providerCalls,
|
|
332
|
+
finalOutput: createReplayTraceFinalOutput(output, finalEvent ?? {
|
|
333
|
+
type: "final",
|
|
334
|
+
runId,
|
|
335
|
+
at: "",
|
|
336
|
+
output,
|
|
337
|
+
cost: totalCost,
|
|
338
|
+
transcript: createTranscriptLink(transcript)
|
|
339
|
+
}),
|
|
340
|
+
events,
|
|
341
|
+
transcript
|
|
342
|
+
};
|
|
292
343
|
|
|
293
344
|
return {
|
|
294
345
|
output,
|
|
295
346
|
eventLog: createRunEventLog(runId, "broadcast", events),
|
|
296
|
-
trace
|
|
297
|
-
schemaVersion: "1.0",
|
|
298
|
-
runId,
|
|
299
|
-
protocol: "broadcast",
|
|
300
|
-
tier: options.tier,
|
|
301
|
-
modelProviderId: options.model.id,
|
|
302
|
-
agentsUsed: options.agents,
|
|
303
|
-
inputs: createReplayTraceRunInputs({
|
|
304
|
-
intent: options.intent,
|
|
305
|
-
protocol: options.protocol,
|
|
306
|
-
tier: options.tier,
|
|
307
|
-
modelProviderId: options.model.id,
|
|
308
|
-
agents: options.agents,
|
|
309
|
-
temperature: options.temperature
|
|
310
|
-
}),
|
|
311
|
-
budget: createReplayTraceBudget({
|
|
312
|
-
tier: options.tier,
|
|
313
|
-
...(options.budget ? { caps: options.budget } : {}),
|
|
314
|
-
...(options.terminate ? { termination: options.terminate } : {})
|
|
315
|
-
}),
|
|
316
|
-
budgetStateChanges: createReplayTraceBudgetStateChanges(events),
|
|
317
|
-
seed: createReplayTraceSeed(options.seed),
|
|
318
|
-
protocolDecisions,
|
|
319
|
-
providerCalls,
|
|
320
|
-
finalOutput: createReplayTraceFinalOutput(output, finalEvent ?? {
|
|
321
|
-
type: "final",
|
|
322
|
-
runId,
|
|
323
|
-
at: "",
|
|
324
|
-
output,
|
|
325
|
-
cost: totalCost,
|
|
326
|
-
transcript: createTranscriptLink(transcript)
|
|
327
|
-
}),
|
|
328
|
-
events,
|
|
329
|
-
transcript
|
|
330
|
-
},
|
|
347
|
+
trace,
|
|
331
348
|
transcript,
|
|
332
349
|
usage: createRunUsage(totalCost),
|
|
333
350
|
metadata: createRunMetadata({
|
|
@@ -345,7 +362,8 @@ export async function runBroadcast(options: BroadcastRunOptions): Promise<RunRes
|
|
|
345
362
|
cost: totalCost,
|
|
346
363
|
events
|
|
347
364
|
}),
|
|
348
|
-
cost: totalCost
|
|
365
|
+
cost: totalCost,
|
|
366
|
+
health: computeHealth(trace, DEFAULT_HEALTH_THRESHOLDS)
|
|
349
367
|
};
|
|
350
368
|
|
|
351
369
|
function stopIfNeeded(): boolean {
|
|
@@ -441,3 +459,72 @@ function responseCost(response: ModelResponse): CostSummary {
|
|
|
441
459
|
};
|
|
442
460
|
}
|
|
443
461
|
|
|
462
|
+
interface FanoutAbortController {
|
|
463
|
+
readonly signal: AbortSignal;
|
|
464
|
+
abort(reason: unknown): void;
|
|
465
|
+
cleanup(): void;
|
|
466
|
+
}
|
|
467
|
+
|
|
468
|
+
function createFanoutAbortController(parentSignal: AbortSignal | undefined): FanoutAbortController {
|
|
469
|
+
const controller = new AbortController();
|
|
470
|
+
let removeParentListener = (): void => {};
|
|
471
|
+
|
|
472
|
+
if (parentSignal?.aborted) {
|
|
473
|
+
controller.abort(parentSignal.reason);
|
|
474
|
+
} else if (parentSignal !== undefined) {
|
|
475
|
+
const abortFromParent = (): void => {
|
|
476
|
+
controller.abort(parentSignal.reason);
|
|
477
|
+
};
|
|
478
|
+
parentSignal.addEventListener("abort", abortFromParent, { once: true });
|
|
479
|
+
removeParentListener = (): void => {
|
|
480
|
+
parentSignal.removeEventListener("abort", abortFromParent);
|
|
481
|
+
};
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
return {
|
|
485
|
+
signal: controller.signal,
|
|
486
|
+
abort(reason: unknown): void {
|
|
487
|
+
if (!controller.signal.aborted) {
|
|
488
|
+
controller.abort(reason);
|
|
489
|
+
}
|
|
490
|
+
},
|
|
491
|
+
cleanup(): void {
|
|
492
|
+
removeParentListener();
|
|
493
|
+
}
|
|
494
|
+
};
|
|
495
|
+
}
|
|
496
|
+
|
|
497
|
+
async function mapWithConcurrency<T, R>(
|
|
498
|
+
items: readonly T[],
|
|
499
|
+
maxConcurrent: number,
|
|
500
|
+
fanout: FanoutAbortController,
|
|
501
|
+
mapper: (item: T, index: number) => Promise<R>
|
|
502
|
+
): Promise<R[]> {
|
|
503
|
+
if (items.length === 0) {
|
|
504
|
+
return [];
|
|
505
|
+
}
|
|
506
|
+
|
|
507
|
+
const results: R[] = new Array(items.length);
|
|
508
|
+
let nextIndex = 0;
|
|
509
|
+
let firstError: unknown;
|
|
510
|
+
const workerCount = Math.min(maxConcurrent, items.length);
|
|
511
|
+
|
|
512
|
+
await Promise.all(Array.from({ length: workerCount }, async () => {
|
|
513
|
+
while (nextIndex < items.length && firstError === undefined) {
|
|
514
|
+
const index = nextIndex;
|
|
515
|
+
nextIndex += 1;
|
|
516
|
+
try {
|
|
517
|
+
results[index] = await mapper(items[index]!, index);
|
|
518
|
+
} catch (error) {
|
|
519
|
+
firstError ??= error;
|
|
520
|
+
fanout.abort(error);
|
|
521
|
+
}
|
|
522
|
+
}
|
|
523
|
+
}));
|
|
524
|
+
|
|
525
|
+
if (firstError !== undefined) {
|
|
526
|
+
throw firstError;
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
return results;
|
|
530
|
+
}
|