@purista/harness 1.2.6 → 1.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -0
- package/dist/agents/index.d.ts +7 -1
- package/dist/agents/index.js +126 -44
- package/dist/errors/catalog.d.ts +18 -2
- package/dist/errors/catalog.js +10 -0
- package/dist/eval/index.d.ts +3 -3
- package/dist/eval/index.js +15 -1
- package/dist/harness/defineHarness.d.ts +149 -3
- package/dist/harness/defineHarness.js +110 -1
- package/dist/index.d.ts +38 -18
- package/dist/index.js +30 -16
- package/dist/local/index.d.ts +36 -0
- package/dist/local/index.js +24 -0
- package/dist/local/local-sandbox.d.ts +25 -0
- package/dist/local/local-sandbox.js +368 -0
- package/dist/local/local-workspace.d.ts +56 -0
- package/dist/local/local-workspace.js +496 -0
- package/dist/local/ref-hash.d.ts +6 -0
- package/dist/local/ref-hash.js +9 -0
- package/dist/local/sqlite-storage.d.ts +106 -0
- package/dist/local/sqlite-storage.js +680 -0
- package/dist/models/adapter-utils.d.ts +52 -0
- package/dist/models/adapter-utils.js +81 -0
- package/dist/models/registry.js +28 -37
- package/dist/models/stream-pump.d.ts +16 -0
- package/dist/models/stream-pump.js +77 -0
- package/dist/ports/base-model-provider.d.ts +7 -1
- package/dist/ports/base-model-provider.js +384 -87
- package/dist/ports/capabilities.d.ts +16 -2
- package/dist/ports/context-checkpoints.d.ts +63 -0
- package/dist/ports/context-checkpoints.js +33 -0
- package/dist/ports/index.d.ts +1 -0
- package/dist/ports/index.js +1 -0
- package/dist/ports/model-provider.d.ts +94 -0
- package/dist/runtime/durable.d.ts +11 -0
- package/dist/runtime/durable.js +15 -2
- package/dist/runtime/sessionDurable.js +47 -21
- package/dist/runtime/steps.d.ts +22 -1
- package/dist/runtime/steps.js +53 -2
- package/dist/sessions/index.d.ts +17 -6
- package/dist/sessions/index.js +345 -84
- package/dist/skills/index.d.ts +0 -2
- package/dist/skills/index.js +0 -8
- package/dist/state/in-memory.js +6 -6
- package/dist/telemetry/shim.js +2 -6
- package/dist/telemetry/span-attrs.d.ts +9 -0
- package/dist/telemetry/span-attrs.js +27 -0
- package/dist/testing/durableWorkspaceStoreContract.js +69 -0
- package/dist/testing/fakeLogger.d.ts +29 -0
- package/dist/testing/fakeLogger.js +47 -0
- package/dist/testing/fakeSandbox.d.ts +27 -0
- package/dist/testing/fakeSandbox.js +153 -0
- package/dist/testing/fakeStateStore.d.ts +36 -0
- package/dist/testing/fakeStateStore.js +66 -0
- package/dist/testing/index.d.ts +10 -4
- package/dist/testing/index.js +14 -4
- package/dist/testing/loggerContract.d.ts +9 -0
- package/dist/testing/loggerContract.js +62 -0
- package/dist/testing/modelProviderContract.d.ts +12 -0
- package/dist/testing/modelProviderContract.js +222 -0
- package/dist/testing/recordEvents.d.ts +3 -0
- package/dist/testing/recordEvents.js +8 -0
- package/dist/testing/stateStoreContract.js +27 -0
- package/dist/tools/index.js +26 -1
- package/dist/tools/mcp/http.d.ts +2 -0
- package/dist/tools/mcp/http.js +34 -21
- package/dist/tools/mcp/runner.d.ts +4 -0
- package/dist/tools/mcp/runner.js +75 -21
- package/dist/tools/mcp/stdio.d.ts +7 -1
- package/dist/tools/mcp/stdio.js +102 -23
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/workspace/in-memory.d.ts +1 -0
- package/dist/workspace/in-memory.js +47 -12
- package/package.json +5 -4
|
@@ -1,4 +1,21 @@
|
|
|
1
1
|
import { ModelCapabilityError, ModelError, OperationCancelledError, OperationTimeoutError, HarnessError, sanitizeForLog, sanitizeProviderBody, sanitizeProviderMessage } from '../errors/index.js';
|
|
2
|
+
import { redactProviderContent } from '../models/adapter-utils.js';
|
|
3
|
+
import { pumpStreamThroughSpan } from '../models/stream-pump.js';
|
|
4
|
+
const DEFAULT_RETRY_POLICY = {
|
|
5
|
+
maxAttempts: 3,
|
|
6
|
+
maxActiveElapsedMs: 60_000,
|
|
7
|
+
maxActiveDelayMs: 20_000,
|
|
8
|
+
respectRetryAfter: true,
|
|
9
|
+
minDelayMs: 500,
|
|
10
|
+
maxDelayMs: 8_000,
|
|
11
|
+
longRetry: 'error',
|
|
12
|
+
retryOn: {
|
|
13
|
+
network: true,
|
|
14
|
+
timeout: true,
|
|
15
|
+
rateLimit: true,
|
|
16
|
+
serverError: true
|
|
17
|
+
}
|
|
18
|
+
};
|
|
2
19
|
/**
|
|
3
20
|
* Base class for model adapters.
|
|
4
21
|
*
|
|
@@ -71,8 +88,17 @@ export class BaseModelProvider {
|
|
|
71
88
|
return this.logger;
|
|
72
89
|
}
|
|
73
90
|
normalizeError(error, method, req) {
|
|
91
|
+
if (error instanceof ModelError)
|
|
92
|
+
return withRedactedProviderBody(error);
|
|
74
93
|
if (error instanceof HarnessError)
|
|
75
94
|
return error;
|
|
95
|
+
// A base-enforced timeout aborts the request signal with the timeout error
|
|
96
|
+
// as reason. Provider SDKs surface that as a generic abort error, so the
|
|
97
|
+
// timeout classification must win over the cancellation classification to
|
|
98
|
+
// keep stream timeouts retry-eligible per spec 23.
|
|
99
|
+
if (req.signal.reason instanceof OperationTimeoutError) {
|
|
100
|
+
return req.signal.reason;
|
|
101
|
+
}
|
|
76
102
|
if (req.signal.aborted || isAbortError(error)) {
|
|
77
103
|
return new OperationCancelledError('Model call was cancelled.', { scope: 'model' }, error);
|
|
78
104
|
}
|
|
@@ -81,9 +107,13 @@ export class BaseModelProvider {
|
|
|
81
107
|
const code = details.providerCode;
|
|
82
108
|
const reason = code === 'context_length_exceeded'
|
|
83
109
|
? 'context_length_exceeded'
|
|
84
|
-
: status
|
|
85
|
-
? '
|
|
86
|
-
: '
|
|
110
|
+
: status === 429
|
|
111
|
+
? 'rate_limited'
|
|
112
|
+
: typeof status === 'number' && status >= 500
|
|
113
|
+
? 'provider_unavailable'
|
|
114
|
+
: status !== undefined
|
|
115
|
+
? 'http_error'
|
|
116
|
+
: 'network';
|
|
87
117
|
return new ModelError(modelErrorMessage(details), {
|
|
88
118
|
provider: this.id,
|
|
89
119
|
model: req.model,
|
|
@@ -98,28 +128,51 @@ export class BaseModelProvider {
|
|
|
98
128
|
const attrs = this.attrs(method, req);
|
|
99
129
|
const started = Date.now();
|
|
100
130
|
const execute = async (span) => {
|
|
101
|
-
const
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
const
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
131
|
+
const retry = resolveRetryPolicy(req);
|
|
132
|
+
let attempt = 1;
|
|
133
|
+
while (true) {
|
|
134
|
+
const next = this.withTimeout(req, method);
|
|
135
|
+
try {
|
|
136
|
+
const operation = fn(next.req);
|
|
137
|
+
const result = await (next.timeoutPromise ? Promise.race([operation, next.timeoutPromise]) : operation);
|
|
138
|
+
this.telemetry?.recordHistogram('harness.model.duration', (Date.now() - started) / 1000, attrs);
|
|
139
|
+
this.recordUsage(method, req.model, result);
|
|
140
|
+
return result;
|
|
141
|
+
}
|
|
142
|
+
catch (error) {
|
|
143
|
+
const normalized = this.normalizeError(error, method, next.req);
|
|
144
|
+
const decision = retryDecision(normalized, retry, attempt, started);
|
|
145
|
+
if (decision.action === 'retry') {
|
|
146
|
+
this.telemetry?.recordCounter('harness.model.retries', 1, { ...attrs, 'harness.model.retry.reason': decision.reason });
|
|
147
|
+
this.telemetry?.recordHistogram('harness.model.retry.delay', decision.delayMs / 1000, attrs);
|
|
148
|
+
this.logger?.warn('Retrying model provider call.', {
|
|
149
|
+
provider: this.id,
|
|
150
|
+
model: req.model,
|
|
151
|
+
method,
|
|
152
|
+
attempt,
|
|
153
|
+
nextAttempt: attempt + 1,
|
|
154
|
+
delayMs: decision.delayMs,
|
|
155
|
+
reason: decision.reason
|
|
156
|
+
});
|
|
157
|
+
next.cleanup();
|
|
158
|
+
await sleep(decision.delayMs, req.signal);
|
|
159
|
+
attempt += 1;
|
|
160
|
+
continue;
|
|
161
|
+
}
|
|
162
|
+
const finalError = decorateRetryMeta(normalized, decision.retryKind, attempt, retry.maxAttempts, decision.delayMs);
|
|
163
|
+
span?.setAttributes?.(modelErrorTelemetryAttrs(finalError));
|
|
164
|
+
this.telemetry?.recordCounter('harness.model.errors', 1, { ...attrs, 'error.code': finalError.code });
|
|
165
|
+
this.logger?.error('Model provider call failed.', {
|
|
166
|
+
provider: this.id,
|
|
167
|
+
model: req.model,
|
|
168
|
+
method,
|
|
169
|
+
error: sanitizeForLog({ code: finalError.code, category: finalError.category, retriable: finalError.retriable, meta: finalError.meta })
|
|
170
|
+
});
|
|
171
|
+
throw finalError;
|
|
172
|
+
}
|
|
173
|
+
finally {
|
|
174
|
+
next.cleanup();
|
|
175
|
+
}
|
|
123
176
|
}
|
|
124
177
|
};
|
|
125
178
|
return this.telemetry ? this.telemetry.span(`harness.model.${method}`, attrs, execute) : execute();
|
|
@@ -129,33 +182,59 @@ export class BaseModelProvider {
|
|
|
129
182
|
const attrs = this.attrs(method, req);
|
|
130
183
|
const started = Date.now();
|
|
131
184
|
const iterate = async function* (span) {
|
|
132
|
-
const
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
185
|
+
const retry = resolveRetryPolicy(req);
|
|
186
|
+
let attempt = 1;
|
|
187
|
+
let emitted = false;
|
|
188
|
+
while (true) {
|
|
189
|
+
const next = this.withTimeout(req, method);
|
|
190
|
+
try {
|
|
191
|
+
for await (const chunk of fn(next.req)) {
|
|
192
|
+
next.req.signal.throwIfAborted();
|
|
193
|
+
emitted = true;
|
|
194
|
+
yield chunk;
|
|
195
|
+
}
|
|
196
|
+
this.telemetry?.recordHistogram('harness.model.duration', (Date.now() - started) / 1000, attrs);
|
|
197
|
+
return;
|
|
198
|
+
}
|
|
199
|
+
catch (error) {
|
|
200
|
+
const normalized = this.normalizeError(error, method, next.req);
|
|
201
|
+
const decision = emitted ? { action: 'fail', retryKind: 'none' } : retryDecision(normalized, retry, attempt, started);
|
|
202
|
+
if (decision.action === 'retry') {
|
|
203
|
+
this.telemetry?.recordCounter('harness.model.retries', 1, { ...attrs, 'harness.model.retry.reason': decision.reason });
|
|
204
|
+
this.telemetry?.recordHistogram('harness.model.retry.delay', decision.delayMs / 1000, attrs);
|
|
205
|
+
this.logger?.warn('Retrying model provider stream before first chunk.', {
|
|
206
|
+
provider: this.id,
|
|
207
|
+
model: req.model,
|
|
208
|
+
method,
|
|
209
|
+
attempt,
|
|
210
|
+
nextAttempt: attempt + 1,
|
|
211
|
+
delayMs: decision.delayMs,
|
|
212
|
+
reason: decision.reason
|
|
213
|
+
});
|
|
214
|
+
next.cleanup();
|
|
215
|
+
await sleep(decision.delayMs, req.signal);
|
|
216
|
+
attempt += 1;
|
|
217
|
+
continue;
|
|
218
|
+
}
|
|
219
|
+
const finalError = decorateRetryMeta(normalized, decision.retryKind, attempt, retry.maxAttempts, 'delayMs' in decision ? decision.delayMs : undefined);
|
|
220
|
+
span?.setAttributes?.(modelErrorTelemetryAttrs(finalError));
|
|
221
|
+
this.telemetry?.recordCounter('harness.model.errors', 1, { ...attrs, 'error.code': finalError.code });
|
|
222
|
+
this.logger?.error('Model provider stream failed.', {
|
|
223
|
+
provider: this.id,
|
|
224
|
+
model: req.model,
|
|
225
|
+
method,
|
|
226
|
+
error: sanitizeForLog({ code: finalError.code, category: finalError.category, retriable: finalError.retriable, meta: finalError.meta })
|
|
227
|
+
});
|
|
228
|
+
throw finalError;
|
|
229
|
+
}
|
|
230
|
+
finally {
|
|
231
|
+
next.cleanup();
|
|
137
232
|
}
|
|
138
|
-
this.telemetry?.recordHistogram('harness.model.duration', (Date.now() - started) / 1000, attrs);
|
|
139
|
-
}
|
|
140
|
-
catch (error) {
|
|
141
|
-
const normalized = this.normalizeError(error, method, next.req);
|
|
142
|
-
span?.setAttributes?.(modelErrorTelemetryAttrs(normalized));
|
|
143
|
-
this.telemetry?.recordCounter('harness.model.errors', 1, { ...attrs, 'error.code': normalized.code });
|
|
144
|
-
this.logger?.error('Model provider stream failed.', {
|
|
145
|
-
provider: this.id,
|
|
146
|
-
model: req.model,
|
|
147
|
-
method,
|
|
148
|
-
error: sanitizeForLog({ code: normalized.code, category: normalized.category, retriable: normalized.retriable, meta: normalized.meta })
|
|
149
|
-
});
|
|
150
|
-
throw normalized;
|
|
151
|
-
}
|
|
152
|
-
finally {
|
|
153
|
-
next.cleanup();
|
|
154
233
|
}
|
|
155
234
|
}.bind(this);
|
|
156
235
|
if (!this.telemetry)
|
|
157
236
|
return iterate();
|
|
158
|
-
return
|
|
237
|
+
return pumpStreamThroughSpan(this.telemetry, `harness.model.${method}`, attrs, iterate);
|
|
159
238
|
}
|
|
160
239
|
withTimeout(req, method) {
|
|
161
240
|
if (!this.timeoutMs || this.timeoutMs <= 0) {
|
|
@@ -168,6 +247,11 @@ export class BaseModelProvider {
|
|
|
168
247
|
relay();
|
|
169
248
|
let rejectTimeout;
|
|
170
249
|
const timeoutPromise = new Promise((_, reject) => { rejectTimeout = reject; });
|
|
250
|
+
// Streams never race against this promise (they rely on the relayed signal
|
|
251
|
+
// abort), and unary calls may settle through the operation branch of the
|
|
252
|
+
// race. Mark the rejection as handled so a firing timer can never surface
|
|
253
|
+
// as an unhandled promise rejection.
|
|
254
|
+
timeoutPromise.catch(() => undefined);
|
|
171
255
|
const timeout = setTimeout(() => {
|
|
172
256
|
const error = new OperationTimeoutError('Model call timed out.', { scope: 'model', timeout_ms: this.timeoutMs });
|
|
173
257
|
controller.abort(error);
|
|
@@ -207,40 +291,6 @@ export class BaseModelProvider {
|
|
|
207
291
|
this.telemetry?.recordCounter('harness.model.tokens.total', usage.totalTokens, attrs);
|
|
208
292
|
}
|
|
209
293
|
}
|
|
210
|
-
async function* streamWithSpan(telemetry, name, attrs, iterate) {
|
|
211
|
-
const queue = [];
|
|
212
|
-
let done = false;
|
|
213
|
-
let failure;
|
|
214
|
-
let notify;
|
|
215
|
-
const wake = () => {
|
|
216
|
-
notify?.();
|
|
217
|
-
notify = undefined;
|
|
218
|
-
};
|
|
219
|
-
const producer = telemetry.span(name, attrs, async (span) => {
|
|
220
|
-
for await (const chunk of iterate(span)) {
|
|
221
|
-
queue.push(chunk);
|
|
222
|
-
wake();
|
|
223
|
-
}
|
|
224
|
-
}).catch((error) => {
|
|
225
|
-
failure = error;
|
|
226
|
-
}).finally(() => {
|
|
227
|
-
done = true;
|
|
228
|
-
wake();
|
|
229
|
-
});
|
|
230
|
-
while (!done || queue.length > 0) {
|
|
231
|
-
const next = queue.shift();
|
|
232
|
-
if (next !== undefined) {
|
|
233
|
-
yield next;
|
|
234
|
-
continue;
|
|
235
|
-
}
|
|
236
|
-
if (failure)
|
|
237
|
-
throw failure;
|
|
238
|
-
await new Promise((resolve) => { notify = resolve; });
|
|
239
|
-
}
|
|
240
|
-
await producer;
|
|
241
|
-
if (failure)
|
|
242
|
-
throw failure;
|
|
243
|
-
}
|
|
244
294
|
function isAbortError(error) {
|
|
245
295
|
const value = error;
|
|
246
296
|
return value?.name === 'AbortError' || value?.code === 'ABORT_ERR';
|
|
@@ -270,7 +320,11 @@ function modelErrorTelemetryAttrs(error) {
|
|
|
270
320
|
'harness.error.model_provider_param': stringTelemetryAttr(meta?.['providerParam']),
|
|
271
321
|
'harness.error.model_provider_request_id': stringTelemetryAttr(meta?.['providerRequestId']),
|
|
272
322
|
'harness.error.model_provider_message': stringTelemetryAttr(meta?.['providerMessage']),
|
|
273
|
-
'harness.error.model_provider_body': jsonTelemetryAttr(meta?.['providerBody'])
|
|
323
|
+
'harness.error.model_provider_body': jsonTelemetryAttr(redactProviderContent(meta?.['providerBody'])),
|
|
324
|
+
'harness.error.model_retry_kind': stringTelemetryAttr(meta?.['retryKind']),
|
|
325
|
+
'harness.error.model_retry_after_ms': numberTelemetryAttr(meta?.['retryAfterMs']),
|
|
326
|
+
'harness.error.model_retry_attempt': numberTelemetryAttr(meta?.['retryAttempt']),
|
|
327
|
+
'harness.error.model_retry_max_attempts': numberTelemetryAttr(meta?.['retryMaxAttempts'])
|
|
274
328
|
};
|
|
275
329
|
}
|
|
276
330
|
function stringTelemetryAttr(value) {
|
|
@@ -295,13 +349,22 @@ function extractProviderErrorDetails(error) {
|
|
|
295
349
|
return {};
|
|
296
350
|
const response = asRecord(record['response']);
|
|
297
351
|
const errorBody = asRecord(record['error']);
|
|
298
|
-
|
|
352
|
+
// AWS SDK v3 errors carry status/headers under `$metadata`/`$response`.
|
|
353
|
+
const awsMetadata = asRecord(record['$metadata']);
|
|
354
|
+
const awsResponse = asRecord(record['$response']);
|
|
355
|
+
const headers = normalizeHeaders(record['headers'] ?? response?.['headers'] ?? awsResponse?.['headers']);
|
|
356
|
+
const retryAfterMs = headers ? parseRetryAfterMs(headers) : undefined;
|
|
357
|
+
const rateLimit = headers ? parseRateLimit(headers) : undefined;
|
|
299
358
|
const providerBody = sanitizeJsonLike(record['body'] ?? response?.['body'] ?? response?.['data'] ?? record['error']);
|
|
300
359
|
const status = numberField(record, 'status')
|
|
301
360
|
?? numberField(record, 'statusCode')
|
|
302
361
|
?? numberField(response, 'status')
|
|
303
|
-
?? numberField(response, 'statusCode')
|
|
304
|
-
|
|
362
|
+
?? numberField(response, 'statusCode')
|
|
363
|
+
?? numberField(awsMetadata, 'httpStatusCode')
|
|
364
|
+
?? (awsMetadata ? statusFromAwsErrorName(stringField(record, 'name')) : undefined);
|
|
365
|
+
const providerCode = stringField(record, 'code')
|
|
366
|
+
?? stringField(errorBody, 'code')
|
|
367
|
+
?? (awsMetadata ? stringField(record, 'name') : undefined);
|
|
305
368
|
const providerType = stringField(record, 'type') ?? stringField(errorBody, 'type');
|
|
306
369
|
const providerParam = stringField(record, 'param') ?? stringField(errorBody, 'param');
|
|
307
370
|
const providerRequestId = stringField(record, 'request_id')
|
|
@@ -317,9 +380,40 @@ function extractProviderErrorDetails(error) {
|
|
|
317
380
|
...(providerParam ? { providerParam } : {}),
|
|
318
381
|
...(providerRequestId ? { providerRequestId } : {}),
|
|
319
382
|
...(providerMessage ? { providerMessage } : {}),
|
|
320
|
-
...(providerBody !== undefined ? { providerBody } : {})
|
|
383
|
+
...(providerBody !== undefined ? { providerBody } : {}),
|
|
384
|
+
...(headers ? { providerHeaders: headers } : {}),
|
|
385
|
+
...(retryAfterMs !== undefined ? { retryAfterMs } : {}),
|
|
386
|
+
...(rateLimit ? { rateLimit } : {})
|
|
321
387
|
};
|
|
322
388
|
}
|
|
389
|
+
/**
|
|
390
|
+
* Well-known AWS SDK error names mapped to their HTTP-equivalent status so
|
|
391
|
+
* retry classification works even when `$metadata.httpStatusCode` is absent.
|
|
392
|
+
*/
|
|
393
|
+
const AWS_ERROR_NAME_STATUS = {
|
|
394
|
+
ThrottlingException: 429,
|
|
395
|
+
TooManyRequestsException: 429,
|
|
396
|
+
ModelNotReadyException: 429,
|
|
397
|
+
ServiceUnavailableException: 503,
|
|
398
|
+
InternalServerException: 500
|
|
399
|
+
};
|
|
400
|
+
function statusFromAwsErrorName(name) {
|
|
401
|
+
return name ? AWS_ERROR_NAME_STATUS[name] : undefined;
|
|
402
|
+
}
|
|
403
|
+
/**
|
|
404
|
+
* Adapter-built errors may carry raw provider/model content in
|
|
405
|
+
* `meta.providerBody`. Re-issue the error with a content-redacted body so logs,
|
|
406
|
+
* spans, and serialized errors never see raw output (POR-07).
|
|
407
|
+
*/
|
|
408
|
+
function withRedactedProviderBody(error) {
|
|
409
|
+
const meta = asRecord(error.meta);
|
|
410
|
+
if (!meta || meta['providerBody'] === undefined)
|
|
411
|
+
return error;
|
|
412
|
+
return new ModelError(error.message, {
|
|
413
|
+
...meta,
|
|
414
|
+
providerBody: redactProviderContent(meta['providerBody'])
|
|
415
|
+
}, error.cause ?? error);
|
|
416
|
+
}
|
|
323
417
|
function asRecord(value) {
|
|
324
418
|
return value !== null && typeof value === 'object' ? value : undefined;
|
|
325
419
|
}
|
|
@@ -332,13 +426,35 @@ function numberField(record, key) {
|
|
|
332
426
|
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
333
427
|
}
|
|
334
428
|
function normalizeHeaders(value) {
|
|
429
|
+
const headersLike = value;
|
|
430
|
+
if (headersLike?.forEach) {
|
|
431
|
+
const headers = {};
|
|
432
|
+
headersLike.forEach((headerValue, key) => {
|
|
433
|
+
const normalizedKey = key.toLowerCase();
|
|
434
|
+
if (!isSensitiveHeader(normalizedKey))
|
|
435
|
+
headers[normalizedKey] = String(headerValue).slice(0, 2000);
|
|
436
|
+
});
|
|
437
|
+
return Object.keys(headers).length > 0 ? headers : undefined;
|
|
438
|
+
}
|
|
439
|
+
if (headersLike?.entries) {
|
|
440
|
+
const headers = {};
|
|
441
|
+
for (const [key, headerValue] of headersLike.entries()) {
|
|
442
|
+
const normalizedKey = key.toLowerCase();
|
|
443
|
+
if (!isSensitiveHeader(normalizedKey))
|
|
444
|
+
headers[normalizedKey] = String(headerValue).slice(0, 2000);
|
|
445
|
+
}
|
|
446
|
+
return Object.keys(headers).length > 0 ? headers : undefined;
|
|
447
|
+
}
|
|
335
448
|
const record = asRecord(value);
|
|
336
449
|
if (!record)
|
|
337
450
|
return undefined;
|
|
338
451
|
const headers = {};
|
|
339
452
|
for (const [key, headerValue] of Object.entries(record)) {
|
|
453
|
+
const normalizedKey = key.toLowerCase();
|
|
454
|
+
if (isSensitiveHeader(normalizedKey))
|
|
455
|
+
continue;
|
|
340
456
|
if (typeof headerValue === 'string' || typeof headerValue === 'number' || typeof headerValue === 'boolean') {
|
|
341
|
-
headers[
|
|
457
|
+
headers[normalizedKey] = String(headerValue).slice(0, 2000);
|
|
342
458
|
}
|
|
343
459
|
}
|
|
344
460
|
return Object.keys(headers).length > 0 ? headers : undefined;
|
|
@@ -346,3 +462,184 @@ function normalizeHeaders(value) {
|
|
|
346
462
|
function sanitizeJsonLike(value) {
|
|
347
463
|
return sanitizeProviderBody(value);
|
|
348
464
|
}
|
|
465
|
+
function resolveRetryPolicy(req) {
|
|
466
|
+
const setting = req.call?.retry ?? ('defaults' in req ? req.defaults?.retry : undefined) ?? true;
|
|
467
|
+
if (setting === false) {
|
|
468
|
+
return { ...DEFAULT_RETRY_POLICY, maxAttempts: 1 };
|
|
469
|
+
}
|
|
470
|
+
if (setting === true || setting === undefined) {
|
|
471
|
+
return DEFAULT_RETRY_POLICY;
|
|
472
|
+
}
|
|
473
|
+
return {
|
|
474
|
+
...DEFAULT_RETRY_POLICY,
|
|
475
|
+
...setting,
|
|
476
|
+
maxAttempts: setting.maxAttempts ?? DEFAULT_RETRY_POLICY.maxAttempts,
|
|
477
|
+
maxActiveElapsedMs: setting.maxActiveElapsedMs ?? DEFAULT_RETRY_POLICY.maxActiveElapsedMs,
|
|
478
|
+
maxActiveDelayMs: setting.maxActiveDelayMs ?? DEFAULT_RETRY_POLICY.maxActiveDelayMs,
|
|
479
|
+
minDelayMs: setting.minDelayMs ?? DEFAULT_RETRY_POLICY.minDelayMs,
|
|
480
|
+
maxDelayMs: setting.maxDelayMs ?? DEFAULT_RETRY_POLICY.maxDelayMs,
|
|
481
|
+
respectRetryAfter: setting.respectRetryAfter ?? DEFAULT_RETRY_POLICY.respectRetryAfter,
|
|
482
|
+
longRetry: setting.longRetry ?? DEFAULT_RETRY_POLICY.longRetry,
|
|
483
|
+
retryOn: {
|
|
484
|
+
...DEFAULT_RETRY_POLICY.retryOn,
|
|
485
|
+
...(setting.retryOn ?? {})
|
|
486
|
+
}
|
|
487
|
+
};
|
|
488
|
+
}
|
|
489
|
+
function isSensitiveHeader(key) {
|
|
490
|
+
return key === 'authorization'
|
|
491
|
+
|| key === 'proxy-authorization'
|
|
492
|
+
|| key === 'x-api-key'
|
|
493
|
+
|| key === 'api-key'
|
|
494
|
+
|| key === 'openai-api-key'
|
|
495
|
+
|| key.endsWith('-api-key');
|
|
496
|
+
}
|
|
497
|
+
function retryDecision(error, policy, attempt, startedAt) {
|
|
498
|
+
if (attempt >= policy.maxAttempts)
|
|
499
|
+
return { action: 'fail', retryKind: 'none' };
|
|
500
|
+
const reason = retryReason(error, policy);
|
|
501
|
+
if (!reason)
|
|
502
|
+
return { action: 'fail', retryKind: 'none' };
|
|
503
|
+
const providerDelay = policy.respectRetryAfter ? retryAfterFromError(error) : undefined;
|
|
504
|
+
const delayMs = providerDelay ?? computedBackoffMs(policy, attempt);
|
|
505
|
+
const elapsed = Date.now() - startedAt;
|
|
506
|
+
if (delayMs > policy.maxActiveDelayMs || elapsed + delayMs > policy.maxActiveElapsedMs) {
|
|
507
|
+
// `longRetry: 'defer'` opts into the deferred classification for
|
|
508
|
+
// provider-instructed delays beyond the active budget; the default
|
|
509
|
+
// `'error'` fails immediately with `retryKind: 'none'`.
|
|
510
|
+
const deferredAllowed = policy.longRetry === 'defer'
|
|
511
|
+
&& providerDelay !== undefined
|
|
512
|
+
&& (policy.maxDeferredDelayMs === undefined || providerDelay <= policy.maxDeferredDelayMs);
|
|
513
|
+
if (deferredAllowed)
|
|
514
|
+
return { action: 'fail', retryKind: 'deferred', delayMs: providerDelay };
|
|
515
|
+
return { action: 'fail', retryKind: 'none' };
|
|
516
|
+
}
|
|
517
|
+
return { action: 'retry', delayMs, reason };
|
|
518
|
+
}
|
|
519
|
+
function retryReason(error, policy) {
|
|
520
|
+
if (error instanceof OperationTimeoutError)
|
|
521
|
+
return policy.retryOn.timeout ? 'timeout' : undefined;
|
|
522
|
+
if (!(error instanceof ModelError))
|
|
523
|
+
return undefined;
|
|
524
|
+
const meta = asRecord(error.meta);
|
|
525
|
+
const status = typeof meta?.['status'] === 'number' ? meta['status'] : undefined;
|
|
526
|
+
const reason = typeof meta?.['reason'] === 'string' ? meta['reason'] : undefined;
|
|
527
|
+
if ((reason === 'network' || status === 408 || status === 409) && policy.retryOn.network)
|
|
528
|
+
return reason ?? `http_${status}`;
|
|
529
|
+
if ((reason === 'rate_limited' || status === 429) && policy.retryOn.rateLimit)
|
|
530
|
+
return 'rate_limited';
|
|
531
|
+
if ((reason === 'provider_unavailable' || (typeof status === 'number' && status >= 500)) && policy.retryOn.serverError)
|
|
532
|
+
return 'provider_unavailable';
|
|
533
|
+
return undefined;
|
|
534
|
+
}
|
|
535
|
+
function retryAfterFromError(error) {
|
|
536
|
+
const meta = asRecord(error.meta);
|
|
537
|
+
const value = meta?.['retryAfterMs'];
|
|
538
|
+
return typeof value === 'number' && Number.isFinite(value) && value >= 0 ? value : undefined;
|
|
539
|
+
}
|
|
540
|
+
function computedBackoffMs(policy, attempt) {
|
|
541
|
+
const base = Math.min(policy.maxDelayMs, policy.minDelayMs * (2 ** Math.max(0, attempt - 1)));
|
|
542
|
+
const jitter = 0.75 + Math.random() * 0.25;
|
|
543
|
+
return Math.max(0, Math.floor(base * jitter));
|
|
544
|
+
}
|
|
545
|
+
function decorateRetryMeta(error, retryKind, attempt, maxAttempts, delayMs) {
|
|
546
|
+
if (!(error instanceof ModelError))
|
|
547
|
+
return error;
|
|
548
|
+
const meta = {
|
|
549
|
+
...(error.meta ?? {}),
|
|
550
|
+
retryKind,
|
|
551
|
+
retryAttempt: attempt,
|
|
552
|
+
retryMaxAttempts: maxAttempts,
|
|
553
|
+
// `retryAfterMs` is only ever the provider-instructed delay carried by a
|
|
554
|
+
// deferred classification; synthetic harness backoff is never written here.
|
|
555
|
+
...(retryKind === 'deferred' && delayMs !== undefined ? { retryAfterMs: delayMs } : {})
|
|
556
|
+
};
|
|
557
|
+
return new ModelError(error.message, meta, error.cause ?? error);
|
|
558
|
+
}
|
|
559
|
+
function parseRetryAfterMs(headers) {
|
|
560
|
+
const retryAfterMs = parsePositiveNumber(headers['retry-after-ms']);
|
|
561
|
+
if (retryAfterMs !== undefined)
|
|
562
|
+
return retryAfterMs;
|
|
563
|
+
const retryAfter = headers['retry-after'];
|
|
564
|
+
if (!retryAfter)
|
|
565
|
+
return undefined;
|
|
566
|
+
const seconds = parsePositiveNumber(retryAfter);
|
|
567
|
+
if (seconds !== undefined)
|
|
568
|
+
return seconds * 1000;
|
|
569
|
+
const date = Date.parse(retryAfter);
|
|
570
|
+
if (Number.isFinite(date))
|
|
571
|
+
return Math.max(0, date - Date.now());
|
|
572
|
+
return undefined;
|
|
573
|
+
}
|
|
574
|
+
/**
|
|
575
|
+
* Parses the OpenAI/Azure (`x-ratelimit-*`) and Anthropic
|
|
576
|
+
* (`anthropic-ratelimit-*`) request and token header families. When several
|
|
577
|
+
* buckets are present the exhausted bucket (`remaining === 0`) wins so the
|
|
578
|
+
* reported scope identifies what was actually rate limited.
|
|
579
|
+
*/
|
|
580
|
+
function parseRateLimit(headers) {
|
|
581
|
+
const buckets = [
|
|
582
|
+
parseRateLimitBucket('requests', headers['x-ratelimit-limit-requests'] ?? headers['anthropic-ratelimit-requests-limit'], headers['x-ratelimit-remaining-requests'] ?? headers['anthropic-ratelimit-requests-remaining'], headers['anthropic-ratelimit-requests-reset'] ?? headers['x-ratelimit-reset-requests']),
|
|
583
|
+
parseRateLimitBucket('tokens', headers['x-ratelimit-limit-tokens'] ?? headers['anthropic-ratelimit-tokens-limit'], headers['x-ratelimit-remaining-tokens'] ?? headers['anthropic-ratelimit-tokens-remaining'], headers['anthropic-ratelimit-tokens-reset'] ?? headers['x-ratelimit-reset-tokens']),
|
|
584
|
+
parseRateLimitBucket('input_tokens', headers['anthropic-ratelimit-input-tokens-limit'], headers['anthropic-ratelimit-input-tokens-remaining'], headers['anthropic-ratelimit-input-tokens-reset']),
|
|
585
|
+
parseRateLimitBucket('output_tokens', headers['anthropic-ratelimit-output-tokens-limit'], headers['anthropic-ratelimit-output-tokens-remaining'], headers['anthropic-ratelimit-output-tokens-reset'])
|
|
586
|
+
].filter((bucket) => bucket !== undefined);
|
|
587
|
+
if (buckets.length === 0)
|
|
588
|
+
return undefined;
|
|
589
|
+
return buckets.find((bucket) => bucket.remaining === 0) ?? buckets[0];
|
|
590
|
+
}
|
|
591
|
+
function parseRateLimitBucket(scope, limitHeader, remainingHeader, resetHeader) {
|
|
592
|
+
const limit = parsePositiveNumber(limitHeader);
|
|
593
|
+
const remaining = parsePositiveNumber(remainingHeader);
|
|
594
|
+
const resetAt = parseResetAt(resetHeader);
|
|
595
|
+
if (limit === undefined && remaining === undefined && resetAt === undefined)
|
|
596
|
+
return undefined;
|
|
597
|
+
return {
|
|
598
|
+
scope,
|
|
599
|
+
...(limit !== undefined ? { limit } : {}),
|
|
600
|
+
...(remaining !== undefined ? { remaining } : {}),
|
|
601
|
+
...(resetAt ? { resetAt } : {})
|
|
602
|
+
};
|
|
603
|
+
}
|
|
604
|
+
function parsePositiveNumber(value) {
|
|
605
|
+
if (!value)
|
|
606
|
+
return undefined;
|
|
607
|
+
const parsed = Number.parseFloat(value);
|
|
608
|
+
return Number.isFinite(parsed) && parsed >= 0 ? parsed : undefined;
|
|
609
|
+
}
|
|
610
|
+
function parseResetAt(value) {
|
|
611
|
+
if (!value)
|
|
612
|
+
return undefined;
|
|
613
|
+
const parsed = Date.parse(value);
|
|
614
|
+
return Number.isFinite(parsed) ? new Date(parsed).toISOString() : undefined;
|
|
615
|
+
}
|
|
616
|
+
/**
|
|
617
|
+
* Shared adapter helpers (`models/adapter-utils.js`) are part of the public
|
|
618
|
+
* model-provider surface so first-party adapter packages consume a single
|
|
619
|
+
* implementation. They are re-exported next to the adapter base class because
|
|
620
|
+
* the ports barrel is the public path for adapter authors.
|
|
621
|
+
*/
|
|
622
|
+
export * from '../models/adapter-utils.js';
|
|
623
|
+
async function sleep(delayMs, signal) {
|
|
624
|
+
if (delayMs <= 0)
|
|
625
|
+
return;
|
|
626
|
+
if (signal.aborted)
|
|
627
|
+
throw new OperationCancelledError('Model retry was cancelled.', { scope: 'model' }, signal.reason);
|
|
628
|
+
await new Promise((resolve, reject) => {
|
|
629
|
+
let timeout;
|
|
630
|
+
const cleanup = () => {
|
|
631
|
+
if (timeout)
|
|
632
|
+
clearTimeout(timeout);
|
|
633
|
+
signal.removeEventListener('abort', onAbort);
|
|
634
|
+
};
|
|
635
|
+
const onAbort = () => {
|
|
636
|
+
cleanup();
|
|
637
|
+
reject(new OperationCancelledError('Model retry was cancelled.', { scope: 'model' }, signal.reason));
|
|
638
|
+
};
|
|
639
|
+
timeout = setTimeout(() => {
|
|
640
|
+
cleanup();
|
|
641
|
+
resolve();
|
|
642
|
+
}, delayMs);
|
|
643
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
644
|
+
});
|
|
645
|
+
}
|
|
@@ -31,8 +31,12 @@ export type AdapterCapability =
|
|
|
31
31
|
| 'runtime.workspace_checkpoint'
|
|
32
32
|
/** Runtime exposes checkpoint retention and expiry metadata. */
|
|
33
33
|
| 'runtime.checkpoint_retention'
|
|
34
|
-
/**
|
|
34
|
+
/** Runtime checkpoints, leases, and terminal state survive process exit. */
|
|
35
|
+
| 'runtime.persistent'
|
|
36
|
+
/** Workspace store implements durable lifecycle and opaque checkpoint refs. */
|
|
35
37
|
| 'workspace_store.durable'
|
|
38
|
+
/** Workspace store persists state beyond process exit. */
|
|
39
|
+
| 'workspace_store.persistent'
|
|
36
40
|
/** Workspace store can produce stable checkpoints. */
|
|
37
41
|
| 'workspace_store.checkpoint'
|
|
38
42
|
/** Workspace store can resume committed checkpoints. */
|
|
@@ -49,6 +53,16 @@ export type AdapterCapability =
|
|
|
49
53
|
| 'workspace_store.quota'
|
|
50
54
|
/** Workspace store encrypts checkpoint, snapshot, file, and metadata storage. */
|
|
51
55
|
| 'workspace_store.encrypted_storage'
|
|
56
|
+
/** Context checkpoint store can write checkpoints. */
|
|
57
|
+
| 'context_checkpoint.write'
|
|
58
|
+
/** Context checkpoint store can read checkpoints. */
|
|
59
|
+
| 'context_checkpoint.read'
|
|
60
|
+
/** Context checkpoint store can list checkpoints. */
|
|
61
|
+
| 'context_checkpoint.list'
|
|
62
|
+
/** Context checkpoint store can delete checkpoints. */
|
|
63
|
+
| 'context_checkpoint.delete'
|
|
64
|
+
/** Context checkpoint store survives adapter close/reopen. */
|
|
65
|
+
| 'context_checkpoint.persistent'
|
|
52
66
|
/** Adapter can record feedback. */
|
|
53
67
|
| 'feedback.record'
|
|
54
68
|
/** Memory adapter supports key/value reads and writes. */
|
|
@@ -79,7 +93,7 @@ export interface AdapterCapabilities {
|
|
|
79
93
|
}
|
|
80
94
|
/** Adapter descriptor surfaced through `harness.inspect()`. */
|
|
81
95
|
export interface AdapterInspection {
|
|
82
|
-
readonly kind: 'state' | 'sandbox' | 'runtime' | 'workspace_store' | 'feedback' | 'model' | 'memory';
|
|
96
|
+
readonly kind: 'state' | 'sandbox' | 'runtime' | 'workspace_store' | 'context_checkpoint' | 'feedback' | 'model' | 'memory';
|
|
83
97
|
readonly id: string;
|
|
84
98
|
readonly capabilities: readonly AdapterCapability[];
|
|
85
99
|
readonly metadata?: Record<string, unknown>;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import type { JsonValue } from '../models/json.js';
|
|
2
|
+
import type { AdapterCapabilities, AdapterCapability } from './capabilities.js';
|
|
3
|
+
import type { HarnessAdapterContext } from './harness-context.js';
|
|
4
|
+
/** Typed long-horizon record written explicitly by workflow or agent code. */
|
|
5
|
+
export interface ContextCheckpoint {
|
|
6
|
+
/** Stable run id this checkpoint belongs to. */
|
|
7
|
+
runId: string;
|
|
8
|
+
/** Stable session id this checkpoint belongs to. */
|
|
9
|
+
sessionId: string;
|
|
10
|
+
/** Workflow id when written from a workflow context. */
|
|
11
|
+
workflowId?: string;
|
|
12
|
+
/** Agent id when written from an agent context. */
|
|
13
|
+
agentId?: string;
|
|
14
|
+
/** Monotonic sequence selected by caller or policy. */
|
|
15
|
+
sequence: number;
|
|
16
|
+
/** Checkpoint purpose. */
|
|
17
|
+
kind: 'summary' | 'handoff' | 'goal_state';
|
|
18
|
+
/** JSON payload. Never emitted in harness telemetry. */
|
|
19
|
+
payload: JsonValue;
|
|
20
|
+
/** UTF-8 JSON payload size. */
|
|
21
|
+
payloadSizeBytes: number;
|
|
22
|
+
/** ISO timestamp. */
|
|
23
|
+
createdAt: string;
|
|
24
|
+
/** Privacy-safe operational metadata only. */
|
|
25
|
+
metadata?: Record<string, JsonValue>;
|
|
26
|
+
}
|
|
27
|
+
/** Stable reference to one context checkpoint. */
|
|
28
|
+
export interface ContextCheckpointRef {
|
|
29
|
+
runId: string;
|
|
30
|
+
sessionId: string;
|
|
31
|
+
sequence: number;
|
|
32
|
+
kind: ContextCheckpoint['kind'];
|
|
33
|
+
}
|
|
34
|
+
/** Query accepted by context checkpoint stores. */
|
|
35
|
+
export interface ContextCheckpointQuery {
|
|
36
|
+
runId?: string;
|
|
37
|
+
sessionId?: string;
|
|
38
|
+
workflowId?: string;
|
|
39
|
+
agentId?: string;
|
|
40
|
+
kind?: ContextCheckpoint['kind'];
|
|
41
|
+
limit?: number;
|
|
42
|
+
signal?: AbortSignal;
|
|
43
|
+
}
|
|
44
|
+
/** Data-only descriptor for a context checkpoint store. */
|
|
45
|
+
export interface ContextCheckpointStoreInfo {
|
|
46
|
+
id: string;
|
|
47
|
+
packageName: string;
|
|
48
|
+
capabilities: readonly AdapterCapability[];
|
|
49
|
+
}
|
|
50
|
+
/** Adapter port for explicit context checkpoint persistence. */
|
|
51
|
+
export interface ContextCheckpointStore extends AdapterCapabilities {
|
|
52
|
+
readonly info: ContextCheckpointStoreInfo;
|
|
53
|
+
configureHarnessContext?(context: HarnessAdapterContext): void;
|
|
54
|
+
write(checkpoint: ContextCheckpoint, opts?: {
|
|
55
|
+
signal?: AbortSignal;
|
|
56
|
+
}): Promise<void>;
|
|
57
|
+
list(query: ContextCheckpointQuery): Promise<readonly ContextCheckpoint[]>;
|
|
58
|
+
read(ref: ContextCheckpointRef): Promise<ContextCheckpoint | undefined>;
|
|
59
|
+
delete(ref: ContextCheckpointRef): Promise<void>;
|
|
60
|
+
close?(): Promise<void>;
|
|
61
|
+
}
|
|
62
|
+
/** Validates the context checkpoint adapter descriptor at harness build time. */
|
|
63
|
+
export declare function validateContextCheckpointStore(adapter: ContextCheckpointStore): void;
|