@orkestrel/agent 0.0.26 → 0.0.27
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/src/core/index.cjs +735 -21
- package/dist/src/core/index.cjs.map +1 -1
- package/dist/src/core/index.d.cts +640 -0
- package/dist/src/core/index.d.ts +640 -0
- package/dist/src/core/index.js +719 -24
- package/dist/src/core/index.js.map +1 -1
- package/package.json +7 -7
package/dist/src/core/index.cjs
CHANGED
|
@@ -146,6 +146,8 @@ var INVALID_RELAY_STATUS = 400;
|
|
|
146
146
|
* carried with no body and answered for an aborted inbound read as well.
|
|
147
147
|
*/
|
|
148
148
|
var OVERSIZED_RELAY_STATUS = 413;
|
|
149
|
+
/** Names the System One decision endpoint shared by compatible servers. */
|
|
150
|
+
var SYSTEM_ONE_PATH = "/v1/systemone";
|
|
149
151
|
//#endregion
|
|
150
152
|
//#region src/core/errors.ts
|
|
151
153
|
/**
|
|
@@ -361,6 +363,75 @@ var ProviderError = class extends Error {
|
|
|
361
363
|
function isProviderError(value) {
|
|
362
364
|
return (0, _orkestrel_contract.isInstance)(value, ProviderError);
|
|
363
365
|
}
|
|
366
|
+
/**
|
|
367
|
+
* Reports a judge call cancelled by the caller's signal or its deadline, carrying the
|
|
368
|
+
* {@link JudgeResult} merged from the calls that completed before the cancel and the
|
|
369
|
+
* machine-readable `code` `'ABORT'`.
|
|
370
|
+
*
|
|
371
|
+
* @remarks
|
|
372
|
+
* `partial` keeps the answers and the usage of every completed call, so usage reported by
|
|
373
|
+
* completed calls is retained; a cancel before the first call carries an empty partial. `cause` holds the failure the
|
|
374
|
+
* cancel superseded when a throw raced the abort, and is undefined when the cancel was the only
|
|
375
|
+
* failure.
|
|
376
|
+
*/
|
|
377
|
+
var JudgeAbortError = class extends Error {
|
|
378
|
+
/** Names the machine-readable condition — `'ABORT'`: a judge call cancelled mid-flight. */
|
|
379
|
+
code = "ABORT";
|
|
380
|
+
partial;
|
|
381
|
+
constructor(partial, options) {
|
|
382
|
+
super("judge call aborted", options);
|
|
383
|
+
this.name = "JudgeAbortError";
|
|
384
|
+
this.partial = partial;
|
|
385
|
+
}
|
|
386
|
+
};
|
|
387
|
+
/**
|
|
388
|
+
* Narrows a caught value to a {@link JudgeAbortError} through `instanceof`, so a `catch` can
|
|
389
|
+
* recover its `partial` result.
|
|
390
|
+
*
|
|
391
|
+
* @param value - The caught value
|
|
392
|
+
* @returns True if the value is a {@link JudgeAbortError}; false otherwise
|
|
393
|
+
* @example
|
|
394
|
+
* ```ts
|
|
395
|
+
* isJudgeAbortError(new JudgeAbortError({ model: 'tev1:0.8b', answers: {} })) // true
|
|
396
|
+
* ```
|
|
397
|
+
*/
|
|
398
|
+
function isJudgeAbortError(value) {
|
|
399
|
+
return (0, _orkestrel_contract.isInstance)(value, JudgeAbortError);
|
|
400
|
+
}
|
|
401
|
+
/**
|
|
402
|
+
* Reports a coded judge failure with its HTTP status and underlying cause when available.
|
|
403
|
+
*
|
|
404
|
+
* @remarks
|
|
405
|
+
* The judge engine throws this error for a non-OK HTTP response, a missing or unparsable
|
|
406
|
+
* response body, and a request refused before inference. Concrete wire decoders also use it for
|
|
407
|
+
* a response they cannot read and for a wire limit. `status` is present only for an `HTTP`
|
|
408
|
+
* failure; other codes leave it undefined.
|
|
409
|
+
*/
|
|
410
|
+
var JudgeError = class extends Error {
|
|
411
|
+
/** Names the machine-readable condition — `'HTTP'`: a non-OK response; `'PROTOCOL'`: a missing, unparsable, or unreadable response body; `'QUESTION'`: a request refused before inference. */
|
|
412
|
+
code;
|
|
413
|
+
/** Holds the response status for an HTTP failure, or undefined for other codes. */
|
|
414
|
+
status;
|
|
415
|
+
constructor(code, message, options) {
|
|
416
|
+
super(message, options);
|
|
417
|
+
this.name = "JudgeError";
|
|
418
|
+
this.code = code;
|
|
419
|
+
this.status = options?.status;
|
|
420
|
+
}
|
|
421
|
+
};
|
|
422
|
+
/**
|
|
423
|
+
* Narrows a caught value to the judge failure class through `instanceof`.
|
|
424
|
+
*
|
|
425
|
+
* @param value - The caught value
|
|
426
|
+
* @returns True if the value is a {@link JudgeError}; false otherwise
|
|
427
|
+
* @example
|
|
428
|
+
* ```ts
|
|
429
|
+
* isJudgeError(new JudgeError('HTTP', 'judge error: 429', { status: 429 })) // true
|
|
430
|
+
* ```
|
|
431
|
+
*/
|
|
432
|
+
function isJudgeError(value) {
|
|
433
|
+
return (0, _orkestrel_contract.isInstance)(value, JudgeError);
|
|
434
|
+
}
|
|
364
435
|
//#endregion
|
|
365
436
|
//#region src/core/shapers.ts
|
|
366
437
|
/**
|
|
@@ -1260,6 +1331,273 @@ async function* readChunks(body, signal) {
|
|
|
1260
1331
|
}
|
|
1261
1332
|
}
|
|
1262
1333
|
}
|
|
1334
|
+
/**
|
|
1335
|
+
* Derives the winner, its probability, the published confidence, and a score answer's expected
|
|
1336
|
+
* level from a judge answer's distribution.
|
|
1337
|
+
*
|
|
1338
|
+
* @remarks
|
|
1339
|
+
* The winner is the first strictly greatest candidate in enumeration order: an option name in
|
|
1340
|
+
* criteria order, a level index, or `'false'` then `'true'` for a noul, so a noul of exactly 0.5
|
|
1341
|
+
* names `'false'`. Confidence follows the TypeSafe formulas, whatever confidence a server sent: a
|
|
1342
|
+
* choice reads `(max(p) - 1/n) / (1 - 1/n)`, a noul reads `|2p - 1|` (the choice formula at
|
|
1343
|
+
* n = 2), and a score reads `max(0, 1 - spread / even_spread)`, where `spread` sums each level's
|
|
1344
|
+
* probability times its distance from the winning level and `even_spread` is the mean distance
|
|
1345
|
+
* from the middle level. `score` is the expected level `sum(i * p_i)` and is set only for a score
|
|
1346
|
+
* answer.
|
|
1347
|
+
*
|
|
1348
|
+
* @param answer - The answer whose distribution is read
|
|
1349
|
+
* @returns The derived measures
|
|
1350
|
+
* @throws JudgeError Thrown with code `PROTOCOL` when a choice or score answer has fewer than 2
|
|
1351
|
+
* candidates, because the confidence formulas divide by the candidate count
|
|
1352
|
+
* @example
|
|
1353
|
+
* ```ts
|
|
1354
|
+
* computeReading({ form: 'choice', probabilities: { billing: 0.88, technical: 0.12, sales: 0 } })
|
|
1355
|
+
* // { winner: 'billing', probability: 0.88, confidence: 0.82 } to two decimals
|
|
1356
|
+
* computeReading({ form: 'score', probabilities: [0, 0.57, 0.43] })
|
|
1357
|
+
* // { winner: '1', probability: 0.57, confidence: 0.355, score: 1.43 } to three decimals
|
|
1358
|
+
* computeReading({ form: 'noul', noul: 0.5 }) // { winner: 'false', probability: 0.5, confidence: 0 }
|
|
1359
|
+
* ```
|
|
1360
|
+
*/
|
|
1361
|
+
function computeReading(answer) {
|
|
1362
|
+
if (answer.form === "noul") {
|
|
1363
|
+
const yes = answer.noul > .5;
|
|
1364
|
+
return {
|
|
1365
|
+
winner: yes ? "true" : "false",
|
|
1366
|
+
probability: yes ? answer.noul : 1 - answer.noul,
|
|
1367
|
+
confidence: Math.abs(2 * answer.noul - 1)
|
|
1368
|
+
};
|
|
1369
|
+
}
|
|
1370
|
+
const candidates = answer.form === "choice" ? Object.entries(answer.probabilities) : answer.probabilities.map((probability, level) => [String(level), probability]);
|
|
1371
|
+
const count = candidates.length;
|
|
1372
|
+
if (count < 2) throw new JudgeError("PROTOCOL", "judge error: an answer needs at least 2 candidates");
|
|
1373
|
+
let winner = "";
|
|
1374
|
+
let index = 0;
|
|
1375
|
+
let probability = -Infinity;
|
|
1376
|
+
for (const [position, [name, candidate]] of candidates.entries()) if (candidate > probability) {
|
|
1377
|
+
winner = name;
|
|
1378
|
+
index = position;
|
|
1379
|
+
probability = candidate;
|
|
1380
|
+
}
|
|
1381
|
+
if (answer.form === "choice") return {
|
|
1382
|
+
winner,
|
|
1383
|
+
probability,
|
|
1384
|
+
confidence: (probability - 1 / count) / (1 - 1 / count)
|
|
1385
|
+
};
|
|
1386
|
+
let spread = 0;
|
|
1387
|
+
let even = 0;
|
|
1388
|
+
let score = 0;
|
|
1389
|
+
for (const [level, candidate] of answer.probabilities.entries()) {
|
|
1390
|
+
spread += candidate * Math.abs(level - index);
|
|
1391
|
+
even += Math.abs(level - (count - 1) / 2);
|
|
1392
|
+
score += level * candidate;
|
|
1393
|
+
}
|
|
1394
|
+
return {
|
|
1395
|
+
winner,
|
|
1396
|
+
probability,
|
|
1397
|
+
confidence: Math.max(0, 1 - spread / (even / count)),
|
|
1398
|
+
score
|
|
1399
|
+
};
|
|
1400
|
+
}
|
|
1401
|
+
/**
|
|
1402
|
+
* Merges the results of a judge request's calls into one result.
|
|
1403
|
+
*
|
|
1404
|
+
* @remarks
|
|
1405
|
+
* Pure and total. Answers and refusals are joined by question id; `refusals` is omitted when no
|
|
1406
|
+
* call refused a question. Each call's usage passes through {@link sanitizeUsage} and the totals
|
|
1407
|
+
* add through {@link sumUsage}; `usage` is omitted when no call reported one. The model is the
|
|
1408
|
+
* first call's, or the given model when the list is empty, as the empty partial of a cancel
|
|
1409
|
+
* before the first call requires.
|
|
1410
|
+
*
|
|
1411
|
+
* @param model - The judge's configured model, reported when no call completed
|
|
1412
|
+
* @param results - The completed calls' results in call order
|
|
1413
|
+
* @returns The merged result
|
|
1414
|
+
* @example
|
|
1415
|
+
* ```ts
|
|
1416
|
+
* buildJudgeResult('tev1:0.8b', []) // { model: 'tev1:0.8b', answers: {} }
|
|
1417
|
+
* buildJudgeResult('jev-latest', [
|
|
1418
|
+
* { model: 'jev-1.13.0', answers: { urgent: { form: 'noul', noul: 0.95 } } },
|
|
1419
|
+
* { model: 'jev-1.13.0', answers: {}, refusals: { team: { missing: ['sales'] } } },
|
|
1420
|
+
* ])
|
|
1421
|
+
* // { model: 'jev-1.13.0', answers: { urgent: ... }, refusals: { team: { missing: ['sales'] } } }
|
|
1422
|
+
* ```
|
|
1423
|
+
*/
|
|
1424
|
+
function buildJudgeResult(model, results) {
|
|
1425
|
+
let answers = {};
|
|
1426
|
+
let refusals = {};
|
|
1427
|
+
let usage;
|
|
1428
|
+
for (const result of results) {
|
|
1429
|
+
answers = {
|
|
1430
|
+
...answers,
|
|
1431
|
+
...result.answers
|
|
1432
|
+
};
|
|
1433
|
+
refusals = {
|
|
1434
|
+
...refusals,
|
|
1435
|
+
...result.refusals
|
|
1436
|
+
};
|
|
1437
|
+
if (result.usage !== void 0) usage = sumUsage(usage, sanitizeUsage(result.usage));
|
|
1438
|
+
}
|
|
1439
|
+
return {
|
|
1440
|
+
model: results[0]?.model ?? model,
|
|
1441
|
+
answers,
|
|
1442
|
+
...Object.keys(refusals).length === 0 ? {} : { refusals },
|
|
1443
|
+
...usage === void 0 ? {} : { usage }
|
|
1444
|
+
};
|
|
1445
|
+
}
|
|
1446
|
+
/**
|
|
1447
|
+
* Builds a request's JSON headers, awaiting the caller's header hook inside the call's
|
|
1448
|
+
* cancellation bound.
|
|
1449
|
+
*
|
|
1450
|
+
* @remarks
|
|
1451
|
+
* The headers start with `Content-Type: application/json`; each entry the hook returns is set
|
|
1452
|
+
* over them, so the hook overrides the content type only when it returns that header. The hook
|
|
1453
|
+
* receives `signal` and races its abort, and the abort listener is removed on every exit. Both
|
|
1454
|
+
* HTTP engines, the provider and the judge, read their headers here.
|
|
1455
|
+
*
|
|
1456
|
+
* @param hook - The caller's header hook, or undefined for the JSON content type alone
|
|
1457
|
+
* @param signal - The call's combined caller and deadline signal
|
|
1458
|
+
* @returns The request headers
|
|
1459
|
+
* @throws Thrown when the signal aborts before the hook settles, with the signal's reason, and
|
|
1460
|
+
* when the hook throws or rejects, with the hook's own failure
|
|
1461
|
+
* @example
|
|
1462
|
+
* ```ts
|
|
1463
|
+
* const signal = new AbortController().signal
|
|
1464
|
+
* const headers = await readHeaders(() => ({ authorization: 'Bearer KEY' }), signal)
|
|
1465
|
+
* headers.get('authorization') // 'Bearer KEY'
|
|
1466
|
+
* headers.get('content-type') // 'application/json'
|
|
1467
|
+
* ```
|
|
1468
|
+
*/
|
|
1469
|
+
async function readHeaders(hook, signal) {
|
|
1470
|
+
const headers = new Headers({ "Content-Type": "application/json" });
|
|
1471
|
+
if (hook === void 0) return headers;
|
|
1472
|
+
const cleanup = new AbortController();
|
|
1473
|
+
const aborted = Promise.withResolvers();
|
|
1474
|
+
signal.addEventListener("abort", () => aborted.reject(signal.reason), {
|
|
1475
|
+
once: true,
|
|
1476
|
+
signal: cleanup.signal
|
|
1477
|
+
});
|
|
1478
|
+
try {
|
|
1479
|
+
signal.throwIfAborted();
|
|
1480
|
+
const entries = await Promise.race([Promise.resolve().then(() => hook(signal)), aborted.promise]);
|
|
1481
|
+
for (const [key, value] of Object.entries(entries)) headers.set(key, value);
|
|
1482
|
+
return headers;
|
|
1483
|
+
} finally {
|
|
1484
|
+
cleanup.abort();
|
|
1485
|
+
}
|
|
1486
|
+
}
|
|
1487
|
+
/**
|
|
1488
|
+
* Owns a value by serializing it to JSON and parsing the text, so the copy shares nothing with its source.
|
|
1489
|
+
*
|
|
1490
|
+
* @remarks
|
|
1491
|
+
* A value that JSON cannot carry, such as a cycle or a bigint, and a value that serializes to
|
|
1492
|
+
* nothing, such as a function, both return undefined, so a guard over the result refuses them.
|
|
1493
|
+
* A proxied value serializes through its traps, where a structured clone refuses it.
|
|
1494
|
+
*
|
|
1495
|
+
* @param value - The value to own
|
|
1496
|
+
* @returns The owned JSON copy, or undefined when the value is not JSON
|
|
1497
|
+
* @example
|
|
1498
|
+
* ```ts
|
|
1499
|
+
* copyJSON({ state: 'A ticket.' }) // { state: 'A ticket.' }
|
|
1500
|
+
* copyJSON(() => 1) // undefined
|
|
1501
|
+
* ```
|
|
1502
|
+
*/
|
|
1503
|
+
function copyJSON(value) {
|
|
1504
|
+
const text = (0, _orkestrel_contract.attempt)(() => JSON.stringify(value));
|
|
1505
|
+
if (!text.success || typeof text.value !== "string") return void 0;
|
|
1506
|
+
return (0, _orkestrel_contract.parseJSON)(text.value);
|
|
1507
|
+
}
|
|
1508
|
+
/**
|
|
1509
|
+
* Projects a judge question onto the System One wire while preserving omitted members.
|
|
1510
|
+
*
|
|
1511
|
+
* @param question - The domain question to send
|
|
1512
|
+
* @returns The wire question with `type` replacing `form`
|
|
1513
|
+
* @example
|
|
1514
|
+
* ```ts
|
|
1515
|
+
* questionToSystemOne({ form: 'noul' }) // { type: 'noul' }
|
|
1516
|
+
* ```
|
|
1517
|
+
*/
|
|
1518
|
+
function questionToSystemOne(question) {
|
|
1519
|
+
return {
|
|
1520
|
+
type: question.form,
|
|
1521
|
+
...question.instructions === void 0 ? {} : { instructions: question.instructions },
|
|
1522
|
+
...question.criteria === void 0 ? {} : { criteria: question.criteria }
|
|
1523
|
+
};
|
|
1524
|
+
}
|
|
1525
|
+
/**
|
|
1526
|
+
* Extracts a System One distribution in question criteria order and drops server measures.
|
|
1527
|
+
*
|
|
1528
|
+
* @remarks
|
|
1529
|
+
* Every requested candidate must have a finite probability in [0, 1]; the helper checks each
|
|
1530
|
+
* requested candidate itself and reads no other member. Score maps and arrays project onto the
|
|
1531
|
+
* requested levels. Additional candidates are ignored, sums are unconstrained, and probabilities
|
|
1532
|
+
* are preserved without normalization.
|
|
1533
|
+
*
|
|
1534
|
+
* @param answer - The wire answer to decode
|
|
1535
|
+
* @param question - The question defining the form and candidate order
|
|
1536
|
+
* @returns The domain answer, or undefined for a mismatched form or a requested candidate whose
|
|
1537
|
+
* probability is missing, not finite, or outside [0, 1]
|
|
1538
|
+
* @example
|
|
1539
|
+
* ```ts
|
|
1540
|
+
* extractSystemOneAnswer({ type: 'noul', noul: 0.9 }, { form: 'noul' })
|
|
1541
|
+
* // { form: 'noul', noul: 0.9 }
|
|
1542
|
+
* ```
|
|
1543
|
+
*/
|
|
1544
|
+
function extractSystemOneAnswer(answer, question) {
|
|
1545
|
+
const bounded = (0, _orkestrel_contract.boundsOf)(0, 1);
|
|
1546
|
+
if (answer.type === "noul" && question.form === "noul") return bounded(answer.noul) ? {
|
|
1547
|
+
form: "noul",
|
|
1548
|
+
noul: answer.noul
|
|
1549
|
+
} : void 0;
|
|
1550
|
+
if (answer.type === "choice" && question.form === "choice") {
|
|
1551
|
+
const entries = [];
|
|
1552
|
+
for (const label of Object.keys(question.criteria)) {
|
|
1553
|
+
if (!Object.hasOwn(answer.probabilities, label)) return void 0;
|
|
1554
|
+
const probability = answer.probabilities[label];
|
|
1555
|
+
if (!bounded(probability)) return void 0;
|
|
1556
|
+
entries.push([label, probability]);
|
|
1557
|
+
}
|
|
1558
|
+
return {
|
|
1559
|
+
form: "choice",
|
|
1560
|
+
probabilities: Object.fromEntries(entries)
|
|
1561
|
+
};
|
|
1562
|
+
}
|
|
1563
|
+
if (answer.type !== "score" || question.form !== "score") return void 0;
|
|
1564
|
+
const probabilities = [];
|
|
1565
|
+
for (let level = 0; level < question.criteria.length; level++) {
|
|
1566
|
+
if (!Object.hasOwn(answer.probabilities, level)) return void 0;
|
|
1567
|
+
const probability = (0, _orkestrel_contract.isArray)(answer.probabilities) ? answer.probabilities[level] : answer.probabilities[String(level)];
|
|
1568
|
+
if (!bounded(probability)) return void 0;
|
|
1569
|
+
probabilities.push(probability);
|
|
1570
|
+
}
|
|
1571
|
+
return {
|
|
1572
|
+
form: "score",
|
|
1573
|
+
probabilities
|
|
1574
|
+
};
|
|
1575
|
+
}
|
|
1576
|
+
/**
|
|
1577
|
+
* Maps complete System One token counts onto validated token usage.
|
|
1578
|
+
*
|
|
1579
|
+
* @param usage - The optional wire token counts
|
|
1580
|
+
* @returns Token usage, or undefined for missing, null, negative, or non-finite counts
|
|
1581
|
+
* @example
|
|
1582
|
+
* ```ts
|
|
1583
|
+
* extractSystemOneUsage({ input_tokens: 975, output_tokens: 4 })
|
|
1584
|
+
* // { prompt: 975, completion: 4, total: 979 }
|
|
1585
|
+
* ```
|
|
1586
|
+
*/
|
|
1587
|
+
function extractSystemOneUsage(usage) {
|
|
1588
|
+
const extracted = (0, _orkestrel_contract.attempt)(() => {
|
|
1589
|
+
if (usage === void 0) return void 0;
|
|
1590
|
+
const { input_tokens: prompt, output_tokens: completion } = usage;
|
|
1591
|
+
if (!(0, _orkestrel_contract.isNumber)(prompt) || !(0, _orkestrel_contract.isNumber)(completion)) return void 0;
|
|
1592
|
+
const result = {
|
|
1593
|
+
prompt,
|
|
1594
|
+
completion,
|
|
1595
|
+
total: prompt + completion
|
|
1596
|
+
};
|
|
1597
|
+
return (0, _orkestrel_budget.isTokenUsage)(result) ? result : void 0;
|
|
1598
|
+
});
|
|
1599
|
+
return extracted.success ? extracted.value : void 0;
|
|
1600
|
+
}
|
|
1263
1601
|
//#endregion
|
|
1264
1602
|
//#region src/core/conversations/Conversation.ts
|
|
1265
1603
|
/**
|
|
@@ -2190,7 +2528,7 @@ var Agent = class {
|
|
|
2190
2528
|
};
|
|
2191
2529
|
const toolMessage = this.#context.messages.add({
|
|
2192
2530
|
role: "tool",
|
|
2193
|
-
content: outcomeResult.success ? JSON.stringify(outcomeResult.value) : outcomeResult.error
|
|
2531
|
+
content: outcomeResult.success ? typeof outcomeResult.value === "string" ? outcomeResult.value : JSON.stringify(outcomeResult.value) : outcomeResult.error
|
|
2194
2532
|
});
|
|
2195
2533
|
messages.push(toolMessage);
|
|
2196
2534
|
}
|
|
@@ -2550,6 +2888,117 @@ function isConversationSnapshot(value) {
|
|
|
2550
2888
|
if (!(0, _orkestrel_contract.isArray)(value.sections) || !value.sections.every(isSection)) return false;
|
|
2551
2889
|
return (0, _orkestrel_contract.isArray)(value.messages) && value.messages.every(isMessage);
|
|
2552
2890
|
}
|
|
2891
|
+
/**
|
|
2892
|
+
* Checks whether a value is a judge entry: a string, a JSON record, or a JSON array.
|
|
2893
|
+
*
|
|
2894
|
+
* @remarks
|
|
2895
|
+
* Total: a cycle, a non-JSON member, a class instance, and a hostile input return false. `null`
|
|
2896
|
+
* is not an entry; a criteria guard admits it where the protocol keeps it.
|
|
2897
|
+
*
|
|
2898
|
+
* @param value - The unknown entry candidate
|
|
2899
|
+
* @returns True if the value is a string or JSON structure the model can read; false otherwise
|
|
2900
|
+
* @example
|
|
2901
|
+
* ```ts
|
|
2902
|
+
* isJudgeEntry('Customer asks for a refund') // true
|
|
2903
|
+
* isJudgeEntry({ ticket: 4182, tags: ['billing'] }) // true
|
|
2904
|
+
* isJudgeEntry(null) // false
|
|
2905
|
+
* isJudgeEntry({ opened: new Date() }) // false
|
|
2906
|
+
* ```
|
|
2907
|
+
*/
|
|
2908
|
+
function isJudgeEntry(value) {
|
|
2909
|
+
if ((0, _orkestrel_contract.isString)(value)) return true;
|
|
2910
|
+
return ((0, _orkestrel_contract.isRecord)(value) || (0, _orkestrel_contract.isArray)(value)) && (0, _orkestrel_contract.isJSONValue)(value);
|
|
2911
|
+
}
|
|
2912
|
+
/**
|
|
2913
|
+
* Checks whether a value is a well-formed judge question of the choice, score, or noul form.
|
|
2914
|
+
*
|
|
2915
|
+
* @remarks
|
|
2916
|
+
* Total: a hostile input returns false. A choice needs at least 2 options and a score at least 2
|
|
2917
|
+
* levels, because the confidence formula divides by the candidate count; a description or a level
|
|
2918
|
+
* can be `null`. `instructions` and noul `criteria` are omitted when absent and never `null`.
|
|
2919
|
+
* Server limits on option, level, and question counts are left to the server.
|
|
2920
|
+
*
|
|
2921
|
+
* @param value - The unknown question candidate
|
|
2922
|
+
* @returns True if the value is a question the judge engine can send; false otherwise
|
|
2923
|
+
* @example
|
|
2924
|
+
* ```ts
|
|
2925
|
+
* isJudgeQuestion({ form: 'choice', criteria: { billing: null, bug: 'Software defect' } }) // true
|
|
2926
|
+
* isJudgeQuestion({ form: 'score', criteria: ['Cosmetic', null, 'Blocking'] }) // true
|
|
2927
|
+
* isJudgeQuestion({ form: 'noul', instructions: 'Is a refund owed?' }) // true
|
|
2928
|
+
* isJudgeQuestion({ form: 'choice', criteria: { billing: null } }) // false
|
|
2929
|
+
* ```
|
|
2930
|
+
*/
|
|
2931
|
+
function isJudgeQuestion(value) {
|
|
2932
|
+
const checked = (0, _orkestrel_contract.attempt)(() => {
|
|
2933
|
+
if (!(0, _orkestrel_contract.isRecord)(value)) return false;
|
|
2934
|
+
const { form, instructions, criteria } = value;
|
|
2935
|
+
if (instructions !== void 0 && !isJudgeEntry(instructions)) return false;
|
|
2936
|
+
if (form === "choice") return (0, _orkestrel_contract.isRecord)(criteria) && Object.keys(criteria).length >= 2 && Object.values(criteria).every((0, _orkestrel_contract.nullableOf)(isJudgeEntry));
|
|
2937
|
+
if (form === "score") return (0, _orkestrel_contract.arrayOf)((0, _orkestrel_contract.nullableOf)(isJudgeEntry))(criteria) && criteria.length >= 2;
|
|
2938
|
+
if (form !== "noul") return false;
|
|
2939
|
+
if (criteria === void 0) return true;
|
|
2940
|
+
if (!(0, _orkestrel_contract.isRecord)(criteria)) return false;
|
|
2941
|
+
return (0, _orkestrel_contract.optionalOf)(isJudgeEntry)(criteria.true) && (0, _orkestrel_contract.optionalOf)(isJudgeEntry)(criteria.false);
|
|
2942
|
+
});
|
|
2943
|
+
return checked.success && checked.value;
|
|
2944
|
+
}
|
|
2945
|
+
/**
|
|
2946
|
+
* Checks whether a value is a System One response envelope with optional model and usage.
|
|
2947
|
+
*
|
|
2948
|
+
* @remarks
|
|
2949
|
+
* Answers remain unknown until checked against their questions. Missing and null usage counts
|
|
2950
|
+
* are accepted. Extra members are ignored, and unreadable fields return false.
|
|
2951
|
+
*
|
|
2952
|
+
* @param value - The unknown response candidate
|
|
2953
|
+
* @returns True if the envelope fields have their wire types; false otherwise
|
|
2954
|
+
* @example
|
|
2955
|
+
* ```ts
|
|
2956
|
+
* isSystemOneResponse({ answers: {}, usage: { input_tokens: null } }) // true
|
|
2957
|
+
* ```
|
|
2958
|
+
*/
|
|
2959
|
+
function isSystemOneResponse(value) {
|
|
2960
|
+
const checked = (0, _orkestrel_contract.attempt)(() => {
|
|
2961
|
+
if (!(0, _orkestrel_contract.isRecord)(value)) return false;
|
|
2962
|
+
const { model, answers, usage } = value;
|
|
2963
|
+
if (!(0, _orkestrel_contract.optionalOf)(_orkestrel_contract.isString)(model) || !(0, _orkestrel_contract.isRecord)(answers)) return false;
|
|
2964
|
+
if (usage === void 0) return true;
|
|
2965
|
+
if (!(0, _orkestrel_contract.isRecord)(usage)) return false;
|
|
2966
|
+
return (0, _orkestrel_contract.optionalOf)((0, _orkestrel_contract.nullableOf)(_orkestrel_contract.isNumber))(usage.input_tokens) && (0, _orkestrel_contract.optionalOf)((0, _orkestrel_contract.nullableOf)(_orkestrel_contract.isNumber))(usage.output_tokens);
|
|
2967
|
+
});
|
|
2968
|
+
return checked.success && checked.value;
|
|
2969
|
+
}
|
|
2970
|
+
/**
|
|
2971
|
+
* Checks whether a value is a System One answer whose type and distribution the wire can read.
|
|
2972
|
+
*
|
|
2973
|
+
* @remarks
|
|
2974
|
+
* The guard checks `type` and the distribution the wire dereferences: the `noul` number for a
|
|
2975
|
+
* noul, the `probabilities` map for a choice, and the `probabilities` map or dense array for a
|
|
2976
|
+
* score, each probability a finite number in [0, 1]. The `choice`, `score`, `confidence`, and
|
|
2977
|
+
* `legend` members are carried unchecked, because the wire never reads them, so a value this
|
|
2978
|
+
* guard accepts can hold any value in those members. Distribution sums are not constrained;
|
|
2979
|
+
* values are never normalized. Hostile reads return false.
|
|
2980
|
+
*
|
|
2981
|
+
* @param value - The unknown answer candidate
|
|
2982
|
+
* @returns True if the type and the distribution satisfy the wire contract; false otherwise
|
|
2983
|
+
* @example
|
|
2984
|
+
* ```ts
|
|
2985
|
+
* isSystemOneAnswer({ type: 'noul', noul: 0.9 }) // true
|
|
2986
|
+
* isSystemOneAnswer({ type: 'score', probabilities: [0.2, 0.8] }) // true
|
|
2987
|
+
* isSystemOneAnswer({ type: 'noul', noul: 1.1 }) // false
|
|
2988
|
+
* ```
|
|
2989
|
+
*/
|
|
2990
|
+
function isSystemOneAnswer(value) {
|
|
2991
|
+
const checked = (0, _orkestrel_contract.attempt)(() => {
|
|
2992
|
+
if (!(0, _orkestrel_contract.isRecord)(value)) return false;
|
|
2993
|
+
const { type } = value;
|
|
2994
|
+
if (type === "noul") return (0, _orkestrel_contract.boundsOf)(0, 1)(value.noul);
|
|
2995
|
+
if (type !== "choice" && type !== "score") return false;
|
|
2996
|
+
const { probabilities } = value;
|
|
2997
|
+
if (type === "score" && (0, _orkestrel_contract.isArray)(probabilities)) return (0, _orkestrel_contract.arrayOf)((0, _orkestrel_contract.boundsOf)(0, 1))(probabilities);
|
|
2998
|
+
return (0, _orkestrel_contract.isRecord)(probabilities) && Object.values(probabilities).every((0, _orkestrel_contract.boundsOf)(0, 1));
|
|
2999
|
+
});
|
|
3000
|
+
return checked.success && checked.value;
|
|
3001
|
+
}
|
|
2553
3002
|
//#endregion
|
|
2554
3003
|
//#region src/core/conversations/stores/DatabaseConversationStore.ts
|
|
2555
3004
|
/**
|
|
@@ -2901,7 +3350,7 @@ var AgentProvider = class {
|
|
|
2901
3350
|
};
|
|
2902
3351
|
}
|
|
2903
3352
|
async #request(request, signal) {
|
|
2904
|
-
const headers = await this.#
|
|
3353
|
+
const headers = await readHeaders(this.#headers, signal);
|
|
2905
3354
|
signal.throwIfAborted();
|
|
2906
3355
|
const response = await this.#transport(this.#url + this.#path, {
|
|
2907
3356
|
method: "POST",
|
|
@@ -2924,25 +3373,6 @@ var AgentProvider = class {
|
|
|
2924
3373
|
}
|
|
2925
3374
|
return response;
|
|
2926
3375
|
}
|
|
2927
|
-
async #requestHeaders(signal) {
|
|
2928
|
-
const headers = new Headers({ "Content-Type": "application/json" });
|
|
2929
|
-
if (this.#headers === void 0) return headers;
|
|
2930
|
-
const cleanup = new AbortController();
|
|
2931
|
-
const aborted = Promise.withResolvers();
|
|
2932
|
-
signal.addEventListener("abort", () => aborted.reject(signal.reason), {
|
|
2933
|
-
once: true,
|
|
2934
|
-
signal: cleanup.signal
|
|
2935
|
-
});
|
|
2936
|
-
try {
|
|
2937
|
-
signal.throwIfAborted();
|
|
2938
|
-
const hook = this.#headers;
|
|
2939
|
-
const entries = await Promise.race([Promise.resolve().then(() => hook(signal)), aborted.promise]);
|
|
2940
|
-
for (const [key, value] of Object.entries(entries)) headers.set(key, value);
|
|
2941
|
-
return headers;
|
|
2942
|
-
} finally {
|
|
2943
|
-
cleanup.abort();
|
|
2944
|
-
}
|
|
2945
|
-
}
|
|
2946
3376
|
};
|
|
2947
3377
|
//#endregion
|
|
2948
3378
|
//#region src/core/providers/RelayProvider.ts
|
|
@@ -3089,6 +3519,258 @@ var RelayProvider = class extends AgentProvider {
|
|
|
3089
3519
|
}
|
|
3090
3520
|
};
|
|
3091
3521
|
//#endregion
|
|
3522
|
+
//#region src/core/AgentJudge.ts
|
|
3523
|
+
/**
|
|
3524
|
+
* Implements the bounded HTTP calls, validation, and result merging of a judge behind concrete
|
|
3525
|
+
* wire seams.
|
|
3526
|
+
*
|
|
3527
|
+
* @remarks
|
|
3528
|
+
* A subclass fills `name`, `body`, and `read`; the constructor takes the `batch` switch that
|
|
3529
|
+
* decides whether one call carries every question or each question gets its own call in key
|
|
3530
|
+
* order. `ask` validates the request before any call, builds every body before the first call so
|
|
3531
|
+
* a wire limit refuses the request before inference, and runs the calls one after another, each
|
|
3532
|
+
* under its own deadline folded with the caller's signal. A cancel throws a `JudgeAbortError`
|
|
3533
|
+
* whose `partial` merges the calls that completed. A transport error reaches the caller unchanged.
|
|
3534
|
+
*
|
|
3535
|
+
* @example Writing a judge wire
|
|
3536
|
+
* ```ts
|
|
3537
|
+
* import type { JudgeRequest, JudgeResult } from '@orkestrel/agent'
|
|
3538
|
+
* import { AgentJudge, JudgeError } from '@orkestrel/agent'
|
|
3539
|
+
* import { isFiniteNumber, isRecord } from '@orkestrel/contract'
|
|
3540
|
+
*
|
|
3541
|
+
* // A wire whose server answers one yes/no question per call as { "yes": 0.93 }.
|
|
3542
|
+
* class YesJudge extends AgentJudge {
|
|
3543
|
+
* readonly name = 'yes'
|
|
3544
|
+
* body(request: JudgeRequest): object {
|
|
3545
|
+
* return { model: this.model, state: request.state, questions: request.questions }
|
|
3546
|
+
* }
|
|
3547
|
+
* read(value: unknown, request: JudgeRequest): JudgeResult {
|
|
3548
|
+
* const [id] = Object.keys(request.questions)
|
|
3549
|
+
* if (id === undefined || !isRecord(value) || !isFiniteNumber(value.yes)) {
|
|
3550
|
+
* throw new JudgeError('PROTOCOL', 'judge error: unreadable answer')
|
|
3551
|
+
* }
|
|
3552
|
+
* return { model: this.model, answers: { [id]: { form: 'noul', noul: value.yes } } }
|
|
3553
|
+
* }
|
|
3554
|
+
* }
|
|
3555
|
+
*
|
|
3556
|
+
* const judge = new YesJudge({
|
|
3557
|
+
* url: 'http://localhost:8010',
|
|
3558
|
+
* path: '/v1/yes',
|
|
3559
|
+
* model: 'yes-1',
|
|
3560
|
+
* batch: false,
|
|
3561
|
+
* })
|
|
3562
|
+
* ```
|
|
3563
|
+
*/
|
|
3564
|
+
var AgentJudge = class {
|
|
3565
|
+
#id;
|
|
3566
|
+
#url;
|
|
3567
|
+
#path;
|
|
3568
|
+
#model;
|
|
3569
|
+
#timeout;
|
|
3570
|
+
#transport;
|
|
3571
|
+
#headers;
|
|
3572
|
+
#batch;
|
|
3573
|
+
constructor(input) {
|
|
3574
|
+
this.#id = crypto.randomUUID();
|
|
3575
|
+
this.#url = input.url;
|
|
3576
|
+
this.#path = input.path ?? "";
|
|
3577
|
+
this.#model = input.model;
|
|
3578
|
+
this.#timeout = input.timeout ?? 12e4;
|
|
3579
|
+
this.#transport = input.fetch ?? globalThis.fetch.bind(globalThis);
|
|
3580
|
+
this.#headers = input.headers;
|
|
3581
|
+
this.#batch = input.batch ?? true;
|
|
3582
|
+
}
|
|
3583
|
+
/** Exposes the instance's minted UUID. */
|
|
3584
|
+
get id() {
|
|
3585
|
+
return this.#id;
|
|
3586
|
+
}
|
|
3587
|
+
/** Exposes the configured model identity. */
|
|
3588
|
+
get model() {
|
|
3589
|
+
return this.#model;
|
|
3590
|
+
}
|
|
3591
|
+
/**
|
|
3592
|
+
* Asks every question of the request about its state and merges the answers of every call.
|
|
3593
|
+
*
|
|
3594
|
+
* @param request - The state and the questions keyed by caller id
|
|
3595
|
+
* @param signal - The caller's cancellation bound
|
|
3596
|
+
* @returns The merged answers, refusals, and usage of every call
|
|
3597
|
+
* @throws JudgeAbortError Thrown when the caller's signal or a call's deadline fires, carrying
|
|
3598
|
+
* the merged result of the completed calls
|
|
3599
|
+
* @throws JudgeError Thrown with code `QUESTION` for an empty question map, a malformed question
|
|
3600
|
+
* or state, or a wire refusal before any call; with code `HTTP` for a non-OK response; and with
|
|
3601
|
+
* code `PROTOCOL` for a missing or unparsable response body
|
|
3602
|
+
*/
|
|
3603
|
+
async ask(request, signal) {
|
|
3604
|
+
if (signal.aborted) throw new JudgeAbortError(buildJudgeResult(this.#model, []));
|
|
3605
|
+
const state = copyJSON(request.state);
|
|
3606
|
+
if (!isJudgeEntry(state)) throw new JudgeError("QUESTION", "judge error: state is not a judge entry");
|
|
3607
|
+
const questions = (0, _orkestrel_contract.isRecord)(request.questions) ? Object.entries(request.questions) : [];
|
|
3608
|
+
if (questions.length === 0) throw new JudgeError("QUESTION", "judge error: no questions");
|
|
3609
|
+
const entries = [];
|
|
3610
|
+
for (const [id, question] of questions) {
|
|
3611
|
+
const copy = copyJSON(question);
|
|
3612
|
+
if (!isJudgeQuestion(copy)) throw new JudgeError("QUESTION", `judge error: question ${id} is malformed`);
|
|
3613
|
+
entries.push([id, copy]);
|
|
3614
|
+
}
|
|
3615
|
+
const owned = {
|
|
3616
|
+
state,
|
|
3617
|
+
questions: Object.fromEntries(entries)
|
|
3618
|
+
};
|
|
3619
|
+
const calls = (this.#batch ? [owned] : entries.map(([id, question]) => ({
|
|
3620
|
+
state,
|
|
3621
|
+
questions: { [id]: question }
|
|
3622
|
+
}))).map((part) => ({
|
|
3623
|
+
request: part,
|
|
3624
|
+
body: JSON.stringify(this.body(part))
|
|
3625
|
+
}));
|
|
3626
|
+
const results = [];
|
|
3627
|
+
for (const call of calls) results.push(await this.#call(call.request, call.body, signal, results));
|
|
3628
|
+
return buildJudgeResult(this.#model, results);
|
|
3629
|
+
}
|
|
3630
|
+
async #call(request, body, signal, completed) {
|
|
3631
|
+
const timeout = new _orkestrel_timeout.Timeout({ ms: this.#timeout });
|
|
3632
|
+
timeout.start();
|
|
3633
|
+
const combined = AbortSignal.any([timeout.signal, signal]);
|
|
3634
|
+
try {
|
|
3635
|
+
const headers = await readHeaders(this.#headers, combined);
|
|
3636
|
+
combined.throwIfAborted();
|
|
3637
|
+
const response = await this.#transport(this.#url + this.#path, {
|
|
3638
|
+
method: "POST",
|
|
3639
|
+
headers,
|
|
3640
|
+
body,
|
|
3641
|
+
signal: combined
|
|
3642
|
+
});
|
|
3643
|
+
if (!response.ok) {
|
|
3644
|
+
let detail;
|
|
3645
|
+
try {
|
|
3646
|
+
detail = response.body === null ? "" : (await readText(response.body, MAX_ERROR_BODY_LENGTH, combined)).text;
|
|
3647
|
+
} catch (cause) {
|
|
3648
|
+
throw new JudgeError("HTTP", `judge error: ${response.status} - (error body unavailable)`, {
|
|
3649
|
+
status: response.status,
|
|
3650
|
+
cause
|
|
3651
|
+
});
|
|
3652
|
+
}
|
|
3653
|
+
combined.throwIfAborted();
|
|
3654
|
+
throw new JudgeError("HTTP", `judge error: ${response.status}${detail === "" ? "" : ` - ${detail}`}`, { status: response.status });
|
|
3655
|
+
}
|
|
3656
|
+
if (response.body === null) throw new JudgeError("PROTOCOL", "judge error: no response body");
|
|
3657
|
+
const text = await readText(response.body, void 0, combined);
|
|
3658
|
+
combined.throwIfAborted();
|
|
3659
|
+
const value = (0, _orkestrel_contract.parseJSON)(text.text);
|
|
3660
|
+
if (value === void 0) throw new JudgeError("PROTOCOL", "judge error: invalid JSON body");
|
|
3661
|
+
const result = this.read(value, request);
|
|
3662
|
+
combined.throwIfAborted();
|
|
3663
|
+
return result;
|
|
3664
|
+
} catch (error) {
|
|
3665
|
+
if (combined.aborted) throw new JudgeAbortError(buildJudgeResult(this.#model, completed), error === combined.reason ? void 0 : { cause: error });
|
|
3666
|
+
throw error;
|
|
3667
|
+
} finally {
|
|
3668
|
+
timeout.clear();
|
|
3669
|
+
}
|
|
3670
|
+
}
|
|
3671
|
+
};
|
|
3672
|
+
//#endregion
|
|
3673
|
+
//#region src/core/providers/SystemOneJudge.ts
|
|
3674
|
+
/**
|
|
3675
|
+
* Carries judge questions over the System One protocol and derives answers from server distributions.
|
|
3676
|
+
*
|
|
3677
|
+
* @remarks
|
|
3678
|
+
* The caller supplies the server origin and model. Every question travels in one request.
|
|
3679
|
+
* Server measures are ignored; the response model is preserved.
|
|
3680
|
+
* `headers` supplies authentication through the shared judge engine.
|
|
3681
|
+
*
|
|
3682
|
+
* @example Asking a System One server a choice, a noul, and a score
|
|
3683
|
+
* ```ts
|
|
3684
|
+
* import { computeReading, createSystemOneJudge } from '@orkestrel/agent'
|
|
3685
|
+
*
|
|
3686
|
+
* const judge = createSystemOneJudge({ url: 'http://localhost:11434', model: 'tev1:0.8b' })
|
|
3687
|
+
* const result = await judge.ask(
|
|
3688
|
+
* {
|
|
3689
|
+
* state: 'Our checkout has returned 500 errors since 9am. I want a refund for today.',
|
|
3690
|
+
* questions: {
|
|
3691
|
+
* label: {
|
|
3692
|
+
* form: 'choice',
|
|
3693
|
+
* instructions: 'Which label fits this ticket?',
|
|
3694
|
+
* criteria: { billing: 'Payments and refunds', bug: 'Software errors', account: null },
|
|
3695
|
+
* },
|
|
3696
|
+
* refund: {
|
|
3697
|
+
* form: 'noul',
|
|
3698
|
+
* instructions: 'Does the customer ask for money back?',
|
|
3699
|
+
* criteria: {
|
|
3700
|
+
* true: 'The customer asks for a refund or for money back.',
|
|
3701
|
+
* false: 'The customer does not ask for money back.',
|
|
3702
|
+
* },
|
|
3703
|
+
* },
|
|
3704
|
+
* severity: {
|
|
3705
|
+
* form: 'score',
|
|
3706
|
+
* instructions: 'How severe is the reported issue?',
|
|
3707
|
+
* criteria: ['Cosmetic; no impact', 'Degraded, workaround exists', 'Blocking; no workaround'],
|
|
3708
|
+
* },
|
|
3709
|
+
* },
|
|
3710
|
+
* },
|
|
3711
|
+
* AbortSignal.timeout(30_000),
|
|
3712
|
+
* )
|
|
3713
|
+
* const readings = Object.fromEntries(
|
|
3714
|
+
* Object.entries(result.answers).map(([id, answer]) => [id, computeReading(answer)]),
|
|
3715
|
+
* )
|
|
3716
|
+
* result.model // 'tev1:0.8b' — the model the server named
|
|
3717
|
+
* result.usage // { prompt: 975, completion: 4, total: 979 }
|
|
3718
|
+
* readings.label // { winner: 'bug', probability: 0.9691, confidence: 0.9536 } to four decimals
|
|
3719
|
+
* readings.refund // { winner: 'true', probability: 0.9979, confidence: 0.9958 } to four decimals
|
|
3720
|
+
* readings.severity // { winner: '1', probability: 0.9494, confidence: 0.9241, score: 0.9919 } to four decimals
|
|
3721
|
+
* ```
|
|
3722
|
+
*/
|
|
3723
|
+
var SystemOneJudge = class extends AgentJudge {
|
|
3724
|
+
constructor(options) {
|
|
3725
|
+
super({
|
|
3726
|
+
...options,
|
|
3727
|
+
path: SYSTEM_ONE_PATH,
|
|
3728
|
+
batch: true
|
|
3729
|
+
});
|
|
3730
|
+
}
|
|
3731
|
+
/** Identifies the System One protocol. */
|
|
3732
|
+
name = "systemone";
|
|
3733
|
+
/**
|
|
3734
|
+
* Projects the state and questions onto the System One request with the configured model.
|
|
3735
|
+
*
|
|
3736
|
+
* @param request - The state and questions keyed by caller id
|
|
3737
|
+
* @returns The System One request body
|
|
3738
|
+
*/
|
|
3739
|
+
body(request) {
|
|
3740
|
+
return {
|
|
3741
|
+
state: request.state,
|
|
3742
|
+
model: this.model,
|
|
3743
|
+
questions: Object.fromEntries(Object.entries(request.questions).map(([id, question]) => [id, questionToSystemOne(question)]))
|
|
3744
|
+
};
|
|
3745
|
+
}
|
|
3746
|
+
/**
|
|
3747
|
+
* Decodes requested System One answers and reports the server model and available usage.
|
|
3748
|
+
*
|
|
3749
|
+
* @param value - The parsed response body
|
|
3750
|
+
* @param request - The questions defining the expected answers
|
|
3751
|
+
* @returns The decoded distributions, model, and available usage
|
|
3752
|
+
* @throws JudgeError Thrown with code `PROTOCOL` for a malformed envelope or an invalid answer naming its question id
|
|
3753
|
+
*/
|
|
3754
|
+
read(value, request) {
|
|
3755
|
+
if (!isSystemOneResponse(value)) throw new JudgeError("PROTOCOL", "judge error: invalid System One response envelope");
|
|
3756
|
+
const entries = [];
|
|
3757
|
+
for (const [id, question] of Object.entries(request.questions)) {
|
|
3758
|
+
if (!Object.hasOwn(value.answers, id)) throw new JudgeError("PROTOCOL", `judge error: question ${id} has no System One answer`);
|
|
3759
|
+
const wire = value.answers[id];
|
|
3760
|
+
if (!isSystemOneAnswer(wire)) throw new JudgeError("PROTOCOL", `judge error: question ${id} has an invalid System One answer`);
|
|
3761
|
+
const answer = extractSystemOneAnswer(wire, question);
|
|
3762
|
+
if (answer === void 0) throw new JudgeError("PROTOCOL", `judge error: question ${id} has a mismatched or incomplete System One answer`);
|
|
3763
|
+
entries.push([id, answer]);
|
|
3764
|
+
}
|
|
3765
|
+
const usage = extractSystemOneUsage(value.usage);
|
|
3766
|
+
return {
|
|
3767
|
+
model: value.model ?? this.model,
|
|
3768
|
+
answers: Object.fromEntries(entries),
|
|
3769
|
+
...usage === void 0 ? {} : { usage }
|
|
3770
|
+
};
|
|
3771
|
+
}
|
|
3772
|
+
};
|
|
3773
|
+
//#endregion
|
|
3092
3774
|
//#region src/core/RelayStream.ts
|
|
3093
3775
|
/**
|
|
3094
3776
|
* Streams a provider call as validated NDJSON frames under response backpressure.
|
|
@@ -4126,11 +4808,25 @@ function createAgentRunner(options) {
|
|
|
4126
4808
|
handler: handleAgentRunnerJob.bind(void 0, registry, partial)
|
|
4127
4809
|
});
|
|
4128
4810
|
}
|
|
4811
|
+
/**
|
|
4812
|
+
* Creates a judge that sends every question through the configured System One server.
|
|
4813
|
+
*
|
|
4814
|
+
* @param options - The server origin, model, and optional transport, headers, and timeout
|
|
4815
|
+
* @returns The configured judge behind its shared interface
|
|
4816
|
+
* @example
|
|
4817
|
+
* ```ts
|
|
4818
|
+
* const judge = createSystemOneJudge({ url: 'http://localhost:11434', model: 'tev1:0.8b' })
|
|
4819
|
+
* ```
|
|
4820
|
+
*/
|
|
4821
|
+
function createSystemOneJudge(options) {
|
|
4822
|
+
return new SystemOneJudge(options);
|
|
4823
|
+
}
|
|
4129
4824
|
//#endregion
|
|
4130
4825
|
exports.Agent = Agent;
|
|
4131
4826
|
exports.AgentContext = AgentContext;
|
|
4132
4827
|
exports.AgentError = AgentError;
|
|
4133
4828
|
exports.AgentJobError = AgentJobError;
|
|
4829
|
+
exports.AgentJudge = AgentJudge;
|
|
4134
4830
|
exports.AgentProvider = AgentProvider;
|
|
4135
4831
|
exports.AgentRegistry = AgentRegistry;
|
|
4136
4832
|
exports.Authority = Authority;
|
|
@@ -4149,6 +4845,8 @@ exports.IMAGE_TOKEN_ESTIMATE = IMAGE_TOKEN_ESTIMATE;
|
|
|
4149
4845
|
exports.INVALID_RELAY_STATUS = INVALID_RELAY_STATUS;
|
|
4150
4846
|
exports.Instruction = Instruction;
|
|
4151
4847
|
exports.InstructionManager = InstructionManager;
|
|
4848
|
+
exports.JudgeAbortError = JudgeAbortError;
|
|
4849
|
+
exports.JudgeError = JudgeError;
|
|
4152
4850
|
exports.MAX_ERROR_BODY_LENGTH = MAX_ERROR_BODY_LENGTH;
|
|
4153
4851
|
exports.MESSAGE_TOKEN_OVERHEAD = MESSAGE_TOKEN_OVERHEAD;
|
|
4154
4852
|
exports.MemoryConversationStore = MemoryConversationStore;
|
|
@@ -4159,8 +4857,10 @@ exports.RELAY_CONTENT_TYPE = RELAY_CONTENT_TYPE;
|
|
|
4159
4857
|
exports.RELAY_PROVIDER_MESSAGE = RELAY_PROVIDER_MESSAGE;
|
|
4160
4858
|
exports.RelayProvider = RelayProvider;
|
|
4161
4859
|
exports.RelayStream = RelayStream;
|
|
4860
|
+
exports.SYSTEM_ONE_PATH = SYSTEM_ONE_PATH;
|
|
4162
4861
|
exports.Scope = Scope;
|
|
4163
4862
|
exports.ScopeManager = ScopeManager;
|
|
4863
|
+
exports.SystemOneJudge = SystemOneJudge;
|
|
4164
4864
|
exports.THINK_CLOSE = THINK_CLOSE;
|
|
4165
4865
|
exports.THINK_OPEN = THINK_OPEN;
|
|
4166
4866
|
exports.ThinkSplitter = ThinkSplitter;
|
|
@@ -4171,10 +4871,13 @@ exports.agentResultToJSON = agentResultToJSON;
|
|
|
4171
4871
|
exports.assembleResult = assembleResult;
|
|
4172
4872
|
exports.attachImages = attachImages;
|
|
4173
4873
|
exports.attachUserImages = attachUserImages;
|
|
4874
|
+
exports.buildJudgeResult = buildJudgeResult;
|
|
4174
4875
|
exports.buildProviderResult = buildProviderResult;
|
|
4175
4876
|
exports.buildRecapMessage = buildRecapMessage;
|
|
4176
4877
|
exports.buildSummaryMessage = buildSummaryMessage;
|
|
4177
4878
|
exports.collectImageData = collectImageData;
|
|
4879
|
+
exports.computeReading = computeReading;
|
|
4880
|
+
exports.copyJSON = copyJSON;
|
|
4178
4881
|
exports.createAgent = createAgent;
|
|
4179
4882
|
exports.createAgentContext = createAgentContext;
|
|
4180
4883
|
exports.createAgentQueue = createAgentQueue;
|
|
@@ -4192,10 +4895,13 @@ exports.createRelay = createRelay;
|
|
|
4192
4895
|
exports.createRelayProvider = createRelayProvider;
|
|
4193
4896
|
exports.createScope = createScope;
|
|
4194
4897
|
exports.createScopeManager = createScopeManager;
|
|
4898
|
+
exports.createSystemOneJudge = createSystemOneJudge;
|
|
4195
4899
|
exports.createThinkSplitter = createThinkSplitter;
|
|
4196
4900
|
exports.denyCall = denyCall;
|
|
4197
4901
|
exports.estimateMessages = estimateMessages;
|
|
4198
4902
|
exports.estimateTokens = estimateTokens;
|
|
4903
|
+
exports.extractSystemOneAnswer = extractSystemOneAnswer;
|
|
4904
|
+
exports.extractSystemOneUsage = extractSystemOneUsage;
|
|
4199
4905
|
exports.filterAllowList = filterAllowList;
|
|
4200
4906
|
exports.handleAgentQueueJob = handleAgentQueueJob;
|
|
4201
4907
|
exports.handleAgentRunnerJob = handleAgentRunnerJob;
|
|
@@ -4204,10 +4910,16 @@ exports.isAgentError = isAgentError;
|
|
|
4204
4910
|
exports.isAgentJobError = isAgentJobError;
|
|
4205
4911
|
exports.isConversationError = isConversationError;
|
|
4206
4912
|
exports.isConversationSnapshot = isConversationSnapshot;
|
|
4913
|
+
exports.isJudgeAbortError = isJudgeAbortError;
|
|
4914
|
+
exports.isJudgeEntry = isJudgeEntry;
|
|
4915
|
+
exports.isJudgeError = isJudgeError;
|
|
4916
|
+
exports.isJudgeQuestion = isJudgeQuestion;
|
|
4207
4917
|
exports.isMessage = isMessage;
|
|
4208
4918
|
exports.isProviderAbortError = isProviderAbortError;
|
|
4209
4919
|
exports.isProviderError = isProviderError;
|
|
4210
4920
|
exports.isSection = isSection;
|
|
4921
|
+
exports.isSystemOneAnswer = isSystemOneAnswer;
|
|
4922
|
+
exports.isSystemOneResponse = isSystemOneResponse;
|
|
4211
4923
|
exports.joinThinking = joinThinking;
|
|
4212
4924
|
exports.messageContract = messageContract;
|
|
4213
4925
|
exports.messageShape = messageShape;
|
|
@@ -4215,7 +4927,9 @@ exports.providerRequestContract = providerRequestContract;
|
|
|
4215
4927
|
exports.providerRequestShape = providerRequestShape;
|
|
4216
4928
|
exports.providerResultContract = providerResultContract;
|
|
4217
4929
|
exports.providerResultShape = providerResultShape;
|
|
4930
|
+
exports.questionToSystemOne = questionToSystemOne;
|
|
4218
4931
|
exports.readChunks = readChunks;
|
|
4932
|
+
exports.readHeaders = readHeaders;
|
|
4219
4933
|
exports.readText = readText;
|
|
4220
4934
|
exports.relayFrameContract = relayFrameContract;
|
|
4221
4935
|
exports.relayFrameShape = relayFrameShape;
|