@aiwg/cli 2026.10.0 → 2026.10.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/providers/capability-matrix.yaml +270 -0
- package/agentic/code/providers/model-capabilities.v1.json +3091 -24
- package/bin/aiwg.mjs +86 -21
- package/dist/src/a2a/codecs.js +9 -0
- package/dist/src/artifacts/corpus-tools/vision-extract.js +105 -15
- package/dist/src/artifacts/repair.js +9 -4
- package/dist/src/channel/manager.mjs +59 -10
- package/dist/src/cli/child-process-result.js +40 -0
- package/dist/src/cli/env.js +38 -16
- package/dist/src/cli/flag-parsing.js +236 -0
- package/dist/src/cli/handlers/decision.js +297 -2
- package/dist/src/cli/handlers/diagnose.js +9 -2
- package/dist/src/cli/handlers/feedback.js +14 -5
- package/dist/src/cli/handlers/init.js +245 -30
- package/dist/src/cli/handlers/mc.js +23 -0
- package/dist/src/cli/handlers/models.js +69 -6
- package/dist/src/cli/handlers/ralph.js +3 -0
- package/dist/src/cli/handlers/refresh.js +16 -0
- package/dist/src/cli/handlers/regenerate.js +14 -0
- package/dist/src/cli/handlers/runtime-info.js +24 -9
- package/dist/src/cli/handlers/script-runner.js +16 -6
- package/dist/src/cli/handlers/session.js +396 -62
- package/dist/src/cli/handlers/sessions.js +146 -40
- package/dist/src/cli/handlers/setup-manifest.js +439 -39
- package/dist/src/cli/handlers/setup.js +139 -39
- package/dist/src/cli/handlers/subcommands.js +3 -1
- package/dist/src/cli/handlers/team.js +18 -8
- package/dist/src/cli/handlers/use.js +13 -0
- package/dist/src/cli/handlers/utilities.js +106 -23
- package/dist/src/cli/prompt-utils.js +72 -0
- package/dist/src/cli/provider-resolution.js +11 -0
- package/dist/src/config/aiwg-config.js +268 -4
- package/dist/src/config/cli.js +146 -19
- package/dist/src/config/dispatch-thresholds-runtime.mjs +38 -0
- package/dist/src/config/project-artifacts-health.mjs +14 -3
- package/dist/src/config/user-config.js +36 -10
- package/dist/src/decision/adapters/jev.js +80 -28
- package/dist/src/decision/ask.js +352 -0
- package/dist/src/decision/driver.js +4 -0
- package/dist/src/decision/entry.js +38 -6
- package/dist/src/decision/heldout/collector.js +10 -1
- package/dist/src/decision/index.js +3 -0
- package/dist/src/decision/interop.js +60 -8
- package/dist/src/decision/jev-credentials.js +121 -0
- package/dist/src/decision/ledger.js +839 -0
- package/dist/src/decision/validate.js +6 -3
- package/dist/src/effects/dispatch-verifier.js +252 -0
- package/dist/src/effects/index.js +1 -0
- package/dist/src/features/installer.js +2 -2
- package/dist/src/flow/graph-metadata.js +5 -0
- package/dist/src/gates/providers/loader.js +3 -1
- package/dist/src/installation/manager-command.mjs +1 -1
- package/dist/src/installation/manager.mjs +17 -2
- package/dist/src/lint/runner.js +2 -2
- package/dist/src/mcp/adapters/codex-runtime.js +360 -56
- package/dist/src/mcp/cli.mjs +587 -77
- package/dist/src/mcp/config-file.mjs +279 -0
- package/dist/src/mcp/config-layers.mjs +348 -0
- package/dist/src/mcp/credentials.mjs +484 -0
- package/dist/src/mcp/grok-build-config.mjs +16 -26
- package/dist/src/mcp/helpers.mjs +106 -2
- package/dist/src/mcp/omp-config.mjs +50 -29
- package/dist/src/mcp/profiles.js +97 -23
- package/dist/src/mcp/profiles.mjs +99 -27
- package/dist/src/mcp/registry.js +264 -57
- package/dist/src/mcp/registry.mjs +198 -45
- package/dist/src/mcp/server.mjs +403 -20
- package/dist/src/mcp/toml-string.mjs +9 -0
- package/dist/src/mcp/toml-strip-mcp.mjs +135 -0
- package/dist/src/mcp/tool-filters.mjs +357 -0
- package/dist/src/mcp/tools/agentic-sandbox.mjs +3 -1
- package/dist/src/mcp/tools/decision.mjs +58 -0
- package/dist/src/mcp/tools/orchestration.mjs +38 -0
- package/dist/src/mcp/tools/subsystems.mjs +39 -8
- package/dist/src/mcp/tools/worker-invoke.mjs +1539 -0
- package/dist/src/mcp/tools/worker-shell-control.mjs +258 -0
- package/dist/src/mcp/tools/worker-stdin-runtime.mjs +79 -0
- package/dist/src/memory/project-registry.js +2 -2
- package/dist/src/mission-protocol/codecs.js +639 -1
- package/dist/src/mission-protocol/types.js +110 -0
- package/dist/src/mission-protocol/worker-status.js +2 -0
- package/dist/src/models/config-loader.js +70 -13
- package/dist/src/models/grok-build-models.js +93 -41
- package/dist/src/models/model-capabilities.v1.json +3091 -24
- package/dist/src/models/model-discovery.js +862 -63
- package/dist/src/models/provider-policy.js +625 -11
- package/dist/src/models/qualification-digest.mjs +17 -0
- package/dist/src/models/resolver.js +259 -0
- package/dist/src/models/worker-direct-readiness.js +529 -0
- package/dist/src/models/worker-dispatch-offer.js +169 -0
- package/dist/src/models/worker-health-readiness.js +131 -0
- package/dist/src/models/worker-installation.mjs +879 -0
- package/dist/src/models/wrapper-route.js +18 -0
- package/dist/src/ops/registry.js +9 -14
- package/dist/src/plugins/standalone-packager.js +4 -3
- package/dist/src/process/bounded-child-process.mjs +387 -0
- package/dist/src/process/cli-command.mjs +75 -0
- package/dist/src/providers/capability-matrix.js +8 -0
- package/dist/src/providers/capability-matrix.yaml +270 -0
- package/dist/src/providers/grok-build-headless.js +43 -21
- package/dist/src/providers/provider-definitions.js +9 -1
- package/dist/src/providers/provider-definitions.mjs +9 -2
- package/dist/src/serve/executor-registry.js +16 -24
- package/dist/src/serve/fleet-mission-conductor.js +9 -0
- package/dist/src/serve/mission-conductor.js +9 -0
- package/dist/src/serve/shared-host-scheduler.js +2 -0
- package/dist/src/serve/worker-invocation.js +538 -0
- package/dist/src/serve/worker-stdin-frame.mjs +107 -0
- package/dist/src/sessions/adapter-registry.js +45 -0
- package/dist/src/sessions/adapters/generic.js +37 -3
- package/dist/src/sessions/adapters/grokbot-locator.js +111 -0
- package/dist/src/sessions/adapters/grokbot.js +423 -24
- package/dist/src/sessions/adapters/muse-workspace-scan.js +108 -0
- package/dist/src/sessions/adapters/muse.js +68 -29
- package/dist/src/sessions/adapters/pi.js +1 -0
- package/dist/src/sessions/batch-contracts.js +13 -1
- package/dist/src/sessions/batch-import.js +23 -42
- package/dist/src/sessions/candidates.js +47 -11
- package/dist/src/sessions/contracts.js +35 -0
- package/dist/src/sessions/coverage-preflight.js +134 -0
- package/dist/src/sessions/discovery.js +3 -0
- package/dist/src/sessions/grok-build-cli.js +67 -5
- package/dist/src/sessions/importer.js +24 -8
- package/dist/src/sessions/index.js +2 -0
- package/dist/src/sessions/repository.js +54 -13
- package/dist/src/sessions/workspace-discovery.js +99 -37
- package/dist/src/skills/adapters/clawhub.js +10 -8
- package/dist/src/skills/adapters/openclaw.js +10 -7
- package/dist/src/smiths/context-pipeline/parallelism-section.js +192 -3
- package/dist/src/smiths/context-pipeline/workspace-context.js +116 -9
- package/dist/src/smiths/hook-bridge/codex-translator.js +6 -5
- package/dist/src/smiths/mcpsmith/example.js +1 -1
- package/dist/src/smiths/toolsmith/runtime-discovery.mjs +10 -14
- package/dist/src/storage/backends/logseq.js +68 -7
- package/dist/src/storage/fortemi-qualification-receipt.js +2 -2
- package/dist/src/testing/generators/test-code-generator.js +57 -30
- package/dist/src/uhp/client.js +11 -1
- package/dist/src/uhp/mission.js +5 -0
- package/dist/src/update/checker.mjs +5 -5
- package/dist/src/update/notifier.mjs +54 -2
- package/dist/src/writing/fidelity.js +18 -1
- package/package.json +18 -1
- package/schemas/mission-v1.schema.json +2429 -0
- package/schemas/models/model-catalog.v1.schema.json +301 -0
- package/schemas/models/model-config-compatibility.v1.schema.json +281 -0
- package/schemas/models/model-policy.v1.schema.json +14 -0
- package/schemas/models/provider-model-capabilities.v1.schema.json +1066 -0
- package/tools/_resolve-impl.mjs +39 -0
- package/tools/agents/deploy-agents.mjs +15 -2
- package/tools/agents/providers/base.mjs +63 -43
- package/tools/agents/providers/grokbot.mjs +11 -12
- package/tools/agents/providers/hermes.mjs +6 -3
- package/tools/agents/providers/omp.mjs +5 -2
- package/tools/agents/providers/openclaw-translator.mjs +19 -5
- package/tools/providers/deepseek-harness-transport.mjs +358 -0
- package/tools/providers/omp-transport.mjs +192 -0
- package/tools/ralph-external/lib/claude-adapter.mjs +389 -0
- package/tools/ralph-external/lib/codex-adapter.mjs +432 -0
- package/tools/ralph-external/lib/deepseek-harness-adapter.mjs +397 -0
- package/tools/ralph-external/lib/factory-adapter.mjs +503 -0
- package/tools/ralph-external/lib/muse-adapter.mjs +364 -0
- package/tools/ralph-external/lib/omp-adapter.mjs +315 -0
- package/tools/ralph-external/lib/opencode-adapter.mjs +575 -0
- package/tools/ralph-external/lib/openhuman-adapter.mjs +75 -0
- package/tools/ralph-external/lib/pi-adapter.mjs +333 -0
- package/tools/ralph-external/lib/provider-adapter.mjs +690 -0
- package/tools/ralph-external/lib/worker-health.mjs +439 -0
- package/vscode-extension/schemas/aiwg.config.v1.json +1953 -0
|
@@ -2,7 +2,6 @@ import { DecisionValidationError, validateDecisionValue, validateDistribution }
|
|
|
2
2
|
import { canonicalJson } from '../../security/artifact-trust.js';
|
|
3
3
|
import { partitionProjectedState } from '../projection.js';
|
|
4
4
|
import { isTraceparent } from '../telemetry/context.js';
|
|
5
|
-
import { runInNewContext } from 'node:vm';
|
|
6
5
|
import { lookup } from 'node:dns/promises';
|
|
7
6
|
import { isIP } from 'node:net';
|
|
8
7
|
import { request as httpsRequest } from 'node:https';
|
|
@@ -12,7 +11,7 @@ export const JEV_ENDPOINT = 'https://api.typesafe.ai/v1/systemone';
|
|
|
12
11
|
const MAX_HEADER_BYTES = 32 * 1024;
|
|
13
12
|
const MAX_BODY_BYTES = 1024 * 1024;
|
|
14
13
|
const MAX_REQUEST_ID_LENGTH = 128;
|
|
15
|
-
const
|
|
14
|
+
const MAX_PARSE_DEPTH = 64;
|
|
16
15
|
/** Safe resolver taxonomy; messages are intentionally discarded at this boundary. */
|
|
17
16
|
export class JevCredentialError extends Error {
|
|
18
17
|
category;
|
|
@@ -45,6 +44,7 @@ export class JevDecisionAdapter {
|
|
|
45
44
|
this.region = options.region?.trim() ? options.region : null;
|
|
46
45
|
this.observeResponseBody = options.observeResponseBody;
|
|
47
46
|
}
|
|
47
|
+
/** Used only for non-2xx error bodies, where there is no separate successful parse to stay consistent with. */
|
|
48
48
|
observeResponse(body) {
|
|
49
49
|
if (!this.observeResponseBody)
|
|
50
50
|
return;
|
|
@@ -61,6 +61,30 @@ export class JevDecisionAdapter {
|
|
|
61
61
|
this.observeResponseBody(body, { actualModel: typeof parsed.model === 'string' && parsed.model ? parsed.model : null,
|
|
62
62
|
usage });
|
|
63
63
|
}
|
|
64
|
+
/**
|
|
65
|
+
* Derives the observer's model/usage from a value `parseBoundedJson` has already produced (or `undefined`
|
|
66
|
+
* on a failed parse) -- never from a second, independent parse of the same bytes. `parseBoundedJson` runs
|
|
67
|
+
* inside a bounded-time VM context (`MAX_PARSE_MS`); calling it twice on identical bytes let one call
|
|
68
|
+
* succeed while the other spuriously hit that wall-clock bound under host contention, so the reported
|
|
69
|
+
* metadata could disagree with the actual parsed answer even though both described one response.
|
|
70
|
+
*/
|
|
71
|
+
observationMetadataOf(parsed) {
|
|
72
|
+
let record = {};
|
|
73
|
+
try {
|
|
74
|
+
record = asRecord(parsed);
|
|
75
|
+
}
|
|
76
|
+
catch { /* A non-object (or absent) top-level value has no model; usage stays unknown. */ }
|
|
77
|
+
let usage = emptyUsage();
|
|
78
|
+
try {
|
|
79
|
+
usage = normalizeUsage(record.usage);
|
|
80
|
+
}
|
|
81
|
+
catch { /* Missing usage is unknown, including on auth errors. */ }
|
|
82
|
+
return { actualModel: typeof record.model === 'string' && record.model ? record.model : null, usage };
|
|
83
|
+
}
|
|
84
|
+
/** Observes exactly once, from the single parse attempt `parsed` already represents (successful or not). */
|
|
85
|
+
observeParsed(body, parsed) {
|
|
86
|
+
this.observeResponseBody?.(body, this.observationMetadataOf(parsed));
|
|
87
|
+
}
|
|
64
88
|
async capabilities() {
|
|
65
89
|
return {
|
|
66
90
|
answerKinds: ['choice', 'ordinal-score', 'truth-probability'],
|
|
@@ -191,9 +215,7 @@ export class JevDecisionAdapter {
|
|
|
191
215
|
try {
|
|
192
216
|
parsedError = parseBoundedJson(errorBody);
|
|
193
217
|
}
|
|
194
|
-
catch
|
|
195
|
-
if (error && typeof error === 'object' && 'code' in error && error.code === 'ERR_SCRIPT_EXECUTION_TIMEOUT')
|
|
196
|
-
throw error;
|
|
218
|
+
catch {
|
|
197
219
|
parsedError = null;
|
|
198
220
|
}
|
|
199
221
|
if (parsedError && typeof parsedError === 'object' && !Array.isArray(parsedError)) {
|
|
@@ -217,8 +239,15 @@ export class JevDecisionAdapter {
|
|
|
217
239
|
let parsed;
|
|
218
240
|
try {
|
|
219
241
|
const encoded = await readBoundedBody(response, signal);
|
|
220
|
-
|
|
221
|
-
|
|
242
|
+
// One parse attempt; the observer reads its outcome (success or failure) before answer validation,
|
|
243
|
+
// so an invalid answer still gets model/usage checked, and a parse failure is reported as unknown
|
|
244
|
+
// to both the observer and the main path -- never divergently.
|
|
245
|
+
try {
|
|
246
|
+
parsed = parseBoundedJson(encoded);
|
|
247
|
+
}
|
|
248
|
+
finally {
|
|
249
|
+
this.observeParsed(encoded, parsed);
|
|
250
|
+
}
|
|
222
251
|
if (request.signal.aborted)
|
|
223
252
|
return externalInterruption(request, 'terminal-response', metadata);
|
|
224
253
|
if (deadline.aborted)
|
|
@@ -334,8 +363,14 @@ export class JevDecisionAdapter {
|
|
|
334
363
|
}
|
|
335
364
|
try {
|
|
336
365
|
const encoded = await readBoundedBody(response, signal);
|
|
337
|
-
|
|
338
|
-
|
|
366
|
+
// One parse attempt; see the single-request `evaluate()` for why observation never re-parses.
|
|
367
|
+
let parsed;
|
|
368
|
+
try {
|
|
369
|
+
parsed = parseBoundedJson(encoded);
|
|
370
|
+
}
|
|
371
|
+
finally {
|
|
372
|
+
this.observeParsed(encoded, parsed);
|
|
373
|
+
}
|
|
339
374
|
const normalized = normalizeBatchResponse(requests, parsed).map(answer => ({
|
|
340
375
|
questionId: answer.questionId,
|
|
341
376
|
observation: { ...answer.observation, ...metadata },
|
|
@@ -453,7 +488,11 @@ function normalizeAnswer(request, body, questionId) {
|
|
|
453
488
|
throw new DecisionValidationError('Score legend does not match declared levels');
|
|
454
489
|
});
|
|
455
490
|
const mean = Object.entries(distribution).reduce((sum, [index, probability]) => sum + Number(index) * probability, 0);
|
|
456
|
-
|
|
491
|
+
// Jev reports probabilities and the score to two decimals, so each probability carries up to
|
|
492
|
+
// 0.005 of rounding error, weighted by its index, plus 0.005 on the score itself.
|
|
493
|
+
const indexSum = Object.keys(distribution).reduce((sum, index) => sum + Number(index), 0);
|
|
494
|
+
const tolerance = Math.max(0.02, 0.005 * indexSum + 0.005) + 1e-9;
|
|
495
|
+
if (Math.abs(mean - answer.score) > tolerance)
|
|
457
496
|
throw new DecisionValidationError('Score is not the distribution weighted mean');
|
|
458
497
|
validateDecisionValue(request.definition, answer.score);
|
|
459
498
|
return success(answer.score, model, usage, uncertainty(answer.confidence, distribution, 'typesafe-distribution-v1'));
|
|
@@ -570,19 +609,21 @@ function safeRequestId(value) {
|
|
|
570
609
|
return typeof value === 'string' && value.length > 0 && value.length <= MAX_REQUEST_ID_LENGTH
|
|
571
610
|
&& /^[\x21-\x7e]+$/.test(value) && !value.includes(',') ? value : null;
|
|
572
611
|
}
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
612
|
+
/** @internal Exported from this module for direct admission and CPU-cost evidence, not the public barrel. */
|
|
613
|
+
export function parseBoundedJson(text) {
|
|
614
|
+
// Bound work, not elapsed time: a VM timeout includes host scheduling delay.
|
|
615
|
+
// The stream reader also enforces this bound, before UTF-8 decoding.
|
|
616
|
+
if (text.length > MAX_BODY_BYTES || Buffer.byteLength(text, 'utf8') > MAX_BODY_BYTES) {
|
|
617
|
+
throw new DecisionValidationError('Jev response too large');
|
|
618
|
+
}
|
|
619
|
+
// Scan iteratively before allocation by JSON.parse. Total key text/count is
|
|
620
|
+
// bounded by body bytes; no recursive walk or suffix copying is needed.
|
|
621
|
+
// String/key spans are disjoint; skip the whitespace consumed by lookahead.
|
|
622
|
+
// Each code unit is visited once. Key slicing/decoding/hashing sum to
|
|
623
|
+
// O(body bytes), as does native parsing.
|
|
624
|
+
const containers = [];
|
|
625
|
+
let index = 0;
|
|
626
|
+
while (index < text.length) {
|
|
586
627
|
const char = text[index];
|
|
587
628
|
if (char === '"') {
|
|
588
629
|
const start = index++;
|
|
@@ -596,20 +637,31 @@ function rejectDuplicateAnswerKeys(text) {
|
|
|
596
637
|
else if (next === '"')
|
|
597
638
|
break;
|
|
598
639
|
}
|
|
599
|
-
|
|
640
|
+
let next = index;
|
|
641
|
+
while (next < text.length && /[\x20\t\r\n]/.test(text[next]))
|
|
642
|
+
next += 1;
|
|
643
|
+
const keys = containers.at(-1);
|
|
644
|
+
if (keys && text[next] === ':') {
|
|
600
645
|
const key = JSON.parse(text.slice(start, index));
|
|
601
646
|
if (keys.has(key))
|
|
602
|
-
throw new DecisionValidationError('duplicate Jev
|
|
647
|
+
throw new DecisionValidationError('duplicate Jev response key');
|
|
603
648
|
keys.add(key);
|
|
604
649
|
}
|
|
650
|
+
index = next;
|
|
605
651
|
continue;
|
|
606
652
|
}
|
|
607
|
-
if (char === '{' || char === '[')
|
|
608
|
-
|
|
653
|
+
if (char === '{' || char === '[') {
|
|
654
|
+
if (containers.length >= MAX_PARSE_DEPTH)
|
|
655
|
+
throw new DecisionValidationError('Jev response nesting too deep');
|
|
656
|
+
containers.push(char === '{' ? new Set() : null);
|
|
657
|
+
}
|
|
609
658
|
else if (char === '}' || char === ']')
|
|
610
|
-
|
|
659
|
+
containers.pop();
|
|
611
660
|
index += 1;
|
|
612
661
|
}
|
|
662
|
+
// Native parsing checks the complete JSON grammar, with no reviver or code
|
|
663
|
+
// execution. Input bytes and nesting bound its work and allocated graph.
|
|
664
|
+
return JSON.parse(text);
|
|
613
665
|
}
|
|
614
666
|
function headersWithinLimit(headers) {
|
|
615
667
|
let bytes = 0;
|
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
import { readFile } from 'node:fs/promises';
|
|
2
|
+
import { JEV_ENDPOINT, JevDecisionAdapter } from './adapters/jev.js';
|
|
3
|
+
import { evaluateDecisionRuleset } from './evaluate.js';
|
|
4
|
+
import { artifactPin } from './validate.js';
|
|
5
|
+
import { JEV_CREDENTIAL_REF, JEV_DEFAULT_REGION, lookupJevCredential, removeJevCredentialFile, resolveJevCredentialBytes, writeJevCredentialFile, } from './jev-credentials.js';
|
|
6
|
+
const CONTEXT_LIMIT_BYTES = 32 * 1024;
|
|
7
|
+
const MODEL = 'jev-latest';
|
|
8
|
+
const PURPOSE = 'agent-bounded-decision';
|
|
9
|
+
const INPUT_SCHEMA = {
|
|
10
|
+
type: 'object',
|
|
11
|
+
properties: { message: { type: 'string', minLength: 1 } },
|
|
12
|
+
required: ['message'],
|
|
13
|
+
additionalProperties: false,
|
|
14
|
+
};
|
|
15
|
+
export class DecisionAskUsageError extends Error {
|
|
16
|
+
reason;
|
|
17
|
+
constructor(reason, message) {
|
|
18
|
+
super(message);
|
|
19
|
+
this.reason = reason;
|
|
20
|
+
this.name = 'DecisionAskUsageError';
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
function usageError(reason, message) {
|
|
24
|
+
throw new DecisionAskUsageError(reason, message);
|
|
25
|
+
}
|
|
26
|
+
function byteLength(value) {
|
|
27
|
+
return Buffer.byteLength(value, 'utf8');
|
|
28
|
+
}
|
|
29
|
+
function validateQuestion(question) {
|
|
30
|
+
if (!question.trim())
|
|
31
|
+
usageError('question-required', 'decision ask requires --question <q>');
|
|
32
|
+
}
|
|
33
|
+
export function validateDecisionAskInput(input) {
|
|
34
|
+
validateQuestion(input.question);
|
|
35
|
+
if (input.context !== undefined && byteLength(input.context) > CONTEXT_LIMIT_BYTES) {
|
|
36
|
+
usageError('context-too-large', 'decision ask context exceeds 32 KiB');
|
|
37
|
+
}
|
|
38
|
+
const threshold = input.threshold ?? 0.8;
|
|
39
|
+
if (!Number.isFinite(threshold) || threshold < 0 || threshold > 1) {
|
|
40
|
+
usageError('invalid-threshold', '--threshold must be a probability from 0 to 1');
|
|
41
|
+
}
|
|
42
|
+
const timeoutMs = input.timeoutMs ?? 15_000;
|
|
43
|
+
if (!Number.isInteger(timeoutMs) || timeoutMs <= 0)
|
|
44
|
+
usageError('invalid-timeout', '--timeout-ms must be a positive integer');
|
|
45
|
+
if (input.mode.kind === 'choices') {
|
|
46
|
+
if (input.mode.choices.length < 2 || input.mode.choices.length > 255)
|
|
47
|
+
usageError('invalid-choices', '--choices requires 2..255 ids');
|
|
48
|
+
const seen = new Set();
|
|
49
|
+
for (const { id, description } of input.mode.choices.map(parseChoice)) {
|
|
50
|
+
if (!/^[A-Za-z0-9_.-]{1,64}$/.test(id))
|
|
51
|
+
usageError('invalid-choices', 'choice ids must match [A-Za-z0-9_.-]{1,64}');
|
|
52
|
+
if (description !== undefined && (!description || description.length > 200)) {
|
|
53
|
+
usageError('invalid-choices', 'choice descriptions after = must be 1..200 characters');
|
|
54
|
+
}
|
|
55
|
+
if (seen.has(id))
|
|
56
|
+
usageError('invalid-choices', 'choice ids must not contain duplicates');
|
|
57
|
+
seen.add(id);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
if (input.mode.kind === 'scale') {
|
|
61
|
+
const count = input.mode.high - input.mode.low + 1;
|
|
62
|
+
if (!Number.isInteger(input.mode.low) || !Number.isInteger(input.mode.high) || input.mode.low > input.mode.high || count < 2 || count > 10) {
|
|
63
|
+
usageError('invalid-scale', '--scale must be an integer range like 1-5 with at most 10 levels');
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
function usage(usageValue) {
|
|
68
|
+
if (!usageValue)
|
|
69
|
+
return null;
|
|
70
|
+
return { inputTokens: usageValue.inputTokens, outputTokens: usageValue.outputTokens };
|
|
71
|
+
}
|
|
72
|
+
function fallback(reason, latencyMs, attempt) {
|
|
73
|
+
return {
|
|
74
|
+
schema: 'aiwg-decision-ask/v1',
|
|
75
|
+
status: 'fallback',
|
|
76
|
+
answer: null,
|
|
77
|
+
confidence: null,
|
|
78
|
+
fallback: 'llm',
|
|
79
|
+
reason,
|
|
80
|
+
model: attempt?.actualModel ?? null,
|
|
81
|
+
usage: usage(attempt?.usage),
|
|
82
|
+
latencyMs,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
/** A choice is `id` or `id=description`; the description tells Jev what the option means. */
|
|
86
|
+
function parseChoice(entry) {
|
|
87
|
+
const separator = entry.indexOf('=');
|
|
88
|
+
if (separator < 0)
|
|
89
|
+
return { id: entry.trim() };
|
|
90
|
+
return { id: entry.slice(0, separator).trim(), description: entry.slice(separator + 1).trim() };
|
|
91
|
+
}
|
|
92
|
+
function scaleLevel(value, low, high) {
|
|
93
|
+
const edge = value === low ? ' (lowest)' : value === high ? ' (highest)' : '';
|
|
94
|
+
return `Level ${value} on a ${low} to ${high} scale${edge}.`;
|
|
95
|
+
}
|
|
96
|
+
function answerKind(mode) {
|
|
97
|
+
if (mode.kind === 'yes-no')
|
|
98
|
+
return {
|
|
99
|
+
kind: 'truth-probability',
|
|
100
|
+
trueDescription: 'Yes.',
|
|
101
|
+
falseDescription: 'No.',
|
|
102
|
+
};
|
|
103
|
+
if (mode.kind === 'choices')
|
|
104
|
+
return {
|
|
105
|
+
kind: 'choice',
|
|
106
|
+
options: mode.choices.map(parseChoice).map(({ id, description }) => ({ id, description: description ?? id })),
|
|
107
|
+
};
|
|
108
|
+
return {
|
|
109
|
+
kind: 'ordinal-score',
|
|
110
|
+
levels: Array.from({ length: mode.high - mode.low + 1 }, (_, index) => scaleLevel(mode.low + index, mode.low, mode.high)),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
function definition(input) {
|
|
114
|
+
return {
|
|
115
|
+
apiVersion: 'decision.aiwg.io/v1alpha2',
|
|
116
|
+
kind: 'DecisionDefinition',
|
|
117
|
+
metadata: { id: 'aiwg.ask.answer', version: '1.0.0', description: 'Bounded advisory ask decision' },
|
|
118
|
+
spec: {
|
|
119
|
+
purpose: PURPOSE,
|
|
120
|
+
inputSchema: INPUT_SCHEMA,
|
|
121
|
+
question: input.question,
|
|
122
|
+
answer: answerKind(input.mode),
|
|
123
|
+
requiredCapabilities: [input.mode.kind === 'yes-no' ? 'truth-probability' : input.mode.kind === 'choices' ? 'choice' : 'ordinal-score'],
|
|
124
|
+
},
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
function ruleset(definitionValue) {
|
|
128
|
+
return {
|
|
129
|
+
apiVersion: 'decision.aiwg.io/v1alpha2',
|
|
130
|
+
kind: 'DecisionRuleset',
|
|
131
|
+
metadata: { id: 'aiwg.ask.ruleset', version: '1.0.0', description: 'Bounded advisory ask ruleset' },
|
|
132
|
+
spec: {
|
|
133
|
+
purpose: PURPOSE,
|
|
134
|
+
inputSchema: INPUT_SCHEMA,
|
|
135
|
+
evaluations: [{ alias: 'answer', decision: artifactPin(definitionValue), inputPointer: '' }],
|
|
136
|
+
rules: [{
|
|
137
|
+
id: 'answered',
|
|
138
|
+
priority: 1,
|
|
139
|
+
when: { op: 'exists', left: { source: 'decision', alias: 'answer', pointer: '/value' } },
|
|
140
|
+
outcome: 'answered',
|
|
141
|
+
}],
|
|
142
|
+
composition: 'first-match',
|
|
143
|
+
conflict: 'review',
|
|
144
|
+
defaultOutcome: 'fallback',
|
|
145
|
+
failureOutcome: 'fallback',
|
|
146
|
+
outputSchema: { enum: ['answered', 'fallback'] },
|
|
147
|
+
},
|
|
148
|
+
};
|
|
149
|
+
}
|
|
150
|
+
function binding(rulesetValue, primitive, timeoutMs) {
|
|
151
|
+
const target = {
|
|
152
|
+
adapter: 'jev',
|
|
153
|
+
adapterVersion: '1.0.0',
|
|
154
|
+
model: MODEL,
|
|
155
|
+
requiredCapabilities: [primitive],
|
|
156
|
+
acceptance: { mode: 'typed-value' },
|
|
157
|
+
timeoutMs,
|
|
158
|
+
retry: { maxRetries: 0, initialDelayMs: 250, maxDelayMs: 2_000 },
|
|
159
|
+
credentialRef: JEV_CREDENTIAL_REF,
|
|
160
|
+
};
|
|
161
|
+
return {
|
|
162
|
+
apiVersion: 'decision.aiwg.io/v1alpha2',
|
|
163
|
+
kind: 'DecisionBinding',
|
|
164
|
+
metadata: { id: 'aiwg.ask.binding', version: '1.0.0', description: 'Bounded advisory ask Jev binding' },
|
|
165
|
+
spec: {
|
|
166
|
+
ruleset: artifactPin(rulesetValue),
|
|
167
|
+
totalTimeoutMs: timeoutMs,
|
|
168
|
+
maxAttempts: 1,
|
|
169
|
+
concurrency: 1,
|
|
170
|
+
evaluations: { answer: { targets: [target], fallbackOn: [] } },
|
|
171
|
+
},
|
|
172
|
+
};
|
|
173
|
+
}
|
|
174
|
+
function projectionPolicy(region, endpoint) {
|
|
175
|
+
const origin = new URL(endpoint ?? JEV_ENDPOINT).origin;
|
|
176
|
+
return {
|
|
177
|
+
version: '1.0.0',
|
|
178
|
+
provider: 'jev',
|
|
179
|
+
model: MODEL,
|
|
180
|
+
origin,
|
|
181
|
+
region,
|
|
182
|
+
purpose: PURPOSE,
|
|
183
|
+
allowIncompleteContext: false,
|
|
184
|
+
fields: [{
|
|
185
|
+
pointer: '/message',
|
|
186
|
+
output: 'message',
|
|
187
|
+
source: 'operator',
|
|
188
|
+
subject: 'aiwg-decision-ask',
|
|
189
|
+
trust: 'untrusted',
|
|
190
|
+
sensitivity: 'internal',
|
|
191
|
+
purpose: PURPOSE,
|
|
192
|
+
retentionClass: 'ephemeral',
|
|
193
|
+
accessScopes: ['decision-runtime'],
|
|
194
|
+
exportPolicy: 'sanitized',
|
|
195
|
+
deletionPolicy: 'erase',
|
|
196
|
+
backupPolicy: 'not-persisted',
|
|
197
|
+
allowedProviders: ['jev'],
|
|
198
|
+
allowedModels: [MODEL],
|
|
199
|
+
allowedOrigins: [origin],
|
|
200
|
+
allowedRegions: [region],
|
|
201
|
+
}],
|
|
202
|
+
};
|
|
203
|
+
}
|
|
204
|
+
function contextMessage(input) {
|
|
205
|
+
return input.context === undefined || input.context === ''
|
|
206
|
+
? input.question
|
|
207
|
+
: `${input.question}\n\nContext:\n${input.context}`;
|
|
208
|
+
}
|
|
209
|
+
function confidenceFor(mode, evaluation) {
|
|
210
|
+
const uncertainty = evaluation.spec.uncertainty;
|
|
211
|
+
const value = evaluation.spec.value;
|
|
212
|
+
if (mode.kind === 'yes-no' && typeof value === 'number')
|
|
213
|
+
return Math.max(value, 1 - value);
|
|
214
|
+
if (uncertainty?.distribution && mode.kind === 'choices' && typeof value === 'string')
|
|
215
|
+
return uncertainty.distribution[value] ?? uncertainty.confidence;
|
|
216
|
+
if (uncertainty?.distribution && mode.kind === 'scale')
|
|
217
|
+
return Math.max(...Object.values(uncertainty.distribution));
|
|
218
|
+
return uncertainty?.confidence ?? null;
|
|
219
|
+
}
|
|
220
|
+
function mostLikelyIndex(distribution) {
|
|
221
|
+
if (!distribution)
|
|
222
|
+
return null;
|
|
223
|
+
let best = null;
|
|
224
|
+
for (const [index, probability] of Object.entries(distribution)) {
|
|
225
|
+
if (best === null || probability > (distribution[String(best)] ?? -1))
|
|
226
|
+
best = Number(index);
|
|
227
|
+
}
|
|
228
|
+
return best;
|
|
229
|
+
}
|
|
230
|
+
function publicAnswer(mode, value, distribution) {
|
|
231
|
+
if (mode.kind === 'yes-no') {
|
|
232
|
+
const probability = typeof value === 'number' ? value : null;
|
|
233
|
+
return probability === null ? { answer: null } : { answer: probability >= 0.5, probability };
|
|
234
|
+
}
|
|
235
|
+
if (mode.kind === 'scale') {
|
|
236
|
+
// Jev returns the distribution-weighted mean index; report the most likely level instead.
|
|
237
|
+
const index = mostLikelyIndex(distribution) ?? (typeof value === 'number' ? Math.round(value) : null);
|
|
238
|
+
return index === null ? { answer: null } : { answer: mode.low + index };
|
|
239
|
+
}
|
|
240
|
+
return typeof value === 'string' ? { answer: value } : { answer: null };
|
|
241
|
+
}
|
|
242
|
+
export async function askDecision(input, options = {}) {
|
|
243
|
+
validateDecisionAskInput(input);
|
|
244
|
+
const started = options.now?.() ?? Date.now();
|
|
245
|
+
let credential;
|
|
246
|
+
try {
|
|
247
|
+
credential = await lookupJevCredential(options);
|
|
248
|
+
}
|
|
249
|
+
catch {
|
|
250
|
+
return fallback('not-configured', 0);
|
|
251
|
+
}
|
|
252
|
+
if (!credential?.enabled || !credential.token)
|
|
253
|
+
return fallback('not-configured', 0);
|
|
254
|
+
const timeoutMs = input.timeoutMs ?? 15_000;
|
|
255
|
+
const def = definition(input);
|
|
256
|
+
const rules = ruleset(def);
|
|
257
|
+
const bind = binding(rules, def.spec.answer.kind, timeoutMs);
|
|
258
|
+
const adapter = options.adapter ?? options.createAdapter?.({ region: credential.region, endpoint: credential.endpoint })
|
|
259
|
+
?? new JevDecisionAdapter({ region: credential.region, ...(credential.endpoint ? {
|
|
260
|
+
endpoint: credential.endpoint,
|
|
261
|
+
allowedOrigins: [new URL(credential.endpoint).origin],
|
|
262
|
+
} : {}) });
|
|
263
|
+
let result;
|
|
264
|
+
try {
|
|
265
|
+
result = await (options.evaluate ?? evaluateDecisionRuleset)({
|
|
266
|
+
ruleset: rules,
|
|
267
|
+
binding: bind,
|
|
268
|
+
definitions: { [def.metadata.id]: def },
|
|
269
|
+
input: { message: contextMessage(input) },
|
|
270
|
+
runId: options.runId ?? 'aiwg-decision-ask',
|
|
271
|
+
invocationId: options.invocationId ?? `ask-${Date.now()}`,
|
|
272
|
+
adapters: { jev: adapter },
|
|
273
|
+
resolveCredential: options.resolveCredential ?? (logicalRef => resolveJevCredentialBytes(logicalRef, options)),
|
|
274
|
+
projection: { resolve: () => projectionPolicy(credential.region, credential.endpoint) },
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
catch {
|
|
278
|
+
return fallback('jev-error', Math.max(0, (options.now?.() ?? Date.now()) - started));
|
|
279
|
+
}
|
|
280
|
+
const latencyMs = Math.max(0, (options.now?.() ?? Date.now()) - started);
|
|
281
|
+
const evaluation = result.spec.evaluations.answer;
|
|
282
|
+
const attempt = evaluation?.spec.attempts.at(-1);
|
|
283
|
+
if (!evaluation || evaluation.spec.status !== 'success')
|
|
284
|
+
return fallback(evaluation?.spec.reason ?? result.spec.reason, latencyMs, attempt);
|
|
285
|
+
const confidence = confidenceFor(input.mode, evaluation);
|
|
286
|
+
const threshold = input.threshold ?? 0.8;
|
|
287
|
+
if (confidence === null || confidence < threshold)
|
|
288
|
+
return fallback('low-confidence', latencyMs, attempt);
|
|
289
|
+
const answer = publicAnswer(input.mode, evaluation.spec.value, evaluation.spec.uncertainty?.distribution);
|
|
290
|
+
return {
|
|
291
|
+
schema: 'aiwg-decision-ask/v1',
|
|
292
|
+
status: 'answered',
|
|
293
|
+
answer: answer.answer,
|
|
294
|
+
confidence,
|
|
295
|
+
...(answer.probability === undefined ? {} : { probability: answer.probability }),
|
|
296
|
+
fallback: null,
|
|
297
|
+
reason: 'none',
|
|
298
|
+
model: attempt?.actualModel ?? null,
|
|
299
|
+
usage: usage(attempt?.usage),
|
|
300
|
+
latencyMs,
|
|
301
|
+
};
|
|
302
|
+
}
|
|
303
|
+
export async function readDecisionContextFile(filePath) {
|
|
304
|
+
const content = await readFile(filePath, 'utf8');
|
|
305
|
+
if (byteLength(content) > CONTEXT_LIMIT_BYTES)
|
|
306
|
+
usageError('context-too-large', 'decision ask context exceeds 32 KiB');
|
|
307
|
+
return content;
|
|
308
|
+
}
|
|
309
|
+
export async function setupJev(input = {}, options = {}) {
|
|
310
|
+
const region = input.region?.trim() || JEV_DEFAULT_REGION;
|
|
311
|
+
if (input.remove) {
|
|
312
|
+
await removeJevCredentialFile(options);
|
|
313
|
+
return {
|
|
314
|
+
schema: 'aiwg-decision-jev-setup/v1',
|
|
315
|
+
configured: false,
|
|
316
|
+
credentialSource: 'file',
|
|
317
|
+
region,
|
|
318
|
+
next: ['Run aiwg decision setup jev --token-stdin to configure Jev again.'],
|
|
319
|
+
};
|
|
320
|
+
}
|
|
321
|
+
const environment = options.env ?? process.env;
|
|
322
|
+
const envToken = environment.JEV_API_KEY?.trim() || environment.AIWG_DECISION_JEV_API_KEY?.trim();
|
|
323
|
+
const token = input.token?.trim() || envToken;
|
|
324
|
+
if (!token)
|
|
325
|
+
usageError('token-required', 'Provide --token-stdin or set JEV_API_KEY or AIWG_DECISION_JEV_API_KEY.');
|
|
326
|
+
await writeJevCredentialFile({
|
|
327
|
+
token,
|
|
328
|
+
region,
|
|
329
|
+
...(input.endpoint?.trim() ? { endpoint: input.endpoint.trim() } : {}),
|
|
330
|
+
enabled: true,
|
|
331
|
+
}, options);
|
|
332
|
+
const result = {
|
|
333
|
+
schema: 'aiwg-decision-jev-setup/v1',
|
|
334
|
+
configured: true,
|
|
335
|
+
credentialSource: input.token?.trim() ? 'file' : 'env',
|
|
336
|
+
region,
|
|
337
|
+
next: ['Run aiwg decision ask --question "<q>" --yes-no --json.'],
|
|
338
|
+
};
|
|
339
|
+
if (input.verify) {
|
|
340
|
+
const verified = await askDecision({
|
|
341
|
+
question: 'Is this Jev decision setup verification request well formed?',
|
|
342
|
+
mode: { kind: 'yes-no' },
|
|
343
|
+
threshold: 0,
|
|
344
|
+
timeoutMs: 15_000,
|
|
345
|
+
}, options);
|
|
346
|
+
result.verified = verified.status === 'answered';
|
|
347
|
+
result.model = verified.model;
|
|
348
|
+
result.usage = verified.usage;
|
|
349
|
+
}
|
|
350
|
+
return result;
|
|
351
|
+
}
|
|
352
|
+
//# sourceMappingURL=ask.js.map
|
|
@@ -2,7 +2,11 @@ import { existsSync } from 'node:fs';
|
|
|
2
2
|
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
|
3
3
|
import path from 'node:path';
|
|
4
4
|
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
5
|
+
import { askDecision, setupJev, DecisionAskUsageError, readDecisionContextFile, validateDecisionAskInput, } from './ask.js';
|
|
6
|
+
import { recordDecisionAsk, decisionLedgerHistory, decisionLedgerStats, resolveDecisionSessionId, parseDecisionLedgerSince, DecisionLedgerSinceError, DECISION_LEDGER_SCHEMA, ATTRIBUTION_FIELD_PATTERN, MAX_ATTRIBUTION_FIELD_LENGTH, MAX_RECORDED_CHOICE_IDS, MAX_RECORD_TEXT_BYTES, MAX_LEDGER_RECORD_BYTES, DECISION_LEDGER_RECORD_TIMEOUT_MS, DECISION_SAVINGS_REASONING_ALLOWANCE_TOKENS, DECISION_SAVINGS_ESTIMATION_METHOD, buildDecisionLedgerRecord, appendDecisionLedgerRecord, filterDecisionLedgerRecords, } from './ledger.js';
|
|
7
|
+
import { resolveActiveProvider } from '../cli/provider-resolution.js';
|
|
5
8
|
import { assertDecisionEvaluateDispatcherConfig, decisionPatternPacks, getDecisionPatternPack, governedPatternArtifacts, listDecisionPatterns, patternInputSchema, patternOutputSchema, planLiveDecisionPattern, runOfflineDecisionPattern, validateAgainstSchema, validateDecisionDocument, validateDefinition, validateRuleset, } from './index.js';
|
|
9
|
+
export { askDecision, setupJev, readDecisionContextFile, validateDecisionAskInput, DecisionAskUsageError, recordDecisionAsk, decisionLedgerHistory, decisionLedgerStats, resolveDecisionSessionId, parseDecisionLedgerSince, DecisionLedgerSinceError, DECISION_LEDGER_SCHEMA, ATTRIBUTION_FIELD_PATTERN, MAX_ATTRIBUTION_FIELD_LENGTH, MAX_RECORDED_CHOICE_IDS, MAX_RECORD_TEXT_BYTES, MAX_LEDGER_RECORD_BYTES, DECISION_LEDGER_RECORD_TIMEOUT_MS, DECISION_SAVINGS_REASONING_ALLOWANCE_TOKENS, DECISION_SAVINGS_ESTIMATION_METHOD, buildDecisionLedgerRecord, appendDecisionLedgerRecord, filterDecisionLedgerRecords, resolveActiveProvider, };
|
|
6
10
|
const DECISION_ROOT = path.dirname(path.dirname(fileURLToPath(import.meta.url)));
|
|
7
11
|
const VALIDATION_TARGETS = ['request', 'definition', 'ruleset', 'binding'];
|
|
8
12
|
const OFFLINE_ADAPTER = 'agentic/code/addons/decision-engine/examples/fixture-jev-adapter.mjs';
|
|
@@ -16,10 +16,42 @@ export class EntryAdmissionError extends Error {
|
|
|
16
16
|
this.name = 'EntryAdmissionError';
|
|
17
17
|
}
|
|
18
18
|
}
|
|
19
|
+
/**
|
|
20
|
+
* Starts an admission-budget clock, feature-detecting
|
|
21
|
+
* `process.threadCpuUsage()` once for this reading. `elapsedAdmissionMs`
|
|
22
|
+
* below reads the *type* of what this returns rather than re-detecting the
|
|
23
|
+
* feature, so one admission check always uses one clock end to end.
|
|
24
|
+
*
|
|
25
|
+
* Per-thread CPU time measures only the work the admitting call itself did
|
|
26
|
+
* on this thread, so it is immune both to host scheduling contention
|
|
27
|
+
* pausing this thread between reads (the root cause of #3047's false
|
|
28
|
+
* `time-budget` rejections of legitimate input under host load) and to
|
|
29
|
+
* other threads' or background GC's CPU time landing on this same process
|
|
30
|
+
* (which a process-wide `process.cpuUsage()` clock would still charge here
|
|
31
|
+
* -- not a safe substitute for the same reason `performance.now()` was not).
|
|
32
|
+
*
|
|
33
|
+
* `process.threadCpuUsage()` was added in Node v22.19.0 (per the installed
|
|
34
|
+
* `@types/node` `@since` annotation) and is absent on this project's
|
|
35
|
+
* `engines.node` floor (`>=20.0.0` in package.json) and on Node
|
|
36
|
+
* 22.0.0-22.18.x. On a runtime without it, this falls back to the
|
|
37
|
+
* pre-#3047 monotonic wall-clock guard as an explicitly documented
|
|
38
|
+
* compatibility fallback: it restores the original host-load sensitivity on
|
|
39
|
+
* those older runtimes, it does not resolve it there.
|
|
40
|
+
*/
|
|
41
|
+
export function admissionClockStart() {
|
|
42
|
+
return typeof process.threadCpuUsage === 'function' ? process.threadCpuUsage() : performance.now();
|
|
43
|
+
}
|
|
44
|
+
/** Elapsed time since `start` (from `admissionClockStart`), in milliseconds. */
|
|
45
|
+
export function elapsedAdmissionMs(start) {
|
|
46
|
+
if (typeof start === 'number')
|
|
47
|
+
return performance.now() - start;
|
|
48
|
+
const diff = process.threadCpuUsage(start);
|
|
49
|
+
return (diff.user + diff.system) / 1000;
|
|
50
|
+
}
|
|
19
51
|
/** Check the object graph before JSON serialization, schema validation or digesting. */
|
|
20
52
|
export function admitEntry(value, limits = DEFAULT_ENTRY_LIMITS) {
|
|
21
53
|
const counts = { entries: 0, properties: 0, bytes: 0, estimatedMemoryBytes: 0 };
|
|
22
|
-
const started =
|
|
54
|
+
const started = admissionClockStart();
|
|
23
55
|
const encoder = new TextEncoder();
|
|
24
56
|
const stack = new Set();
|
|
25
57
|
const fail = (reason) => { throw new EntryAdmissionError(reason, counts); };
|
|
@@ -32,7 +64,7 @@ export function admitEntry(value, limits = DEFAULT_ENTRY_LIMITS) {
|
|
|
32
64
|
fail('memory-budget');
|
|
33
65
|
};
|
|
34
66
|
const visit = (item, depth) => {
|
|
35
|
-
if (
|
|
67
|
+
if (elapsedAdmissionMs(started) > limits.timeMs)
|
|
36
68
|
fail('time-budget');
|
|
37
69
|
if (++counts.entries > limits.entries)
|
|
38
70
|
fail('entry-count');
|
|
@@ -107,7 +139,7 @@ export function parseDecisionJson(source, limits = DEFAULT_ENTRY_LIMITS) {
|
|
|
107
139
|
throw new EntryAdmissionError('serialized-bytes', { bytes });
|
|
108
140
|
if (bytes * 4 > limits.memoryBytes)
|
|
109
141
|
throw new EntryAdmissionError('memory-budget', { bytes });
|
|
110
|
-
const started =
|
|
142
|
+
const started = admissionClockStart();
|
|
111
143
|
let value;
|
|
112
144
|
try {
|
|
113
145
|
value = JSON.parse(source);
|
|
@@ -118,7 +150,7 @@ export function parseDecisionJson(source, limits = DEFAULT_ENTRY_LIMITS) {
|
|
|
118
150
|
const document = parseDocument(source, { uniqueKeys: true });
|
|
119
151
|
if (document.errors.length)
|
|
120
152
|
throw new EntryAdmissionError('duplicate-or-invalid-key', { bytes });
|
|
121
|
-
if (
|
|
153
|
+
if (elapsedAdmissionMs(started) > limits.timeMs)
|
|
122
154
|
throw new EntryAdmissionError('time-budget', { bytes });
|
|
123
155
|
admitEntry(value, limits);
|
|
124
156
|
return value;
|
|
@@ -130,7 +162,7 @@ export function parseDecisionYaml(source, limits = DEFAULT_ENTRY_LIMITS) {
|
|
|
130
162
|
throw new EntryAdmissionError('serialized-bytes', { bytes });
|
|
131
163
|
if (bytes * 4 > limits.memoryBytes)
|
|
132
164
|
throw new EntryAdmissionError('memory-budget', { bytes });
|
|
133
|
-
const started =
|
|
165
|
+
const started = admissionClockStart();
|
|
134
166
|
const document = parseDocument(source, { uniqueKeys: true });
|
|
135
167
|
if (document.errors.length)
|
|
136
168
|
throw new EntryAdmissionError('duplicate-or-invalid-key', { bytes });
|
|
@@ -160,7 +192,7 @@ export function parseDecisionYaml(source, limits = DEFAULT_ENTRY_LIMITS) {
|
|
|
160
192
|
}
|
|
161
193
|
};
|
|
162
194
|
inspect(document.contents, 0);
|
|
163
|
-
if (
|
|
195
|
+
if (elapsedAdmissionMs(started) > limits.timeMs)
|
|
164
196
|
throw new EntryAdmissionError('time-budget', { bytes, entries });
|
|
165
197
|
const value = document.toJS();
|
|
166
198
|
admitEntry(value, limits);
|
|
@@ -250,6 +250,12 @@ export async function collectHeldoutStudy(options) {
|
|
|
250
250
|
// No raw provider exceptions, model IDs or request IDs cross this boundary when a secret matches.
|
|
251
251
|
const raw = observation;
|
|
252
252
|
const metadata = wireMetadata;
|
|
253
|
+
// `metadata` is `jev.ts`'s observer, reading model/usage off the exact same single parse that
|
|
254
|
+
// produced `raw` (never a second, independent parse of the same bytes -- see jev.ts `observeParsed`).
|
|
255
|
+
// The two can therefore no longer disagree on a successful response; preferring `metadata` here
|
|
256
|
+
// only matters when `raw` is a failure (e.g. an otherwise-parseable but invalid answer), so the
|
|
257
|
+
// model/usage bound checks below still see the wire-observed values jev.ts's `failure()` does not
|
|
258
|
+
// carry.
|
|
253
259
|
const observed = raw && metadata ? { ...raw, actualModel: metadata.actualModel, usage: metadata.usage } : raw;
|
|
254
260
|
const completed = receipt;
|
|
255
261
|
const captures = QUALIFICATION_PRIVACY_SURFACES.map(surface => ({ surface,
|
|
@@ -275,7 +281,10 @@ export async function collectHeldoutStudy(options) {
|
|
|
275
281
|
else if (observed?.usage.costUsd !== null && observed?.usage.costUsd !== undefined && observed.usage.costUsd * 1_000_000 > reserveMicros)
|
|
276
282
|
outcome = 'price-bound';
|
|
277
283
|
else if (observed?.status === 'success') {
|
|
278
|
-
|
|
284
|
+
// A null model is unknown, not a mismatch: it must complete neither as 'ok' nor as 'served-model'.
|
|
285
|
+
if (observed.actualModel === null)
|
|
286
|
+
outcome = 'model-unknown';
|
|
287
|
+
else if (observed.actualModel !== a.servedModel)
|
|
279
288
|
outcome = 'served-model';
|
|
280
289
|
else if (observed.usage.inputTokens === null || observed.usage.outputTokens === null)
|
|
281
290
|
outcome = 'usage-unknown';
|