@ggui-ai/negotiator 0.2.0-alpha.1 → 0.2.0-alpha.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/ensure-conforming-contract.d.ts +70 -0
- package/dist/ensure-conforming-contract.d.ts.map +1 -0
- package/dist/ensure-conforming-contract.js +115 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/normalize-draft.d.ts +33 -0
- package/dist/normalize-draft.d.ts.map +1 -0
- package/dist/normalize-draft.js +143 -0
- package/dist/preserve-seed-surfaces.d.ts +40 -0
- package/dist/preserve-seed-surfaces.d.ts.map +1 -0
- package/dist/preserve-seed-surfaces.js +57 -0
- package/dist/synth-bench/cli-llm.d.ts +20 -0
- package/dist/synth-bench/cli-llm.d.ts.map +1 -0
- package/dist/synth-bench/cli-llm.js +97 -0
- package/dist/synth-bench/corpus.d.ts +52 -0
- package/dist/synth-bench/corpus.d.ts.map +1 -1
- package/dist/synth-bench/corpus.js +306 -5
- package/dist/synth-bench/round-trip-score.d.ts +87 -0
- package/dist/synth-bench/round-trip-score.d.ts.map +1 -0
- package/dist/synth-bench/round-trip-score.js +105 -0
- package/dist/synth-bench/run-bench-cli.js +6 -82
- package/dist/synth-bench/run-repair-bench-cli.d.ts +3 -0
- package/dist/synth-bench/run-repair-bench-cli.d.ts.map +1 -0
- package/dist/synth-bench/run-repair-bench-cli.js +86 -0
- package/dist/synth-bench/run-repair-bench.d.ts +94 -0
- package/dist/synth-bench/run-repair-bench.d.ts.map +1 -0
- package/dist/synth-bench/run-repair-bench.js +172 -0
- package/dist/synthesize-contract.d.ts +38 -6
- package/dist/synthesize-contract.d.ts.map +1 -1
- package/dist/synthesize-contract.js +246 -32
- package/package.json +5 -4
- package/src/ensure-conforming-contract.ts +175 -0
- package/src/index.ts +2 -0
- package/src/normalize-draft.ts +156 -0
- package/src/preserve-seed-surfaces.ts +61 -0
- package/src/synth-bench/cli-llm.ts +140 -0
- package/src/synth-bench/corpus.ts +335 -5
- package/src/synth-bench/round-trip-score.ts +169 -0
- package/src/synth-bench/run-bench-cli.ts +13 -115
- package/src/synth-bench/run-repair-bench-cli.ts +119 -0
- package/src/synth-bench/run-repair-bench.ts +266 -0
- package/src/synthesize-contract.ts +299 -37
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `ensureConformingContract` — the negotiator's create-path guarantee.
|
|
3
|
+
*
|
|
4
|
+
* Given the agent's PROPOSED draft, return a contract that is
|
|
5
|
+
* GUARANTEED to pass the deterministic gate (`lintContract` with zero
|
|
6
|
+
* errors), so the handshake backstop (`validateContract`) never throws
|
|
7
|
+
* on it. This is the "Propose vs Commit" forgiving-handshake core
|
|
8
|
+
* shared by every negotiator implementation (OSS llm-backed + cloud
|
|
9
|
+
* bedrock) so the behavior cannot drift between deployments:
|
|
10
|
+
*
|
|
11
|
+
* - draft already conforms → return it verbatim (origin: 'agent')
|
|
12
|
+
* - draft has errors → repair-in-place via the bounded LLM loop,
|
|
13
|
+
* seeded with the draft + the deterministic
|
|
14
|
+
* findings, looping until the gate is green
|
|
15
|
+
* (origin: 'synth')
|
|
16
|
+
* - repair impossible → minimal conforming contract (`{}`) + loud
|
|
17
|
+
* (LLM down / provider error findings; STILL origin 'synth';
|
|
18
|
+
* can't synth / budget NEVER throws.
|
|
19
|
+
* exhausted)
|
|
20
|
+
*
|
|
21
|
+
* Determinism lives in the GATE (`lintContract`), never in the repair.
|
|
22
|
+
* The repair LLM is non-deterministic, but the loop only exits when the
|
|
23
|
+
* deterministic gate is green — the same shape ui-gen uses to tolerate
|
|
24
|
+
* non-deterministic code generation behind a deterministic self_check.
|
|
25
|
+
*
|
|
26
|
+
* Cache/blueprint matching is NOT this function's job — the caller
|
|
27
|
+
* (negotiator `decide()`) runs its deployment-specific cache match
|
|
28
|
+
* FIRST and only falls through to here on a miss. That preserves the
|
|
29
|
+
* "cache-first, repair-second" ordering the negotiator contract
|
|
30
|
+
* mandates.
|
|
31
|
+
*/
|
|
32
|
+
import { type DataContract, type GadgetDescriptor, type SuggestionFinding } from '@ggui-ai/protocol';
|
|
33
|
+
import type { LLMCaller } from './llm-caller.js';
|
|
34
|
+
export interface EnsureConformingResult {
|
|
35
|
+
/** A contract guaranteed to pass `lintContract` with zero errors. */
|
|
36
|
+
readonly contract: DataContract;
|
|
37
|
+
/**
|
|
38
|
+
* - `'agent'` — the draft was already conforming; returned verbatim.
|
|
39
|
+
* - `'synth'` — the draft had errors; this is the repaired result
|
|
40
|
+
* (or the minimal-conforming fallback when repair was impossible).
|
|
41
|
+
*/
|
|
42
|
+
readonly origin: 'agent' | 'synth';
|
|
43
|
+
/**
|
|
44
|
+
* How the conforming contract was produced — finer-grained than
|
|
45
|
+
* `origin`, for telemetry (the efficiency tiers):
|
|
46
|
+
* - `verbatim` — draft was clean; returned as-is (origin agent).
|
|
47
|
+
* - `normalized` — deterministic fix only, NO LLM (origin synth).
|
|
48
|
+
* - `llm-repair` — the bounded LLM repair loop ran (origin synth).
|
|
49
|
+
* - `fallback-empty`— unrepairable; minimal `{}` contract (origin synth).
|
|
50
|
+
*/
|
|
51
|
+
readonly method: 'verbatim' | 'normalized' | 'llm-repair' | 'fallback-empty';
|
|
52
|
+
/**
|
|
53
|
+
* Findings surfaced to the agent. On `origin: 'agent'`, any hygiene
|
|
54
|
+
* warnings on the (valid) draft. On `origin: 'synth'`, the ERROR
|
|
55
|
+
* findings that rejected the agent's draft — so the agent-side model
|
|
56
|
+
* learns what it got wrong, even though we repaired it.
|
|
57
|
+
*/
|
|
58
|
+
readonly findings: readonly SuggestionFinding[];
|
|
59
|
+
/** Operator- + LLM-readable explanation. */
|
|
60
|
+
readonly reasoning: string;
|
|
61
|
+
}
|
|
62
|
+
export declare function ensureConformingContract(deps: {
|
|
63
|
+
readonly llm: LLMCaller;
|
|
64
|
+
}, args: {
|
|
65
|
+
/** Untrusted: the agent's draft may not be a valid DataContract. */
|
|
66
|
+
readonly draft: unknown;
|
|
67
|
+
readonly intent: string;
|
|
68
|
+
readonly appGadgets?: readonly GadgetDescriptor[];
|
|
69
|
+
}): Promise<EnsureConformingResult>;
|
|
70
|
+
//# sourceMappingURL=ensure-conforming-contract.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"ensure-conforming-contract.d.ts","sourceRoot":"","sources":["../src/ensure-conforming-contract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA8BG;AACH,OAAO,EAGL,KAAK,YAAY,EACjB,KAAK,gBAAgB,EACrB,KAAK,iBAAiB,EACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,MAAM,WAAW,sBAAsB;IACrC,qEAAqE;IACrE,QAAQ,CAAC,QAAQ,EAAE,YAAY,CAAC;IAChC;;;;OAIG;IACH,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC;IACnC;;;;;;;OAOG;IACH,QAAQ,CAAC,MAAM,EAAE,UAAU,GAAG,YAAY,GAAG,YAAY,GAAG,gBAAgB,CAAC;IAC7E;;;;;OAKG;IACH,QAAQ,CAAC,QAAQ,EAAE,SAAS,iBAAiB,EAAE,CAAC;IAChD,4CAA4C;IAC5C,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;CAC5B;AAKD,wBAAsB,wBAAwB,CAC5C,IAAI,EAAE;IAAE,QAAQ,CAAC,GAAG,EAAE,SAAS,CAAA;CAAE,EACjC,IAAI,EAAE;IACJ,oEAAoE;IACpE,QAAQ,CAAC,KAAK,EAAE,OAAO,CAAC;IACxB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,UAAU,CAAC,EAAE,SAAS,gBAAgB,EAAE,CAAC;CACnD,GACA,OAAO,CAAC,sBAAsB,CAAC,CA4FjC"}
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `ensureConformingContract` — the negotiator's create-path guarantee.
|
|
3
|
+
*
|
|
4
|
+
* Given the agent's PROPOSED draft, return a contract that is
|
|
5
|
+
* GUARANTEED to pass the deterministic gate (`lintContract` with zero
|
|
6
|
+
* errors), so the handshake backstop (`validateContract`) never throws
|
|
7
|
+
* on it. This is the "Propose vs Commit" forgiving-handshake core
|
|
8
|
+
* shared by every negotiator implementation (OSS llm-backed + cloud
|
|
9
|
+
* bedrock) so the behavior cannot drift between deployments:
|
|
10
|
+
*
|
|
11
|
+
* - draft already conforms → return it verbatim (origin: 'agent')
|
|
12
|
+
* - draft has errors → repair-in-place via the bounded LLM loop,
|
|
13
|
+
* seeded with the draft + the deterministic
|
|
14
|
+
* findings, looping until the gate is green
|
|
15
|
+
* (origin: 'synth')
|
|
16
|
+
* - repair impossible → minimal conforming contract (`{}`) + loud
|
|
17
|
+
* (LLM down / provider error findings; STILL origin 'synth';
|
|
18
|
+
* can't synth / budget NEVER throws.
|
|
19
|
+
* exhausted)
|
|
20
|
+
*
|
|
21
|
+
* Determinism lives in the GATE (`lintContract`), never in the repair.
|
|
22
|
+
* The repair LLM is non-deterministic, but the loop only exits when the
|
|
23
|
+
* deterministic gate is green — the same shape ui-gen uses to tolerate
|
|
24
|
+
* non-deterministic code generation behind a deterministic self_check.
|
|
25
|
+
*
|
|
26
|
+
* Cache/blueprint matching is NOT this function's job — the caller
|
|
27
|
+
* (negotiator `decide()`) runs its deployment-specific cache match
|
|
28
|
+
* FIRST and only falls through to here on a miss. That preserves the
|
|
29
|
+
* "cache-first, repair-second" ordering the negotiator contract
|
|
30
|
+
* mandates.
|
|
31
|
+
*/
|
|
32
|
+
import { lintContract, dataContractSchema, } from '@ggui-ai/protocol';
|
|
33
|
+
import { synthesizeContract } from './synthesize-contract.js';
|
|
34
|
+
import { normalizeDraft } from './normalize-draft.js';
|
|
35
|
+
/** Trivially-valid last-resort contract — all four specs omitted. */
|
|
36
|
+
const EMPTY_CONTRACT = {};
|
|
37
|
+
export async function ensureConformingContract(deps, args) {
|
|
38
|
+
const lint = lintContract(args.draft);
|
|
39
|
+
const warnFindings = lint.warnings.map((w) => ({
|
|
40
|
+
code: w.code,
|
|
41
|
+
severity: 'warn',
|
|
42
|
+
path: w.path,
|
|
43
|
+
message: w.message,
|
|
44
|
+
}));
|
|
45
|
+
// Fast path — draft already conforms. Deterministic, no LLM call.
|
|
46
|
+
// `lint.errors.length === 0` implies the shape phase passed, so the
|
|
47
|
+
// strict parse cannot throw — it just re-derives the typed DataContract
|
|
48
|
+
// from the untrusted input (validator returns the typed shape; no cast).
|
|
49
|
+
if (lint.errors.length === 0) {
|
|
50
|
+
return {
|
|
51
|
+
contract: dataContractSchema.parse(args.draft),
|
|
52
|
+
origin: 'agent',
|
|
53
|
+
method: 'verbatim',
|
|
54
|
+
findings: warnFindings,
|
|
55
|
+
reasoning: 'agent draft passed validateContract; accepted verbatim (origin: agent)',
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
const errorFindings = lint.errors.map((e) => ({
|
|
59
|
+
code: e.code,
|
|
60
|
+
severity: 'error',
|
|
61
|
+
path: e.path,
|
|
62
|
+
message: e.message,
|
|
63
|
+
}));
|
|
64
|
+
// L3 — deterministic normalization tier. Most agent malformations are
|
|
65
|
+
// mechanical (stray illegal wrapper keys, non-canonical schema types).
|
|
66
|
+
// Fix them WITHOUT an LLM: strip + canonicalize, re-lint, and if the
|
|
67
|
+
// draft now conforms, return it verbatim-but-cleaned. Faithful (no
|
|
68
|
+
// reshape risk — the agent's specs are preserved exactly) and free (no
|
|
69
|
+
// LLM call). Semantic deficiencies fall through to the repair loop.
|
|
70
|
+
const normalized = normalizeDraft(args.draft);
|
|
71
|
+
const normLint = lintContract(normalized);
|
|
72
|
+
if (normLint.errors.length === 0) {
|
|
73
|
+
return {
|
|
74
|
+
contract: dataContractSchema.parse(normalized),
|
|
75
|
+
origin: 'synth',
|
|
76
|
+
method: 'normalized',
|
|
77
|
+
findings: errorFindings,
|
|
78
|
+
reasoning: 'normalized the agent draft deterministically (stripped invalid keys / canonicalized schema types, no LLM) to pass validateContract',
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
// Repair loop on the NORMALIZED draft (mechanical errors already
|
|
82
|
+
// fixed) with only the REMAINING (semantic) findings — so the LLM
|
|
83
|
+
// patches what reasoning is genuinely needed for, from a clean start.
|
|
84
|
+
const synth = await synthesizeContract(deps, args.intent, {
|
|
85
|
+
...(args.appGadgets ? { appGadgets: args.appGadgets } : {}),
|
|
86
|
+
draft: normalized,
|
|
87
|
+
draftFindings: normLint.errors.map((e) => ({
|
|
88
|
+
code: e.code,
|
|
89
|
+
path: e.path,
|
|
90
|
+
message: e.message,
|
|
91
|
+
})),
|
|
92
|
+
});
|
|
93
|
+
if (synth.contract !== null &&
|
|
94
|
+
lintContract(synth.contract).errors.length === 0) {
|
|
95
|
+
return {
|
|
96
|
+
contract: synth.contract,
|
|
97
|
+
origin: 'synth',
|
|
98
|
+
method: 'llm-repair',
|
|
99
|
+
findings: errorFindings,
|
|
100
|
+
reasoning: `repaired the agent draft to pass validateContract — ${synth.reason}`,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
// Repair impossible (LLM down, provider can't synthesize, or the
|
|
104
|
+
// repair budget exhausted). We still MUST return a conforming
|
|
105
|
+
// contract — the handshake never hard-fails. Minimal conforming
|
|
106
|
+
// contract + loud findings so the agent can re-issue a corrected
|
|
107
|
+
// contract via ggui_render override if it needs the declared specs.
|
|
108
|
+
return {
|
|
109
|
+
contract: EMPTY_CONTRACT,
|
|
110
|
+
origin: 'synth',
|
|
111
|
+
method: 'fallback-empty',
|
|
112
|
+
findings: errorFindings,
|
|
113
|
+
reasoning: `could not repair the agent draft within budget (${synth.reason}); returning a minimal conforming contract — re-issue a corrected contract via ggui_render override if you need the declared specs`,
|
|
114
|
+
};
|
|
115
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -31,6 +31,8 @@ export { rerankCandidates } from './llm-rerank.js';
|
|
|
31
31
|
export type { RerankCandidate, RerankDecision, RerankQuery, } from './llm-rerank.js';
|
|
32
32
|
export { synthesizeContract } from './synthesize-contract.js';
|
|
33
33
|
export type { SynthesizeContractResult } from './synthesize-contract.js';
|
|
34
|
+
export { ensureConformingContract } from './ensure-conforming-contract.js';
|
|
35
|
+
export type { EnsureConformingResult } from './ensure-conforming-contract.js';
|
|
34
36
|
export { validateContractStructure, validateContractNovelty, formatValidationFindings, } from './contract-validators.js';
|
|
35
37
|
export type { ContractValidationFinding, ContractValidationFindingKind, ContractValidationResult, ContractValidationNoveltyDeps, ContractValidationNoveltyOptions, } from './contract-validators.js';
|
|
36
38
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACtE,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EACV,aAAa,EACb,cAAc,EACd,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC9E,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC5D,YAAY,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EACL,sBAAsB,EACtB,wBAAwB,EACxB,YAAY,GACb,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,YAAY,EACV,aAAa,EACb,cAAc,EACd,eAAe,EACf,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,YAAY,EACV,eAAe,EACf,cAAc,EACd,WAAW,GACZ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,EACL,yBAAyB,EACzB,uBAAuB,EACvB,wBAAwB,GACzB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,yBAAyB,EACzB,6BAA6B,EAC7B,wBAAwB,EACxB,6BAA6B,EAC7B,gCAAgC,GACjC,MAAM,0BAA0B,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAEH,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAChE,OAAO,EAAE,eAAe,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AACxE,OAAO,EAAE,kBAAkB,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACtE,YAAY,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC5D,OAAO,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAC;AACtE,OAAO,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAC5C,YAAY,EACV,aAAa,EACb,cAAc,EACd,eAAe,GAChB,MAAM,iBAAiB,CAAC;AACzB,YAAY,EAAE,gBAAgB,EAAE,MAAM,YAAY,CAAC;AACnD,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAC9E,YAAY,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC5D,YAAY,EAAE,uBAAuB,EAAE,MAAM,qBAAqB,CAAC;AACnE,OAAO,EACL,sBAAsB,EACtB,wBAAwB,EACxB,YAAY,GACb,MAAM,eAAe,CAAC;AACvB,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAC3C,YAAY,EACV,aAAa,EACb,cAAc,EACd,eAAe,EACf,eAAe,GAChB,MAAM,gBAAgB,CAAC;AACxB,OAAO,EAAE,gBAAgB,EAAE,MAAM,iBAAiB,CAAC;AACnD,YAAY,EACV,eAAe,EACf,cAAc,EACd,WAAW,GACZ,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,kBAAkB,EAAE,MAAM,0BAA0B,CAAC;AAC9D,YAAY,EAAE,wBAAwB,EAAE,MAAM,0BAA0B,CAAC;AACzE,OAAO,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAC;AAC3E,YAAY,EAAE,sBAAsB,EAAE,MAAM,iCAAiC,CAAC;AAC9E,OAAO,EACL,yBAAyB,EACzB,uBAAuB,EACvB,wBAAwB,GACzB,MAAM,0BAA0B,CAAC;AAClC,YAAY,EACV,yBAAyB,EACzB,6BAA6B,EAC7B,wBAAwB,EACxB,6BAA6B,EAC7B,gCAAgC,GACjC,MAAM,0BAA0B,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -22,4 +22,5 @@ export { DECISION_SYSTEM_PROMPT, buildDecisionUserMessage, makeDecision, } from
|
|
|
22
22
|
export { negotiate } from './negotiate.js';
|
|
23
23
|
export { rerankCandidates } from './llm-rerank.js';
|
|
24
24
|
export { synthesizeContract } from './synthesize-contract.js';
|
|
25
|
+
export { ensureConformingContract } from './ensure-conforming-contract.js';
|
|
25
26
|
export { validateContractStructure, validateContractNovelty, formatValidationFindings, } from './contract-validators.js';
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic draft normalization — the cheap, faithful repair tier.
|
|
3
|
+
*
|
|
4
|
+
* Most agent-draft malformations are MECHANICAL: a stray illegal key on
|
|
5
|
+
* a spec wrapper (a JSON-Schema-reflex `required: [...]` array on the
|
|
6
|
+
* propsSpec wrapper, an `additionalProperties` key), or a non-canonical
|
|
7
|
+
* schema `type` spelling (`"enum"`, `"integer"`). None of these needs an
|
|
8
|
+
* LLM to fix — and routing them through the LLM repair loop is both
|
|
9
|
+
* wasteful (a full regeneration) and RISKY (the model may re-author and
|
|
10
|
+
* reshape a 95%-correct draft, e.g. drop a propsSpec seed surface).
|
|
11
|
+
*
|
|
12
|
+
* This pass fixes the mechanical classes deterministically, preserving
|
|
13
|
+
* the agent's intent exactly:
|
|
14
|
+
* - strips keys the protocol's `.strict()` spec schemas would reject,
|
|
15
|
+
* keeping only the allowed keys at each wrapper / entry level;
|
|
16
|
+
* - canonicalizes every inner JSON Schema via {@link normalizeSchema}
|
|
17
|
+
* (the same normalizer `buildContract` runs on synth output).
|
|
18
|
+
*
|
|
19
|
+
* The caller (`ensureConformingContract`) re-lints the result: if it now
|
|
20
|
+
* passes the gate, the draft is returned WITHOUT ever calling the LLM.
|
|
21
|
+
* Semantic deficiencies (wrong placement, missing data surface, dangling
|
|
22
|
+
* cross-refs) are deliberately out of scope — those still go to the
|
|
23
|
+
* repair loop, where reasoning earns its keep.
|
|
24
|
+
*/
|
|
25
|
+
/**
|
|
26
|
+
* Return a structurally-normalized copy of an untrusted draft: illegal
|
|
27
|
+
* wrapper/entry keys stripped, inner schemas canonicalized. Pure — never
|
|
28
|
+
* mutates the input, never throws. Unknown top-level fields ride through
|
|
29
|
+
* (the top-level DataContract schema is `.passthrough()`); only the
|
|
30
|
+
* `.strict()` spec wrappers and entries are cleaned.
|
|
31
|
+
*/
|
|
32
|
+
export declare function normalizeDraft(draft: unknown): unknown;
|
|
33
|
+
//# sourceMappingURL=normalize-draft.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"normalize-draft.d.ts","sourceRoot":"","sources":["../src/normalize-draft.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AA8EH;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,OAAO,CA+CtD"}
|
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic draft normalization — the cheap, faithful repair tier.
|
|
3
|
+
*
|
|
4
|
+
* Most agent-draft malformations are MECHANICAL: a stray illegal key on
|
|
5
|
+
* a spec wrapper (a JSON-Schema-reflex `required: [...]` array on the
|
|
6
|
+
* propsSpec wrapper, an `additionalProperties` key), or a non-canonical
|
|
7
|
+
* schema `type` spelling (`"enum"`, `"integer"`). None of these needs an
|
|
8
|
+
* LLM to fix — and routing them through the LLM repair loop is both
|
|
9
|
+
* wasteful (a full regeneration) and RISKY (the model may re-author and
|
|
10
|
+
* reshape a 95%-correct draft, e.g. drop a propsSpec seed surface).
|
|
11
|
+
*
|
|
12
|
+
* This pass fixes the mechanical classes deterministically, preserving
|
|
13
|
+
* the agent's intent exactly:
|
|
14
|
+
* - strips keys the protocol's `.strict()` spec schemas would reject,
|
|
15
|
+
* keeping only the allowed keys at each wrapper / entry level;
|
|
16
|
+
* - canonicalizes every inner JSON Schema via {@link normalizeSchema}
|
|
17
|
+
* (the same normalizer `buildContract` runs on synth output).
|
|
18
|
+
*
|
|
19
|
+
* The caller (`ensureConformingContract`) re-lints the result: if it now
|
|
20
|
+
* passes the gate, the draft is returned WITHOUT ever calling the LLM.
|
|
21
|
+
* Semantic deficiencies (wrong placement, missing data surface, dangling
|
|
22
|
+
* cross-refs) are deliberately out of scope — those still go to the
|
|
23
|
+
* repair loop, where reasoning earns its keep.
|
|
24
|
+
*/
|
|
25
|
+
import { normalizeSchema } from './normalize-schema.js';
|
|
26
|
+
// Allowed-key sets mirror the protocol `.strict()` schemas
|
|
27
|
+
// (schemas/data-contract.ts). Stripping anything outside these is safe:
|
|
28
|
+
// the strict schema would reject it as CTR_SHAPE_UNRECOGNIZED_KEYS.
|
|
29
|
+
const PROPS_WRAPPER_KEYS = new Set(['description', 'properties']);
|
|
30
|
+
const PROP_ENTRY_KEYS = new Set([
|
|
31
|
+
'description',
|
|
32
|
+
'schema',
|
|
33
|
+
'required',
|
|
34
|
+
'default',
|
|
35
|
+
'example',
|
|
36
|
+
'sourceTool',
|
|
37
|
+
]);
|
|
38
|
+
const CONTEXT_ENTRY_KEYS = new Set([
|
|
39
|
+
'description',
|
|
40
|
+
'schema',
|
|
41
|
+
'default',
|
|
42
|
+
'debounceMs',
|
|
43
|
+
'example',
|
|
44
|
+
]);
|
|
45
|
+
const ACTION_ENTRY_KEYS = new Set([
|
|
46
|
+
'description',
|
|
47
|
+
'label',
|
|
48
|
+
'schema',
|
|
49
|
+
'example',
|
|
50
|
+
'icon',
|
|
51
|
+
'confirm',
|
|
52
|
+
'nextStep',
|
|
53
|
+
]);
|
|
54
|
+
const STREAM_ENTRY_KEYS = new Set(['description', 'schema', 'source']);
|
|
55
|
+
const AGENT_TOOL_KEYS = new Set([
|
|
56
|
+
'description',
|
|
57
|
+
'usage',
|
|
58
|
+
'inputSchema',
|
|
59
|
+
'outputSchema',
|
|
60
|
+
'required',
|
|
61
|
+
'example',
|
|
62
|
+
]);
|
|
63
|
+
function isRecord(value) {
|
|
64
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
65
|
+
}
|
|
66
|
+
/** Keep only `allowed` keys; normalize the schema-bearing fields named
|
|
67
|
+
* in `schemaFields`. Non-record entries pass through untouched. */
|
|
68
|
+
function cleanEntry(entry, allowed, schemaFields) {
|
|
69
|
+
if (!isRecord(entry))
|
|
70
|
+
return entry;
|
|
71
|
+
const out = {};
|
|
72
|
+
for (const [key, value] of Object.entries(entry)) {
|
|
73
|
+
if (!allowed.has(key))
|
|
74
|
+
continue; // strip the illegal key
|
|
75
|
+
out[key] =
|
|
76
|
+
schemaFields.includes(key) && value !== undefined
|
|
77
|
+
? normalizeSchema(value)
|
|
78
|
+
: value;
|
|
79
|
+
}
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
/** Apply {@link cleanEntry} across a `Record<name, entry>` spec map. */
|
|
83
|
+
function cleanEntryMap(map, allowed, schemaFields) {
|
|
84
|
+
const out = {};
|
|
85
|
+
for (const [name, entry] of Object.entries(map)) {
|
|
86
|
+
out[name] = cleanEntry(entry, allowed, schemaFields);
|
|
87
|
+
}
|
|
88
|
+
return out;
|
|
89
|
+
}
|
|
90
|
+
/**
|
|
91
|
+
* Return a structurally-normalized copy of an untrusted draft: illegal
|
|
92
|
+
* wrapper/entry keys stripped, inner schemas canonicalized. Pure — never
|
|
93
|
+
* mutates the input, never throws. Unknown top-level fields ride through
|
|
94
|
+
* (the top-level DataContract schema is `.passthrough()`); only the
|
|
95
|
+
* `.strict()` spec wrappers and entries are cleaned.
|
|
96
|
+
*/
|
|
97
|
+
export function normalizeDraft(draft) {
|
|
98
|
+
if (!isRecord(draft))
|
|
99
|
+
return draft;
|
|
100
|
+
const out = { ...draft };
|
|
101
|
+
// propsSpec wrapper: keep {description, properties}; clean each PropEntry.
|
|
102
|
+
if (isRecord(out['propsSpec'])) {
|
|
103
|
+
const ps = {};
|
|
104
|
+
for (const [key, value] of Object.entries(out['propsSpec'])) {
|
|
105
|
+
if (PROPS_WRAPPER_KEYS.has(key))
|
|
106
|
+
ps[key] = value;
|
|
107
|
+
}
|
|
108
|
+
if (isRecord(ps['properties'])) {
|
|
109
|
+
ps['properties'] = cleanEntryMap(ps['properties'], PROP_ENTRY_KEYS, [
|
|
110
|
+
'schema',
|
|
111
|
+
]);
|
|
112
|
+
}
|
|
113
|
+
out['propsSpec'] = ps;
|
|
114
|
+
}
|
|
115
|
+
if (isRecord(out['contextSpec'])) {
|
|
116
|
+
out['contextSpec'] = cleanEntryMap(out['contextSpec'], CONTEXT_ENTRY_KEYS, [
|
|
117
|
+
'schema',
|
|
118
|
+
]);
|
|
119
|
+
}
|
|
120
|
+
if (isRecord(out['actionSpec'])) {
|
|
121
|
+
out['actionSpec'] = cleanEntryMap(out['actionSpec'], ACTION_ENTRY_KEYS, [
|
|
122
|
+
'schema',
|
|
123
|
+
]);
|
|
124
|
+
}
|
|
125
|
+
if (isRecord(out['streamSpec'])) {
|
|
126
|
+
out['streamSpec'] = cleanEntryMap(out['streamSpec'], STREAM_ENTRY_KEYS, [
|
|
127
|
+
'schema',
|
|
128
|
+
]);
|
|
129
|
+
}
|
|
130
|
+
if (isRecord(out['agentCapabilities'])) {
|
|
131
|
+
const ac = out['agentCapabilities'];
|
|
132
|
+
if (isRecord(ac['tools'])) {
|
|
133
|
+
out['agentCapabilities'] = {
|
|
134
|
+
...ac,
|
|
135
|
+
tools: cleanEntryMap(ac['tools'], AGENT_TOOL_KEYS, [
|
|
136
|
+
'inputSchema',
|
|
137
|
+
'outputSchema',
|
|
138
|
+
]),
|
|
139
|
+
};
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
return out;
|
|
143
|
+
}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Seed-surface preservation — the deterministic backstop that keeps the
|
|
3
|
+
* repair loop FAITHFUL.
|
|
4
|
+
*
|
|
5
|
+
* The agent's draft declares its agent-owned render-time data as
|
|
6
|
+
* `propsSpec.properties` (the only agent→client seed channel —
|
|
7
|
+
* `contextSpec` has no runtime seed path). A repair that drops or
|
|
8
|
+
* reshapes one of those keys away from propsSpec (the canonical
|
|
9
|
+
* `propsSpec.X → contextSpec.X` regression) produces a contract that is
|
|
10
|
+
* VALID (`lintContract` passes) yet round-trip-BROKEN: the agent can no
|
|
11
|
+
* longer seed X at render, so the UI renders empty.
|
|
12
|
+
*
|
|
13
|
+
* `lintContract` + the placement validators can't catch this — none of
|
|
14
|
+
* them sees the agent's DRAFT. These helpers do: they compare the
|
|
15
|
+
* repaired candidate against the draft and report which agent-owned seed
|
|
16
|
+
* surfaces went missing, so the synth loop can drive a corrective retry.
|
|
17
|
+
* Model-independent — it works even when a weak model keeps reshaping.
|
|
18
|
+
*/
|
|
19
|
+
import type { DataContract } from '@ggui-ai/protocol';
|
|
20
|
+
/**
|
|
21
|
+
* The `propsSpec.properties` keys an agent declared on a (possibly
|
|
22
|
+
* malformed) draft — its agent-owned render-time SEED surfaces.
|
|
23
|
+
* Defensive: the draft is untrusted, so every level is probed before
|
|
24
|
+
* access. Returns `[]` for any non-propsSpec-bearing draft.
|
|
25
|
+
*/
|
|
26
|
+
export declare function draftSeedPropKeys(draft: unknown): string[];
|
|
27
|
+
/**
|
|
28
|
+
* Agent-owned seed surfaces (propsSpec property keys) present in `draft`
|
|
29
|
+
* that the repaired `candidate` DROPPED — i.e. they are no longer
|
|
30
|
+
* seedable as a propsSpec property. Reshaping `propsSpec.X` to
|
|
31
|
+
* `contextSpec.X`, or dropping it entirely, both surface here (contextSpec
|
|
32
|
+
* is not an agent-seedable home). Returns `[]` when every declared seed
|
|
33
|
+
* surface survived (preservation holds).
|
|
34
|
+
*
|
|
35
|
+
* Preservation-biased on purpose: keeping a seed key the intent turned
|
|
36
|
+
* out not to need is harmless (an unsent optional prop); DROPPING one the
|
|
37
|
+
* agent relies on is the round-trip break. So we only ever flag drops.
|
|
38
|
+
*/
|
|
39
|
+
export declare function findDroppedSeedSurfaces(draft: unknown, candidate: DataContract): string[];
|
|
40
|
+
//# sourceMappingURL=preserve-seed-surfaces.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"preserve-seed-surfaces.d.ts","sourceRoot":"","sources":["../src/preserve-seed-surfaces.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAMtD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,EAAE,CAO1D;AAED;;;;;;;;;;;GAWG;AACH,wBAAgB,uBAAuB,CACrC,KAAK,EAAE,OAAO,EACd,SAAS,EAAE,YAAY,GACtB,MAAM,EAAE,CAKV"}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Seed-surface preservation — the deterministic backstop that keeps the
|
|
3
|
+
* repair loop FAITHFUL.
|
|
4
|
+
*
|
|
5
|
+
* The agent's draft declares its agent-owned render-time data as
|
|
6
|
+
* `propsSpec.properties` (the only agent→client seed channel —
|
|
7
|
+
* `contextSpec` has no runtime seed path). A repair that drops or
|
|
8
|
+
* reshapes one of those keys away from propsSpec (the canonical
|
|
9
|
+
* `propsSpec.X → contextSpec.X` regression) produces a contract that is
|
|
10
|
+
* VALID (`lintContract` passes) yet round-trip-BROKEN: the agent can no
|
|
11
|
+
* longer seed X at render, so the UI renders empty.
|
|
12
|
+
*
|
|
13
|
+
* `lintContract` + the placement validators can't catch this — none of
|
|
14
|
+
* them sees the agent's DRAFT. These helpers do: they compare the
|
|
15
|
+
* repaired candidate against the draft and report which agent-owned seed
|
|
16
|
+
* surfaces went missing, so the synth loop can drive a corrective retry.
|
|
17
|
+
* Model-independent — it works even when a weak model keeps reshaping.
|
|
18
|
+
*/
|
|
19
|
+
function isRecord(value) {
|
|
20
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* The `propsSpec.properties` keys an agent declared on a (possibly
|
|
24
|
+
* malformed) draft — its agent-owned render-time SEED surfaces.
|
|
25
|
+
* Defensive: the draft is untrusted, so every level is probed before
|
|
26
|
+
* access. Returns `[]` for any non-propsSpec-bearing draft.
|
|
27
|
+
*/
|
|
28
|
+
export function draftSeedPropKeys(draft) {
|
|
29
|
+
if (!isRecord(draft))
|
|
30
|
+
return [];
|
|
31
|
+
const propsSpec = draft['propsSpec'];
|
|
32
|
+
if (!isRecord(propsSpec))
|
|
33
|
+
return [];
|
|
34
|
+
const properties = propsSpec['properties'];
|
|
35
|
+
if (!isRecord(properties))
|
|
36
|
+
return [];
|
|
37
|
+
return Object.keys(properties);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Agent-owned seed surfaces (propsSpec property keys) present in `draft`
|
|
41
|
+
* that the repaired `candidate` DROPPED — i.e. they are no longer
|
|
42
|
+
* seedable as a propsSpec property. Reshaping `propsSpec.X` to
|
|
43
|
+
* `contextSpec.X`, or dropping it entirely, both surface here (contextSpec
|
|
44
|
+
* is not an agent-seedable home). Returns `[]` when every declared seed
|
|
45
|
+
* surface survived (preservation holds).
|
|
46
|
+
*
|
|
47
|
+
* Preservation-biased on purpose: keeping a seed key the intent turned
|
|
48
|
+
* out not to need is harmless (an unsent optional prop); DROPPING one the
|
|
49
|
+
* agent relies on is the round-trip break. So we only ever flag drops.
|
|
50
|
+
*/
|
|
51
|
+
export function findDroppedSeedSurfaces(draft, candidate) {
|
|
52
|
+
const seedKeys = draftSeedPropKeys(draft);
|
|
53
|
+
if (seedKeys.length === 0)
|
|
54
|
+
return [];
|
|
55
|
+
const candidateProps = candidate.propsSpec?.properties ?? {};
|
|
56
|
+
return seedKeys.filter((key) => candidateProps[key] === undefined);
|
|
57
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import type { LLMCaller } from '../llm-caller.js';
|
|
2
|
+
/** Default bench model — matches the `ui-gen-default-haiku-4-5` slug. */
|
|
3
|
+
export declare const DEFAULT_MODEL = "claude-haiku-4-5";
|
|
4
|
+
/** Haiku 4.5 token pricing (USD per token) for the cost report. */
|
|
5
|
+
export declare const HAIKU_4_5_PRICE_INPUT_PER_TOKEN: number;
|
|
6
|
+
export declare const HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN: number;
|
|
7
|
+
/**
|
|
8
|
+
* Resolve the Anthropic API key: `ANTHROPIC_API_KEY` env var first, then
|
|
9
|
+
* `~/.ggui/credentials.json` at `apps.global.anthropic`. `scriptName`
|
|
10
|
+
* prefixes the error hints so a failure names the bench that needs it.
|
|
11
|
+
*/
|
|
12
|
+
export declare function resolveAnthropicKey(scriptName: string): string;
|
|
13
|
+
/** Running token usage accumulated across `callStructured` calls in this
|
|
14
|
+
* process — read after a run for the cost line. */
|
|
15
|
+
export declare function getTokenUsage(): {
|
|
16
|
+
readonly input: number;
|
|
17
|
+
readonly output: number;
|
|
18
|
+
};
|
|
19
|
+
export declare function buildAnthropicLlmCaller(apiKey: string, model: string): LLMCaller;
|
|
20
|
+
//# sourceMappingURL=cli-llm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cli-llm.d.ts","sourceRoot":"","sources":["../../src/synth-bench/cli-llm.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,SAAS,EAAc,MAAM,kBAAkB,CAAC;AAI9D,yEAAyE;AACzE,eAAO,MAAM,aAAa,qBAAqB,CAAC;AAEhD,mEAAmE;AACnE,eAAO,MAAM,+BAA+B,QAAkB,CAAC;AAC/D,eAAO,MAAM,gCAAgC,QAAkB,CAAC;AAMhE;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,UAAU,EAAE,MAAM,GAAG,MAAM,CAmB9D;AAmBD;oDACoD;AACpD,wBAAgB,aAAa,IAAI;IAC/B,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;CACzB,CAEA;AAED,wBAAgB,uBAAuB,CACrC,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,GACZ,SAAS,CA2DX"}
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared CLI LLM plumbing for the synth-bench family of live probes
|
|
3
|
+
* (bench-synth, bench-repair). Builds an Anthropic-backed
|
|
4
|
+
* {@link LLMCaller} with forced tool-use, resolves the API key from env
|
|
5
|
+
* or `~/.ggui/credentials.json`, and accumulates token usage for the
|
|
6
|
+
* per-run cost line. Bench-only — not exported from the package index.
|
|
7
|
+
*/
|
|
8
|
+
import { readFileSync } from 'node:fs';
|
|
9
|
+
import { homedir } from 'node:os';
|
|
10
|
+
import { resolve as pathResolve } from 'node:path';
|
|
11
|
+
const ANTHROPIC_API = 'https://api.anthropic.com/v1/messages';
|
|
12
|
+
/** Default bench model — matches the `ui-gen-default-haiku-4-5` slug. */
|
|
13
|
+
export const DEFAULT_MODEL = 'claude-haiku-4-5';
|
|
14
|
+
/** Haiku 4.5 token pricing (USD per token) for the cost report. */
|
|
15
|
+
export const HAIKU_4_5_PRICE_INPUT_PER_TOKEN = 1.0 / 1_000_000;
|
|
16
|
+
export const HAIKU_4_5_PRICE_OUTPUT_PER_TOKEN = 5.0 / 1_000_000;
|
|
17
|
+
/**
|
|
18
|
+
* Resolve the Anthropic API key: `ANTHROPIC_API_KEY` env var first, then
|
|
19
|
+
* `~/.ggui/credentials.json` at `apps.global.anthropic`. `scriptName`
|
|
20
|
+
* prefixes the error hints so a failure names the bench that needs it.
|
|
21
|
+
*/
|
|
22
|
+
export function resolveAnthropicKey(scriptName) {
|
|
23
|
+
const envKey = process.env['ANTHROPIC_API_KEY'];
|
|
24
|
+
if (envKey && envKey.length > 0)
|
|
25
|
+
return envKey;
|
|
26
|
+
const credsPath = pathResolve(homedir(), '.ggui', 'credentials.json');
|
|
27
|
+
let parsed;
|
|
28
|
+
try {
|
|
29
|
+
parsed = JSON.parse(readFileSync(credsPath, 'utf8'));
|
|
30
|
+
}
|
|
31
|
+
catch (err) {
|
|
32
|
+
throw new Error(`${scriptName}: could not read ${credsPath} (${err instanceof Error ? err.message : String(err)}). Set ANTHROPIC_API_KEY env var or run \`ggui auth set anthropic\`.`);
|
|
33
|
+
}
|
|
34
|
+
const key = parsed.apps?.global?.anthropic;
|
|
35
|
+
if (typeof key !== 'string' || key.length === 0) {
|
|
36
|
+
throw new Error(`${scriptName}: no anthropic key found at apps.global.anthropic in ${credsPath}.`);
|
|
37
|
+
}
|
|
38
|
+
return key;
|
|
39
|
+
}
|
|
40
|
+
let totalInputTokens = 0;
|
|
41
|
+
let totalOutputTokens = 0;
|
|
42
|
+
/** Running token usage accumulated across `callStructured` calls in this
|
|
43
|
+
* process — read after a run for the cost line. */
|
|
44
|
+
export function getTokenUsage() {
|
|
45
|
+
return { input: totalInputTokens, output: totalOutputTokens };
|
|
46
|
+
}
|
|
47
|
+
export function buildAnthropicLlmCaller(apiKey, model) {
|
|
48
|
+
return {
|
|
49
|
+
async call() {
|
|
50
|
+
throw new Error('synth-bench: text-mode not exercised — synth uses callStructured');
|
|
51
|
+
},
|
|
52
|
+
async callStructured(systemPrompt, userMessage, tool, maxTokens) {
|
|
53
|
+
// `temperature` deprecated on Haiku 4.5+ — Anthropic rejects with
|
|
54
|
+
// HTTP 400. `tool_choice: { type: 'tool', name }` below already
|
|
55
|
+
// binds output to the input_schema; residual stochasticity stays
|
|
56
|
+
// bounded via canonical-key normalization downstream.
|
|
57
|
+
const body = {
|
|
58
|
+
model,
|
|
59
|
+
max_tokens: maxTokens ?? 1024,
|
|
60
|
+
system: systemPrompt,
|
|
61
|
+
messages: [{ role: 'user', content: userMessage }],
|
|
62
|
+
tools: [
|
|
63
|
+
{
|
|
64
|
+
name: tool.name,
|
|
65
|
+
description: tool.description,
|
|
66
|
+
input_schema: tool.input_schema,
|
|
67
|
+
},
|
|
68
|
+
],
|
|
69
|
+
tool_choice: { type: 'tool', name: tool.name },
|
|
70
|
+
};
|
|
71
|
+
const res = await fetch(ANTHROPIC_API, {
|
|
72
|
+
method: 'POST',
|
|
73
|
+
headers: {
|
|
74
|
+
'content-type': 'application/json',
|
|
75
|
+
'x-api-key': apiKey,
|
|
76
|
+
'anthropic-version': '2023-06-01',
|
|
77
|
+
},
|
|
78
|
+
body: JSON.stringify(body),
|
|
79
|
+
});
|
|
80
|
+
const json = (await res.json());
|
|
81
|
+
if (!res.ok) {
|
|
82
|
+
const errType = json.error?.type ?? 'unknown';
|
|
83
|
+
const errMsg = json.error?.message ?? `HTTP ${res.status}`;
|
|
84
|
+
throw new Error(`anthropic ${errType}: ${errMsg}`);
|
|
85
|
+
}
|
|
86
|
+
if (json.usage) {
|
|
87
|
+
totalInputTokens += json.usage.input_tokens ?? 0;
|
|
88
|
+
totalOutputTokens += json.usage.output_tokens ?? 0;
|
|
89
|
+
}
|
|
90
|
+
const toolBlock = json.content?.find((b) => b.type === 'tool_use');
|
|
91
|
+
if (!toolBlock || toolBlock.input === undefined) {
|
|
92
|
+
throw new Error(`anthropic: no tool_use block in response (stop_reason=${json.stop_reason ?? 'unknown'})`);
|
|
93
|
+
}
|
|
94
|
+
return toolBlock.input;
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
}
|