deepclause-sdk 0.0.87 → 0.0.89
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/judge/index.d.ts +9 -0
- package/dist/judge/index.d.ts.map +1 -0
- package/dist/judge/index.js +6 -0
- package/dist/judge/index.js.map +1 -0
- package/dist/judge/jev.d.ts +21 -0
- package/dist/judge/jev.d.ts.map +1 -0
- package/dist/judge/jev.js +218 -0
- package/dist/judge/jev.js.map +1 -0
- package/dist/judge/llm.d.ts +23 -0
- package/dist/judge/llm.d.ts.map +1 -0
- package/dist/judge/llm.js +120 -0
- package/dist/judge/llm.js.map +1 -0
- package/dist/judge/marshal.d.ts +26 -0
- package/dist/judge/marshal.d.ts.map +1 -0
- package/dist/judge/marshal.js +266 -0
- package/dist/judge/marshal.js.map +1 -0
- package/dist/judge/mock.d.ts +28 -0
- package/dist/judge/mock.d.ts.map +1 -0
- package/dist/judge/mock.js +98 -0
- package/dist/judge/mock.js.map +1 -0
- package/dist/judge/types.d.ts +95 -0
- package/dist/judge/types.d.ts.map +1 -0
- package/dist/judge/types.js +23 -0
- package/dist/judge/types.js.map +1 -0
- package/dist/prolog-src/deepclause_mi.pl +224 -1
- package/dist/runner.d.ts +12 -0
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +158 -0
- package/dist/runner.js.map +1 -1
- package/dist/sdk.d.ts.map +1 -1
- package/dist/sdk.js +55 -0
- package/dist/sdk.js.map +1 -1
- package/dist/system/assets/docs/DML_REFERENCE.md +82 -1
- package/dist/types.d.ts +28 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
- package/src/prolog-src/deepclause_mi.pl +224 -1
package/README.md
CHANGED
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
# DeepClause TUI Agent, CLI and SDK
|
|
2
2
|
|
|
3
|
+
*Update (09/2026): Now also available as an extension for the pi coding agent: [https://github.com/deepclause/deepclause-pi](https://github.com/deepclause/deepclause-pi).*
|
|
4
|
+
|
|
3
5
|
Compile markdown specs into executable logic programs. Guaranteed execution semantics for agentic workflows. Comes with a minimal coding agent incl. a nostalgic Borland-style TUI.
|
|
4
6
|
|
|
5
7
|

|
package/dist/index.d.ts
CHANGED
|
@@ -7,4 +7,6 @@
|
|
|
7
7
|
export { createDeepClause } from './sdk.js';
|
|
8
8
|
export type { CompactionAction, CompactionOptions, CompactionScope, CompactionTrigger, CompactorBinding, CompactorDefinition, CompactorSourceType, CreateOptions, DeepClauseSDK, LLMBackend, LLMBackendMessage, LLMBackendRequest, LLMBackendResponse, LLMBackendTool, LLMBackendToolCall, LLMUsage, MemoryMessage, RunOptions, CompileOptions, CompileResult, CompileTool, DMLEvent, TraceEntry, ToolDefinition, ToolPolicy } from './types.js';
|
|
9
9
|
export { compileToDML, extractParameters, extractToolDependencies, extractDescription, validateWithProlog, analyzeDML, } from './compiler.js';
|
|
10
|
+
export { createMockJudgeBackend, createMockJevJudgeBackend, createJevJudgeBackend, createLLMJudgeBackend, buildJudgePrompt, parseJudgeResponse, } from './judge/index.js';
|
|
11
|
+
export type { JudgeAnswer, JudgeBackend, JudgeBackendRequest, JudgeBackendResponse, JudgeBasis, JudgeCapabilities, JudgeLevel, JudgeOption, JudgeQuestion, JudgeQuestionKind, JudgeSelection, JudgeUsage, JsonValue, JevJudgeBackendOptions, LLMJudgeBackendOptions, MockAnswerSource, MockJudgeBackendOptions, } from './judge/index.js';
|
|
10
12
|
//# sourceMappingURL=index.d.ts.map
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,kBAAkB,EAClB,QAAQ,EACR,aAAa,EACb,UAAU,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,QAAQ,EACR,UAAU,EACV,cAAc,EACd,UAAU,EACX,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,uBAAuB,EACvB,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,GACX,MAAM,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AAC5C,YAAY,EACV,gBAAgB,EAChB,iBAAiB,EACjB,eAAe,EACf,iBAAiB,EACjB,gBAAgB,EAChB,mBAAmB,EACnB,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,UAAU,EACV,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EAClB,cAAc,EACd,kBAAkB,EAClB,QAAQ,EACR,aAAa,EACb,UAAU,EACV,cAAc,EACd,aAAa,EACb,WAAW,EACX,QAAQ,EACR,UAAU,EACV,cAAc,EACd,UAAU,EACX,MAAM,YAAY,CAAC;AAGpB,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,uBAAuB,EACvB,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,GACX,MAAM,eAAe,CAAC;AAGvB,OAAO,EACL,sBAAsB,EACtB,yBAAyB,EACzB,qBAAqB,EACrB,qBAAqB,EACrB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC;AAC1B,YAAY,EACV,WAAW,EACX,YAAY,EACZ,mBAAmB,EACnB,oBAAoB,EACpB,UAAU,EACV,iBAAiB,EACjB,UAAU,EACV,WAAW,EACX,aAAa,EACb,iBAAiB,EACjB,cAAc,EACd,UAAU,EACV,SAAS,EACT,sBAAsB,EACtB,sBAAsB,EACtB,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,kBAAkB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -7,4 +7,6 @@
|
|
|
7
7
|
export { createDeepClause } from './sdk.js';
|
|
8
8
|
// Compiler utilities — also available via the 'deepclause-sdk/compiler' subpath
|
|
9
9
|
export { compileToDML, extractParameters, extractToolDependencies, extractDescription, validateWithProlog, analyzeDML, } from './compiler.js';
|
|
10
|
+
// Judgment layer: generic judge primitives with pluggable backends
|
|
11
|
+
export { createMockJudgeBackend, createMockJevJudgeBackend, createJevJudgeBackend, createLLMJudgeBackend, buildJudgePrompt, parseJudgeResponse, } from './judge/index.js';
|
|
10
12
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AA6B5C,gFAAgF;AAChF,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,uBAAuB,EACvB,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,GACX,MAAM,eAAe,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,UAAU,CAAC;AA6B5C,gFAAgF;AAChF,OAAO,EACL,YAAY,EACZ,iBAAiB,EACjB,uBAAuB,EACvB,kBAAkB,EAClB,kBAAkB,EAClB,UAAU,GACX,MAAM,eAAe,CAAC;AAEvB,mEAAmE;AACnE,OAAO,EACL,sBAAsB,EACtB,yBAAyB,EACzB,qBAAqB,EACrB,qBAAqB,EACrB,gBAAgB,EAChB,kBAAkB,GACnB,MAAM,kBAAkB,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export * from './types.js';
|
|
2
|
+
export { createMockJudgeBackend, createMockJevJudgeBackend } from './mock.js';
|
|
3
|
+
export type { MockJudgeBackendOptions, MockAnswerSource } from './mock.js';
|
|
4
|
+
export { createJevJudgeBackend } from './jev.js';
|
|
5
|
+
export type { JevJudgeBackendOptions } from './jev.js';
|
|
6
|
+
export { createLLMJudgeBackend, buildJudgePrompt, parseJudgeResponse } from './llm.js';
|
|
7
|
+
export type { LLMJudgeBackendOptions } from './llm.js';
|
|
8
|
+
export { normalizePrologValue, normalizeJudgeState, parseJudgeQuestions, validateJudgeAnswers, judgeAnswersToValues, valuesToPrologList, judgeCacheKey, } from './marshal.js';
|
|
9
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/judge/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,MAAM,WAAW,CAAC;AAC9E,YAAY,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAC3E,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AACjD,YAAY,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AACvD,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AACvF,YAAY,EAAE,sBAAsB,EAAE,MAAM,UAAU,CAAC;AACvD,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,GACd,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
export * from './types.js';
|
|
2
|
+
export { createMockJudgeBackend, createMockJevJudgeBackend } from './mock.js';
|
|
3
|
+
export { createJevJudgeBackend } from './jev.js';
|
|
4
|
+
export { createLLMJudgeBackend, buildJudgePrompt, parseJudgeResponse } from './llm.js';
|
|
5
|
+
export { normalizePrologValue, normalizeJudgeState, parseJudgeQuestions, validateJudgeAnswers, judgeAnswersToValues, valuesToPrologList, judgeCacheKey, } from './marshal.js';
|
|
6
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/judge/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,sBAAsB,EAAE,yBAAyB,EAAE,MAAM,WAAW,CAAC;AAE9E,OAAO,EAAE,qBAAqB,EAAE,MAAM,UAAU,CAAC;AAEjD,OAAO,EAAE,qBAAqB,EAAE,gBAAgB,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AAEvF,OAAO,EACL,oBAAoB,EACpB,mBAAmB,EACnB,mBAAmB,EACnB,oBAAoB,EACpB,oBAAoB,EACpB,kBAAkB,EAClB,aAAa,GACd,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jev (TypeSafe System One) judge backend.
|
|
3
|
+
*
|
|
4
|
+
* Calls `POST /v1/systemone` with the explicit state and the typed questions,
|
|
5
|
+
* and maps the typed answers back to `JudgeAnswer`s. Calibration, full
|
|
6
|
+
* distributions, and confidence are available from this backend.
|
|
7
|
+
*
|
|
8
|
+
* The API key is supplied by the caller (for example, the pi extension reading
|
|
9
|
+
* it from an environment variable); the SDK never reads or stores credentials.
|
|
10
|
+
*/
|
|
11
|
+
import { type JudgeBackend } from './types.js';
|
|
12
|
+
export interface JevJudgeBackendOptions {
|
|
13
|
+
apiKey?: string;
|
|
14
|
+
baseUrl?: string;
|
|
15
|
+
model?: string;
|
|
16
|
+
maxRetries?: number;
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
fetch?: typeof fetch;
|
|
19
|
+
}
|
|
20
|
+
export declare function createJevJudgeBackend(options?: JevJudgeBackendOptions): JudgeBackend;
|
|
21
|
+
//# sourceMappingURL=jev.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jev.d.ts","sourceRoot":"","sources":["../../src/judge/jev.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAEL,KAAK,YAAY,EAKlB,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,sBAAsB;IACrC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;CACtB;AAoJD,wBAAgB,qBAAqB,CAAC,OAAO,GAAE,sBAA2B,GAAG,YAAY,CA0ExF"}
|
|
@@ -0,0 +1,218 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Jev (TypeSafe System One) judge backend.
|
|
3
|
+
*
|
|
4
|
+
* Calls `POST /v1/systemone` with the explicit state and the typed questions,
|
|
5
|
+
* and maps the typed answers back to `JudgeAnswer`s. Calibration, full
|
|
6
|
+
* distributions, and confidence are available from this backend.
|
|
7
|
+
*
|
|
8
|
+
* The API key is supplied by the caller (for example, the pi extension reading
|
|
9
|
+
* it from an environment variable); the SDK never reads or stores credentials.
|
|
10
|
+
*/
|
|
11
|
+
const JEV_CAPABILITIES = {
|
|
12
|
+
calibrated: true,
|
|
13
|
+
probability: true,
|
|
14
|
+
confidence: true,
|
|
15
|
+
independentQuestions: true,
|
|
16
|
+
batch: true,
|
|
17
|
+
structuredCriteria: true,
|
|
18
|
+
maxQuestions: 128,
|
|
19
|
+
maxOptions: 255,
|
|
20
|
+
maxLevels: 10,
|
|
21
|
+
stateTokenBudget: 32_000,
|
|
22
|
+
};
|
|
23
|
+
function levelToText(level) {
|
|
24
|
+
if (level && typeof level === 'object' && !Array.isArray(level) && 'what' in level) {
|
|
25
|
+
return level;
|
|
26
|
+
}
|
|
27
|
+
return level;
|
|
28
|
+
}
|
|
29
|
+
function toJevQuestion(question) {
|
|
30
|
+
if (question.kind === 'choose') {
|
|
31
|
+
const criteria = {};
|
|
32
|
+
for (const option of question.options ?? [])
|
|
33
|
+
criteria[option.id] = option.description ?? null;
|
|
34
|
+
return { type: 'choice', instructions: question.instruction, criteria };
|
|
35
|
+
}
|
|
36
|
+
if (question.kind === 'rate') {
|
|
37
|
+
return {
|
|
38
|
+
type: 'score',
|
|
39
|
+
instructions: question.instruction,
|
|
40
|
+
criteria: (question.levels ?? []).map(levelToText),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
const noul = { type: 'noul', instructions: question.instruction };
|
|
44
|
+
if (question.criteria) {
|
|
45
|
+
noul.criteria = {
|
|
46
|
+
true: question.criteria.true ?? null,
|
|
47
|
+
false: question.criteria.false ?? null,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
return noul;
|
|
51
|
+
}
|
|
52
|
+
function distributionForQuestion(question, probabilities) {
|
|
53
|
+
if (!probabilities)
|
|
54
|
+
return undefined;
|
|
55
|
+
if (question.kind === 'choose') {
|
|
56
|
+
const optionIds = (question.options ?? []).map((option) => option.id);
|
|
57
|
+
return optionIds.map((id) => Number(probabilities[id] ?? 0));
|
|
58
|
+
}
|
|
59
|
+
if (question.kind === 'rate') {
|
|
60
|
+
const count = (question.levels ?? []).length;
|
|
61
|
+
return Array.from({ length: count }, (_, index) => Number(probabilities[String(index)] ?? 0));
|
|
62
|
+
}
|
|
63
|
+
return undefined;
|
|
64
|
+
}
|
|
65
|
+
function levelLabel(question, index) {
|
|
66
|
+
const level = question.levels?.[index];
|
|
67
|
+
if (typeof level === 'string')
|
|
68
|
+
return level;
|
|
69
|
+
if (typeof level === 'number' || typeof level === 'boolean')
|
|
70
|
+
return String(level);
|
|
71
|
+
if (level && typeof level === 'object' && typeof level.what === 'string') {
|
|
72
|
+
return level.what;
|
|
73
|
+
}
|
|
74
|
+
return `level_${index}`;
|
|
75
|
+
}
|
|
76
|
+
function argmax(values) {
|
|
77
|
+
let best = 0;
|
|
78
|
+
for (let index = 1; index < values.length; index++) {
|
|
79
|
+
if ((values[index] ?? 0) > (values[best] ?? 0))
|
|
80
|
+
best = index;
|
|
81
|
+
}
|
|
82
|
+
return best;
|
|
83
|
+
}
|
|
84
|
+
function fromJevAnswer(question, raw) {
|
|
85
|
+
const distribution = distributionForQuestion(question, raw.probabilities);
|
|
86
|
+
if (question.kind === 'choose') {
|
|
87
|
+
return {
|
|
88
|
+
id: question.id,
|
|
89
|
+
kind: 'choose',
|
|
90
|
+
value: raw.choice,
|
|
91
|
+
confidence: raw.confidence,
|
|
92
|
+
distribution,
|
|
93
|
+
basis: 'calibrated',
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
if (question.kind === 'rate') {
|
|
97
|
+
const index = distribution && distribution.length > 0 ? argmax(distribution) : Math.round(raw.score ?? 0);
|
|
98
|
+
return {
|
|
99
|
+
id: question.id,
|
|
100
|
+
kind: 'rate',
|
|
101
|
+
value: levelLabel(question, index),
|
|
102
|
+
confidence: raw.confidence,
|
|
103
|
+
distribution,
|
|
104
|
+
basis: 'calibrated',
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
const probability = typeof raw.noul === 'number' ? raw.noul : 0;
|
|
108
|
+
if (question.kind === 'probability') {
|
|
109
|
+
return { id: question.id, kind: 'probability', probability, basis: 'calibrated' };
|
|
110
|
+
}
|
|
111
|
+
return {
|
|
112
|
+
id: question.id,
|
|
113
|
+
kind: 'verify',
|
|
114
|
+
value: probability >= 0.5 ? 'yes' : 'no',
|
|
115
|
+
probability,
|
|
116
|
+
basis: 'calibrated',
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
function retryableStatus(status) {
|
|
120
|
+
return status === 429 || status === 529;
|
|
121
|
+
}
|
|
122
|
+
function sleep(ms) {
|
|
123
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
124
|
+
}
|
|
125
|
+
function combineSignals(signal, timeoutMs) {
|
|
126
|
+
const controller = new AbortController();
|
|
127
|
+
const disposers = [];
|
|
128
|
+
if (signal) {
|
|
129
|
+
if (signal.aborted)
|
|
130
|
+
controller.abort(signal.reason);
|
|
131
|
+
else {
|
|
132
|
+
const onAbort = () => controller.abort(signal.reason);
|
|
133
|
+
signal.addEventListener('abort', onAbort, { once: true });
|
|
134
|
+
disposers.push(() => signal.removeEventListener('abort', onAbort));
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
const timer = setTimeout(() => controller.abort(new Error(`Jev request timed out after ${timeoutMs}ms`)), timeoutMs);
|
|
138
|
+
disposers.push(() => clearTimeout(timer));
|
|
139
|
+
return {
|
|
140
|
+
signal: controller.signal,
|
|
141
|
+
dispose: () => disposers.forEach((dispose) => dispose()),
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
export function createJevJudgeBackend(options = {}) {
|
|
145
|
+
const baseUrl = (options.baseUrl ?? 'https://api.typesafe.ai').replace(/\/$/, '');
|
|
146
|
+
const model = options.model ?? 'jev-latest';
|
|
147
|
+
const maxRetries = options.maxRetries ?? 2;
|
|
148
|
+
const timeoutMs = options.timeoutMs ?? 30_000;
|
|
149
|
+
const fetchImpl = options.fetch ?? fetch;
|
|
150
|
+
return {
|
|
151
|
+
id: 'jev',
|
|
152
|
+
capabilities: JEV_CAPABILITIES,
|
|
153
|
+
async complete(request) {
|
|
154
|
+
if (!options.apiKey)
|
|
155
|
+
throw new Error('Jev backend requires an API key');
|
|
156
|
+
const questions = {};
|
|
157
|
+
for (const question of request.questions)
|
|
158
|
+
questions[question.id] = toJevQuestion(question);
|
|
159
|
+
const body = JSON.stringify({ state: request.state, model, questions });
|
|
160
|
+
let lastError;
|
|
161
|
+
for (let attempt = 0; attempt <= maxRetries; attempt++) {
|
|
162
|
+
const { signal, dispose } = combineSignals(request.signal, timeoutMs);
|
|
163
|
+
try {
|
|
164
|
+
const response = await fetchImpl(`${baseUrl}/v1/systemone`, {
|
|
165
|
+
method: 'POST',
|
|
166
|
+
headers: {
|
|
167
|
+
Authorization: `Bearer ${options.apiKey}`,
|
|
168
|
+
'Content-Type': 'application/json',
|
|
169
|
+
},
|
|
170
|
+
body,
|
|
171
|
+
signal,
|
|
172
|
+
});
|
|
173
|
+
if (!response.ok) {
|
|
174
|
+
const text = await response.text().catch(() => '');
|
|
175
|
+
const error = new Error(`Jev request failed (${response.status}): ${text.slice(0, 500)}`);
|
|
176
|
+
if (retryableStatus(response.status) && attempt < maxRetries) {
|
|
177
|
+
lastError = error;
|
|
178
|
+
await sleep(250 * 2 ** attempt);
|
|
179
|
+
continue;
|
|
180
|
+
}
|
|
181
|
+
throw error;
|
|
182
|
+
}
|
|
183
|
+
const payload = (await response.json());
|
|
184
|
+
const rawAnswers = payload.answers ?? {};
|
|
185
|
+
return {
|
|
186
|
+
answers: request.questions.map((question) => {
|
|
187
|
+
const raw = rawAnswers[question.id];
|
|
188
|
+
if (!raw)
|
|
189
|
+
throw new Error(`Jev returned no answer for question ${question.id}`);
|
|
190
|
+
return fromJevAnswer(question, raw);
|
|
191
|
+
}),
|
|
192
|
+
model: payload.model ?? model,
|
|
193
|
+
usage: payload.usage
|
|
194
|
+
? {
|
|
195
|
+
inputTokens: payload.usage.input_tokens ?? 0,
|
|
196
|
+
outputTokens: payload.usage.output_tokens ?? 0,
|
|
197
|
+
totalTokens: (payload.usage.input_tokens ?? 0) + (payload.usage.output_tokens ?? 0),
|
|
198
|
+
}
|
|
199
|
+
: undefined,
|
|
200
|
+
};
|
|
201
|
+
}
|
|
202
|
+
catch (error) {
|
|
203
|
+
if (attempt < maxRetries && !request.signal?.aborted) {
|
|
204
|
+
lastError = error;
|
|
205
|
+
await sleep(250 * 2 ** attempt);
|
|
206
|
+
continue;
|
|
207
|
+
}
|
|
208
|
+
throw error;
|
|
209
|
+
}
|
|
210
|
+
finally {
|
|
211
|
+
dispose();
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
throw lastError instanceof Error ? lastError : new Error('Jev request failed');
|
|
215
|
+
},
|
|
216
|
+
};
|
|
217
|
+
}
|
|
218
|
+
//# sourceMappingURL=jev.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"jev.js","sourceRoot":"","sources":["../../src/judge/jev.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAoBH,MAAM,gBAAgB,GAAsB;IAC1C,UAAU,EAAE,IAAI;IAChB,WAAW,EAAE,IAAI;IACjB,UAAU,EAAE,IAAI;IAChB,oBAAoB,EAAE,IAAI;IAC1B,KAAK,EAAE,IAAI;IACX,kBAAkB,EAAE,IAAI;IACxB,YAAY,EAAE,GAAG;IACjB,UAAU,EAAE,GAAG;IACf,SAAS,EAAE,EAAE;IACb,gBAAgB,EAAE,MAAM;CACzB,CAAC;AAEF,SAAS,WAAW,CAAC,KAAiB;IACpC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,MAAM,IAAI,KAAK,EAAE,CAAC;QACnF,OAAO,KAAK,CAAC;IACf,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,aAAa,CAAC,QAAuB;IAC5C,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,QAAQ,GAA4B,EAAE,CAAC;QAC7C,KAAK,MAAM,MAAM,IAAI,QAAQ,CAAC,OAAO,IAAI,EAAE;YAAE,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,WAAW,IAAI,IAAI,CAAC;QAC9F,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,QAAQ,CAAC,WAAW,EAAE,QAAQ,EAAE,CAAC;IAC1E,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC7B,OAAO;YACL,IAAI,EAAE,OAAO;YACb,YAAY,EAAE,QAAQ,CAAC,WAAW;YAClC,QAAQ,EAAE,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC;SACnD,CAAC;IACJ,CAAC;IACD,MAAM,IAAI,GAA4B,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,CAAC,WAAW,EAAE,CAAC;IAC3F,IAAI,QAAQ,CAAC,QAAQ,EAAE,CAAC;QACtB,IAAI,CAAC,QAAQ,GAAG;YACd,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC,IAAI,IAAI,IAAI;YACpC,KAAK,EAAE,QAAQ,CAAC,QAAQ,CAAC,KAAK,IAAI,IAAI;SACvC,CAAC;IACJ,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,uBAAuB,CAAC,QAAuB,EAAE,aAAiD;IACzG,IAAI,CAAC,aAAa;QAAE,OAAO,SAAS,CAAC;IACrC,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,SAAS,GAAG,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QACtE,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/D,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;QAC7C,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,SAAS,UAAU,CAAC,QAAuB,EAAE,KAAa;IACxD,MAAM,KAAK,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,KAAK,CAAC,CAAC;IACvC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAClF,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAQ,KAA4B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACjG,OAAQ,KAA0B,CAAC,IAAI,CAAC;IAC1C,CAAC;IACD,OAAO,SAAS,KAAK,EAAE,CAAC;AAC1B,CAAC;AAED,SAAS,MAAM,CAAC,MAAgB;IAC9B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC;QACnD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAAE,IAAI,GAAG,KAAK,CAAC;IAC/D,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAWD,SAAS,aAAa,CAAC,QAAuB,EAAE,GAAiB;IAC/D,MAAM,YAAY,GAAG,uBAAuB,CAAC,QAAQ,EAAE,GAAG,CAAC,aAAa,CAAC,CAAC;IAC1E,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO;YACL,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI,EAAE,QAAQ;YACd,KAAK,EAAE,GAAG,CAAC,MAAM;YACjB,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,YAAY;YACZ,KAAK,EAAE,YAAY;SACpB,CAAC;IACJ,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,YAAY,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC;QAC1G,OAAO;YACL,EAAE,EAAE,QAAQ,CAAC,EAAE;YACf,IAAI,EAAE,MAAM;YACZ,KAAK,EAAE,UAAU,CAAC,QAAQ,EAAE,KAAK,CAAC;YAClC,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,YAAY;YACZ,KAAK,EAAE,YAAY;SACpB,CAAC;IACJ,CAAC;IACD,MAAM,WAAW,GAAG,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAChE,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;QACpC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,WAAW,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC;IACpF,CAAC;IACD,OAAO;QACL,EAAE,EAAE,QAAQ,CAAC,EAAE;QACf,IAAI,EAAE,QAAQ;QACd,KAAK,EAAE,WAAW,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI;QACxC,WAAW;QACX,KAAK,EAAE,YAAY;KACpB,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CAAC,MAAc;IACrC,OAAO,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,CAAC;AAC1C,CAAC;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAED,SAAS,cAAc,CAAC,MAA+B,EAAE,SAAiB;IACxE,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;IACzC,MAAM,SAAS,GAAsB,EAAE,CAAC;IACxC,IAAI,MAAM,EAAE,CAAC;QACX,IAAI,MAAM,CAAC,OAAO;YAAE,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;aAC/C,CAAC;YACJ,MAAM,OAAO,GAAG,GAAS,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC5D,MAAM,CAAC,gBAAgB,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC1D,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC;QACrE,CAAC;IACH,CAAC;IACD,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,+BAA+B,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACrH,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC;IAC1C,OAAO;QACL,MAAM,EAAE,UAAU,CAAC,MAAM;QACzB,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,EAAE,CAAC;KACzD,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,UAAkC,EAAE;IACxE,MAAM,OAAO,GAAG,CAAC,OAAO,CAAC,OAAO,IAAI,yBAAyB,CAAC,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IAClF,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,IAAI,YAAY,CAAC;IAC5C,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,CAAC,CAAC;IAC3C,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,IAAI,MAAM,CAAC;IAC9C,MAAM,SAAS,GAAG,OAAO,CAAC,KAAK,IAAI,KAAK,CAAC;IAEzC,OAAO;QACL,EAAE,EAAE,KAAK;QACT,YAAY,EAAE,gBAAgB;QAC9B,KAAK,CAAC,QAAQ,CAAC,OAA4B;YACzC,IAAI,CAAC,OAAO,CAAC,MAAM;gBAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACxE,MAAM,SAAS,GAA4B,EAAE,CAAC;YAC9C,KAAK,MAAM,QAAQ,IAAI,OAAO,CAAC,SAAS;gBAAE,SAAS,CAAC,QAAQ,CAAC,EAAE,CAAC,GAAG,aAAa,CAAC,QAAQ,CAAC,CAAC;YAC3F,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;YAExE,IAAI,SAAkB,CAAC;YACvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;gBACvD,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBACtE,IAAI,CAAC;oBACH,MAAM,QAAQ,GAAG,MAAM,SAAS,CAAC,GAAG,OAAO,eAAe,EAAE;wBAC1D,MAAM,EAAE,MAAM;wBACd,OAAO,EAAE;4BACP,aAAa,EAAE,UAAU,OAAO,CAAC,MAAM,EAAE;4BACzC,cAAc,EAAE,kBAAkB;yBACnC;wBACD,IAAI;wBACJ,MAAM;qBACP,CAAC,CAAC;oBACH,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;wBACjB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;wBACnD,MAAM,KAAK,GAAG,IAAI,KAAK,CAAC,uBAAuB,QAAQ,CAAC,MAAM,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;wBAC1F,IAAI,eAAe,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,OAAO,GAAG,UAAU,EAAE,CAAC;4BAC7D,SAAS,GAAG,KAAK,CAAC;4BAClB,MAAM,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC;4BAChC,SAAS;wBACX,CAAC;wBACD,MAAM,KAAK,CAAC;oBACd,CAAC;oBACD,MAAM,OAAO,GAAG,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,CAIrC,CAAC;oBACF,MAAM,UAAU,GAAG,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC;oBACzC,OAAO;wBACL,OAAO,EAAE,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE;4BAC1C,MAAM,GAAG,GAAG,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;4BACpC,IAAI,CAAC,GAAG;gCAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,QAAQ,CAAC,EAAE,EAAE,CAAC,CAAC;4BAChF,OAAO,aAAa,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;wBACtC,CAAC,CAAC;wBACF,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,KAAK;wBAC7B,KAAK,EAAE,OAAO,CAAC,KAAK;4BAClB,CAAC,CAAC;gCACE,WAAW,EAAE,OAAO,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC;gCAC5C,YAAY,EAAE,OAAO,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC;gCAC9C,WAAW,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,YAAY,IAAI,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,aAAa,IAAI,CAAC,CAAC;6BACpF;4BACH,CAAC,CAAC,SAAS;qBACd,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAI,OAAO,GAAG,UAAU,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;wBACrD,SAAS,GAAG,KAAK,CAAC;wBAClB,MAAM,KAAK,CAAC,GAAG,GAAG,CAAC,IAAI,OAAO,CAAC,CAAC;wBAChC,SAAS;oBACX,CAAC;oBACD,MAAM,KAAK,CAAC;gBACd,CAAC;wBAAS,CAAC;oBACT,OAAO,EAAE,CAAC;gBACZ,CAAC;YACH,CAAC;YACD,MAAM,SAAS,YAAY,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,oBAAoB,CAAC,CAAC;QACjF,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM judge backend.
|
|
3
|
+
*
|
|
4
|
+
* Answers the same generic questions through an ordinary LLM. When an injected
|
|
5
|
+
* `LLMBackend` is present (for example, pi's active model in the extension) it
|
|
6
|
+
* is used directly; otherwise the SDK provider path is used. Answers are
|
|
7
|
+
* labels or self-reported probabilities, never calibrated ones.
|
|
8
|
+
*/
|
|
9
|
+
import type { LLMBackend } from '../types.js';
|
|
10
|
+
import type { JudgeAnswer, JudgeBackend, JudgeQuestion } from './types.js';
|
|
11
|
+
export interface LLMJudgeBackendOptions {
|
|
12
|
+
llmBackend?: LLMBackend;
|
|
13
|
+
model?: string;
|
|
14
|
+
provider?: string;
|
|
15
|
+
temperature?: number;
|
|
16
|
+
maxTokens?: number;
|
|
17
|
+
baseUrl?: string;
|
|
18
|
+
providerOptions?: Record<string, Record<string, any>>;
|
|
19
|
+
}
|
|
20
|
+
export declare function buildJudgePrompt(state: unknown, questions: JudgeQuestion[]): string;
|
|
21
|
+
export declare function parseJudgeResponse(text: string, questions: JudgeQuestion[]): JudgeAnswer[];
|
|
22
|
+
export declare function createLLMJudgeBackend(options?: LLMJudgeBackendOptions): JudgeBackend;
|
|
23
|
+
//# sourceMappingURL=llm.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm.d.ts","sourceRoot":"","sources":["../../src/judge/llm.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,UAAU,EAAiB,MAAM,aAAa,CAAC;AAC7D,OAAO,KAAK,EACV,WAAW,EACX,YAAY,EAGZ,aAAa,EACd,MAAM,YAAY,CAAC;AAEpB,MAAM,WAAW,sBAAsB;IACrC,UAAU,CAAC,EAAE,UAAU,CAAC;IACxB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;IAEjB,eAAe,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC;CACvD;AAgCD,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,MAAM,CAWnF;AAgBD,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,aAAa,EAAE,GAAG,WAAW,EAAE,CAsB1F;AAED,wBAAgB,qBAAqB,CAAC,OAAO,GAAE,sBAA2B,GAAG,YAAY,CAqCxF"}
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* LLM judge backend.
|
|
3
|
+
*
|
|
4
|
+
* Answers the same generic questions through an ordinary LLM. When an injected
|
|
5
|
+
* `LLMBackend` is present (for example, pi's active model in the extension) it
|
|
6
|
+
* is used directly; otherwise the SDK provider path is used. Answers are
|
|
7
|
+
* labels or self-reported probabilities, never calibrated ones.
|
|
8
|
+
*/
|
|
9
|
+
import { generateLlmReply } from '../prolog/bridge.js';
|
|
10
|
+
const LLM_CAPABILITIES = {
|
|
11
|
+
calibrated: false,
|
|
12
|
+
probability: true,
|
|
13
|
+
confidence: false,
|
|
14
|
+
independentQuestions: false,
|
|
15
|
+
batch: true,
|
|
16
|
+
structuredCriteria: false,
|
|
17
|
+
maxQuestions: 32,
|
|
18
|
+
maxOptions: 64,
|
|
19
|
+
maxLevels: 10,
|
|
20
|
+
stateTokenBudget: 16_000,
|
|
21
|
+
};
|
|
22
|
+
function describeQuestion(question) {
|
|
23
|
+
if (question.kind === 'choose') {
|
|
24
|
+
const options = (question.options ?? [])
|
|
25
|
+
.map((option) => (option.description ? `${option.id} (${JSON.stringify(option.description)})` : option.id))
|
|
26
|
+
.join(', ');
|
|
27
|
+
return `[choose] ${String(question.instruction)}\n options: ${options}`;
|
|
28
|
+
}
|
|
29
|
+
if (question.kind === 'rate') {
|
|
30
|
+
const levels = (question.levels ?? []).map((level) => JSON.stringify(level)).join(', ');
|
|
31
|
+
return `[rate] ${String(question.instruction)}\n levels: ${levels}`;
|
|
32
|
+
}
|
|
33
|
+
if (question.kind === 'verify') {
|
|
34
|
+
return `[verify] ${String(question.instruction)}\n answer yes, no, or unknown, with a probability 0..1`;
|
|
35
|
+
}
|
|
36
|
+
return `[probability] ${String(question.instruction)}\n answer only with a probability 0..1`;
|
|
37
|
+
}
|
|
38
|
+
export function buildJudgePrompt(state, questions) {
|
|
39
|
+
const questionLines = questions.map((question, index) => `${index + 1}. ${describeQuestion(question)}`).join('\n');
|
|
40
|
+
return [
|
|
41
|
+
'You are a precise judge. Answer every question using only the state and the allowed answers.',
|
|
42
|
+
'Return strict JSON of the form {"answers":[...]} with one entry per question, in the same order.',
|
|
43
|
+
'Each entry uses "value" for choose/rate/verify and "probability" for verify/probability.',
|
|
44
|
+
'',
|
|
45
|
+
`STATE:\n${JSON.stringify(state, null, 2)}`,
|
|
46
|
+
'',
|
|
47
|
+
`QUESTIONS:\n${questionLines}`,
|
|
48
|
+
].join('\n');
|
|
49
|
+
}
|
|
50
|
+
function extractJson(text) {
|
|
51
|
+
const fenced = /```(?:json)?\s*([\s\S]*?)```/i.exec(text);
|
|
52
|
+
const candidate = fenced ? fenced[1] : text;
|
|
53
|
+
const start = candidate.indexOf('{');
|
|
54
|
+
const end = candidate.lastIndexOf('}');
|
|
55
|
+
if (start === -1 || end === -1 || end <= start)
|
|
56
|
+
throw new Error('LLM judge response did not contain JSON');
|
|
57
|
+
return JSON.parse(candidate.slice(start, end + 1));
|
|
58
|
+
}
|
|
59
|
+
export function parseJudgeResponse(text, questions) {
|
|
60
|
+
const parsed = extractJson(text);
|
|
61
|
+
const rawAnswers = Array.isArray(parsed.answers) ? parsed.answers : [];
|
|
62
|
+
return questions.map((question, index) => {
|
|
63
|
+
const raw = rawAnswers[index] ?? {};
|
|
64
|
+
const probability = typeof raw.probability === 'number' ? raw.probability : undefined;
|
|
65
|
+
const value = typeof raw.value === 'string' ? raw.value : undefined;
|
|
66
|
+
const basis = probability !== undefined ? 'self_reported' : 'label';
|
|
67
|
+
if (question.kind === 'probability') {
|
|
68
|
+
return { id: question.id, kind: question.kind, probability: probability ?? 0, basis };
|
|
69
|
+
}
|
|
70
|
+
if (question.kind === 'verify') {
|
|
71
|
+
return {
|
|
72
|
+
id: question.id,
|
|
73
|
+
kind: question.kind,
|
|
74
|
+
value: value ?? (probability !== undefined ? (probability >= 0.5 ? 'yes' : 'no') : 'unknown'),
|
|
75
|
+
probability,
|
|
76
|
+
basis,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
return { id: question.id, kind: question.kind, value, basis };
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
export function createLLMJudgeBackend(options = {}) {
|
|
83
|
+
return {
|
|
84
|
+
id: 'llm',
|
|
85
|
+
capabilities: LLM_CAPABILITIES,
|
|
86
|
+
async complete(request) {
|
|
87
|
+
const messages = [
|
|
88
|
+
{
|
|
89
|
+
role: 'system',
|
|
90
|
+
content: 'You are a precise judge. Reply with strict JSON and nothing else.',
|
|
91
|
+
},
|
|
92
|
+
{ role: 'user', content: buildJudgePrompt(request.state, request.questions) },
|
|
93
|
+
];
|
|
94
|
+
const { text, usage } = await generateLlmReply({
|
|
95
|
+
messages,
|
|
96
|
+
modelOptions: {
|
|
97
|
+
provider: options.provider ?? 'openai',
|
|
98
|
+
model: options.model ?? request.model ?? 'gpt-4o-mini',
|
|
99
|
+
temperature: options.temperature ?? 0,
|
|
100
|
+
maxOutputTokens: options.maxTokens,
|
|
101
|
+
baseUrl: options.baseUrl,
|
|
102
|
+
providerOptions: options.providerOptions,
|
|
103
|
+
},
|
|
104
|
+
signal: request.signal,
|
|
105
|
+
llmBackend: options.llmBackend,
|
|
106
|
+
});
|
|
107
|
+
return {
|
|
108
|
+
answers: parseJudgeResponse(text, request.questions),
|
|
109
|
+
usage: usage
|
|
110
|
+
? {
|
|
111
|
+
inputTokens: usage.inputTokens,
|
|
112
|
+
outputTokens: usage.outputTokens,
|
|
113
|
+
totalTokens: usage.totalTokens,
|
|
114
|
+
}
|
|
115
|
+
: undefined,
|
|
116
|
+
};
|
|
117
|
+
},
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
//# sourceMappingURL=llm.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"llm.js","sourceRoot":"","sources":["../../src/judge/llm.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,gBAAgB,EAAE,MAAM,qBAAqB,CAAC;AAqBvD,MAAM,gBAAgB,GAAsB;IAC1C,UAAU,EAAE,KAAK;IACjB,WAAW,EAAE,IAAI;IACjB,UAAU,EAAE,KAAK;IACjB,oBAAoB,EAAE,KAAK;IAC3B,KAAK,EAAE,IAAI;IACX,kBAAkB,EAAE,KAAK;IACzB,YAAY,EAAE,EAAE;IAChB,UAAU,EAAE,EAAE;IACd,SAAS,EAAE,EAAE;IACb,gBAAgB,EAAE,MAAM;CACzB,CAAC;AAEF,SAAS,gBAAgB,CAAC,QAAuB;IAC/C,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC/B,MAAM,OAAO,GAAG,CAAC,QAAQ,CAAC,OAAO,IAAI,EAAE,CAAC;aACrC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,EAAE,KAAK,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;aAC1G,IAAI,CAAC,IAAI,CAAC,CAAC;QACd,OAAO,YAAY,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,gBAAgB,OAAO,EAAE,CAAC;IAC3E,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;QAC7B,MAAM,MAAM,GAAG,CAAC,QAAQ,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACxF,OAAO,UAAU,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,eAAe,MAAM,EAAE,CAAC;IACvE,CAAC;IACD,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC/B,OAAO,YAAY,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,yDAAyD,CAAC;IAC3G,CAAC;IACD,OAAO,iBAAiB,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAC,yCAAyC,CAAC;AAChG,CAAC;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAc,EAAE,SAA0B;IACzE,MAAM,aAAa,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,KAAK,GAAG,CAAC,KAAK,gBAAgB,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACnH,OAAO;QACL,8FAA8F;QAC9F,kGAAkG;QAClG,0FAA0F;QAC1F,EAAE;QACF,WAAW,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE;QAC3C,EAAE;QACF,eAAe,aAAa,EAAE;KAC/B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,MAAM,GAAG,+BAA+B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1D,MAAM,SAAS,GAAG,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC5C,MAAM,KAAK,GAAG,SAAS,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;IACrC,MAAM,GAAG,GAAG,SAAS,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;IACvC,IAAI,KAAK,KAAK,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,CAAC,IAAI,GAAG,IAAI,KAAK;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IAC3G,OAAO,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;AACrD,CAAC;AAOD,MAAM,UAAU,kBAAkB,CAAC,IAAY,EAAE,SAA0B;IACzE,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAiC,CAAC;IACjE,MAAM,UAAU,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IACvE,OAAO,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,EAAE;QACvC,MAAM,GAAG,GAAG,UAAU,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACpC,MAAM,WAAW,GAAG,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC,SAAS,CAAC;QACtF,MAAM,KAAK,GAAG,OAAO,GAAG,CAAC,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS,CAAC;QACpE,MAAM,KAAK,GAAyB,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,CAAC;QAC1F,IAAI,QAAQ,CAAC,IAAI,KAAK,aAAa,EAAE,CAAC;YACpC,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,WAAW,EAAE,WAAW,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC;QACxF,CAAC;QACD,IAAI,QAAQ,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;YAC/B,OAAO;gBACL,EAAE,EAAE,QAAQ,CAAC,EAAE;gBACf,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,KAAK,EAAE,KAAK,IAAI,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,CAAC,WAAW,IAAI,GAAG,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;gBAC7F,WAAW;gBACX,KAAK;aACN,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,EAAE,EAAE,QAAQ,CAAC,EAAE,EAAE,IAAI,EAAE,QAAQ,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC;IAChE,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,qBAAqB,CAAC,UAAkC,EAAE;IACxE,OAAO;QACL,EAAE,EAAE,KAAK;QACT,YAAY,EAAE,gBAAgB;QAC9B,KAAK,CAAC,QAAQ,CAAC,OAA4B;YACzC,MAAM,QAAQ,GAAoB;gBAChC;oBACE,IAAI,EAAE,QAAQ;oBACd,OAAO,EAAE,mEAAmE;iBAC7E;gBACD,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,CAAC,OAAO,CAAC,KAAK,EAAE,OAAO,CAAC,SAAS,CAAC,EAAE;aAC9E,CAAC;YACF,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,gBAAgB,CAAC;gBAC7C,QAAQ;gBACR,YAAY,EAAE;oBACZ,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,QAAQ;oBACtC,KAAK,EAAE,OAAO,CAAC,KAAK,IAAI,OAAO,CAAC,KAAK,IAAI,aAAa;oBACtD,WAAW,EAAE,OAAO,CAAC,WAAW,IAAI,CAAC;oBACrC,eAAe,EAAE,OAAO,CAAC,SAAS;oBAClC,OAAO,EAAE,OAAO,CAAC,OAAO;oBACxB,eAAe,EAAE,OAAO,CAAC,eAAe;iBACzC;gBACD,MAAM,EAAE,OAAO,CAAC,MAAM;gBACtB,UAAU,EAAE,OAAO,CAAC,UAAU;aAC/B,CAAC,CAAC;YACH,OAAO;gBACL,OAAO,EAAE,kBAAkB,CAAC,IAAI,EAAE,OAAO,CAAC,SAAS,CAAC;gBACpD,KAAK,EAAE,KAAK;oBACV,CAAC,CAAC;wBACE,WAAW,EAAE,KAAK,CAAC,WAAW;wBAC9B,YAAY,EAAE,KAAK,CAAC,YAAY;wBAChC,WAAW,EAAE,KAAK,CAAC,WAAW;qBAC/B;oBACH,CAAC,CAAC,SAAS;aACd,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Marshalling between the Prolog wire shapes and the SDK judge types.
|
|
3
|
+
*
|
|
4
|
+
* The engine payload arrives as swipl-wasm values:
|
|
5
|
+
* - a compound `f(a, b)` -> `{ $t: 't', f: [[a, b]] }`
|
|
6
|
+
* - a pair `K-V` -> `{ $t: 't', '-': [[K, V]] }`
|
|
7
|
+
* - a dict `tag{k: v}` -> `{ $tag: 'tag', k: v }`
|
|
8
|
+
* - a list -> an array
|
|
9
|
+
* - atoms / strings / numbers / booleans arrive as plain JS values.
|
|
10
|
+
*/
|
|
11
|
+
import type { JsonValue, JudgeAnswer, JudgeQuestion } from './types.js';
|
|
12
|
+
/** Normalize an arbitrary Prolog value into JSON. A list of `Key-Value` pairs becomes an object. */
|
|
13
|
+
export declare function normalizePrologValue(value: unknown): JsonValue;
|
|
14
|
+
/** The author-facing state term, normalized to JSON. */
|
|
15
|
+
export declare function normalizeJudgeState(value: unknown): JsonValue;
|
|
16
|
+
/** Parse the `questions` list yielded by the meta-interpreter into `JudgeQuestion[]`. */
|
|
17
|
+
export declare function parseJudgeQuestions(raw: unknown): JudgeQuestion[];
|
|
18
|
+
/** Validate backend answers against the questions and preserve question order. */
|
|
19
|
+
export declare function validateJudgeAnswers(questions: JudgeQuestion[], answers: JudgeAnswer[]): JudgeAnswer[];
|
|
20
|
+
/** The ordered primary values bound back into DML. */
|
|
21
|
+
export declare function judgeAnswersToValues(questions: JudgeQuestion[], answers: JudgeAnswer[]): Array<string | number>;
|
|
22
|
+
/** Serialize answer values into a Prolog list, using atoms for identifiers. */
|
|
23
|
+
export declare function valuesToPrologList(values: Array<string | number>): string;
|
|
24
|
+
/** Cache key for a judgment: same backend/model/state/questions returns the same answers. */
|
|
25
|
+
export declare function judgeCacheKey(backendId: string, model: string, state: JsonValue, questions: JudgeQuestion[]): string;
|
|
26
|
+
//# sourceMappingURL=marshal.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"marshal.d.ts","sourceRoot":"","sources":["../../src/judge/marshal.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,WAAW,EAA2B,aAAa,EAAE,MAAM,YAAY,CAAC;AA2BjG,oGAAoG;AACpG,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,OAAO,GAAG,SAAS,CAiD9D;AAED,wDAAwD;AACxD,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,OAAO,GAAG,SAAS,CAE7D;AAkCD,yFAAyF;AACzF,wBAAgB,mBAAmB,CAAC,GAAG,EAAE,OAAO,GAAG,aAAa,EAAE,CAejE;AAqED,kFAAkF;AAClF,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,aAAa,EAAE,EAC1B,OAAO,EAAE,WAAW,EAAE,GACrB,WAAW,EAAE,CAMf;AAED,sDAAsD;AACtD,wBAAgB,oBAAoB,CAClC,SAAS,EAAE,aAAa,EAAE,EAC1B,OAAO,EAAE,WAAW,EAAE,GACrB,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,CAaxB;AAED,+EAA+E;AAC/E,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,GAAG,MAAM,CAEzE;AAkBD,6FAA6F;AAC7F,wBAAgB,aAAa,CAC3B,SAAS,EAAE,MAAM,EACjB,KAAK,EAAE,MAAM,EACb,KAAK,EAAE,SAAS,EAChB,SAAS,EAAE,aAAa,EAAE,GACzB,MAAM,CAER"}
|