@elyracode/jev-tools 0.9.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +99 -0
- package/extensions/index.ts +532 -0
- package/package.json +35 -0
package/README.md
ADDED
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
# @elyracode/jev-tools
|
|
2
|
+
|
|
3
|
+
TypeSafe Jev decisions for Elyra -- calibrated yes/no, choice, and score judgements through OpenRouter.
|
|
4
|
+
|
|
5
|
+
Jev is a decision model, not a chat model. It never generates text. You give it a piece of content
|
|
6
|
+
(`state`) and one or more typed questions, and it returns probabilities. A call takes about one second
|
|
7
|
+
and costs roughly $0.00002.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
elyra install npm:@elyracode/jev-tools
|
|
13
|
+
```
|
|
14
|
+
|
|
15
|
+
## Requirements
|
|
16
|
+
|
|
17
|
+
An OpenRouter API key. Either log in with `/login` and pick OpenRouter, or set `OPENROUTER_API_KEY`.
|
|
18
|
+
The extension reuses whatever key Elyra already has for the `openrouter` provider.
|
|
19
|
+
|
|
20
|
+
## Tools
|
|
21
|
+
|
|
22
|
+
| Tool | Description |
|
|
23
|
+
|------|-------------|
|
|
24
|
+
| `decide` | Ask Jev one or more questions about a `state`. Question types: `noul` (yes/no), `choice` (pick one), `score` (ordinal scale). Returns probabilities, confidence, and usage. |
|
|
25
|
+
|
|
26
|
+
## Commands
|
|
27
|
+
|
|
28
|
+
- `/jev` -- show model, endpoint, key status, and bash gate state
|
|
29
|
+
- `/jev gate on|off` -- toggle the bash gate for the current session
|
|
30
|
+
|
|
31
|
+
## Bash gate (opt-in)
|
|
32
|
+
|
|
33
|
+
When enabled, every `bash` tool call is first sent to Jev with the question "would running this
|
|
34
|
+
command destroy data or make a change that is hard to reverse?". If the probability is at or above
|
|
35
|
+
the threshold, Elyra asks you to confirm before running it. In non-interactive mode (print, RPC)
|
|
36
|
+
the command is blocked instead.
|
|
37
|
+
|
|
38
|
+
Enable with `elyra --jev-gate`, `JEV_BASH_GATE=1`, or `/jev gate on`. Off by default because it adds
|
|
39
|
+
about one second of latency to each bash call.
|
|
40
|
+
|
|
41
|
+
If the key is missing or the API fails, the gate logs a warning and lets the command through rather
|
|
42
|
+
than blocking all shell access. It is a second opinion, not a security boundary.
|
|
43
|
+
|
|
44
|
+
## Usage
|
|
45
|
+
|
|
46
|
+
```
|
|
47
|
+
> Is this diff a breaking change? Ask Jev before you answer.
|
|
48
|
+
> Classify these 20 log lines by severity using decide, then only show me the ones rated high or critical.
|
|
49
|
+
> Before running that migration, use decide to check if the command is destructive.
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
### Question types
|
|
53
|
+
|
|
54
|
+
**noul** -- yes/no with a calibrated probability:
|
|
55
|
+
|
|
56
|
+
```json
|
|
57
|
+
{ "name": "refund", "type": "noul", "instructions": "Is the user asking for a refund?",
|
|
58
|
+
"criteria": { "true": "explicitly wants money back", "false": "anything else" } }
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
**choice** -- pick one option, get the full distribution:
|
|
62
|
+
|
|
63
|
+
```json
|
|
64
|
+
{ "name": "route", "type": "choice", "instructions": "Who should handle this?",
|
|
65
|
+
"criteria": { "code": "write or change code", "research": "needs web lookup", "other": "none of the above" } }
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
**score** -- ordinal scale, 2-10 labels from low to high:
|
|
69
|
+
|
|
70
|
+
```json
|
|
71
|
+
{ "name": "severity", "type": "score", "instructions": "How severe is this log line?",
|
|
72
|
+
"criteria": ["debug", "info", "warning", "error", "critical"] }
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
### Writing good questions
|
|
76
|
+
|
|
77
|
+
Jev reads criteria literally and does not infer intent beyond what is stated.
|
|
78
|
+
|
|
79
|
+
- Describe observable conditions, not goals. Spell out both sides for `noul` via `criteria`.
|
|
80
|
+
- Make `choice` options exhaustive, or add an `other` option.
|
|
81
|
+
- Keep `state` to what the questions need. The context window is about 32K tokens.
|
|
82
|
+
- Ask several questions in one call when they concern the same state; the cost is the same.
|
|
83
|
+
|
|
84
|
+
## Configuration
|
|
85
|
+
|
|
86
|
+
| Variable | Default | Purpose |
|
|
87
|
+
|----------|---------|---------|
|
|
88
|
+
| `OPENROUTER_API_KEY` | -- | API key, if not stored via `/login` |
|
|
89
|
+
| `JEV_MODEL` | `~typesafe/jev-latest` | Model slug sent to the endpoint |
|
|
90
|
+
| `JEV_BASE_URL` | `https://openrouter.ai/api/alpha/decisions` | Override if OpenRouter moves the endpoint out of alpha |
|
|
91
|
+
| `JEV_BASH_GATE` | off | Set to `1` to enable the bash gate at startup |
|
|
92
|
+
| `JEV_BASH_GATE_THRESHOLD` | `0.7` | Probability (0-1) at or above which the gate intervenes |
|
|
93
|
+
|
|
94
|
+
## Notes
|
|
95
|
+
|
|
96
|
+
- The decisions endpoint is an OpenRouter alpha API. The request contract here is derived from public
|
|
97
|
+
community clients, not from an official specification, and may change.
|
|
98
|
+
- Retries with backoff on 429/5xx, up to three attempts. Requests time out after 60 seconds.
|
|
99
|
+
- Unofficial community integration. Not affiliated with TypeSafe or OpenRouter.
|
|
@@ -0,0 +1,532 @@
|
|
|
1
|
+
import type { ExtensionAPI, ExtensionContext } from "@elyracode/coding-agent";
|
|
2
|
+
import { type Static, Type } from "typebox";
|
|
3
|
+
|
|
4
|
+
// TypeSafe Jev is a decision model, not a chat model. It is reached through
|
|
5
|
+
// OpenRouter's decisions endpoint (alpha), which has its own request shape:
|
|
6
|
+
// a `state` to judge plus a map of typed `questions`. It never generates text.
|
|
7
|
+
const DEFAULT_BASE_URL = "https://openrouter.ai/api/alpha/decisions";
|
|
8
|
+
const DEFAULT_MODEL = "~typesafe/jev-latest";
|
|
9
|
+
const ENV_BASE_URL = "JEV_BASE_URL";
|
|
10
|
+
const ENV_MODEL = "JEV_MODEL";
|
|
11
|
+
const PROVIDER = "openrouter";
|
|
12
|
+
const MAX_ATTEMPTS = 3;
|
|
13
|
+
const REQUEST_TIMEOUT_MS = 60_000;
|
|
14
|
+
|
|
15
|
+
// Optional bash gate: ask Jev whether a command is destructive before it runs.
|
|
16
|
+
// Off by default because it adds ~1s latency per bash call.
|
|
17
|
+
const ENV_GATE = "JEV_BASH_GATE";
|
|
18
|
+
const ENV_GATE_THRESHOLD = "JEV_BASH_GATE_THRESHOLD";
|
|
19
|
+
const FLAG_GATE = "jev-gate";
|
|
20
|
+
const DEFAULT_GATE_THRESHOLD = 0.7;
|
|
21
|
+
const GATE_QUESTION_NAME = "destructive";
|
|
22
|
+
const GATE_QUESTION: ApiQuestion = {
|
|
23
|
+
type: "noul",
|
|
24
|
+
instructions: "Would running this shell command destroy data or make a change that is hard to reverse?",
|
|
25
|
+
criteria: {
|
|
26
|
+
true:
|
|
27
|
+
"Deletes files or directories, drops or truncates databases or tables, force-pushes or rewrites shared git history, " +
|
|
28
|
+
"discards uncommitted work (reset --hard, checkout ., clean -fd, stash), overwrites files without backup, " +
|
|
29
|
+
"kills processes, changes permissions or ownership broadly, runs as root, or modifies system configuration.",
|
|
30
|
+
false:
|
|
31
|
+
"Reads, lists, searches, builds, tests, lints, formats, installs or updates dependencies, starts dev servers, " +
|
|
32
|
+
"creates new files, or makes changes that git or a package manager can undo.",
|
|
33
|
+
},
|
|
34
|
+
};
|
|
35
|
+
// Jev has a ~32K context window. Warn well before that so the caller can trim state.
|
|
36
|
+
const SIZE_WARN_TOKENS = 30_000;
|
|
37
|
+
|
|
38
|
+
// ── Parameter schema ──
|
|
39
|
+
|
|
40
|
+
const NoulQuestion = Type.Object({
|
|
41
|
+
name: Type.String({ description: "Identifier for this question, used as the key in the result" }),
|
|
42
|
+
type: Type.Literal("noul"),
|
|
43
|
+
instructions: Type.String({ description: "The yes/no condition to evaluate against the state" }),
|
|
44
|
+
criteria: Type.Optional(
|
|
45
|
+
Type.Object(
|
|
46
|
+
{
|
|
47
|
+
true: Type.String({ description: "What counts as yes" }),
|
|
48
|
+
false: Type.String({ description: "What counts as no" }),
|
|
49
|
+
},
|
|
50
|
+
{ description: "Optional descriptions of both outcomes. If given, both keys are required." },
|
|
51
|
+
),
|
|
52
|
+
),
|
|
53
|
+
});
|
|
54
|
+
|
|
55
|
+
const ChoiceQuestion = Type.Object({
|
|
56
|
+
name: Type.String({ description: "Identifier for this question, used as the key in the result" }),
|
|
57
|
+
type: Type.Literal("choice"),
|
|
58
|
+
instructions: Type.String({ description: "The single-choice question to answer" }),
|
|
59
|
+
criteria: Type.Record(Type.String(), Type.String(), {
|
|
60
|
+
description:
|
|
61
|
+
"Option name -> description. At least two options. Add an 'other' option when the set is not exhaustive.",
|
|
62
|
+
}),
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
const ScoreQuestion = Type.Object({
|
|
66
|
+
name: Type.String({ description: "Identifier for this question, used as the key in the result" }),
|
|
67
|
+
type: Type.Literal("score"),
|
|
68
|
+
instructions: Type.String({ description: "What to rate on the ordinal scale" }),
|
|
69
|
+
criteria: Type.Array(Type.String(), {
|
|
70
|
+
minItems: 2,
|
|
71
|
+
maxItems: 10,
|
|
72
|
+
description: "Ordered scale labels from lowest to highest, 2-10 items",
|
|
73
|
+
}),
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
const Question = Type.Union([NoulQuestion, ChoiceQuestion, ScoreQuestion]);
|
|
77
|
+
|
|
78
|
+
const DecideParams = Type.Object({
|
|
79
|
+
state: Type.String({
|
|
80
|
+
description:
|
|
81
|
+
"The content to judge: a message, diff, log line, file excerpt, or a JSON string. " +
|
|
82
|
+
"Include only what the questions need; keep it well under 32K tokens.",
|
|
83
|
+
}),
|
|
84
|
+
questions: Type.Array(Question, {
|
|
85
|
+
minItems: 1,
|
|
86
|
+
description: "One or more typed questions about the state. Asking several at once costs the same as one.",
|
|
87
|
+
}),
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
type DecideParamsType = Static<typeof DecideParams>;
|
|
91
|
+
type QuestionType = Static<typeof Question>;
|
|
92
|
+
|
|
93
|
+
// ── API contract ──
|
|
94
|
+
|
|
95
|
+
interface NoulAnswer {
|
|
96
|
+
type: "noul";
|
|
97
|
+
noul: number;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
interface ChoiceAnswer {
|
|
101
|
+
type: "choice";
|
|
102
|
+
choice: string;
|
|
103
|
+
probabilities: Record<string, number>;
|
|
104
|
+
confidence: number;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
interface ScoreAnswer {
|
|
108
|
+
type: "score";
|
|
109
|
+
score: number;
|
|
110
|
+
probabilities: Record<string, number>;
|
|
111
|
+
legend: Record<string, string>;
|
|
112
|
+
confidence: number;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
type Answer = NoulAnswer | ChoiceAnswer | ScoreAnswer;
|
|
116
|
+
|
|
117
|
+
interface DecisionsUsage {
|
|
118
|
+
input_tokens?: number;
|
|
119
|
+
output_tokens?: number;
|
|
120
|
+
cost?: number;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
interface DecisionsResponse {
|
|
124
|
+
id?: string;
|
|
125
|
+
model?: string;
|
|
126
|
+
answers: Record<string, Answer>;
|
|
127
|
+
usage?: DecisionsUsage;
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
interface DecideDetails {
|
|
131
|
+
model?: string;
|
|
132
|
+
answers?: Record<string, Answer>;
|
|
133
|
+
usage?: DecisionsUsage;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
type ApiQuestion = Omit<QuestionType, "name">;
|
|
137
|
+
|
|
138
|
+
export default function (elyra: ExtensionAPI): void {
|
|
139
|
+
elyra.registerTool({
|
|
140
|
+
name: "decide",
|
|
141
|
+
label: "Jev Decision",
|
|
142
|
+
description:
|
|
143
|
+
"Ask TypeSafe Jev for calibrated judgements about a piece of content. " +
|
|
144
|
+
"Jev is a fast, cheap decision model (about 1 second, ~$0.00002 per call) that returns " +
|
|
145
|
+
"probabilities instead of prose. Use it to classify, gate, triage, or rank without spending " +
|
|
146
|
+
"main-model tokens: 'is this command destructive?', 'which category does this issue belong to?', " +
|
|
147
|
+
"'how severe is this log line on a 1-5 scale?'. " +
|
|
148
|
+
"Question types: 'noul' (yes/no with probability), 'choice' (pick one option with a distribution), " +
|
|
149
|
+
"'score' (ordinal scale with expected value). " +
|
|
150
|
+
"Write literal, observable criteria; Jev does not infer intent beyond what is stated.",
|
|
151
|
+
promptSnippet: "Get calibrated yes/no, choice, or score judgements from the Jev decision model",
|
|
152
|
+
parameters: DecideParams,
|
|
153
|
+
execute: async (_toolCallId, params, signal, _onUpdate, ctx) => {
|
|
154
|
+
validateQuestions(params.questions);
|
|
155
|
+
|
|
156
|
+
const apiKey = await requireApiKey(ctx);
|
|
157
|
+
const state = parseState(params.state);
|
|
158
|
+
const questions = toApiQuestions(params.questions);
|
|
159
|
+
const warning = sizeWarning(state, questions);
|
|
160
|
+
const response = await requestDecision(apiKey, state, questions, signal);
|
|
161
|
+
const text = formatResponse(params.questions, response);
|
|
162
|
+
|
|
163
|
+
return {
|
|
164
|
+
content: [{ type: "text", text: warning ? `${warning}\n\n${text}` : text }],
|
|
165
|
+
details: {
|
|
166
|
+
model: response.model,
|
|
167
|
+
answers: response.answers,
|
|
168
|
+
usage: response.usage,
|
|
169
|
+
} satisfies DecideDetails,
|
|
170
|
+
};
|
|
171
|
+
},
|
|
172
|
+
});
|
|
173
|
+
|
|
174
|
+
// ── Bash gate ──
|
|
175
|
+
|
|
176
|
+
elyra.registerFlag(FLAG_GATE, {
|
|
177
|
+
description: "Ask the Jev decision model whether each bash command is destructive before running it",
|
|
178
|
+
type: "boolean",
|
|
179
|
+
default: false,
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
let gateEnabled = elyra.getFlag(FLAG_GATE) === true || isTruthy(process.env[ENV_GATE]);
|
|
183
|
+
const gateThreshold = parseThreshold(process.env[ENV_GATE_THRESHOLD]);
|
|
184
|
+
|
|
185
|
+
elyra.on("tool_call", async (event, ctx) => {
|
|
186
|
+
if (!gateEnabled || event.toolName !== "bash") return undefined;
|
|
187
|
+
|
|
188
|
+
const command = event.input.command;
|
|
189
|
+
if (typeof command !== "string" || !command.trim()) return undefined;
|
|
190
|
+
|
|
191
|
+
const apiKey = await ctx.modelRegistry.getApiKeyForProvider(PROVIDER);
|
|
192
|
+
if (!apiKey) {
|
|
193
|
+
ctx.ui.notify("Jev bash gate: no OpenRouter API key, skipping check", "warning");
|
|
194
|
+
return undefined;
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
let probability: number;
|
|
198
|
+
try {
|
|
199
|
+
const response = await requestDecision(
|
|
200
|
+
apiKey,
|
|
201
|
+
{ command, cwd: ctx.cwd },
|
|
202
|
+
{ [GATE_QUESTION_NAME]: GATE_QUESTION },
|
|
203
|
+
ctx.signal,
|
|
204
|
+
);
|
|
205
|
+
const answer = response.answers[GATE_QUESTION_NAME];
|
|
206
|
+
if (!answer || answer.type !== "noul") {
|
|
207
|
+
ctx.ui.notify("Jev bash gate: unexpected response, skipping check", "warning");
|
|
208
|
+
return undefined;
|
|
209
|
+
}
|
|
210
|
+
probability = answer.noul;
|
|
211
|
+
} catch (error) {
|
|
212
|
+
if (ctx.signal?.aborted) return undefined;
|
|
213
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
214
|
+
ctx.ui.notify(`Jev bash gate: ${msg}. Skipping check.`, "warning");
|
|
215
|
+
return undefined;
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
if (probability < gateThreshold) return undefined;
|
|
219
|
+
|
|
220
|
+
const pct = `${Math.round(probability * 100)}%`;
|
|
221
|
+
if (!ctx.hasUI) {
|
|
222
|
+
return {
|
|
223
|
+
block: true,
|
|
224
|
+
reason: `Jev rated this command ${pct} likely destructive (threshold ${gateThreshold})`,
|
|
225
|
+
};
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
const allowed = await ctx.ui.confirm(`Jev: ${pct} likely destructive`, `${command}\n\nRun this command?`);
|
|
229
|
+
return allowed ? undefined : { block: true, reason: "Blocked by user after Jev destructive-command check" };
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
// ── Command ──
|
|
233
|
+
|
|
234
|
+
elyra.registerCommand("jev", {
|
|
235
|
+
description: "Jev decision model: show config, or `gate on|off` to toggle the bash gate",
|
|
236
|
+
handler: async (args: string, ctx) => {
|
|
237
|
+
const [sub, value] = args.trim().split(/\s+/);
|
|
238
|
+
if (sub === "gate") {
|
|
239
|
+
if (value === "on") gateEnabled = true;
|
|
240
|
+
else if (value === "off") gateEnabled = false;
|
|
241
|
+
else if (value) {
|
|
242
|
+
ctx.ui.notify("Usage: /jev gate on|off", "warning");
|
|
243
|
+
return;
|
|
244
|
+
}
|
|
245
|
+
ctx.ui.notify(`Jev bash gate: ${gateEnabled ? "on" : "off"} (threshold ${gateThreshold})`, "info");
|
|
246
|
+
return;
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
const apiKey = await ctx.modelRegistry.getApiKeyForProvider(PROVIDER);
|
|
250
|
+
const lines = [
|
|
251
|
+
`Model: ${process.env[ENV_MODEL] || DEFAULT_MODEL}`,
|
|
252
|
+
`Endpoint: ${process.env[ENV_BASE_URL] || DEFAULT_BASE_URL}`,
|
|
253
|
+
`OpenRouter key: ${apiKey ? "configured" : "missing (run /login or set OPENROUTER_API_KEY)"}`,
|
|
254
|
+
`Bash gate: ${gateEnabled ? "on" : "off"} (threshold ${gateThreshold}, toggle with /jev gate on|off)`,
|
|
255
|
+
];
|
|
256
|
+
ctx.ui.notify(lines.join("\n"), apiKey ? "info" : "warning");
|
|
257
|
+
},
|
|
258
|
+
});
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
// ── Config helpers ──
|
|
262
|
+
|
|
263
|
+
function isTruthy(value: string | undefined): boolean {
|
|
264
|
+
if (!value) return false;
|
|
265
|
+
return ["1", "true", "yes", "on"].includes(value.trim().toLowerCase());
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
function parseThreshold(value: string | undefined): number {
|
|
269
|
+
if (!value) return DEFAULT_GATE_THRESHOLD;
|
|
270
|
+
const parsed = Number.parseFloat(value);
|
|
271
|
+
if (!Number.isFinite(parsed) || parsed <= 0 || parsed > 1) return DEFAULT_GATE_THRESHOLD;
|
|
272
|
+
return parsed;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
async function requireApiKey(ctx: ExtensionContext): Promise<string> {
|
|
276
|
+
const apiKey = await ctx.modelRegistry.getApiKeyForProvider(PROVIDER);
|
|
277
|
+
if (!apiKey) {
|
|
278
|
+
throw new Error("No OpenRouter API key found. Run /login and pick OpenRouter, or set OPENROUTER_API_KEY.");
|
|
279
|
+
}
|
|
280
|
+
return apiKey;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// ── Validation ──
|
|
284
|
+
|
|
285
|
+
function validateQuestions(questions: QuestionType[]): void {
|
|
286
|
+
const seen = new Set<string>();
|
|
287
|
+
for (const q of questions) {
|
|
288
|
+
if (!q.name.trim()) {
|
|
289
|
+
throw new Error("Question name must not be empty");
|
|
290
|
+
}
|
|
291
|
+
if (seen.has(q.name)) {
|
|
292
|
+
throw new Error(`Duplicate question name: ${q.name}`);
|
|
293
|
+
}
|
|
294
|
+
seen.add(q.name);
|
|
295
|
+
if (!q.instructions.trim()) {
|
|
296
|
+
throw new Error(`Question "${q.name}" is missing instructions`);
|
|
297
|
+
}
|
|
298
|
+
switch (q.type) {
|
|
299
|
+
case "choice": {
|
|
300
|
+
if (Object.keys(q.criteria).length < 2) {
|
|
301
|
+
throw new Error(`Question "${q.name}" (choice) needs at least two options`);
|
|
302
|
+
}
|
|
303
|
+
break;
|
|
304
|
+
}
|
|
305
|
+
case "score": {
|
|
306
|
+
if (q.criteria.length < 2 || q.criteria.length > 10) {
|
|
307
|
+
throw new Error(`Question "${q.name}" (score) needs 2-10 scale labels`);
|
|
308
|
+
}
|
|
309
|
+
break;
|
|
310
|
+
}
|
|
311
|
+
case "noul": {
|
|
312
|
+
// Schema already enforces both keys when criteria is present.
|
|
313
|
+
break;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// ── Request building ──
|
|
320
|
+
|
|
321
|
+
function parseState(raw: string): string | Record<string, unknown> | unknown[] {
|
|
322
|
+
const trimmed = raw.trim();
|
|
323
|
+
if (trimmed.startsWith("{") || trimmed.startsWith("[")) {
|
|
324
|
+
try {
|
|
325
|
+
const parsed: unknown = JSON.parse(trimmed);
|
|
326
|
+
if (parsed !== null && typeof parsed === "object") {
|
|
327
|
+
return parsed as Record<string, unknown> | unknown[];
|
|
328
|
+
}
|
|
329
|
+
} catch {
|
|
330
|
+
// Not JSON after all; send as text.
|
|
331
|
+
}
|
|
332
|
+
}
|
|
333
|
+
return trimmed;
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
function toApiQuestions(questions: QuestionType[]): Record<string, ApiQuestion> {
|
|
337
|
+
const out: Record<string, ApiQuestion> = {};
|
|
338
|
+
for (const q of questions) {
|
|
339
|
+
const { name, ...rest } = q;
|
|
340
|
+
out[name] = rest;
|
|
341
|
+
}
|
|
342
|
+
return out;
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
function estimateTokens(text: string): number {
|
|
346
|
+
let ascii = 0;
|
|
347
|
+
for (const ch of text) {
|
|
348
|
+
if (ch.charCodeAt(0) < 128) ascii++;
|
|
349
|
+
}
|
|
350
|
+
const nonAscii = text.length - ascii;
|
|
351
|
+
return ascii / 4 + nonAscii;
|
|
352
|
+
}
|
|
353
|
+
|
|
354
|
+
function sizeWarning(state: unknown, questions: Record<string, ApiQuestion>): string | undefined {
|
|
355
|
+
const stateText = typeof state === "string" ? state : JSON.stringify(state);
|
|
356
|
+
let longest = 0;
|
|
357
|
+
for (const q of Object.values(questions)) {
|
|
358
|
+
longest = Math.max(longest, estimateTokens(JSON.stringify(q)));
|
|
359
|
+
}
|
|
360
|
+
const total = estimateTokens(stateText) + longest;
|
|
361
|
+
if (total > SIZE_WARN_TOKENS) {
|
|
362
|
+
return `Warning: estimated ${Math.round(total)} tokens, which may exceed the model's ~32K context. Trim the state.`;
|
|
363
|
+
}
|
|
364
|
+
return undefined;
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
// ── HTTP ──
|
|
368
|
+
|
|
369
|
+
function requestDecision(
|
|
370
|
+
apiKey: string,
|
|
371
|
+
state: unknown,
|
|
372
|
+
questions: Record<string, ApiQuestion>,
|
|
373
|
+
signal: AbortSignal | undefined,
|
|
374
|
+
): Promise<DecisionsResponse> {
|
|
375
|
+
return callDecisions(apiKey, { model: process.env[ENV_MODEL] || DEFAULT_MODEL, state, questions }, signal);
|
|
376
|
+
}
|
|
377
|
+
|
|
378
|
+
async function callDecisions(
|
|
379
|
+
apiKey: string,
|
|
380
|
+
payload: Record<string, unknown>,
|
|
381
|
+
signal: AbortSignal | undefined,
|
|
382
|
+
): Promise<DecisionsResponse> {
|
|
383
|
+
const url = process.env[ENV_BASE_URL] || DEFAULT_BASE_URL;
|
|
384
|
+
const headers = {
|
|
385
|
+
Authorization: `Bearer ${apiKey}`,
|
|
386
|
+
"Content-Type": "application/json",
|
|
387
|
+
"X-Title": "elyra-jev-tools",
|
|
388
|
+
"HTTP-Referer": "https://github.com/kwhorne/elyra",
|
|
389
|
+
};
|
|
390
|
+
|
|
391
|
+
for (let attempt = 1; ; attempt++) {
|
|
392
|
+
signal?.throwIfAborted();
|
|
393
|
+
const timeout = AbortSignal.timeout(REQUEST_TIMEOUT_MS);
|
|
394
|
+
const combined = signal ? AbortSignal.any([signal, timeout]) : timeout;
|
|
395
|
+
|
|
396
|
+
let response: Response;
|
|
397
|
+
try {
|
|
398
|
+
response = await fetch(url, {
|
|
399
|
+
method: "POST",
|
|
400
|
+
headers,
|
|
401
|
+
body: JSON.stringify(payload),
|
|
402
|
+
signal: combined,
|
|
403
|
+
});
|
|
404
|
+
} catch (error) {
|
|
405
|
+
if (signal?.aborted) throw error;
|
|
406
|
+
if (attempt >= MAX_ATTEMPTS) {
|
|
407
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
408
|
+
throw new Error(`Jev request failed: ${msg}`);
|
|
409
|
+
}
|
|
410
|
+
await sleep(backoffDelay(attempt), signal);
|
|
411
|
+
continue;
|
|
412
|
+
}
|
|
413
|
+
|
|
414
|
+
if (response.ok) {
|
|
415
|
+
const data = (await response.json()) as Partial<DecisionsResponse>;
|
|
416
|
+
if (!data.answers || typeof data.answers !== "object") {
|
|
417
|
+
throw new Error("Jev response is missing 'answers'");
|
|
418
|
+
}
|
|
419
|
+
return data as DecisionsResponse;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
const body = await response.text();
|
|
423
|
+
const retryable = response.status === 429 || response.status === 529 || response.status >= 500;
|
|
424
|
+
if (retryable && attempt < MAX_ATTEMPTS) {
|
|
425
|
+
await sleep(backoffDelay(attempt, response.headers.get("Retry-After")), signal);
|
|
426
|
+
continue;
|
|
427
|
+
}
|
|
428
|
+
throw new Error(formatApiError(response.status, body));
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
function backoffDelay(attempt: number, retryAfter?: string | null): number {
|
|
433
|
+
if (retryAfter) {
|
|
434
|
+
const seconds = Number.parseFloat(retryAfter);
|
|
435
|
+
if (Number.isFinite(seconds)) return seconds * 1000;
|
|
436
|
+
}
|
|
437
|
+
const base = Math.min(20_000, 500 * 2 ** (attempt - 1));
|
|
438
|
+
return base + Math.random() * base * 0.5;
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
function sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {
|
|
442
|
+
return new Promise((resolve, reject) => {
|
|
443
|
+
const timer = setTimeout(() => {
|
|
444
|
+
signal?.removeEventListener("abort", onAbort);
|
|
445
|
+
resolve();
|
|
446
|
+
}, ms);
|
|
447
|
+
const onAbort = () => {
|
|
448
|
+
clearTimeout(timer);
|
|
449
|
+
reject(signal?.reason instanceof Error ? signal.reason : new Error("Aborted"));
|
|
450
|
+
};
|
|
451
|
+
signal?.addEventListener("abort", onAbort, { once: true });
|
|
452
|
+
});
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
function formatApiError(status: number, body: string): string {
|
|
456
|
+
let message = body.trim() || `HTTP ${status}`;
|
|
457
|
+
try {
|
|
458
|
+
const parsed = JSON.parse(body) as { error?: { message?: string } | string };
|
|
459
|
+
if (typeof parsed.error === "string") {
|
|
460
|
+
message = parsed.error;
|
|
461
|
+
} else if (parsed.error?.message) {
|
|
462
|
+
message = parsed.error.message;
|
|
463
|
+
}
|
|
464
|
+
} catch {
|
|
465
|
+
// Non-JSON body; keep raw text.
|
|
466
|
+
}
|
|
467
|
+
const hint = status === 401 ? " -- check that the OpenRouter API key is valid" : "";
|
|
468
|
+
return `Jev API error (HTTP ${status}): ${message}${hint}`;
|
|
469
|
+
}
|
|
470
|
+
|
|
471
|
+
// ── Output formatting ──
|
|
472
|
+
|
|
473
|
+
function formatResponse(questions: QuestionType[], response: DecisionsResponse): string {
|
|
474
|
+
const lines: string[] = [];
|
|
475
|
+
for (const q of questions) {
|
|
476
|
+
const answer = response.answers[q.name];
|
|
477
|
+
if (!answer) {
|
|
478
|
+
lines.push(`${q.name}: (no answer returned)`);
|
|
479
|
+
continue;
|
|
480
|
+
}
|
|
481
|
+
lines.push(...formatAnswer(q.name, answer));
|
|
482
|
+
}
|
|
483
|
+
|
|
484
|
+
const usage = response.usage;
|
|
485
|
+
if (usage) {
|
|
486
|
+
const tokens = (usage.input_tokens ?? 0) + (usage.output_tokens ?? 0);
|
|
487
|
+
const cost = usage.cost !== undefined ? ` · $${usage.cost.toFixed(6)}` : "";
|
|
488
|
+
lines.push("", `[${response.model ?? DEFAULT_MODEL} · ${tokens} tok${cost}]`);
|
|
489
|
+
}
|
|
490
|
+
return lines.join("\n");
|
|
491
|
+
}
|
|
492
|
+
|
|
493
|
+
function formatAnswer(name: string, answer: Answer): string[] {
|
|
494
|
+
switch (answer.type) {
|
|
495
|
+
case "noul": {
|
|
496
|
+
const p = answer.noul ?? 0;
|
|
497
|
+
return [`${name}: ${p >= 0.5 ? "yes" : "no"} (p=${p.toFixed(2)})`];
|
|
498
|
+
}
|
|
499
|
+
case "choice": {
|
|
500
|
+
const probs = answer.probabilities ?? {};
|
|
501
|
+
const p = probs[answer.choice] ?? 0;
|
|
502
|
+
const dist = Object.entries(probs)
|
|
503
|
+
.sort((a, b) => b[1] - a[1])
|
|
504
|
+
.map(([k, v]) => `${k}=${v.toFixed(2)}`)
|
|
505
|
+
.join(", ");
|
|
506
|
+
return [
|
|
507
|
+
`${name}: ${answer.choice} (p=${p.toFixed(2)}, conf=${(answer.confidence ?? 0).toFixed(2)})`,
|
|
508
|
+
` ${dist}`,
|
|
509
|
+
];
|
|
510
|
+
}
|
|
511
|
+
case "score": {
|
|
512
|
+
const legend = answer.legend ?? {};
|
|
513
|
+
const probs = answer.probabilities ?? {};
|
|
514
|
+
const maxIdx = Math.max(0, Object.keys(legend).length - 1);
|
|
515
|
+
const argmax = Object.keys(probs).sort((a, b) => (probs[b] ?? 0) - (probs[a] ?? 0))[0];
|
|
516
|
+
const label = argmax !== undefined ? (legend[argmax] ?? argmax) : "";
|
|
517
|
+
const dist = Object.keys(legend)
|
|
518
|
+
.sort((a, b) => Number(a) - Number(b))
|
|
519
|
+
.map((k) => `${legend[k]}=${(probs[k] ?? 0).toFixed(2)}`)
|
|
520
|
+
.join(", ");
|
|
521
|
+
return [
|
|
522
|
+
`${name}: ${(answer.score ?? 0).toFixed(2)}/${maxIdx} -> ${label} (conf=${(answer.confidence ?? 0).toFixed(2)})`,
|
|
523
|
+
` ${dist}`,
|
|
524
|
+
];
|
|
525
|
+
}
|
|
526
|
+
default: {
|
|
527
|
+
return [`${name}: ${JSON.stringify(answer)}`];
|
|
528
|
+
}
|
|
529
|
+
}
|
|
530
|
+
}
|
|
531
|
+
|
|
532
|
+
export type { DecideDetails, DecideParamsType as DecideParams };
|
package/package.json
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@elyracode/jev-tools",
|
|
3
|
+
"version": "0.9.39",
|
|
4
|
+
"description": "Elyra extension for TypeSafe Jev decisions via OpenRouter -- calibrated yes/no, choice, and score judgements",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"keywords": [
|
|
7
|
+
"elyra-package",
|
|
8
|
+
"jev",
|
|
9
|
+
"typesafe",
|
|
10
|
+
"openrouter",
|
|
11
|
+
"decisions",
|
|
12
|
+
"classification"
|
|
13
|
+
],
|
|
14
|
+
"license": "MIT",
|
|
15
|
+
"author": "Knut W. Horne",
|
|
16
|
+
"repository": {
|
|
17
|
+
"type": "git",
|
|
18
|
+
"url": "git+https://github.com/kwhorne/elyra.git",
|
|
19
|
+
"directory": "packages/jev-tools"
|
|
20
|
+
},
|
|
21
|
+
"elyra": {
|
|
22
|
+
"extensions": [
|
|
23
|
+
"./extensions/index.ts"
|
|
24
|
+
]
|
|
25
|
+
},
|
|
26
|
+
"peerDependencies": {
|
|
27
|
+
"@elyracode/coding-agent": "*",
|
|
28
|
+
"typebox": "*"
|
|
29
|
+
},
|
|
30
|
+
"scripts": {
|
|
31
|
+
"clean": "echo 'nothing to clean'",
|
|
32
|
+
"build": "echo 'nothing to build'",
|
|
33
|
+
"check": "echo 'nothing to check'"
|
|
34
|
+
}
|
|
35
|
+
}
|