@bobobowis/jev-kit 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +311 -2
- package/dist/capabilities.d.ts +33 -0
- package/dist/capabilities.js +81 -0
- package/dist/cli/ask.d.ts +9 -0
- package/dist/cli/ask.js +210 -0
- package/dist/cli/common.d.ts +38 -0
- package/dist/cli/common.js +40 -0
- package/dist/cli/hook.d.ts +3 -0
- package/dist/cli/hook.js +85 -0
- package/dist/cli/index.d.ts +2 -0
- package/dist/cli/index.js +21 -0
- package/dist/cli/main.d.ts +8 -0
- package/dist/cli/main.js +41 -0
- package/dist/cli/mcp.d.ts +3 -0
- package/dist/cli/mcp.js +47 -0
- package/dist/cli/setup.d.ts +3 -0
- package/dist/cli/setup.js +84 -0
- package/dist/compact/compact-on-demand.d.ts +22 -0
- package/dist/compact/compact-on-demand.js +29 -0
- package/dist/compact/evaluate.d.ts +36 -0
- package/dist/compact/evaluate.js +31 -0
- package/dist/compact/index.d.ts +7 -0
- package/dist/compact/index.js +7 -0
- package/dist/compact/pick-cut-point.d.ts +18 -0
- package/dist/compact/pick-cut-point.js +29 -0
- package/dist/compact/should-compact.d.ts +57 -0
- package/dist/compact/should-compact.js +87 -0
- package/dist/core/client.d.ts +93 -0
- package/dist/core/client.js +295 -0
- package/dist/core/helpers.d.ts +14 -0
- package/dist/core/helpers.js +9 -0
- package/dist/core/mock.d.ts +20 -0
- package/dist/core/mock.js +132 -0
- package/dist/core/types.d.ts +102 -0
- package/dist/core/types.js +82 -0
- package/dist/events/index.d.ts +44 -0
- package/dist/events/index.js +77 -0
- package/dist/files/ask-file-bool.d.ts +12 -0
- package/dist/files/ask-file-bool.js +15 -0
- package/dist/files/ask-file-choice.d.ts +10 -0
- package/dist/files/ask-file-choice.js +17 -0
- package/dist/files/ask-file-score.d.ts +12 -0
- package/dist/files/ask-file-score.js +15 -0
- package/dist/files/ask-files.d.ts +23 -0
- package/dist/files/ask-files.js +56 -0
- package/dist/files/ask.d.ts +12 -0
- package/dist/files/ask.js +28 -0
- package/dist/files/assemble.d.ts +52 -0
- package/dist/files/assemble.js +129 -0
- package/dist/files/decide.d.ts +10 -0
- package/dist/files/decide.js +1 -0
- package/dist/files/index.d.ts +14 -0
- package/dist/files/index.js +14 -0
- package/dist/files/pick-first.d.ts +16 -0
- package/dist/files/pick-first.js +24 -0
- package/dist/files/prune.d.ts +13 -0
- package/dist/files/prune.js +87 -0
- package/dist/files/read-state.d.ts +17 -0
- package/dist/files/read-state.js +47 -0
- package/dist/guards/bash-gate.d.ts +65 -0
- package/dist/guards/bash-gate.js +78 -0
- package/dist/guards/index.d.ts +7 -0
- package/dist/guards/index.js +7 -0
- package/dist/guards/mcp-gate.d.ts +23 -0
- package/dist/guards/mcp-gate.js +52 -0
- package/dist/guards/result-screen.d.ts +20 -0
- package/dist/guards/result-screen.js +31 -0
- package/dist/guards/write-gate.d.ts +18 -0
- package/dist/guards/write-gate.js +44 -0
- package/dist/hooks/handle.d.ts +76 -0
- package/dist/hooks/handle.js +61 -0
- package/dist/hooks/harness.d.ts +45 -0
- package/dist/hooks/harness.js +124 -0
- package/dist/hooks/index.d.ts +7 -0
- package/dist/hooks/index.js +7 -0
- package/dist/hooks/state.d.ts +8 -0
- package/dist/hooks/state.js +25 -0
- package/dist/hooks/transcript.d.ts +8 -0
- package/dist/hooks/transcript.js +52 -0
- package/dist/index.d.ts +8 -0
- package/dist/index.js +8 -0
- package/dist/ledger.d.ts +22 -0
- package/dist/ledger.js +26 -0
- package/dist/mcp/server.d.ts +28 -0
- package/dist/mcp/server.js +124 -0
- package/dist/policy/index.d.ts +17 -0
- package/dist/policy/index.js +30 -0
- package/dist/setup/index.d.ts +76 -0
- package/dist/setup/index.js +221 -0
- package/dist/tools/ask-jev.d.ts +7 -0
- package/dist/tools/ask-jev.js +42 -0
- package/dist/tools/compact-tool.d.ts +2 -0
- package/dist/tools/compact-tool.js +33 -0
- package/dist/tools/description.d.ts +5 -0
- package/dist/tools/description.js +20 -0
- package/dist/tools/file-tools.d.ts +7 -0
- package/dist/tools/file-tools.js +127 -0
- package/dist/tools/index.d.ts +10 -0
- package/dist/tools/index.js +10 -0
- package/dist/tools/run-command.d.ts +7 -0
- package/dist/tools/run-command.js +27 -0
- package/dist/tools/sets.d.ts +9 -0
- package/dist/tools/sets.js +32 -0
- package/dist/tools/types.d.ts +51 -0
- package/dist/tools/types.js +4 -0
- package/package.json +96 -3
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Question builders, matching the @typesafe-ai/sdk helper style.
|
|
3
|
+
* Answer types are inferred from the question you build.
|
|
4
|
+
*/
|
|
5
|
+
import type { ChoiceAnswer, ChoiceCriteria, ChoiceQuestion, Instructions, NoulAnswer, NoulCriteria, NoulQuestion, Question, ScoreAnswer, ScoreCriteria, ScoreQuestion } from "./types.ts";
|
|
6
|
+
export declare function noul(instructions: Instructions, criteria?: NoulCriteria): NoulQuestion;
|
|
7
|
+
export declare function choice(instructions: Instructions, criteria: ChoiceCriteria): ChoiceQuestion;
|
|
8
|
+
export declare function score(instructions: Instructions, criteria: ScoreCriteria): ScoreQuestion;
|
|
9
|
+
/** Infer the answer type a question returns. */
|
|
10
|
+
export type AnswerOf<Q> = Q extends NoulQuestion ? NoulAnswer : Q extends ChoiceQuestion ? ChoiceAnswer : Q extends ScoreQuestion ? ScoreAnswer : never;
|
|
11
|
+
/** Typed question map -> typed answer map. */
|
|
12
|
+
export type AnswersOf<Q extends Record<string, Question>> = {
|
|
13
|
+
[K in keyof Q]: AnswerOf<Q[K]>;
|
|
14
|
+
};
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export function noul(instructions, criteria) {
|
|
2
|
+
return criteria ? { type: "noul", instructions, criteria } : { type: "noul", instructions };
|
|
3
|
+
}
|
|
4
|
+
export function choice(instructions, criteria) {
|
|
5
|
+
return { type: "choice", instructions, criteria };
|
|
6
|
+
}
|
|
7
|
+
export function score(instructions, criteria) {
|
|
8
|
+
return { type: "score", instructions, criteria };
|
|
9
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* MockJev — a deterministic, offline stand-in for the real System One endpoint.
|
|
3
|
+
*
|
|
4
|
+
* It exists so every level in this codebase runs (and its tests pass) without an
|
|
5
|
+
* API key. It mimics the exact wire contract: typed answers, probability
|
|
6
|
+
* distributions that sum to 1, legends on scores, and confidence derived from
|
|
7
|
+
* how peaked the distribution is.
|
|
8
|
+
*
|
|
9
|
+
* How it decides: token overlap between the flattened state and each option /
|
|
10
|
+
* level / criteria description, run through a softmax. It is a stand-in for
|
|
11
|
+
* *shape*, not for intelligence — with a real key, the same code paths hit
|
|
12
|
+
* https://api.typesafe.ai/v1/systemone and nothing else changes.
|
|
13
|
+
*/
|
|
14
|
+
import type { State, SystemOneRequest, SystemOneResponse } from "./types.ts";
|
|
15
|
+
/** Flatten any state shape into text the overlap engine can read. */
|
|
16
|
+
export declare function flattenState(state: State): string;
|
|
17
|
+
export declare class MockJev {
|
|
18
|
+
readonly model = "jev-1.13.0-mock";
|
|
19
|
+
systemOne(req: SystemOneRequest): SystemOneResponse;
|
|
20
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
const STOPWORDS = new Set([
|
|
2
|
+
"the", "a", "an", "and", "or", "but", "if", "then", "than", "of", "to", "in", "on", "at", "for",
|
|
3
|
+
"with", "without", "is", "are", "was", "were", "be", "been", "being", "do", "does", "did", "done",
|
|
4
|
+
"this", "that", "these", "those", "it", "its", "as", "by", "from", "into", "about", "against",
|
|
5
|
+
"between", "through", "during", "before", "after", "above", "below", "up", "down", "out", "off",
|
|
6
|
+
"over", "under", "again", "further", "once", "here", "there", "when", "where", "why", "how",
|
|
7
|
+
"all", "any", "both", "each", "few", "more", "most", "other", "some", "such", "no", "nor", "not",
|
|
8
|
+
"only", "own", "same", "so", "too", "very", "s", "t", "can", "will", "just", "don", "should",
|
|
9
|
+
"now", "d", "ll", "m", "o", "re", "ve", "y", "i", "you", "we", "they", "he", "she", "what",
|
|
10
|
+
"which", "who", "whom", "am", "has", "have", "had", "having", "message", "text", "given",
|
|
11
|
+
]);
|
|
12
|
+
function stem(t) {
|
|
13
|
+
if (t.length > 4 && t.endsWith("ing"))
|
|
14
|
+
return t.slice(0, -3);
|
|
15
|
+
if (t.length > 4 && t.endsWith("ed"))
|
|
16
|
+
return t.slice(0, -2);
|
|
17
|
+
if (t.length > 3 && t.endsWith("es"))
|
|
18
|
+
return t.slice(0, -2);
|
|
19
|
+
if (t.length > 3 && t.endsWith("s") && !t.endsWith("ss"))
|
|
20
|
+
return t.slice(0, -1);
|
|
21
|
+
return t;
|
|
22
|
+
}
|
|
23
|
+
function tokenize(text) {
|
|
24
|
+
return new Set(text
|
|
25
|
+
.toLowerCase()
|
|
26
|
+
.replace(/[`_*#>/\[\]{}()"',.;:!?\\-]/g, " ")
|
|
27
|
+
.split(/\s+/)
|
|
28
|
+
.filter((t) => t.length > 1 && !STOPWORDS.has(t))
|
|
29
|
+
.map(stem));
|
|
30
|
+
}
|
|
31
|
+
/** Flatten any state shape into text the overlap engine can read. */
|
|
32
|
+
export function flattenState(state) {
|
|
33
|
+
if (typeof state === "string")
|
|
34
|
+
return state;
|
|
35
|
+
return JSON.stringify(state, null, 2);
|
|
36
|
+
}
|
|
37
|
+
function instructionsText(q) {
|
|
38
|
+
if (typeof q.instructions === "string")
|
|
39
|
+
return q.instructions;
|
|
40
|
+
return JSON.stringify(q.instructions);
|
|
41
|
+
}
|
|
42
|
+
function softmax(values, temperature = 0.6) {
|
|
43
|
+
if (values.every((v) => v === 0)) {
|
|
44
|
+
return values.map(() => 1 / values.length);
|
|
45
|
+
}
|
|
46
|
+
const max = Math.max(...values);
|
|
47
|
+
const exps = values.map((v) => Math.exp((v - max) / temperature));
|
|
48
|
+
const sum = exps.reduce((a, b) => a + b, 0);
|
|
49
|
+
return exps.map((e) => e / sum);
|
|
50
|
+
}
|
|
51
|
+
/** Affinity: how many meaningful tokens the state shares with a description, weighted by rarity-free overlap. */
|
|
52
|
+
function affinity(stateTokens, description) {
|
|
53
|
+
const descTokens = tokenize(description);
|
|
54
|
+
let shared = 0;
|
|
55
|
+
for (const t of descTokens)
|
|
56
|
+
if (stateTokens.has(t))
|
|
57
|
+
shared++;
|
|
58
|
+
return shared;
|
|
59
|
+
}
|
|
60
|
+
function round(n, places = 4) {
|
|
61
|
+
const f = 10 ** places;
|
|
62
|
+
return Math.round(n * f) / f;
|
|
63
|
+
}
|
|
64
|
+
function noulAnswer(q, stateTokens) {
|
|
65
|
+
const yesText = instructionsText(q) + " " + (q.criteria?.true ?? "");
|
|
66
|
+
const noText = q.criteria?.false ?? "";
|
|
67
|
+
const yes = affinity(stateTokens, yesText);
|
|
68
|
+
const no = affinity(stateTokens, noText);
|
|
69
|
+
// Smoothed signal in [-1, 1]; logistic to [0, 1]. No signal -> 0.5.
|
|
70
|
+
const k = 2.5;
|
|
71
|
+
const signal = (yes - no) / (yes + no + k);
|
|
72
|
+
const p = 1 / (1 + Math.exp(-4 * signal));
|
|
73
|
+
return { type: "noul", noul: round(p) };
|
|
74
|
+
}
|
|
75
|
+
function choiceAnswer(q, stateTokens) {
|
|
76
|
+
const entries = Object.entries(q.criteria);
|
|
77
|
+
const affinities = entries.map(([option, desc]) => {
|
|
78
|
+
const keyText = option.replace(/_/g, " ");
|
|
79
|
+
return affinity(stateTokens, keyText) * 1.5 + affinity(stateTokens, desc ?? "") * 1.0;
|
|
80
|
+
});
|
|
81
|
+
const probs = softmax(affinities);
|
|
82
|
+
const probabilities = {};
|
|
83
|
+
entries.forEach(([option], i) => (probabilities[option] = round(probs[i])));
|
|
84
|
+
let best = 0;
|
|
85
|
+
entries.forEach((_, i) => {
|
|
86
|
+
if (probs[i] > probs[best])
|
|
87
|
+
best = i;
|
|
88
|
+
});
|
|
89
|
+
return {
|
|
90
|
+
type: "choice",
|
|
91
|
+
choice: entries[best][0],
|
|
92
|
+
probabilities,
|
|
93
|
+
confidence: round(probs[best]),
|
|
94
|
+
};
|
|
95
|
+
}
|
|
96
|
+
function scoreAnswer(q, stateTokens) {
|
|
97
|
+
const affinities = q.criteria.map((level) => affinity(stateTokens, level) + 1e-9);
|
|
98
|
+
const probs = softmax(affinities);
|
|
99
|
+
const legend = {};
|
|
100
|
+
q.criteria.forEach((level, i) => (legend[String(i)] = level));
|
|
101
|
+
const probabilities = {};
|
|
102
|
+
q.criteria.forEach((_, i) => (probabilities[String(i)] = round(probs[i])));
|
|
103
|
+
const scoreValue = probs.reduce((acc, p, i) => acc + p * i, 0);
|
|
104
|
+
const confidence = Math.max(...probs);
|
|
105
|
+
return {
|
|
106
|
+
type: "score",
|
|
107
|
+
score: round(scoreValue, 2),
|
|
108
|
+
legend,
|
|
109
|
+
probabilities,
|
|
110
|
+
confidence: round(confidence),
|
|
111
|
+
};
|
|
112
|
+
}
|
|
113
|
+
export class MockJev {
|
|
114
|
+
model = "jev-1.13.0-mock";
|
|
115
|
+
systemOne(req) {
|
|
116
|
+
const stateTokens = tokenize(flattenState(req.state));
|
|
117
|
+
const answers = {};
|
|
118
|
+
for (const [id, q] of Object.entries(req.questions)) {
|
|
119
|
+
if (q.type === "noul")
|
|
120
|
+
answers[id] = noulAnswer(q, stateTokens);
|
|
121
|
+
else if (q.type === "choice")
|
|
122
|
+
answers[id] = choiceAnswer(q, stateTokens);
|
|
123
|
+
else
|
|
124
|
+
answers[id] = scoreAnswer(q, stateTokens);
|
|
125
|
+
}
|
|
126
|
+
const usage = {
|
|
127
|
+
input_tokens: Math.ceil(flattenState(req.state).length / 4) + 20 * Object.keys(req.questions).length,
|
|
128
|
+
output_tokens: 20 * Object.keys(req.questions).length,
|
|
129
|
+
};
|
|
130
|
+
return { model: this.model, answers, usage };
|
|
131
|
+
}
|
|
132
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TypeSafe System One API types — the exact wire contract for Jev.
|
|
3
|
+
*
|
|
4
|
+
* One endpoint: POST https://api.typesafe.ai/v1/systemone
|
|
5
|
+
* You send a `state` and a map of typed `questions`; you get back one typed
|
|
6
|
+
* `answer` per question, keyed by the IDs you chose.
|
|
7
|
+
*/
|
|
8
|
+
/** The content being evaluated. A plain string, or structured data. */
|
|
9
|
+
export type State = string | Record<string, unknown> | unknown[];
|
|
10
|
+
/**
|
|
11
|
+
* What to ask about the state. Can be a plain string, or a structured object
|
|
12
|
+
* that puts the question in one field and the data it refers to in others.
|
|
13
|
+
*/
|
|
14
|
+
export type Instructions = string | Record<string, unknown>;
|
|
15
|
+
/** Optional clarification of what yes and no mean. */
|
|
16
|
+
export interface NoulCriteria {
|
|
17
|
+
true?: string;
|
|
18
|
+
false?: string;
|
|
19
|
+
}
|
|
20
|
+
/** A map of option -> rubric description. Use null when an option needs no detail. Max 255 options. */
|
|
21
|
+
export type ChoiceCriteria = Record<string, string | null>;
|
|
22
|
+
/** An ordered array of level descriptions, low to high. 2-10 levels. */
|
|
23
|
+
export type ScoreCriteria = string[];
|
|
24
|
+
export interface NoulQuestion {
|
|
25
|
+
type: "noul";
|
|
26
|
+
instructions: Instructions;
|
|
27
|
+
criteria?: NoulCriteria;
|
|
28
|
+
}
|
|
29
|
+
export interface ChoiceQuestion {
|
|
30
|
+
type: "choice";
|
|
31
|
+
instructions: Instructions;
|
|
32
|
+
criteria: ChoiceCriteria;
|
|
33
|
+
}
|
|
34
|
+
export interface ScoreQuestion {
|
|
35
|
+
type: "score";
|
|
36
|
+
instructions: Instructions;
|
|
37
|
+
criteria: ScoreCriteria;
|
|
38
|
+
}
|
|
39
|
+
export type Question = NoulQuestion | ChoiceQuestion | ScoreQuestion;
|
|
40
|
+
/** Question IDs are for your code. They are not sent to the model and are not used in inference. */
|
|
41
|
+
export type Questions = Record<string, Question>;
|
|
42
|
+
export interface NoulAnswer {
|
|
43
|
+
type: "noul";
|
|
44
|
+
/** The probability the answer is yes. 0 = strong no, 1 = strong yes, 0.5 = uncertain. */
|
|
45
|
+
noul: number;
|
|
46
|
+
}
|
|
47
|
+
export interface ChoiceAnswer {
|
|
48
|
+
type: "choice";
|
|
49
|
+
/** The highest-probability option. Always one you defined. */
|
|
50
|
+
choice: string;
|
|
51
|
+
/** Every option mapped to its probability. Floats that sum to 1. */
|
|
52
|
+
probabilities: Record<string, number>;
|
|
53
|
+
/** How certain the model is, derived from the shape of the distribution. */
|
|
54
|
+
confidence: number;
|
|
55
|
+
}
|
|
56
|
+
export interface ScoreAnswer {
|
|
57
|
+
type: "score";
|
|
58
|
+
/** Probability-weighted position along the levels. Can land between levels. */
|
|
59
|
+
score: number;
|
|
60
|
+
/** Each level number mapped back to its description. */
|
|
61
|
+
legend: Record<string, string>;
|
|
62
|
+
/** Each level mapped to its probability. Floats that sum to 1. */
|
|
63
|
+
probabilities: Record<string, number>;
|
|
64
|
+
confidence: number;
|
|
65
|
+
}
|
|
66
|
+
export type Answer = NoulAnswer | ChoiceAnswer | ScoreAnswer;
|
|
67
|
+
export interface SystemOneRequest {
|
|
68
|
+
/** Defaults to "jev-latest". */
|
|
69
|
+
model?: string;
|
|
70
|
+
state: State;
|
|
71
|
+
questions: Questions;
|
|
72
|
+
}
|
|
73
|
+
export interface SystemOneUsage {
|
|
74
|
+
input_tokens: number;
|
|
75
|
+
output_tokens: number;
|
|
76
|
+
/** Provider-reported USD cost. Only finite, nonnegative numbers are trusted. */
|
|
77
|
+
cost?: unknown;
|
|
78
|
+
[key: string]: unknown;
|
|
79
|
+
}
|
|
80
|
+
export interface SystemOneResponse {
|
|
81
|
+
/** The versioned model that answered. Log it. */
|
|
82
|
+
model: string;
|
|
83
|
+
answers: Record<string, Answer>;
|
|
84
|
+
usage: SystemOneUsage;
|
|
85
|
+
/** Provider extensions are retained, not stripped during validation. */
|
|
86
|
+
[key: string]: unknown;
|
|
87
|
+
}
|
|
88
|
+
/** Client-side validation limits, mirrored from the published API. */
|
|
89
|
+
export declare const LIMITS: {
|
|
90
|
+
readonly MAX_CHOICE_OPTIONS: 255;
|
|
91
|
+
readonly MIN_SCORE_LEVELS: 2;
|
|
92
|
+
readonly MAX_SCORE_LEVELS: 10;
|
|
93
|
+
/** Approximate shared token budget for state + all questions. */
|
|
94
|
+
readonly TOTAL_TOKEN_BUDGET: 64000;
|
|
95
|
+
};
|
|
96
|
+
export declare class QuestionValidationError extends Error {
|
|
97
|
+
constructor(message: string);
|
|
98
|
+
}
|
|
99
|
+
/** Validate runtime input as well as TypeScript callers, before any transport. */
|
|
100
|
+
export declare function validateRequest(request: unknown): asserts request is SystemOneRequest;
|
|
101
|
+
/** Validate a request's question shapes before it is ever sent. */
|
|
102
|
+
export declare function validateQuestions(questions: unknown): asserts questions is Questions;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* TypeSafe System One API types — the exact wire contract for Jev.
|
|
3
|
+
*
|
|
4
|
+
* One endpoint: POST https://api.typesafe.ai/v1/systemone
|
|
5
|
+
* You send a `state` and a map of typed `questions`; you get back one typed
|
|
6
|
+
* `answer` per question, keyed by the IDs you chose.
|
|
7
|
+
*/
|
|
8
|
+
/** Client-side validation limits, mirrored from the published API. */
|
|
9
|
+
export const LIMITS = {
|
|
10
|
+
MAX_CHOICE_OPTIONS: 255,
|
|
11
|
+
MIN_SCORE_LEVELS: 2,
|
|
12
|
+
MAX_SCORE_LEVELS: 10,
|
|
13
|
+
/** Approximate shared token budget for state + all questions. */
|
|
14
|
+
TOTAL_TOKEN_BUDGET: 64_000,
|
|
15
|
+
};
|
|
16
|
+
export class QuestionValidationError extends Error {
|
|
17
|
+
constructor(message) {
|
|
18
|
+
super(message);
|
|
19
|
+
this.name = "QuestionValidationError";
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
const isRecord = (value) => value !== null && typeof value === "object" &&
|
|
23
|
+
(Object.getPrototypeOf(value) === Object.prototype || Object.getPrototypeOf(value) === null);
|
|
24
|
+
/** Validate runtime input as well as TypeScript callers, before any transport. */
|
|
25
|
+
export function validateRequest(request) {
|
|
26
|
+
if (!isRecord(request))
|
|
27
|
+
throw new QuestionValidationError("Expected a request object.");
|
|
28
|
+
if (typeof request.state !== "string" && !isRecord(request.state) && !Array.isArray(request.state)) {
|
|
29
|
+
throw new QuestionValidationError("State must be a string, object, or array.");
|
|
30
|
+
}
|
|
31
|
+
if (request.model !== undefined && (typeof request.model !== "string" || !request.model.trim())) {
|
|
32
|
+
throw new QuestionValidationError("Model must be a nonblank string.");
|
|
33
|
+
}
|
|
34
|
+
validateQuestions(request.questions);
|
|
35
|
+
}
|
|
36
|
+
/** Validate a request's question shapes before it is ever sent. */
|
|
37
|
+
export function validateQuestions(questions) {
|
|
38
|
+
if (!isRecord(questions) || Object.keys(questions).length === 0) {
|
|
39
|
+
throw new QuestionValidationError("Questions must be a nonempty object.");
|
|
40
|
+
}
|
|
41
|
+
for (const [id, q] of Object.entries(questions)) {
|
|
42
|
+
if (!isRecord(q))
|
|
43
|
+
throw new QuestionValidationError(`Question "${id}" must be an object.`);
|
|
44
|
+
if (q.type !== "noul" && q.type !== "choice" && q.type !== "score") {
|
|
45
|
+
throw new QuestionValidationError(`Question "${id}" has a missing or unknown type.`);
|
|
46
|
+
}
|
|
47
|
+
if (typeof q.instructions === "string" ? !q.instructions.trim() : !isRecord(q.instructions)) {
|
|
48
|
+
throw new QuestionValidationError(`Question "${id}" needs nonblank string or object instructions.`);
|
|
49
|
+
}
|
|
50
|
+
if (q.type === "noul" && q.criteria !== undefined) {
|
|
51
|
+
if (!isRecord(q.criteria) || Object.entries(q.criteria).some(([key, value]) => !["true", "false"].includes(key) || (value !== undefined && typeof value !== "string"))) {
|
|
52
|
+
throw new QuestionValidationError(`Noul "${id}" criteria must map true/false to descriptions.`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
if (q.type === "choice") {
|
|
56
|
+
if (!isRecord(q.criteria)) {
|
|
57
|
+
throw new QuestionValidationError(`Choice "${id}" criteria must be an object.`);
|
|
58
|
+
}
|
|
59
|
+
const options = Object.keys(q.criteria);
|
|
60
|
+
if (options.length === 0) {
|
|
61
|
+
throw new QuestionValidationError(`Choice "${id}" has no options.`);
|
|
62
|
+
}
|
|
63
|
+
if (options.length > LIMITS.MAX_CHOICE_OPTIONS) {
|
|
64
|
+
throw new QuestionValidationError(`Choice "${id}" has ${options.length} options; the maximum is ${LIMITS.MAX_CHOICE_OPTIONS}.`);
|
|
65
|
+
}
|
|
66
|
+
if (Object.values(q.criteria).some((value) => value !== null && typeof value !== "string")) {
|
|
67
|
+
throw new QuestionValidationError(`Choice "${id}" descriptions must be strings or null.`);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
if (q.type === "score") {
|
|
71
|
+
if (!Array.isArray(q.criteria)) {
|
|
72
|
+
throw new QuestionValidationError(`Score "${id}" criteria must be an array.`);
|
|
73
|
+
}
|
|
74
|
+
if (q.criteria.length < LIMITS.MIN_SCORE_LEVELS || q.criteria.length > LIMITS.MAX_SCORE_LEVELS) {
|
|
75
|
+
throw new QuestionValidationError(`Score "${id}" must have between ${LIMITS.MIN_SCORE_LEVELS} and ${LIMITS.MAX_SCORE_LEVELS} levels; got ${q.criteria.length}.`);
|
|
76
|
+
}
|
|
77
|
+
if (q.criteria.some((level) => typeof level !== "string" || !level.trim())) {
|
|
78
|
+
throw new QuestionValidationError(`Score "${id}" levels must be nonblank strings.`);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { type JevClient } from "../core/client.ts";
|
|
2
|
+
import { type Answer, type Questions, type State, type SystemOneUsage } from "../core/types.ts";
|
|
3
|
+
export declare const EVENT_PREFIX = "JEV_EVENT ";
|
|
4
|
+
export interface JevEventLine {
|
|
5
|
+
kind: string;
|
|
6
|
+
at: number;
|
|
7
|
+
harness: string;
|
|
8
|
+
[field: string]: unknown;
|
|
9
|
+
}
|
|
10
|
+
export interface Reporter {
|
|
11
|
+
readonly harness: string;
|
|
12
|
+
/** Where events go: the file path, "stderr", or "off". */
|
|
13
|
+
readonly target: string;
|
|
14
|
+
report(kind: string, payload?: Record<string, unknown>): void;
|
|
15
|
+
}
|
|
16
|
+
export interface ReporterOptions {
|
|
17
|
+
/** Names who produced the events: pi, claude, codex, mcp, cli, ... */
|
|
18
|
+
harness: string;
|
|
19
|
+
/** Defaults to JEV_EVENT_DIR. */
|
|
20
|
+
dir?: string;
|
|
21
|
+
/** No event directory means no events, instead of stderr lines. */
|
|
22
|
+
quiet?: boolean;
|
|
23
|
+
/** Also called with every event, for a harness's own record (pi session entries). Errors are ignored. */
|
|
24
|
+
onEvent?: (kind: string, payload: Record<string, unknown>) => void;
|
|
25
|
+
stderr?: {
|
|
26
|
+
write(chunk: string): unknown;
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
export declare function createReporter(opts: ReporterOptions): Reporter;
|
|
30
|
+
export interface Decision {
|
|
31
|
+
answers: Record<string, Answer>;
|
|
32
|
+
usage: SystemOneUsage;
|
|
33
|
+
model: string;
|
|
34
|
+
ms: number;
|
|
35
|
+
}
|
|
36
|
+
/** One Jev call that validates, calls, times, and reports. `source` names who asked, a hook or a tool. */
|
|
37
|
+
export type Decider = (source: string, state: State, questions: Questions, extra?: Record<string, unknown>) => Promise<Decision>;
|
|
38
|
+
/**
|
|
39
|
+
* `extra` must not use the keys the report already sets: source, state, questions, answers, usage, model, ms.
|
|
40
|
+
* The client defaults to the shared, auto-selected one.
|
|
41
|
+
*/
|
|
42
|
+
export declare function createDecider(reporter: Reporter, client?: () => Pick<JevClient, "systemOne">): Decider;
|
|
43
|
+
/** Parse one `JEV_EVENT ` stderr line, or null when the line is something else. */
|
|
44
|
+
export declare function parseEventLine(line: string): JevEventLine | null;
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The event channel every tool, hook, and CLI run reports on.
|
|
3
|
+
*
|
|
4
|
+
* With JEV_EVENT_DIR set, each process appends JSON lines to its own file, `<dir>/<harness>-<pid>.jsonl`,
|
|
5
|
+
* so no two processes share a file and large events never tear. Without it, events go to stderr as
|
|
6
|
+
* `JEV_EVENT {json}` lines, unless the reporter is quiet (the general CLI keeps stderr readable).
|
|
7
|
+
* A failed write falls back to stderr and never fails the caller.
|
|
8
|
+
*/
|
|
9
|
+
import { appendFileSync, mkdirSync } from "node:fs";
|
|
10
|
+
import { join } from "node:path";
|
|
11
|
+
import { sharedJev } from "../core/client.js";
|
|
12
|
+
import { validateQuestions } from "../core/types.js";
|
|
13
|
+
export const EVENT_PREFIX = "JEV_EVENT ";
|
|
14
|
+
export function createReporter(opts) {
|
|
15
|
+
const stderr = opts.stderr ?? process.stderr;
|
|
16
|
+
const dir = opts.dir ?? process.env.JEV_EVENT_DIR?.trim() ?? "";
|
|
17
|
+
let file = dir ? join(dir, `${opts.harness}-${process.pid}.jsonl`) : "";
|
|
18
|
+
let ready = false;
|
|
19
|
+
const toStderr = (line) => {
|
|
20
|
+
try {
|
|
21
|
+
stderr.write(EVENT_PREFIX + line + "\n");
|
|
22
|
+
}
|
|
23
|
+
catch { /* nowhere left to report */ }
|
|
24
|
+
};
|
|
25
|
+
return {
|
|
26
|
+
harness: opts.harness,
|
|
27
|
+
get target() { return file || (opts.quiet ? "off" : "stderr"); },
|
|
28
|
+
report(kind, payload = {}) {
|
|
29
|
+
const line = JSON.stringify({ kind, at: Date.now(), harness: opts.harness, ...payload });
|
|
30
|
+
try {
|
|
31
|
+
opts.onEvent?.(kind, payload);
|
|
32
|
+
}
|
|
33
|
+
catch { /* optional record */ }
|
|
34
|
+
if (file) {
|
|
35
|
+
try {
|
|
36
|
+
if (!ready) {
|
|
37
|
+
mkdirSync(dir, { recursive: true });
|
|
38
|
+
ready = true;
|
|
39
|
+
}
|
|
40
|
+
appendFileSync(file, line + "\n");
|
|
41
|
+
return;
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
file = "";
|
|
45
|
+
toStderr(JSON.stringify({ kind: "error", at: Date.now(), harness: opts.harness, hook: "events", message: `cannot write ${dir}: ${err.message}; reporting on stderr` }));
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
if (!opts.quiet)
|
|
49
|
+
toStderr(line);
|
|
50
|
+
},
|
|
51
|
+
};
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* `extra` must not use the keys the report already sets: source, state, questions, answers, usage, model, ms.
|
|
55
|
+
* The client defaults to the shared, auto-selected one.
|
|
56
|
+
*/
|
|
57
|
+
export function createDecider(reporter, client = sharedJev) {
|
|
58
|
+
return async (source, state, questions, extra = {}) => {
|
|
59
|
+
validateQuestions(questions);
|
|
60
|
+
const started = performance.now();
|
|
61
|
+
const result = await client().systemOne(state, questions);
|
|
62
|
+
const out = { answers: result.answers, usage: result.usage, model: result.model, ms: Math.round(performance.now() - started) };
|
|
63
|
+
reporter.report("jev", { source, state, questions, answers: out.answers, usage: out.usage, model: out.model, ms: out.ms, ...extra });
|
|
64
|
+
return out;
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
/** Parse one `JEV_EVENT ` stderr line, or null when the line is something else. */
|
|
68
|
+
export function parseEventLine(line) {
|
|
69
|
+
if (!line.startsWith(EVENT_PREFIX))
|
|
70
|
+
return null;
|
|
71
|
+
try {
|
|
72
|
+
return JSON.parse(line.slice(EVENT_PREFIX.length));
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return null;
|
|
76
|
+
}
|
|
77
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Decide } from "./decide.ts";
|
|
2
|
+
export interface FileBool {
|
|
3
|
+
path: string;
|
|
4
|
+
answer: boolean;
|
|
5
|
+
noul: number;
|
|
6
|
+
usage?: unknown;
|
|
7
|
+
}
|
|
8
|
+
/** A: the question is written against `content`; the tool description says so. */
|
|
9
|
+
export declare function askFileBool(path: string, question: string, cwd: string, criteria?: {
|
|
10
|
+
yes?: string;
|
|
11
|
+
no?: string;
|
|
12
|
+
}, decide?: Decide): Promise<FileBool>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Level 8, option A: ask_jev_file_bool.
|
|
3
|
+
* One file, one yes or no question, flat parameters. The agent asks "does src/auth/session.ts validate tokens?" and gets a Noul back without the file ever entering its context.
|
|
4
|
+
*/
|
|
5
|
+
import { jev } from "../core/client.js";
|
|
6
|
+
import { noul } from "../core/helpers.js";
|
|
7
|
+
import { readFileState } from "./read-state.js";
|
|
8
|
+
/** A: the question is written against `content`; the tool description says so. */
|
|
9
|
+
export async function askFileBool(path, question, cwd, criteria = {}, decide = (s, q) => jev.systemOne(s, q)) {
|
|
10
|
+
const state = await readFileState(path, cwd);
|
|
11
|
+
const questions = { answer: noul(question, criteria.yes || criteria.no ? { true: criteria.yes, false: criteria.no } : undefined) };
|
|
12
|
+
const { answers, usage } = await decide(state, questions);
|
|
13
|
+
const a = answers.answer;
|
|
14
|
+
return { path, answer: a.noul > 0.5, noul: a.noul, usage };
|
|
15
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Decide } from "./decide.ts";
|
|
2
|
+
export interface FileChoice {
|
|
3
|
+
path: string;
|
|
4
|
+
choice: string;
|
|
5
|
+
confidence: number;
|
|
6
|
+
probabilities: Record<string, number>;
|
|
7
|
+
usage?: unknown;
|
|
8
|
+
}
|
|
9
|
+
/** B: options are name to description. An `other` entry is added when the agent leaves none, so the model has an exit. */
|
|
10
|
+
export declare function askFileChoice(path: string, question: string, options: Record<string, string>, cwd: string, decide?: Decide): Promise<FileChoice>;
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Level 8, option B: ask_jev_file_choice.
|
|
3
|
+
* One file, one pick from options the agent names. "Which layer is this file?" comes back as one of the agent's own keys, with the whole distribution.
|
|
4
|
+
*/
|
|
5
|
+
import { jev } from "../core/client.js";
|
|
6
|
+
import { choice } from "../core/helpers.js";
|
|
7
|
+
import { readFileState } from "./read-state.js";
|
|
8
|
+
/** B: options are name to description. An `other` entry is added when the agent leaves none, so the model has an exit. */
|
|
9
|
+
export async function askFileChoice(path, question, options, cwd, decide = (s, q) => jev.systemOne(s, q)) {
|
|
10
|
+
const state = await readFileState(path, cwd);
|
|
11
|
+
const criteria = { ...options };
|
|
12
|
+
if (!("other" in criteria) && !("none" in criteria) && !("none_of_the_above" in criteria))
|
|
13
|
+
criteria.other = "None of the above";
|
|
14
|
+
const { answers, usage } = await decide(state, { answer: choice(question, criteria) });
|
|
15
|
+
const a = answers.answer;
|
|
16
|
+
return { path, choice: a.choice, confidence: a.confidence, probabilities: a.probabilities, usage };
|
|
17
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Decide } from "./decide.ts";
|
|
2
|
+
export interface FileScore {
|
|
3
|
+
path: string;
|
|
4
|
+
score: number;
|
|
5
|
+
top: number;
|
|
6
|
+
nearest: string;
|
|
7
|
+
confidence: number;
|
|
8
|
+
legend: Record<string, string>;
|
|
9
|
+
usage?: unknown;
|
|
10
|
+
}
|
|
11
|
+
/** C: levels are situations, not degrees. The nearest level's text comes back so the agent can quote it. */
|
|
12
|
+
export declare function askFileScore(path: string, question: string, levels: string[], cwd: string, decide?: Decide): Promise<FileScore>;
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Level 8, option C: ask_jev_file_score.
|
|
3
|
+
* One file, one scale the agent writes, low to high. "How risky is a refactor of this file?" comes back as a position on the agent's own levels.
|
|
4
|
+
*/
|
|
5
|
+
import { jev } from "../core/client.js";
|
|
6
|
+
import { score } from "../core/helpers.js";
|
|
7
|
+
import { readFileState } from "./read-state.js";
|
|
8
|
+
/** C: levels are situations, not degrees. The nearest level's text comes back so the agent can quote it. */
|
|
9
|
+
export async function askFileScore(path, question, levels, cwd, decide = (s, q) => jev.systemOne(s, q)) {
|
|
10
|
+
const state = await readFileState(path, cwd);
|
|
11
|
+
const { answers, usage } = await decide(state, { answer: score(question, levels) });
|
|
12
|
+
const a = answers.answer;
|
|
13
|
+
const top = levels.length - 1;
|
|
14
|
+
return { path, score: a.score, top, nearest: a.legend[String(Math.round(a.score))] ?? "", confidence: a.confidence, legend: a.legend, usage };
|
|
15
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type Answer, type Questions } from "../core/types.ts";
|
|
2
|
+
import type { Decide } from "./decide.ts";
|
|
3
|
+
import { type Skipped } from "./prune.ts";
|
|
4
|
+
export interface FileAnswers {
|
|
5
|
+
path: string;
|
|
6
|
+
answers: Record<string, Answer>;
|
|
7
|
+
usage?: unknown;
|
|
8
|
+
}
|
|
9
|
+
export interface AskFilesResult {
|
|
10
|
+
results: FileAnswers[];
|
|
11
|
+
skipped: Skipped[];
|
|
12
|
+
calls: number;
|
|
13
|
+
}
|
|
14
|
+
/** The agent writes Jev's question shape as JSON. Malformed blocks fail here, with the reason, before any file is read. */
|
|
15
|
+
export declare function parseQuestions(questionsJson: string): Questions;
|
|
16
|
+
/** Run `fn` over `items` with at most `limit` in flight. Results keep the input order. */
|
|
17
|
+
export declare function parallel<T, R>(items: T[], limit: number, fn: (item: T) => Promise<R>): Promise<R[]>;
|
|
18
|
+
export declare function askFiles(patterns: string[], questionsJson: string, cwd: string, opts?: {
|
|
19
|
+
recursive?: boolean;
|
|
20
|
+
concurrency?: number;
|
|
21
|
+
decide?: Decide;
|
|
22
|
+
onResult?: (r: FileAnswers) => void;
|
|
23
|
+
}): Promise<AskFilesResult>;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Level 9, option A: ask_jev_files.
|
|
3
|
+
* The same questions, raw Jev question JSON, over many files. One call per file, every question in each, all in parallel with a cap. Answers come back per path. No file enters the agent's context.
|
|
4
|
+
*/
|
|
5
|
+
import { jev } from "../core/client.js";
|
|
6
|
+
import { validateQuestions } from "../core/types.js";
|
|
7
|
+
import { FileStateError, readFileState } from "./read-state.js";
|
|
8
|
+
import { expandPatterns, pruneFiles } from "./prune.js";
|
|
9
|
+
/** The agent writes Jev's question shape as JSON. Malformed blocks fail here, with the reason, before any file is read. */
|
|
10
|
+
export function parseQuestions(questionsJson) {
|
|
11
|
+
let parsed;
|
|
12
|
+
try {
|
|
13
|
+
parsed = JSON.parse(questionsJson);
|
|
14
|
+
}
|
|
15
|
+
catch (err) {
|
|
16
|
+
throw new Error(`questions_json is not valid JSON: ${err.message}`);
|
|
17
|
+
}
|
|
18
|
+
if (!parsed || typeof parsed !== "object" || Array.isArray(parsed))
|
|
19
|
+
throw new Error("questions_json must be an object keyed by question id");
|
|
20
|
+
validateQuestions(parsed);
|
|
21
|
+
return parsed;
|
|
22
|
+
}
|
|
23
|
+
/** Run `fn` over `items` with at most `limit` in flight. Results keep the input order. */
|
|
24
|
+
export async function parallel(items, limit, fn) {
|
|
25
|
+
const out = new Array(items.length);
|
|
26
|
+
let next = 0;
|
|
27
|
+
const worker = async () => {
|
|
28
|
+
while (next < items.length) {
|
|
29
|
+
const i = next++;
|
|
30
|
+
out[i] = await fn(items[i]);
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
await Promise.all(Array.from({ length: Math.min(limit, items.length) }, worker));
|
|
34
|
+
return out;
|
|
35
|
+
}
|
|
36
|
+
export async function askFiles(patterns, questionsJson, cwd, opts = {}) {
|
|
37
|
+
const questions = parseQuestions(questionsJson);
|
|
38
|
+
const decide = opts.decide ?? ((s, q) => jev.systemOne(s, q));
|
|
39
|
+
const expanded = await expandPatterns(patterns, cwd, opts.recursive ?? false);
|
|
40
|
+
const { files, skipped } = await pruneFiles(expanded, cwd);
|
|
41
|
+
const results = [];
|
|
42
|
+
await parallel(files, opts.concurrency ?? 16, async (path) => {
|
|
43
|
+
try {
|
|
44
|
+
const state = await readFileState(path, cwd);
|
|
45
|
+
const { answers, usage } = await decide(state, questions);
|
|
46
|
+
const r = { path, answers, usage };
|
|
47
|
+
results.push(r);
|
|
48
|
+
opts.onResult?.(r);
|
|
49
|
+
}
|
|
50
|
+
catch (err) {
|
|
51
|
+
skipped.push({ path, reason: err instanceof FileStateError ? err.message : `call failed: ${err?.message ?? err}` });
|
|
52
|
+
}
|
|
53
|
+
});
|
|
54
|
+
results.sort((a, b) => a.path.localeCompare(b.path));
|
|
55
|
+
return { results, skipped, calls: results.length };
|
|
56
|
+
}
|