@wyattjoh/demur 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +53 -16
- package/extensions/demur/index.ts +99 -15
- package/extensions/demur/training-store.ts +213 -35
- package/package.json +2 -1
- package/src/adapters/pi-worker.ts +21 -7
- package/src/cli.ts +154 -5
- package/src/guard.internal.ts +83 -33
- package/src/guard.ts +42 -2
- package/src/judge.ts +53 -7
- package/src/policy.ts +10 -1
- package/src/questions.ts +9 -1
- package/src/state.ts +14 -16
- package/src/training-evaluation.ts +408 -0
- package/src/training-review-model.ts +17 -0
- package/src/training-review-tui.tsx +110 -30
- package/src/types.ts +56 -0
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
} from "../extensions/demur/cost-tracker.ts";
|
|
15
15
|
import type { DemurSettings } from "../extensions/demur/settings.ts";
|
|
16
16
|
import type {
|
|
17
|
+
TrainingCorrectionReason,
|
|
17
18
|
TrainingReview,
|
|
18
19
|
TrainingReviewInput,
|
|
19
20
|
} from "../extensions/demur/training-store.ts";
|
|
@@ -26,6 +27,7 @@ import {
|
|
|
26
27
|
createTrainingReviewInput,
|
|
27
28
|
filterTrainingReviewEntries,
|
|
28
29
|
getLatestTrainingReview,
|
|
30
|
+
getTrainingCorrectionReason,
|
|
29
31
|
getTrainingReviewFilter,
|
|
30
32
|
type TrainingReviewEntry,
|
|
31
33
|
type TrainingReviewFilter,
|
|
@@ -63,6 +65,21 @@ const FILTERS: ReadonlyArray<{
|
|
|
63
65
|
{ value: "deny", label: "deny" },
|
|
64
66
|
];
|
|
65
67
|
|
|
68
|
+
const CORRECTION_REASONS: ReadonlyArray<{
|
|
69
|
+
value: TrainingCorrectionReason;
|
|
70
|
+
label: string;
|
|
71
|
+
}> = [
|
|
72
|
+
{ value: "inert-or-read-only", label: "Command is inert or read-only" },
|
|
73
|
+
{ value: "sensitive-data", label: "Sensitive-data judgment" },
|
|
74
|
+
{ value: "security-boundary", label: "Security-boundary judgment" },
|
|
75
|
+
{ value: "recoverability", label: "Recoverability judgment" },
|
|
76
|
+
{ value: "shared-infrastructure", label: "Local versus shared target" },
|
|
77
|
+
{ value: "blast-radius", label: "Blast-radius judgment" },
|
|
78
|
+
{ value: "static-uncertainty", label: "Static uncertainty gate" },
|
|
79
|
+
{ value: "missing-context", label: "Model was missing objective context" },
|
|
80
|
+
{ value: "service-failure", label: "TypeSafe or guard service failure" },
|
|
81
|
+
];
|
|
82
|
+
|
|
66
83
|
const SECTIONS: ReadonlyArray<{ value: AppSection; label: string }> = [
|
|
67
84
|
{ value: "reviews", label: "Reviews" },
|
|
68
85
|
{ value: "settings", label: "Settings" },
|
|
@@ -112,6 +129,7 @@ type TrainingReviewAppProps = {
|
|
|
112
129
|
type NoteEditorState = {
|
|
113
130
|
recordId: string;
|
|
114
131
|
expectedDecision: Decision;
|
|
132
|
+
correctionReason: TrainingCorrectionReason | undefined;
|
|
115
133
|
};
|
|
116
134
|
|
|
117
135
|
type AppSection = "reviews" | "settings";
|
|
@@ -277,11 +295,16 @@ export function TrainingReviewApp(
|
|
|
277
295
|
const persistDecision = async (
|
|
278
296
|
entry: TrainingReviewEntry,
|
|
279
297
|
expectedDecision: Decision,
|
|
298
|
+
correctionReason: TrainingCorrectionReason | undefined,
|
|
280
299
|
note: string | undefined,
|
|
281
300
|
) => {
|
|
282
301
|
if (saving) return;
|
|
283
302
|
const previous = getLatestTrainingReview(entry);
|
|
284
|
-
if (
|
|
303
|
+
if (
|
|
304
|
+
previous?.expectedDecision === expectedDecision &&
|
|
305
|
+
correctionReason === undefined &&
|
|
306
|
+
note === undefined
|
|
307
|
+
) {
|
|
285
308
|
setNoteEditor(undefined);
|
|
286
309
|
return;
|
|
287
310
|
}
|
|
@@ -291,7 +314,12 @@ export function TrainingReviewApp(
|
|
|
291
314
|
const corrected = expectedDecision !== entry.record.verdict.decision;
|
|
292
315
|
try {
|
|
293
316
|
const review = await props.recordReview(
|
|
294
|
-
createTrainingReviewInput(
|
|
317
|
+
createTrainingReviewInput(
|
|
318
|
+
entry.record,
|
|
319
|
+
expectedDecision,
|
|
320
|
+
correctionReason,
|
|
321
|
+
note,
|
|
322
|
+
),
|
|
295
323
|
);
|
|
296
324
|
setSnapshot((current) =>
|
|
297
325
|
current.reviews.some((existing) =>
|
|
@@ -322,13 +350,14 @@ export function TrainingReviewApp(
|
|
|
322
350
|
const chooseDecision = (decision: Decision) => {
|
|
323
351
|
if (selected === undefined || saving) return;
|
|
324
352
|
if (decision === selected.record.verdict.decision) {
|
|
325
|
-
void persistDecision(selected, decision, undefined);
|
|
353
|
+
void persistDecision(selected, decision, undefined, undefined);
|
|
326
354
|
return;
|
|
327
355
|
}
|
|
328
356
|
setError(undefined);
|
|
329
357
|
setNoteEditor({
|
|
330
358
|
recordId: selected.record.id,
|
|
331
359
|
expectedDecision: decision,
|
|
360
|
+
correctionReason: undefined,
|
|
332
361
|
});
|
|
333
362
|
};
|
|
334
363
|
|
|
@@ -387,9 +416,19 @@ export function TrainingReviewApp(
|
|
|
387
416
|
key.preventDefault();
|
|
388
417
|
setNoteEditor(undefined);
|
|
389
418
|
setError(undefined);
|
|
390
|
-
|
|
391
|
-
|
|
419
|
+
return;
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
if (noteEditor.correctionReason === undefined) {
|
|
423
|
+
const reason = correctionReasonForKey(key.name, key.sequence);
|
|
424
|
+
if (reason !== undefined) {
|
|
425
|
+
key.preventDefault();
|
|
426
|
+
setNoteEditor({ ...noteEditor, correctionReason: reason });
|
|
427
|
+
}
|
|
428
|
+
return;
|
|
392
429
|
}
|
|
430
|
+
|
|
431
|
+
if (key.name === "tab") key.preventDefault();
|
|
393
432
|
return;
|
|
394
433
|
}
|
|
395
434
|
|
|
@@ -847,6 +886,13 @@ export function TrainingReviewApp(
|
|
|
847
886
|
<text fg={decisionColor(review.expectedDecision)}>
|
|
848
887
|
{`${index + 1}. ${review.expectedDecision.toUpperCase()} · ${review.reviewedAt}`}
|
|
849
888
|
</text>
|
|
889
|
+
{getTrainingCorrectionReason(review) === undefined
|
|
890
|
+
? null
|
|
891
|
+
: (
|
|
892
|
+
<text fg={COLORS.muted} wrapMode="word">
|
|
893
|
+
{`Reason: ${correctionReasonLabel(getTrainingCorrectionReason(review)!)}`}
|
|
894
|
+
</text>
|
|
895
|
+
)}
|
|
850
896
|
{review.note === undefined
|
|
851
897
|
? null
|
|
852
898
|
: <text fg={COLORS.text} wrapMode="word">{review.note}</text>}
|
|
@@ -860,31 +906,52 @@ export function TrainingReviewApp(
|
|
|
860
906
|
</box>
|
|
861
907
|
|
|
862
908
|
{noteEditor !== undefined && editingEntry !== undefined
|
|
863
|
-
?
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
909
|
+
? noteEditor.correctionReason === undefined
|
|
910
|
+
? (
|
|
911
|
+
<box
|
|
912
|
+
title={` Why ${statusLabel(getTrainingReviewFilter(editingEntry))} → ${noteEditor.expectedDecision}? `}
|
|
913
|
+
titleColor={decisionColor(noteEditor.expectedDecision)}
|
|
914
|
+
border
|
|
915
|
+
borderColor={decisionColor(noteEditor.expectedDecision)}
|
|
916
|
+
height={CORRECTION_REASONS.length + 3}
|
|
917
|
+
paddingLeft={1}
|
|
918
|
+
paddingRight={1}
|
|
919
|
+
flexDirection="column"
|
|
920
|
+
>
|
|
921
|
+
{CORRECTION_REASONS.map((reason, index) => (
|
|
922
|
+
<text key={reason.value} fg={COLORS.text}>
|
|
923
|
+
{`${index + 1}. ${reason.label}`}
|
|
924
|
+
</text>
|
|
925
|
+
))}
|
|
926
|
+
<text fg={COLORS.muted}>1–9 select reason · Esc cancel</text>
|
|
927
|
+
</box>
|
|
928
|
+
)
|
|
929
|
+
: (
|
|
930
|
+
<box
|
|
931
|
+
title={` ${correctionReasonLabel(noteEditor.correctionReason)} `}
|
|
932
|
+
titleColor={decisionColor(noteEditor.expectedDecision)}
|
|
933
|
+
border
|
|
934
|
+
borderColor={decisionColor(noteEditor.expectedDecision)}
|
|
935
|
+
height={5}
|
|
936
|
+
paddingLeft={1}
|
|
937
|
+
paddingRight={1}
|
|
938
|
+
flexDirection="column"
|
|
939
|
+
>
|
|
940
|
+
<input
|
|
941
|
+
placeholder="Optional correction note — Enter saves, Esc cancels"
|
|
942
|
+
focused
|
|
943
|
+
onSubmit={(note) => {
|
|
944
|
+
void persistDecision(
|
|
945
|
+
editingEntry,
|
|
946
|
+
noteEditor.expectedDecision,
|
|
947
|
+
noteEditor.correctionReason,
|
|
948
|
+
typeof note === "string" ? note : undefined,
|
|
949
|
+
);
|
|
950
|
+
}}
|
|
951
|
+
/>
|
|
952
|
+
<text fg={COLORS.muted}>Enter save revision · Esc cancel</text>
|
|
953
|
+
</box>
|
|
954
|
+
)
|
|
888
955
|
: null}
|
|
889
956
|
|
|
890
957
|
{error === undefined
|
|
@@ -1130,6 +1197,19 @@ function decisionForKey(
|
|
|
1130
1197
|
return undefined;
|
|
1131
1198
|
}
|
|
1132
1199
|
|
|
1200
|
+
function correctionReasonForKey(
|
|
1201
|
+
name: string,
|
|
1202
|
+
sequence: string,
|
|
1203
|
+
): TrainingCorrectionReason | undefined {
|
|
1204
|
+
const index = Number(sequence || name) - 1;
|
|
1205
|
+
return Number.isInteger(index) ? CORRECTION_REASONS[index]?.value : undefined;
|
|
1206
|
+
}
|
|
1207
|
+
|
|
1208
|
+
function correctionReasonLabel(reason: TrainingCorrectionReason): string {
|
|
1209
|
+
return CORRECTION_REASONS.find((candidate) => candidate.value === reason)
|
|
1210
|
+
?.label ?? reason;
|
|
1211
|
+
}
|
|
1212
|
+
|
|
1133
1213
|
function statusLabel(filter: TrainingReviewFilter): string {
|
|
1134
1214
|
return {
|
|
1135
1215
|
all: "ALL",
|
package/src/types.ts
CHANGED
|
@@ -5,6 +5,24 @@ import type { CommandAnalysis } from "./analyze.ts";
|
|
|
5
5
|
*/
|
|
6
6
|
export type Host = "pi" | "claude-code" | "cli";
|
|
7
7
|
|
|
8
|
+
/**
|
|
9
|
+
* JSON-compatible value accepted by the TypeSafe state boundary.
|
|
10
|
+
*/
|
|
11
|
+
export type JsonValue =
|
|
12
|
+
| string
|
|
13
|
+
| number
|
|
14
|
+
| boolean
|
|
15
|
+
| null
|
|
16
|
+
| ReadonlyArray<JsonValue>
|
|
17
|
+
| { readonly [key: string]: JsonValue };
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Exact JSON object submitted as TypeSafe state for one command.
|
|
21
|
+
*/
|
|
22
|
+
export type RenderedCommandState = {
|
|
23
|
+
readonly [key: string]: JsonValue;
|
|
24
|
+
};
|
|
25
|
+
|
|
8
26
|
/**
|
|
9
27
|
* Everything demur knows about a command before judging it.
|
|
10
28
|
*
|
|
@@ -160,3 +178,41 @@ export type Verdict = {
|
|
|
160
178
|
*/
|
|
161
179
|
usage: { inputTokens: number; outputTokens: number } | undefined;
|
|
162
180
|
};
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Reproducible inputs and implementation versions behind one guard verdict.
|
|
184
|
+
*/
|
|
185
|
+
export type GuardEvidence = {
|
|
186
|
+
/**
|
|
187
|
+
* Exact JSON state submitted to TypeSafe.
|
|
188
|
+
*/
|
|
189
|
+
modelState: RenderedCommandState;
|
|
190
|
+
/**
|
|
191
|
+
* Deterministic command analysis used only by the local policy gate.
|
|
192
|
+
*/
|
|
193
|
+
analysis: CommandAnalysis | undefined;
|
|
194
|
+
/**
|
|
195
|
+
* TypeSafe model identifier used for the judgment.
|
|
196
|
+
*/
|
|
197
|
+
model: string;
|
|
198
|
+
/**
|
|
199
|
+
* Version of the question definitions used for the judgment.
|
|
200
|
+
*/
|
|
201
|
+
questionSetVersion: number;
|
|
202
|
+
/**
|
|
203
|
+
* Version of the policy composition used for the verdict.
|
|
204
|
+
*/
|
|
205
|
+
policyVersion: number;
|
|
206
|
+
/**
|
|
207
|
+
* Exact numeric thresholds used for the verdict.
|
|
208
|
+
*/
|
|
209
|
+
policyThresholds: { readonly [key: string]: number };
|
|
210
|
+
};
|
|
211
|
+
|
|
212
|
+
/**
|
|
213
|
+
* Guard verdict paired with replayable evidence when state collection occurred.
|
|
214
|
+
*/
|
|
215
|
+
export type GuardEvaluation = {
|
|
216
|
+
verdict: Verdict;
|
|
217
|
+
evidence: GuardEvidence | undefined;
|
|
218
|
+
};
|