instar 1.3.1248 → 1.3.1249
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +20 -0
- package/dist/commands/server.js.map +1 -1
- package/dist/core/JevSignalShadow.d.ts +98 -0
- package/dist/core/JevSignalShadow.d.ts.map +1 -0
- package/dist/core/JevSignalShadow.js +230 -0
- package/dist/core/JevSignalShadow.js.map +1 -0
- package/dist/core/MessagingToneGate.d.ts +10 -0
- package/dist/core/MessagingToneGate.d.ts.map +1 -1
- package/dist/core/MessagingToneGate.js +18 -0
- package/dist/core/MessagingToneGate.js.map +1 -1
- package/dist/core/PostUpdateMigrator.d.ts.map +1 -1
- package/dist/core/PostUpdateMigrator.js +15 -0
- package/dist/core/PostUpdateMigrator.js.map +1 -1
- package/dist/core/types.d.ts +18 -0
- package/dist/core/types.d.ts.map +1 -1
- package/dist/core/types.js.map +1 -1
- package/dist/data/standards-guard-index.json +1 -1
- package/dist/data/standards-guard-index.meta.json +2 -2
- package/dist/data/standards-registry.meta.json +1 -1
- package/package.json +1 -1
- package/scripts/jev-shadow-report.mjs +89 -0
- package/src/data/builtin-manifest.json +19 -19
- package/src/data/standards-guard-index.json +1 -1
- package/src/data/standards-guard-index.meta.json +2 -2
- package/src/data/standards-registry.meta.json +1 -1
- package/upgrades/{1.3.1248.md → 1.3.1249.md} +26 -0
- package/upgrades/side-effects/jev-signal-shadow.md +138 -0
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* jev-shadow-report.mjs — read the Jev signal-layer shadow log and report the
|
|
4
|
+
* soak as PER-RULE CONFUSION MATRICES plus COVERAGE. Never one blended
|
|
5
|
+
* agreement percentage: 99% agreement can hide total failure on rare positives.
|
|
6
|
+
*
|
|
7
|
+
* Spec: docs/specs/jev-signal-layer-shadow.md (graduation criterion).
|
|
8
|
+
*
|
|
9
|
+
* Usage (from the agent home):
|
|
10
|
+
* node scripts/jev-shadow-report.mjs [logPath] [--json]
|
|
11
|
+
* Default logPath: logs/jev-signal-shadow.jsonl
|
|
12
|
+
*
|
|
13
|
+
* Rows are deduplicated by sha256 (a message re-entering the gate on retry is
|
|
14
|
+
* one candidate, not two). Detector truth = the B1–B7 deterministic signal
|
|
15
|
+
* kinds recorded in the row; Jev "positive" = p > 0.5 (the frozen threshold).
|
|
16
|
+
*/
|
|
17
|
+
import fs from 'node:fs';
|
|
18
|
+
|
|
19
|
+
const RULE_TO_KIND = {
|
|
20
|
+
raw_path: 'file-path', cli_command: 'cli-command', config_key: 'config-key',
|
|
21
|
+
api_endpoint: 'api-endpoint', copy_paste_code: 'copy-paste-code', env_var: 'env-var',
|
|
22
|
+
cron_or_slug: 'cron-or-slug',
|
|
23
|
+
};
|
|
24
|
+
const THRESHOLD = 0.5;
|
|
25
|
+
const MIN_POSITIVES = 20;
|
|
26
|
+
|
|
27
|
+
const args = process.argv.slice(2);
|
|
28
|
+
const asJson = args.includes('--json');
|
|
29
|
+
const logPath = args.find((a) => !a.startsWith('--')) ?? 'logs/jev-signal-shadow.jsonl';
|
|
30
|
+
|
|
31
|
+
export function buildReport(lines) {
|
|
32
|
+
const seen = new Set();
|
|
33
|
+
const compared = [];
|
|
34
|
+
const notCompared = {};
|
|
35
|
+
for (const line of lines) {
|
|
36
|
+
if (!line.trim()) continue;
|
|
37
|
+
let row; try { row = JSON.parse(line); } catch { continue; }
|
|
38
|
+
if (row.kind === 'not-compared') { notCompared[row.reason] = (notCompared[row.reason] ?? 0) + 1; continue; }
|
|
39
|
+
if (row.kind !== 'compared' || seen.has(row.sha256)) continue;
|
|
40
|
+
seen.add(row.sha256);
|
|
41
|
+
compared.push(row);
|
|
42
|
+
}
|
|
43
|
+
const perRule = {};
|
|
44
|
+
for (const [rule, kind] of Object.entries(RULE_TO_KIND)) {
|
|
45
|
+
const m = { hits: 0, misses: 0, falseAlarms: 0, trueNegatives: 0, answered: 0 };
|
|
46
|
+
for (const r of compared) {
|
|
47
|
+
const p = r.jev?.[rule];
|
|
48
|
+
if (typeof p !== 'number') continue;
|
|
49
|
+
m.answered++;
|
|
50
|
+
const det = (r.detectorSignals ?? []).includes(kind);
|
|
51
|
+
const jev = p > THRESHOLD;
|
|
52
|
+
if (det && jev) m.hits++; else if (det && !jev) m.misses++;
|
|
53
|
+
else if (!det && jev) m.falseAlarms++; else m.trueNegatives++;
|
|
54
|
+
}
|
|
55
|
+
const positives = m.hits + m.misses;
|
|
56
|
+
m.positiveCases = positives;
|
|
57
|
+
m.agreement = m.answered ? +((m.hits + m.trueNegatives) / m.answered).toFixed(4) : null;
|
|
58
|
+
m.verdict = positives < MIN_POSITIVES ? 'insufficient-evidence'
|
|
59
|
+
: (m.agreement >= 0.99 && m.misses === 0 ? 'meets-bar' : 'below-bar');
|
|
60
|
+
perRule[rule] = m;
|
|
61
|
+
}
|
|
62
|
+
const notComparedTotal = Object.values(notCompared).reduce((a, b) => a + b, 0);
|
|
63
|
+
const candidates = compared.length + notComparedTotal;
|
|
64
|
+
return {
|
|
65
|
+
candidates,
|
|
66
|
+
comparedDistinct: compared.length,
|
|
67
|
+
coverage: candidates ? +(compared.length / candidates).toFixed(4) : null,
|
|
68
|
+
notCompared,
|
|
69
|
+
perRule,
|
|
70
|
+
graduationEligible: candidates > 0
|
|
71
|
+
&& compared.length / candidates >= 0.9
|
|
72
|
+
&& Object.values(perRule).every((m) => m.verdict !== 'below-bar')
|
|
73
|
+
&& Object.values(perRule).some((m) => m.verdict === 'meets-bar'),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const isMain = import.meta.url === `file://${process.argv[1]}`;
|
|
78
|
+
if (isMain) {
|
|
79
|
+
if (!fs.existsSync(logPath)) { console.error(`no shadow log at ${logPath}`); process.exit(1); }
|
|
80
|
+
const report = buildReport(fs.readFileSync(logPath, 'utf8').split('\n'));
|
|
81
|
+
if (asJson) { console.log(JSON.stringify(report, null, 2)); process.exit(0); }
|
|
82
|
+
console.log(`candidates ${report.candidates} · compared (distinct) ${report.comparedDistinct} · coverage ${report.coverage}`);
|
|
83
|
+
console.log(`not compared: ${JSON.stringify(report.notCompared)}`);
|
|
84
|
+
console.log('rule positives hits misses falseAlarms agreement verdict');
|
|
85
|
+
for (const [rule, m] of Object.entries(report.perRule)) {
|
|
86
|
+
console.log(`${rule.padEnd(17)} ${String(m.positiveCases).padStart(9)} ${String(m.hits).padStart(5)} ${String(m.misses).padStart(7)} ${String(m.falseAlarms).padStart(12)} ${String(m.agreement).padStart(9)} ${m.verdict}`);
|
|
87
|
+
}
|
|
88
|
+
console.log(`graduation-eligible: ${report.graduationEligible}`);
|
|
89
|
+
}
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
{
|
|
2
2
|
"$schema": "./builtin-manifest.schema.json",
|
|
3
3
|
"schemaVersion": 1,
|
|
4
|
-
"generatedAt": "2026-09-21T17:
|
|
5
|
-
"instarVersion": "1.3.
|
|
4
|
+
"generatedAt": "2026-09-21T17:48:31.257Z",
|
|
5
|
+
"instarVersion": "1.3.1249",
|
|
6
6
|
"entryCount": 203,
|
|
7
7
|
"entries": {
|
|
8
8
|
"hook:session-start": {
|
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
"domain": "identity",
|
|
12
12
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
13
13
|
"installedPath": ".instar/hooks/instar/session-start.sh",
|
|
14
|
-
"contentHash": "
|
|
14
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
15
15
|
"since": "2025-01-01"
|
|
16
16
|
},
|
|
17
17
|
"hook:dangerous-command-guard": {
|
|
@@ -20,7 +20,7 @@
|
|
|
20
20
|
"domain": "safety",
|
|
21
21
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
22
22
|
"installedPath": ".instar/hooks/instar/dangerous-command-guard.sh",
|
|
23
|
-
"contentHash": "
|
|
23
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
24
24
|
"since": "2025-01-01"
|
|
25
25
|
},
|
|
26
26
|
"hook:grounding-before-messaging": {
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
"domain": "safety",
|
|
30
30
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
31
31
|
"installedPath": ".instar/hooks/instar/grounding-before-messaging.sh",
|
|
32
|
-
"contentHash": "
|
|
32
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
33
33
|
"since": "2025-01-01"
|
|
34
34
|
},
|
|
35
35
|
"hook:compaction-recovery": {
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
"domain": "identity",
|
|
39
39
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
40
40
|
"installedPath": ".instar/hooks/instar/compaction-recovery.sh",
|
|
41
|
-
"contentHash": "
|
|
41
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
42
42
|
"since": "2025-01-01"
|
|
43
43
|
},
|
|
44
44
|
"hook:external-operation-gate": {
|
|
@@ -47,7 +47,7 @@
|
|
|
47
47
|
"domain": "safety",
|
|
48
48
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
49
49
|
"installedPath": ".instar/hooks/instar/external-operation-gate.js",
|
|
50
|
-
"contentHash": "
|
|
50
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
51
51
|
"since": "2025-01-01"
|
|
52
52
|
},
|
|
53
53
|
"hook:deferral-detector": {
|
|
@@ -56,7 +56,7 @@
|
|
|
56
56
|
"domain": "safety",
|
|
57
57
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
58
58
|
"installedPath": ".instar/hooks/instar/deferral-detector.js",
|
|
59
|
-
"contentHash": "
|
|
59
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
60
60
|
"since": "2025-01-01"
|
|
61
61
|
},
|
|
62
62
|
"hook:self-stop-guard": {
|
|
@@ -65,7 +65,7 @@
|
|
|
65
65
|
"domain": "coherence",
|
|
66
66
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
67
67
|
"installedPath": ".instar/hooks/instar/self-stop-guard.js",
|
|
68
|
-
"contentHash": "
|
|
68
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
69
69
|
"since": "2025-01-01"
|
|
70
70
|
},
|
|
71
71
|
"hook:post-action-reflection": {
|
|
@@ -74,7 +74,7 @@
|
|
|
74
74
|
"domain": "evolution",
|
|
75
75
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
76
76
|
"installedPath": ".instar/hooks/instar/post-action-reflection.js",
|
|
77
|
-
"contentHash": "
|
|
77
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
78
78
|
"since": "2025-01-01"
|
|
79
79
|
},
|
|
80
80
|
"hook:external-communication-guard": {
|
|
@@ -83,7 +83,7 @@
|
|
|
83
83
|
"domain": "safety",
|
|
84
84
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
85
85
|
"installedPath": ".instar/hooks/instar/external-communication-guard.js",
|
|
86
|
-
"contentHash": "
|
|
86
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
87
87
|
"since": "2025-01-01"
|
|
88
88
|
},
|
|
89
89
|
"hook:telegram-origin-guard": {
|
|
@@ -101,7 +101,7 @@
|
|
|
101
101
|
"domain": "coherence",
|
|
102
102
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
103
103
|
"installedPath": ".instar/hooks/instar/scope-coherence-collector.js",
|
|
104
|
-
"contentHash": "
|
|
104
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
105
105
|
"since": "2025-01-01"
|
|
106
106
|
},
|
|
107
107
|
"hook:scope-coherence-checkpoint": {
|
|
@@ -110,7 +110,7 @@
|
|
|
110
110
|
"domain": "coherence",
|
|
111
111
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
112
112
|
"installedPath": ".instar/hooks/instar/scope-coherence-checkpoint.js",
|
|
113
|
-
"contentHash": "
|
|
113
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
114
114
|
"since": "2025-01-01"
|
|
115
115
|
},
|
|
116
116
|
"hook:free-text-guard": {
|
|
@@ -119,7 +119,7 @@
|
|
|
119
119
|
"domain": "safety",
|
|
120
120
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
121
121
|
"installedPath": ".instar/hooks/instar/free-text-guard.sh",
|
|
122
|
-
"contentHash": "
|
|
122
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
123
123
|
"since": "2025-01-01"
|
|
124
124
|
},
|
|
125
125
|
"hook:claim-intercept": {
|
|
@@ -128,7 +128,7 @@
|
|
|
128
128
|
"domain": "coherence",
|
|
129
129
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
130
130
|
"installedPath": ".instar/hooks/instar/claim-intercept.js",
|
|
131
|
-
"contentHash": "
|
|
131
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
132
132
|
"since": "2025-01-01"
|
|
133
133
|
},
|
|
134
134
|
"hook:claim-intercept-response": {
|
|
@@ -137,7 +137,7 @@
|
|
|
137
137
|
"domain": "coherence",
|
|
138
138
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
139
139
|
"installedPath": ".instar/hooks/instar/claim-intercept-response.js",
|
|
140
|
-
"contentHash": "
|
|
140
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
141
141
|
"since": "2025-01-01"
|
|
142
142
|
},
|
|
143
143
|
"hook:stop-gate-router": {
|
|
@@ -146,7 +146,7 @@
|
|
|
146
146
|
"domain": "safety",
|
|
147
147
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
148
148
|
"installedPath": ".instar/hooks/instar/stop-gate-router.js",
|
|
149
|
-
"contentHash": "
|
|
149
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
150
150
|
"since": "2025-01-01"
|
|
151
151
|
},
|
|
152
152
|
"hook:auto-approve-permissions": {
|
|
@@ -155,7 +155,7 @@
|
|
|
155
155
|
"domain": "safety",
|
|
156
156
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
157
157
|
"installedPath": ".instar/hooks/instar/auto-approve-permissions.js",
|
|
158
|
-
"contentHash": "
|
|
158
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
159
159
|
"since": "2025-01-01"
|
|
160
160
|
},
|
|
161
161
|
"job:health-check": {
|
|
@@ -1571,7 +1571,7 @@
|
|
|
1571
1571
|
"type": "subsystem",
|
|
1572
1572
|
"domain": "updates",
|
|
1573
1573
|
"sourcePath": "src/core/PostUpdateMigrator.ts",
|
|
1574
|
-
"contentHash": "
|
|
1574
|
+
"contentHash": "5f7b43d410a9570065bdb15fd007a5fcbf938ec0c9ba210bf4db698c106bed80",
|
|
1575
1575
|
"since": "2025-01-01"
|
|
1576
1576
|
},
|
|
1577
1577
|
"subsystem:scheduler": {
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
"schemaVersion": 1,
|
|
3
3
|
"generatedFrom": "source-tree",
|
|
4
4
|
"registrySha256": "890eec3dcf80e4388c396f43001a0bd3fabc80c88f79aa9525885d6402a6dfdf",
|
|
5
|
-
"packageVersion": "1.3.
|
|
5
|
+
"packageVersion": "1.3.1249",
|
|
6
6
|
"guards": [
|
|
7
7
|
{
|
|
8
8
|
"ref": "docs/audits/phase-b/f10-triage.md",
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"sha256": "
|
|
2
|
+
"sha256": "da9c464af31b76d291aca99a5c3c89118da1ff2a46d113e7a4af5409206a9f5c",
|
|
3
3
|
"registrySha256": "890eec3dcf80e4388c396f43001a0bd3fabc80c88f79aa9525885d6402a6dfdf",
|
|
4
|
-
"packageVersion": "1.3.
|
|
4
|
+
"packageVersion": "1.3.1249"
|
|
5
5
|
}
|
|
@@ -23,6 +23,20 @@ Two honesty fixes ride along. An account that genuinely reports no usage window
|
|
|
23
23
|
|
|
24
24
|
This is a reader and presentation fix. No routing, placement, load-shedding or proactive-swap threshold changed. Their input improves — two exhausted accounts now present a real number where they previously presented nothing — which makes them more likely to route work away from a full account, not less.
|
|
25
25
|
|
|
26
|
+
A new, dark research instrument: the Jev signal-layer shadow. When turned on, every
|
|
27
|
+
message reaching the outbound tone gate is also shown to TypeSafe's Jev model with
|
|
28
|
+
seven yes/no questions — one for each existing deterministic check for jargon a
|
|
29
|
+
user shouldn't see (raw paths, terminal commands, config keys, internal endpoints,
|
|
30
|
+
paste-and-run code, environment variables, cron strings and internal ids). Each
|
|
31
|
+
answer is compared with what the existing check found and a content-free row is
|
|
32
|
+
written to `logs/jev-signal-shadow.jsonl`. It decides nothing, is never waited on,
|
|
33
|
+
and never stores message text.
|
|
34
|
+
|
|
35
|
+
It ships OFF (`intelligence.jevSignalShadow.enabled: false`), and it stays inert
|
|
36
|
+
even when enabled unless a future `soakEndsAt` is set and a `typesafe_api_key` is in
|
|
37
|
+
the vault. `scripts/jev-shadow-report.mjs` turns the log into per-check agreement
|
|
38
|
+
tables. Existing agents receive the dark default block on update.
|
|
39
|
+
|
|
26
40
|
The outbound live-credential wall — the one check that stops a message carrying an
|
|
27
41
|
API key or token from being sent, and the one check that cannot be overridden — did
|
|
28
42
|
not recognise the key formats providers issue today. Modern OpenAI keys
|
|
@@ -62,6 +76,10 @@ Two things will look a little different. An account that genuinely has no weekly
|
|
|
62
76
|
|
|
63
77
|
Nothing about how your work is spread across accounts changed.
|
|
64
78
|
|
|
79
|
+
Nothing changes for you. This adds a measuring tool I can switch on for a limited
|
|
80
|
+
time to test whether a small, cheap model can take over some of the checks I run
|
|
81
|
+
on my own messages. It never changes, delays or blocks what I send.
|
|
82
|
+
|
|
65
83
|
If I ever try to send you a message containing one of today's API keys or access
|
|
66
84
|
tokens — the current OpenAI and OpenRouter key formats, or GitHub's newer
|
|
67
85
|
fine-grained tokens — it is now stopped before it reaches you, the same way older key
|
|
@@ -86,6 +104,10 @@ Your Subscriptions dashboard now shows each account's quota bar in green, amber
|
|
|
86
104
|
- Quota readings older than six hours are labelled with their age on the Subscriptions card.
|
|
87
105
|
- Back-compatible: rollouts with no `limit_id` are still read as codex-family; the new snapshot field is optional and absent on existing stored state.
|
|
88
106
|
|
|
107
|
+
- Dark, time-bounded comparison of Jev against the B1–B7 outbound jargon checks.
|
|
108
|
+
- Content-free comparison log plus a per-check confusion-matrix report script.
|
|
109
|
+
- Shadow calls metered under feature `jev-signal-shadow` in the LLM metrics.
|
|
110
|
+
|
|
89
111
|
- The non-overridable outbound credential wall covers modern OpenAI, OpenRouter and GitHub fine-grained key formats.
|
|
90
112
|
- Invisible-character interleaving no longer hides a key from the wall.
|
|
91
113
|
- Proxy replies and Agent-Health-lane attention items are covered by the wall.
|
|
@@ -112,6 +134,10 @@ Your Subscriptions dashboard now shows each account's quota bar in green, amber
|
|
|
112
134
|
- Unit (`tests/unit/quota-poller.test.ts`, 26 passing): both sides of the boundary — a windowless reading sets `noQuotaWindow`, a windowed one does not.
|
|
113
135
|
- Unit (`tests/unit/subscriptions-render.test.ts`, 97 passing): the three distinct card states (a bar, "reports no usage window", "No quota reading yet") and the age label appearing above the six-hour threshold and not below it.
|
|
114
136
|
|
|
137
|
+
- Unit `tests/unit/JevSignalShadow.test.ts` (17 passing): question set covers every detector kind; inert when disabled, with no soak window, past it, or without a key; compared and every not-compared reason reachable; no message text in any row or from a vendor error body; synchronous, never-throwing `observe()`; key cached; metering recorded.
|
|
138
|
+
- Integration `tests/integration/jev-signal-shadow-gate.test.ts` (6 passing): real gate verdict identical with a disabled shadow; verdict returns while the Jev call hangs; a throwing observer or a network failure cannot break the gate; report script builds deduplicated confusion matrices.
|
|
139
|
+
- E2E `tests/e2e/jev-signal-shadow-lifecycle.test.ts` (3 passing): the update path installs the dark default idempotently; the shipped state makes no call and writes no row; flipping the config with no restart produces a compared row.
|
|
140
|
+
|
|
115
141
|
- Unit `tests/unit/outbound-credential-wall-modern-formats.test.ts` (65 passing): every modern issued shape refused (generated at test time, nothing committed); no regression on older shapes; drift guard mapping every messaging pattern type to a wall kind or a stated exemption; cross-file ratchet over the eight copies; no false positives on prose, bare prefixes and kebab slugs; invisible-character strip across nine classes plus the separated-key regression guard; oversize still fails closed; scrubber offsets stay valid; per-pattern linear-time budget on a 64 KB unbroken run with interior word boundaries (no exemptions); capped `jwt`/`url` detection equivalence.
|
|
116
142
|
- Integration `tests/integration/telegram-reply-advisory-migration.test.ts` (18 passing): modern keys refused on a normal reply, on a proxy reply, and in a laned attention item; a clean proxy reply still sends.
|
|
117
143
|
- E2E `tests/e2e/tone-gate-advisory-migration-alive.test.ts` (12 passing): modern keys refused on the real boot.
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
# Side-Effects Review — Jev signal-layer shadow (dark, measure-only)
|
|
2
|
+
|
|
3
|
+
**Version / slug:** `jev-signal-shadow`
|
|
4
|
+
**Date:** `2026-09-21`
|
|
5
|
+
**Author:** `echo`
|
|
6
|
+
**Second-pass reviewer:** `independent reviewer subagent (see appended section)`
|
|
7
|
+
|
|
8
|
+
## Summary of the change
|
|
9
|
+
|
|
10
|
+
Adds `JevSignalShadow` (`src/core/JevSignalShadow.ts`), a research instrument that
|
|
11
|
+
sends each message reaching the outbound tone gate to TypeSafe's Jev model with
|
|
12
|
+
seven yes/no questions — one per deterministic B1–B7 artefact detector — and logs a
|
|
13
|
+
content-free comparison row to `logs/jev-signal-shadow.jsonl`. `MessagingToneGate`
|
|
14
|
+
gains `setSignalShadow()` and calls `observe(text)` once at the top of `review()`
|
|
15
|
+
inside a try/catch; `observe()` is synchronous, never throws, and never awaited.
|
|
16
|
+
`server.ts` constructs it after the tone gate with a live config read, the vault key
|
|
17
|
+
`typesafe_api_key`, and the feature-metrics recorder. `PostUpdateMigrator.migrateConfig`
|
|
18
|
+
adds the dark default `intelligence.jevSignalShadow` block (enabled:false,
|
|
19
|
+
soakEndsAt:null). `scripts/jev-shadow-report.mjs` reads the log into per-rule
|
|
20
|
+
confusion matrices plus coverage.
|
|
21
|
+
|
|
22
|
+
Spec: `docs/specs/jev-signal-layer-shadow.md` (converged; approved by Justin
|
|
23
|
+
2026-09-21, including the soak egress approval).
|
|
24
|
+
|
|
25
|
+
## Decision-point inventory
|
|
26
|
+
|
|
27
|
+
- *(none)* — the shadow writes an audit row and nothing reads it on any decision
|
|
28
|
+
path. `MessagingToneGate.review()` gains one guarded fire-and-forget call before
|
|
29
|
+
its existing logic; the verdict computation is untouched.
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## 1. Over-block
|
|
34
|
+
|
|
35
|
+
No issue identified. The shadow holds no authority and cannot refuse, delay or alter
|
|
36
|
+
a message. Proven by the integration test that runs the real gate against a Jev stub
|
|
37
|
+
that never settles: the verdict returns while the call is still pending.
|
|
38
|
+
|
|
39
|
+
## 2. Under-block
|
|
40
|
+
|
|
41
|
+
Not applicable — it blocks nothing. What it can MISS as an instrument is recorded,
|
|
42
|
+
not hidden: every candidate that is not compared produces a row with a closed-enum
|
|
43
|
+
reason (timeout, http-error, oversize, skipped-concurrent, skipped-sample,
|
|
44
|
+
model-mismatch, disabled-no-key, soak-expired), so the report's coverage figure is
|
|
45
|
+
honest. Standing conditions (no key, soak expired) write one row per process, not
|
|
46
|
+
one per message.
|
|
47
|
+
|
|
48
|
+
## 3. Level-of-abstraction fit
|
|
49
|
+
|
|
50
|
+
The tone gate is the one place every outbound message already passes with its
|
|
51
|
+
detector signals available, so observing there is the correct layer. The shadow
|
|
52
|
+
re-runs `detectGateSignals` itself (cheap, deterministic) rather than threading
|
|
53
|
+
state out of the gate, keeping zero shared state with the decision path.
|
|
54
|
+
|
|
55
|
+
## 4. Signal vs authority compliance
|
|
56
|
+
|
|
57
|
+
Pure signal, and not even a consumed one yet: the output is a log for a later human
|
|
58
|
+
decision. Per `docs/signal-vs-authority.md`, adding an unconsumed detector is the
|
|
59
|
+
safest possible shape. Graduation into any authority is explicitly a separate spec.
|
|
60
|
+
|
|
61
|
+
## 5. Interactions
|
|
62
|
+
|
|
63
|
+
- **Gate latency:** synchronous cost per candidate is one config read, one sha256
|
|
64
|
+
(over at most the first 1 MB), one small append for any not-compared row
|
|
65
|
+
(skipped-sample, oversize, skipped-concurrent), and (when dispatching) one
|
|
66
|
+
detector pass and one JSON body. The vault key — a lookup that can block on a
|
|
67
|
+
macOS keychain subprocess — is read at construction (boot); after a miss or a 401/403 it is re-read at most once per 10 minutes, so a
|
|
68
|
+
revoked key costs one lookup per 10 minutes, not one per message. That rate-limited
|
|
69
|
+
re-read does run synchronously on the gate path, and only while the shadow is
|
|
70
|
+
enabled with a missing or revoked key. A key added to
|
|
71
|
+
the vault later is picked up within 10 minutes, no restart.
|
|
72
|
+
Network runs detached. Single-flight: at most one call in flight per process;
|
|
73
|
+
overflow is a row, never a queue, so a slow vendor cannot build memory pressure.
|
|
74
|
+
- **Feature metrics:** each call records under feature `jev-signal-shadow`,
|
|
75
|
+
framework `typesafe-api`, so token spend appears in `/metrics/features` beside
|
|
76
|
+
other LLM features. No double-counting with the gate's own LLM call.
|
|
77
|
+
- **Credential wall:** unaffected; the shadow sees the same text the gate sees and
|
|
78
|
+
sends it only to TypeSafe, and only while enabled.
|
|
79
|
+
- **Tests:** the shadow is optional (`signalShadow?`); every existing tone-gate test
|
|
80
|
+
constructs the gate without one and is unchanged.
|
|
81
|
+
|
|
82
|
+
## 6. External surfaces
|
|
83
|
+
|
|
84
|
+
While ENABLED, message text reaching the gate is sent to one more third party
|
|
85
|
+
(TypeSafe, under the MCA accepted 2026-09-20). This is the egress the operator
|
|
86
|
+
approved explicitly in the spec (`soak-egress-approved: true`). Ships DISABLED; the
|
|
87
|
+
14-day bound is mechanical (`soakEndsAt`, checked per candidate, survives restarts).
|
|
88
|
+
Log rows never contain message text: sha256, byte length, detector kinds, Jev
|
|
89
|
+
probabilities, reason enum. A vendor error body is never recorded. The vault key is
|
|
90
|
+
read once and cached, re-read only after a 401/403, and never logged.
|
|
91
|
+
|
|
92
|
+
## 7. Multi-machine posture
|
|
93
|
+
|
|
94
|
+
Machine-local BY DESIGN. The log is this machine's observation of its own gate
|
|
95
|
+
traffic (observation locality). Config is ordinary per-machine config; the soak is
|
|
96
|
+
enabled on this dev agent only. No user-facing notices, no durable state that must
|
|
97
|
+
follow a topic, no URLs.
|
|
98
|
+
|
|
99
|
+
## 8. Rollback cost
|
|
100
|
+
|
|
101
|
+
Operational: set `intelligence.jevSignalShadow.enabled` false — read live per
|
|
102
|
+
candidate, no restart. Code: revert the commit; the config block left behind is
|
|
103
|
+
inert (nothing else reads it). The log is inert data.
|
|
104
|
+
|
|
105
|
+
## Second-pass review
|
|
106
|
+
|
|
107
|
+
**Reviewer:** independent reviewer subagent
|
|
108
|
+
**Verdict:** Concern raised: the vault key read runs synchronously on the gate path, and after a 401/403 it runs again on every dispatched candidate.
|
|
109
|
+
|
|
110
|
+
The concern:
|
|
111
|
+
- `observeInner()` calls `readKey()` inline, inside `MessagingToneGate.review()`, whenever `cachedKey === undefined`. In production `readKey` builds a `SecretStore` and calls `.get()`. On macOS that resolves the master key with `execFileSync('security', …, { timeout: 5000 })`, which is a synchronous subprocess that blocks the whole event loop. It can take up to 5 s per lookup, and the resolution order may try two keychain accounts. This cost lands on the first enabled candidate.
|
|
112
|
+
- After a 401/403, `dispatch` resets `cachedKey = undefined`. The next candidate then re-reads the key synchronously. The key read happens before the single-flight check, so a revoked key that stays in the vault leads to one blocking keychain spawn per dispatched candidate, for the whole soak. The spec forbids exactly this ("never a per-call decrypt on the event loop", "never a gate delay").
|
|
113
|
+
- Suggested fix: resolve the key off the message path. Either read it inside the detached `dispatch()`, or warm it at construction or on a `setImmediate`. Also latch an auth failure, for example to a `disabled-no-key`-class status with a bounded re-read interval, instead of re-reading on every candidate.
|
|
114
|
+
|
|
115
|
+
Smaller accuracy points for this artifact (not blocking):
|
|
116
|
+
- §5 "Interactions / Gate latency" lists the synchronous cost as config read, sha256, detector pass and JSON body. It leaves out two synchronous costs: the keychain read above, and the `mkdirSync` + `appendFileSync` for the `not-compared` rows (`skipped-sample`, `oversize`, `skipped-concurrent`, status rows). Those rows are written on the gate path.
|
|
117
|
+
- The sha256 is computed over the full text before the 1 MB size guard, so hashing an oversize message is not bounded by the guard.
|
|
118
|
+
- A missing key is cached as `null` for the life of the process, so a key added to the vault after enabling needs a restart. That is consistent with "read once", but the operator should be told.
|
|
119
|
+
- The e2e test wires the real gate to a shadow built in the test. It does not exercise the `server.ts` construction: the live `intelligence.jevSignalShadow` read, the `SecretStore` key read and the `logs/` path. The tone-gate lifecycle standard asks for a test of the production initialization path.
|
|
120
|
+
|
|
121
|
+
What I checked and found sound:
|
|
122
|
+
- Verdict isolation: `observe()` is wrapped in try/catch both in the module and at the call site. `dispatch` is async and chained with `.catch().finally()`, so no promise rejection goes unhandled. `review()` never awaits the dispatch. The abort timer is bounded by `timeoutMs` and cleared in `finally`, so it cannot keep the process alive past that bound.
|
|
123
|
+
- Single-flight: `inFlight` is set only after every early return, and it is reset in `.finally()` on every path. `write()` and the metrics call each swallow their own errors, so neither can skip the reset.
|
|
124
|
+
- Leakage: rows carry only the sha256, byte length, detector kinds, probabilities, model and a closed-enum reason. The vendor response body is never read on a non-OK status. The API key appears only in the request header and never reaches a row, a metric or the console. The construction-failure log line prints only the error message.
|
|
125
|
+
- Disabled means inert: `enabled !== true` returns before hashing, any key read, any network call or any row. The integration and e2e tests assert zero rows.
|
|
126
|
+
- The soak bound is mechanical: a missing, unparseable or past `soakEndsAt` is inert and checked on every candidate. The block is read from live config, so the bound survives restarts. It writes one status row per process.
|
|
127
|
+
- Spec coverage: the closed reason set is complete and each reason has a test. Model pinning and `model-mismatch` exclusion, the migration (existence-checked, idempotent), the feature-metrics metering and the report script (dedupe by hash, per-rule confusion matrices, coverage, insufficient-evidence under 20 positives) are all present. All 26 tests in the three new files pass. The enabled-vs-disabled latency measurement belongs to the test-agent-live stage and is not claimed as done here.
|
|
128
|
+
|
|
129
|
+
### Re-review after fixes
|
|
130
|
+
|
|
131
|
+
**Verdict:** Concur with the review
|
|
132
|
+
|
|
133
|
+
- **Key read:** the constructor now reads the key once, at boot, through `refreshKey()`. A missing key, or a 401/403 (which clears both the key and the timestamp), is re-read at most once per `KEY_REREAD_MS` (10 min). The earlier failure mode, one blocking keychain spawn per dispatched message with a revoked key, is gone.
|
|
134
|
+
- **One wording point, not blocking:** a rate-limited re-read still runs synchronously inside `observeInner()`, so on the message path. The cost is at most one lookup per 10 minutes, and only while the shadow is enabled with a missing or revoked key. That bound is acceptable. §5's "never on the message path" should be read as applying to the first read only.
|
|
135
|
+
- **Boot-time read on dark agents:** construction now does a vault read at boot on every agent, including ones where the shadow is disabled. That is harmless. `SecretStore.read()` returns `{}` without touching the keychain when no vault file exists, it never generates a key on this path, and other boot code already reads the vault.
|
|
136
|
+
- **Hash bound:** sha256 is now computed over at most the first `maxScanBytes`, so hash cost is bounded.
|
|
137
|
+
- **Production wiring:** it is extracted into `buildJevSignalShadow()`. `server.ts` and the e2e test both build through it (live `intelligence` read, `readSecret('typesafe_api_key')`, the `logs/` path), so Tier 3 now covers the production construction path.
|
|
138
|
+
- **Everything else still holds:** single-flight reset, no rejection left unhandled, no leakage, disabled is inert, and the mechanical soak bound are unchanged. The three test files now hold 29 tests (up from 26), and all pass.
|