@onlooker-community/ecosystem 0.43.2 → 0.43.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/settings.json +1 -1
- package/.claude/skills/writing-tests/SKILL.md +18 -0
- package/.claude-plugin/plugin.json +1 -1
- package/.release-please-manifest.json +1 -1
- package/AGENTS.md +19 -6
- package/CHANGELOG.md +14 -0
- package/CLAUDE.md +18 -3
- package/biome.json +7 -1
- package/docs/architecture.md +43 -1
- package/docs/superpowers/plans/2026-08-21-schema-emission-harness.md +1115 -0
- package/docs/superpowers/specs/2026-08-21-schema-emission-harness-design.md +255 -0
- package/package.json +6 -4
- package/scripts/hooks/prompt-rule-injector.sh +10 -2
- package/scripts/lib/onlooker-event.mjs +38 -1
- package/scripts/lib/prompt-rules.sh +43 -12
- package/scripts/lint/check-bus-coverage.mjs +153 -0
- package/scripts/lint/check-managed-blocks.mjs +206 -0
- package/test/bats/assayer-events.bats +1 -1
- package/test/bats/bursar-events.bats +1 -1
- package/test/bats/cartographer-events.bats +2 -2
- package/test/bats/echo-events.bats +1 -1
- package/test/bats/emission-report-optout.bats +41 -0
- package/test/bats/governor-events.bats +1 -1
- package/test/bats/inspector-events.bats +1 -1
- package/test/bats/librarian-session-end.bats +1 -1
- package/test/bats/lineage-events.bats +2 -2
- package/test/bats/prompt-rules.bats +61 -24
- package/test/bats/tribunal-events.bats +1 -1
- package/test/bats/warden-events.bats +1 -1
- package/test/bus-coverage.json +131 -0
- package/test/helpers/setup.bash +22 -0
- package/test/node/check-bus-coverage.test.mjs +160 -0
- package/test/node/check-managed-blocks.test.mjs +179 -0
- package/test/node/emission-report.test.mjs +97 -0
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// Managed-block fence checker.
|
|
3
|
+
//
|
|
4
|
+
// Tools like `bd setup` own the text between a `<!-- BEGIN ... -->` and
|
|
5
|
+
// `<!-- END ... -->` marker pair and rewrite it verbatim on every run. Any
|
|
6
|
+
// `markdownlint --fix` we apply inside such a block is silently undone the next
|
|
7
|
+
// time anyone regenerates it. Because CI runs `lint:check` (markdownlint with
|
|
8
|
+
// no `--fix`), that churn lands as a red build that looks unrelated to whatever
|
|
9
|
+
// the person was actually doing.
|
|
10
|
+
//
|
|
11
|
+
// The fix is to fence each block with markdownlint-disable/enable comments
|
|
12
|
+
// placed *outside* the BEGIN/END markers, so the generator cannot clobber them.
|
|
13
|
+
// This script asserts that every managed block in every tracked markdown file
|
|
14
|
+
// carries such a fence, covering the rules the generated text is known to trip.
|
|
15
|
+
//
|
|
16
|
+
// Exit codes:
|
|
17
|
+
// 0 ok
|
|
18
|
+
// 1 one or more unfenced or malformed managed blocks
|
|
19
|
+
// 2 setup/usage error
|
|
20
|
+
//
|
|
21
|
+
// Flags:
|
|
22
|
+
// --root <path> override the repo root (used by the tests)
|
|
23
|
+
|
|
24
|
+
import { execFileSync } from 'node:child_process';
|
|
25
|
+
import { existsSync, readFileSync } from 'node:fs';
|
|
26
|
+
import { dirname, join, resolve } from 'node:path';
|
|
27
|
+
import { fileURLToPath } from 'node:url';
|
|
28
|
+
|
|
29
|
+
// Rules the `bd`-generated blocks trip. MD034 fires because the generator
|
|
30
|
+
// writes bare URLs where markdownlint wants angle brackets; MD012 because it
|
|
31
|
+
// leaves consecutive blank lines; MD024 because two different `bd` subcommands
|
|
32
|
+
// each emit a "Beads Issue Tracker" heading into the same file.
|
|
33
|
+
const REQUIRED_RULES = ['MD012', 'MD024', 'MD034'];
|
|
34
|
+
|
|
35
|
+
const BEGIN_MARKER = /^<!--\s*BEGIN\s+\S.*-->\s*$/;
|
|
36
|
+
const END_MARKER = /^<!--\s*END\s+\S.*-->\s*$/;
|
|
37
|
+
const DISABLE_COMMENT = /^<!--\s*markdownlint-disable\s+(.*?)\s*-->\s*$/;
|
|
38
|
+
const ENABLE_COMMENT = /^<!--\s*markdownlint-enable\s+(.*?)\s*-->\s*$/;
|
|
39
|
+
|
|
40
|
+
function parseArgs(argv) {
|
|
41
|
+
const args = { root: null };
|
|
42
|
+
for (let i = 0; i < argv.length; i++) {
|
|
43
|
+
if (argv[i] === '--root') {
|
|
44
|
+
args.root = argv[++i];
|
|
45
|
+
if (!args.root) usageError('--root requires a path');
|
|
46
|
+
} else {
|
|
47
|
+
usageError(`unknown argument: ${argv[i]}`);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
return args;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
function usageError(message) {
|
|
54
|
+
process.stderr.write(`check-managed-blocks: ${message}\n`);
|
|
55
|
+
process.exit(2);
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Minimal .markdownlintignore support, matching the two pattern shapes the file
|
|
59
|
+
// actually uses: a directory prefix ("node_modules/") and a recursive basename
|
|
60
|
+
// glob ("**/CHANGELOG.md"). Anything else is compared literally. Kept
|
|
61
|
+
// deliberately small -- if the ignore file grows richer patterns, reach for a
|
|
62
|
+
// real glob matcher rather than extending this.
|
|
63
|
+
function loadIgnorePatterns(root) {
|
|
64
|
+
const ignorePath = join(root, '.markdownlintignore');
|
|
65
|
+
if (!existsSync(ignorePath)) return [];
|
|
66
|
+
return readFileSync(ignorePath, 'utf8')
|
|
67
|
+
.split('\n')
|
|
68
|
+
.map((line) => line.trim())
|
|
69
|
+
.filter((line) => line !== '' && !line.startsWith('#'));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function isIgnored(relPath, patterns) {
|
|
73
|
+
for (const pattern of patterns) {
|
|
74
|
+
if (pattern.endsWith('/')) {
|
|
75
|
+
if (relPath === pattern.slice(0, -1) || relPath.startsWith(pattern)) return true;
|
|
76
|
+
} else if (pattern.startsWith('**/')) {
|
|
77
|
+
const basename = pattern.slice(3);
|
|
78
|
+
if (relPath === basename || relPath.endsWith(`/${basename}`)) return true;
|
|
79
|
+
} else if (relPath === pattern) {
|
|
80
|
+
return true;
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return false;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
function trackedMarkdownFiles(root) {
|
|
87
|
+
let stdout;
|
|
88
|
+
try {
|
|
89
|
+
stdout = execFileSync('git', ['ls-files', '-z', '*.md'], { cwd: root, encoding: 'utf8' });
|
|
90
|
+
} catch (err) {
|
|
91
|
+
usageError(`could not list tracked files under ${root}: ${err.message}`);
|
|
92
|
+
}
|
|
93
|
+
return stdout.split('\0').filter((path) => path !== '');
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
function rulesFrom(match) {
|
|
97
|
+
return match[1].split(/\s+/).filter((rule) => rule !== '');
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
function missingRules(declared) {
|
|
101
|
+
return REQUIRED_RULES.filter((rule) => !declared.includes(rule));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
// Walks one file and reports every managed block that is not correctly fenced.
|
|
105
|
+
function checkFile(relPath, contents, errors) {
|
|
106
|
+
const lines = contents.split('\n');
|
|
107
|
+
|
|
108
|
+
for (let i = 0; i < lines.length; i++) {
|
|
109
|
+
if (!BEGIN_MARKER.test(lines[i])) continue;
|
|
110
|
+
|
|
111
|
+
const beginLine = lines[i];
|
|
112
|
+
const beginNo = i + 1;
|
|
113
|
+
const label = `${relPath}:${beginNo}`;
|
|
114
|
+
|
|
115
|
+
// Locate the matching END before judging the fence, so an unterminated
|
|
116
|
+
// block is reported as exactly one error rather than cascading.
|
|
117
|
+
let endIndex = -1;
|
|
118
|
+
for (let j = i + 1; j < lines.length; j++) {
|
|
119
|
+
if (BEGIN_MARKER.test(lines[j])) break; // nested/overlapping: stop looking
|
|
120
|
+
if (END_MARKER.test(lines[j])) {
|
|
121
|
+
endIndex = j;
|
|
122
|
+
break;
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
if (endIndex === -1) {
|
|
127
|
+
errors.push(`${label}: managed block "${beginLine.trim()}" has no matching <!-- END ... --> marker`);
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
const before = i > 0 ? lines[i - 1] : '';
|
|
132
|
+
const disable = before.match(DISABLE_COMMENT);
|
|
133
|
+
if (!disable) {
|
|
134
|
+
errors.push(
|
|
135
|
+
`${label}: managed block is not fenced -- the line directly above it must be ` +
|
|
136
|
+
`"<!-- markdownlint-disable ${REQUIRED_RULES.join(' ')} -->", found ${JSON.stringify(before)}`,
|
|
137
|
+
);
|
|
138
|
+
} else {
|
|
139
|
+
const missing = missingRules(rulesFrom(disable));
|
|
140
|
+
if (missing.length > 0) {
|
|
141
|
+
errors.push(`${label}: markdownlint-disable above this block omits ${missing.join(', ')}`);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
const after = endIndex + 1 < lines.length ? lines[endIndex + 1] : '';
|
|
146
|
+
const enable = after.match(ENABLE_COMMENT);
|
|
147
|
+
if (!enable) {
|
|
148
|
+
errors.push(
|
|
149
|
+
`${relPath}:${endIndex + 1}: managed block is not closed -- the line directly below its END marker ` +
|
|
150
|
+
`must be "<!-- markdownlint-enable ${REQUIRED_RULES.join(' ')} -->", found ${JSON.stringify(after)}`,
|
|
151
|
+
);
|
|
152
|
+
} else {
|
|
153
|
+
const missing = missingRules(rulesFrom(enable));
|
|
154
|
+
if (missing.length > 0) {
|
|
155
|
+
errors.push(`${relPath}:${endIndex + 2}: markdownlint-enable below this block omits ${missing.join(', ')}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
i = endIndex;
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function main() {
|
|
164
|
+
const args = parseArgs(process.argv.slice(2));
|
|
165
|
+
const here = dirname(fileURLToPath(import.meta.url));
|
|
166
|
+
const root = resolve(args.root ?? join(here, '..', '..'));
|
|
167
|
+
|
|
168
|
+
if (!existsSync(root)) usageError(`root does not exist: ${root}`);
|
|
169
|
+
|
|
170
|
+
const ignorePatterns = loadIgnorePatterns(root);
|
|
171
|
+
const errors = [];
|
|
172
|
+
let blockCount = 0;
|
|
173
|
+
let fileCount = 0;
|
|
174
|
+
|
|
175
|
+
for (const relPath of trackedMarkdownFiles(root)) {
|
|
176
|
+
if (isIgnored(relPath, ignorePatterns)) continue;
|
|
177
|
+
|
|
178
|
+
const absPath = join(root, relPath);
|
|
179
|
+
if (!existsSync(absPath)) continue; // staged deletion; nothing to check
|
|
180
|
+
|
|
181
|
+
const contents = readFileSync(absPath, 'utf8');
|
|
182
|
+
if (!contents.includes('<!--')) continue; // fast path: no comments, no markers
|
|
183
|
+
|
|
184
|
+
checkFile(relPath, contents, errors);
|
|
185
|
+
|
|
186
|
+
const found = (contents.match(new RegExp(BEGIN_MARKER.source, 'gm')) ?? []).length;
|
|
187
|
+
if (found > 0) {
|
|
188
|
+
fileCount++;
|
|
189
|
+
blockCount += found;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
for (const e of errors) process.stderr.write(`error: ${e}\n`);
|
|
194
|
+
|
|
195
|
+
if (errors.length > 0) {
|
|
196
|
+
process.stderr.write(
|
|
197
|
+
`check-managed-blocks: ${errors.length} error(s) across ${blockCount} managed block(s)\n` +
|
|
198
|
+
'Fence generated blocks so `bd setup` cannot undo `markdownlint --fix`. See ecosystem-55g.\n',
|
|
199
|
+
);
|
|
200
|
+
process.exit(1);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
process.stdout.write(`check-managed-blocks: ok (${blockCount} managed block(s) in ${fileCount} file(s))\n`);
|
|
204
|
+
}
|
|
205
|
+
|
|
206
|
+
main();
|
|
@@ -89,7 +89,7 @@ AUDIT_ID="01J0000000000000000000AB34"
|
|
|
89
89
|
}
|
|
90
90
|
|
|
91
91
|
@test "emission fails on unknown event type" {
|
|
92
|
-
|
|
92
|
+
expect_emission_rejected assayer_emit_event "assayer.no.such.event" '{"audit_id":"x"}'
|
|
93
93
|
[ "$status" -ne 0 ]
|
|
94
94
|
}
|
|
95
95
|
|
|
@@ -68,6 +68,6 @@ _validate_latest_event() {
|
|
|
68
68
|
}
|
|
69
69
|
|
|
70
70
|
@test "bursar_emit_event returns nonzero for an unknown event type" {
|
|
71
|
-
|
|
71
|
+
expect_emission_rejected bursar_emit_event "bursar.no_such_event" '{"project_key":"x"}' "$SID"
|
|
72
72
|
[ "$status" -ne 0 ]
|
|
73
73
|
}
|
|
@@ -147,7 +147,7 @@ _finding() {
|
|
|
147
147
|
_require_cartographer_schema
|
|
148
148
|
# Guards the direction of the fix: if someone "restores" the old schema,
|
|
149
149
|
# this is the test that objects.
|
|
150
|
-
|
|
150
|
+
expect_emission_rejected cartographer_emit_event "cartographer.issue.found" \
|
|
151
151
|
'{"issue_type":"orphaned_plugin","file_path":"CLAUDE.md","severity":"warning"}'
|
|
152
152
|
[ "$status" -ne 0 ]
|
|
153
153
|
}
|
|
@@ -257,7 +257,7 @@ _finding() {
|
|
|
257
257
|
}
|
|
258
258
|
|
|
259
259
|
@test "emission fails on unknown event type" {
|
|
260
|
-
|
|
260
|
+
expect_emission_rejected cartographer_emit_event "cartographer.no.such.event" '{"audit_id":"x"}'
|
|
261
261
|
[ "$status" -ne 0 ]
|
|
262
262
|
}
|
|
263
263
|
|
|
@@ -111,7 +111,7 @@ TEST_ID="01J000000000000000000000TT"
|
|
|
111
111
|
}
|
|
112
112
|
|
|
113
113
|
@test "emission fails on unknown event type" {
|
|
114
|
-
|
|
114
|
+
expect_emission_rejected echo_emit_event "echo.no.such.event" '{"suite_id":"x"}'
|
|
115
115
|
[ "$status" -ne 0 ]
|
|
116
116
|
}
|
|
117
117
|
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
#!/usr/bin/env bats
|
|
2
|
+
|
|
3
|
+
setup() {
|
|
4
|
+
source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
|
|
5
|
+
setup_test_env
|
|
6
|
+
|
|
7
|
+
PLUGIN_ROOT="${REPO_ROOT}/plugins/warden"
|
|
8
|
+
export CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT"
|
|
9
|
+
export ONLOOKER_ECOSYSTEM_ROOT="$REPO_ROOT"
|
|
10
|
+
source "${PLUGIN_ROOT}/scripts/lib/warden-events.sh"
|
|
11
|
+
|
|
12
|
+
export ONLOOKER_TEST_REPORT_DIR="${BATS_TEST_TMPDIR}/report"
|
|
13
|
+
mkdir -p "$ONLOOKER_TEST_REPORT_DIR"
|
|
14
|
+
REPORT="${ONLOOKER_TEST_REPORT_DIR}/emissions.jsonl"
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
_valid_payload() {
|
|
18
|
+
jq -cn '{source_type:"web_fetch", threat_type:"prompt_injection", confidence:0.5}'
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
# Positive control. Without this, the opt-out test below could pass because
|
|
22
|
+
# nothing writes a report at all, rather than because the helper suppressed it.
|
|
23
|
+
@test "a normal emission does write a report line" {
|
|
24
|
+
run warden_emit_event "warden.threat.detected" "$(_valid_payload)"
|
|
25
|
+
[ "$status" -eq 0 ] || return 1
|
|
26
|
+
[ -s "$REPORT" ]
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
@test "expect_emission_rejected keeps a deliberate rejection out of the report" {
|
|
30
|
+
expect_emission_rejected warden_emit_event "warden.bogus.event" "$(_valid_payload)"
|
|
31
|
+
[ "$status" -ne 0 ] || return 1
|
|
32
|
+
[ ! -f "$REPORT" ]
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
@test "expect_emission_rejected restores the report dir afterward" {
|
|
36
|
+
expect_emission_rejected warden_emit_event "warden.bogus.event" "$(_valid_payload)"
|
|
37
|
+
[ "$ONLOOKER_TEST_REPORT_DIR" = "${BATS_TEST_TMPDIR}/report" ] || return 1
|
|
38
|
+
run warden_emit_event "warden.threat.detected" "$(_valid_payload)"
|
|
39
|
+
[ "$status" -eq 0 ] || return 1
|
|
40
|
+
[ -s "$REPORT" ]
|
|
41
|
+
}
|
|
@@ -233,6 +233,6 @@ AID="bats-agent-000"
|
|
|
233
233
|
}
|
|
234
234
|
|
|
235
235
|
@test "governor_emit_event returns nonzero for unknown event type" {
|
|
236
|
-
|
|
236
|
+
expect_emission_rejected governor_emit_event "governor.no_such_event" '{"session_id":"x"}'
|
|
237
237
|
[ "$status" -ne 0 ]
|
|
238
238
|
}
|
|
@@ -146,7 +146,7 @@ _validate_latest_event() {
|
|
|
146
146
|
}
|
|
147
147
|
|
|
148
148
|
@test "emission rejects an unknown event type" {
|
|
149
|
-
|
|
149
|
+
expect_emission_rejected inspector_emit_event "inspector.no.such.event" '{"file_path":"x"}'
|
|
150
150
|
[ "$status" -ne 0 ]
|
|
151
151
|
}
|
|
152
152
|
|
|
@@ -329,7 +329,7 @@ _scan_complete() {
|
|
|
329
329
|
source "${PLUGIN_ROOT}/scripts/lib/librarian-emit.sh"
|
|
330
330
|
mkdir -p "$(dirname "$ONLOOKER_EVENTS_LOG")"
|
|
331
331
|
|
|
332
|
-
librarian_emit "librarian.scan.complete" "sess-bad" \
|
|
332
|
+
expect_emission_rejected librarian_emit "librarian.scan.complete" "sess-bad" \
|
|
333
333
|
'{"outcome":"gave_up","duration_ms":5}'
|
|
334
334
|
|
|
335
335
|
# A refused payload never reaches the log, so the file may not exist at all.
|
|
@@ -71,11 +71,11 @@ _validate_latest_event() {
|
|
|
71
71
|
local p
|
|
72
72
|
p=$(jq -n --arg pk "$PK" --arg sid "$SID" \
|
|
73
73
|
'{project_key:$pk, session_id:$sid, file_path:"x", tool:"NotebookEdit", operation:"edit"}')
|
|
74
|
-
|
|
74
|
+
expect_emission_rejected lineage_emit_event "lineage.change.recorded" "$p" "$SID"
|
|
75
75
|
[ "$status" -ne 0 ]
|
|
76
76
|
}
|
|
77
77
|
|
|
78
78
|
@test "lineage_emit_event returns nonzero for an unknown event type" {
|
|
79
|
-
|
|
79
|
+
expect_emission_rejected lineage_emit_event "lineage.no_such_event" '{"project_key":"x"}' "$SID"
|
|
80
80
|
[ "$status" -ne 0 ]
|
|
81
81
|
}
|
|
@@ -295,9 +295,13 @@ write_project_rules() {
|
|
|
295
295
|
[ "$path" = "$expected" ]
|
|
296
296
|
}
|
|
297
297
|
|
|
298
|
+
# The canonical prompt_rule.matched payload. Both extra fields are required by
|
|
299
|
+
# the schema, so every positive matched-event test has to carry them.
|
|
300
|
+
MATCHED_PAYLOAD='{"rule_id":"rule-1","match_type":"regex","trigger_source":"prompt"}'
|
|
301
|
+
|
|
298
302
|
@test "emit: appends a JSON event line with type, session, payload, and plugin" {
|
|
299
303
|
: >"$ONLOOKER_EVENTS_LOG"
|
|
300
|
-
run prompt_rules_emit "sess-emit" "prompt_rule.matched"
|
|
304
|
+
run prompt_rules_emit "sess-emit" "prompt_rule.matched" "$MATCHED_PAYLOAD"
|
|
301
305
|
[ "$status" -eq 0 ]
|
|
302
306
|
|
|
303
307
|
local line
|
|
@@ -309,59 +313,92 @@ write_project_rules() {
|
|
|
309
313
|
[ "$(echo "$line" | jq -r '.plugin')" = "onlooker" ]
|
|
310
314
|
}
|
|
311
315
|
|
|
312
|
-
@test "emit:
|
|
316
|
+
@test "emit: writes a full canonical envelope, not a hand-built subset" {
|
|
313
317
|
: >"$ONLOOKER_EVENTS_LOG"
|
|
314
|
-
|
|
318
|
+
prompt_rules_emit "sess-envelope" "prompt_rule.matched" "$MATCHED_PAYLOAD"
|
|
315
319
|
|
|
316
320
|
local line
|
|
317
321
|
line=$(tail -1 "$ONLOOKER_EVENTS_LOG")
|
|
318
|
-
|
|
322
|
+
# The five fields the old jq-built envelope omitted. Each is required by
|
|
323
|
+
# event.v1.json, so their absence is what made those lines unvalidatable.
|
|
324
|
+
[ "$(echo "$line" | jq -r 'has("id")')" = "true" ]
|
|
325
|
+
[ "$(echo "$line" | jq -r 'has("schema_version")')" = "true" ]
|
|
326
|
+
[ "$(echo "$line" | jq -r 'has("runtime")')" = "true" ]
|
|
327
|
+
[ "$(echo "$line" | jq -r 'has("machine_id")')" = "true" ]
|
|
328
|
+
[ "$(echo "$line" | jq -r 'has("sequence")')" = "true" ]
|
|
329
|
+
[ "$(echo "$line" | jq -r '.sequence | type')" = "number" ]
|
|
319
330
|
}
|
|
320
331
|
|
|
321
|
-
@test "emit:
|
|
332
|
+
@test "emit: honors ONLOOKER_PLUGIN_NAME for the plugin field" {
|
|
322
333
|
: >"$ONLOOKER_EVENTS_LOG"
|
|
323
|
-
|
|
324
|
-
[ "$status" -eq 0 ]
|
|
334
|
+
ONLOOKER_PLUGIN_NAME="prompt-rules" prompt_rules_emit "sess-plugin" "prompt_rule.applied" '{"rule_id":"r9"}'
|
|
325
335
|
|
|
326
336
|
local line
|
|
327
337
|
line=$(tail -1 "$ONLOOKER_EVENTS_LOG")
|
|
328
|
-
[ "$(echo "$line" | jq -
|
|
338
|
+
[ "$(echo "$line" | jq -r '.plugin')" = "prompt-rules" ]
|
|
329
339
|
}
|
|
330
340
|
|
|
331
341
|
@test "emit: defaults session id to 'unknown' when none is given" {
|
|
332
342
|
: >"$ONLOOKER_EVENTS_LOG"
|
|
333
|
-
prompt_rules_emit "" "prompt_rule.matched"
|
|
343
|
+
prompt_rules_emit "" "prompt_rule.matched" "$MATCHED_PAYLOAD"
|
|
334
344
|
|
|
335
345
|
local line
|
|
336
346
|
line=$(tail -1 "$ONLOOKER_EVENTS_LOG")
|
|
337
347
|
[ "$(echo "$line" | jq -r '.session_id')" = "unknown" ]
|
|
338
348
|
}
|
|
339
349
|
|
|
340
|
-
|
|
350
|
+
# The old hand-built envelope added `turn` from ONLOOKER_TURN_NUMBER. event.v1.json
|
|
351
|
+
# is additionalProperties:false and has no `turn`, so that field was one of the
|
|
352
|
+
# reasons those lines could never validate. The canonical emitter carries a turn
|
|
353
|
+
# as `turn_number` inside the payload, and neither prompt_rule payload schema
|
|
354
|
+
# declares that property, so the turn has nowhere valid to live on these events.
|
|
355
|
+
@test "emit: does not add a turn field the envelope schema forbids" {
|
|
341
356
|
: >"$ONLOOKER_EVENTS_LOG"
|
|
342
|
-
ONLOOKER_TURN_NUMBER=7 prompt_rules_emit "sess-turn" "prompt_rule.matched"
|
|
357
|
+
ONLOOKER_TURN_NUMBER=7 prompt_rules_emit "sess-turn" "prompt_rule.matched" "$MATCHED_PAYLOAD"
|
|
343
358
|
|
|
344
359
|
local line
|
|
345
360
|
line=$(tail -1 "$ONLOOKER_EVENTS_LOG")
|
|
346
|
-
[ "$(echo "$line" | jq -r '
|
|
347
|
-
[ "$(echo "$line" | jq -r '.
|
|
361
|
+
[ "$(echo "$line" | jq -r 'has("turn")')" = "false" ]
|
|
362
|
+
[ "$(echo "$line" | jq -r '.payload | has("turn_number")')" = "false" ]
|
|
348
363
|
}
|
|
349
364
|
|
|
350
|
-
@test "emit:
|
|
365
|
+
@test "emit: returns 1 and writes nothing when event_type is empty" {
|
|
351
366
|
: >"$ONLOOKER_EVENTS_LOG"
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
367
|
+
run prompt_rules_emit "sess-empty" ""
|
|
368
|
+
[ "$status" -eq 1 ]
|
|
369
|
+
[ ! -s "$ONLOOKER_EVENTS_LOG" ]
|
|
370
|
+
}
|
|
355
371
|
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
372
|
+
# Fail-closed is the whole point of routing through the emitter: an event that
|
|
373
|
+
# cannot be made valid must not reach the bus at all. Before this change the
|
|
374
|
+
# same call wrote a line regardless.
|
|
375
|
+
@test "emit: rejects a matched payload missing the schema's required fields" {
|
|
376
|
+
: >"$ONLOOKER_EVENTS_LOG"
|
|
377
|
+
expect_emission_rejected prompt_rules_emit "sess-bad" "prompt_rule.matched" '{"rule_id":"only-id"}'
|
|
378
|
+
[ "$status" -ne 0 ]
|
|
379
|
+
[ ! -s "$ONLOOKER_EVENTS_LOG" ]
|
|
360
380
|
}
|
|
361
381
|
|
|
362
|
-
@test "emit:
|
|
382
|
+
@test "emit: rejects an empty payload for an event type that requires fields" {
|
|
363
383
|
: >"$ONLOOKER_EVENTS_LOG"
|
|
364
|
-
|
|
365
|
-
[ "$status" -
|
|
384
|
+
expect_emission_rejected prompt_rules_emit "sess-nopayload" "prompt_rule.matched"
|
|
385
|
+
[ "$status" -ne 0 ]
|
|
386
|
+
[ ! -s "$ONLOOKER_EVENTS_LOG" ]
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
@test "emit: rejects an unknown event type" {
|
|
390
|
+
: >"$ONLOOKER_EVENTS_LOG"
|
|
391
|
+
expect_emission_rejected prompt_rules_emit "sess-unknown" "prompt_rule.invented" '{}'
|
|
392
|
+
[ "$status" -ne 0 ]
|
|
366
393
|
[ ! -s "$ONLOOKER_EVENTS_LOG" ]
|
|
367
394
|
}
|
|
395
|
+
|
|
396
|
+
@test "emit: accepts prompt_rule.applied with the fields the injector sends" {
|
|
397
|
+
: >"$ONLOOKER_EVENTS_LOG"
|
|
398
|
+
run prompt_rules_emit "sess-applied" "prompt_rule.applied" '{"rule_id":"r9","guidance_chars":42}'
|
|
399
|
+
[ "$status" -eq 0 ]
|
|
400
|
+
|
|
401
|
+
local line
|
|
402
|
+
line=$(tail -1 "$ONLOOKER_EVENTS_LOG")
|
|
403
|
+
[ "$(echo "$line" | jq -r '.payload.guidance_chars')" = "42" ]
|
|
404
|
+
}
|
|
@@ -204,6 +204,6 @@ JUDGE_ID="01J000000000000000000000JJ"
|
|
|
204
204
|
}
|
|
205
205
|
|
|
206
206
|
@test "emission fails loudly on bogus event_type (schema rejects)" {
|
|
207
|
-
|
|
207
|
+
expect_emission_rejected tribunal_emit_event "tribunal.no.such.event" '{"task_id":"x"}'
|
|
208
208
|
[ "$status" -ne 0 ]
|
|
209
209
|
}
|
|
@@ -80,6 +80,6 @@ _validate_latest_event() {
|
|
|
80
80
|
# warden.* type must be rejected so typos never reach the log.
|
|
81
81
|
local p
|
|
82
82
|
p=$(jq -n '{source_type:"web_fetch", threat_type:"prompt_injection", confidence:0.5}')
|
|
83
|
-
|
|
83
|
+
expect_emission_rejected warden_emit_event "warden.bogus.event" "$p"
|
|
84
84
|
[ "$status" -ne 0 ]
|
|
85
85
|
}
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
{
|
|
2
|
+
"expected": [
|
|
3
|
+
"assayer.audit.complete",
|
|
4
|
+
"assayer.audit.started",
|
|
5
|
+
"assayer.claim.contradicted",
|
|
6
|
+
"assayer.claim.unverified",
|
|
7
|
+
"bursar.rollup.skipped",
|
|
8
|
+
"bursar.rollup.surfaced",
|
|
9
|
+
"bursar.session.recorded",
|
|
10
|
+
"cartographer.audit.complete",
|
|
11
|
+
"cartographer.issue.found",
|
|
12
|
+
"cartographer.issue.resolved",
|
|
13
|
+
"compass.check.passed",
|
|
14
|
+
"compass.check.skipped",
|
|
15
|
+
"counsel.brief.generated",
|
|
16
|
+
"curator.finding.broken_index",
|
|
17
|
+
"curator.finding.date_decayed",
|
|
18
|
+
"curator.finding.orphaned_memory",
|
|
19
|
+
"curator.finding.path_broken",
|
|
20
|
+
"curator.scan.complete",
|
|
21
|
+
"curator.scan.started",
|
|
22
|
+
"echo.improvement.detected",
|
|
23
|
+
"echo.regression.detected",
|
|
24
|
+
"echo.suite.complete",
|
|
25
|
+
"echo.suite.started",
|
|
26
|
+
"governor.budget.exceeded",
|
|
27
|
+
"governor.budget.warning",
|
|
28
|
+
"governor.call.recorded",
|
|
29
|
+
"governor.child.allocated",
|
|
30
|
+
"governor.child.returned",
|
|
31
|
+
"governor.gate.checked",
|
|
32
|
+
"governor.ledger.write_failed",
|
|
33
|
+
"governor.lock.stale_cleared",
|
|
34
|
+
"governor.session.complete",
|
|
35
|
+
"historian.chunk.dropped",
|
|
36
|
+
"historian.chunk.sanitized",
|
|
37
|
+
"historian.embedder.unavailable",
|
|
38
|
+
"historian.indexing.complete",
|
|
39
|
+
"historian.indexing.started",
|
|
40
|
+
"historian.retrieval.complete",
|
|
41
|
+
"historian.retrieval.started",
|
|
42
|
+
"historian.retrieval.surfaced",
|
|
43
|
+
"inspector.check.failed",
|
|
44
|
+
"inspector.check.passed",
|
|
45
|
+
"inspector.check.skipped",
|
|
46
|
+
"inspector.run.completed",
|
|
47
|
+
"librarian.candidate.dropped",
|
|
48
|
+
"librarian.candidate.proposed",
|
|
49
|
+
"librarian.proposal.accepted",
|
|
50
|
+
"librarian.proposal.rejected",
|
|
51
|
+
"librarian.scan.complete",
|
|
52
|
+
"librarian.scan.started",
|
|
53
|
+
"librarian.tombstone.created",
|
|
54
|
+
"lineage.change.recorded",
|
|
55
|
+
"lineage.query.answered",
|
|
56
|
+
"memory.recalled",
|
|
57
|
+
"onlooker.artifact.ready",
|
|
58
|
+
"prompt_rule.applied",
|
|
59
|
+
"prompt_rule.matched",
|
|
60
|
+
"session.compact",
|
|
61
|
+
"session.end",
|
|
62
|
+
"session.prompt",
|
|
63
|
+
"session.start",
|
|
64
|
+
"skill.invoked",
|
|
65
|
+
"task.complete",
|
|
66
|
+
"task.start",
|
|
67
|
+
"tool.agent.spawn",
|
|
68
|
+
"tool.file.read",
|
|
69
|
+
"tool.shell.exec",
|
|
70
|
+
"tribunal.actor.complete",
|
|
71
|
+
"tribunal.actor.start",
|
|
72
|
+
"tribunal.consensus.reached",
|
|
73
|
+
"tribunal.dissent.recorded",
|
|
74
|
+
"tribunal.gate.blocked",
|
|
75
|
+
"tribunal.gate.passed",
|
|
76
|
+
"tribunal.iteration.start",
|
|
77
|
+
"tribunal.judge.start",
|
|
78
|
+
"tribunal.jury.empaneled",
|
|
79
|
+
"tribunal.meta.complete",
|
|
80
|
+
"tribunal.meta.start",
|
|
81
|
+
"tribunal.session.complete",
|
|
82
|
+
"tribunal.session.start",
|
|
83
|
+
"tribunal.verdict",
|
|
84
|
+
"warden.gate.blocked",
|
|
85
|
+
"warden.threat.cleared",
|
|
86
|
+
"warden.threat.detected"
|
|
87
|
+
],
|
|
88
|
+
"excluded": {
|
|
89
|
+
"archivist.compact.complete": "archivist-extract.sh (the PreCompact hook) never calls archivist_emit_event with this type; only onlooker.artifact.ready is wired up",
|
|
90
|
+
"archivist.compact.started": "archivist-extract.sh (the PreCompact hook) never calls archivist_emit_event with this type; only onlooker.artifact.ready is wired up",
|
|
91
|
+
"archivist.extract.complete": "archivist-extract.sh only emits onlooker.artifact.ready; no code path emits archivist.extract.complete yet",
|
|
92
|
+
"archivist.inject.complete": "archivist-inject.sh (the SessionStart hook) has no archivist_emit_event call at all",
|
|
93
|
+
"archivist.inject.started": "archivist-inject.sh (the SessionStart hook) has no archivist_emit_event call at all",
|
|
94
|
+
"compass.check.canceled": "compass has no implementation yet; design phase",
|
|
95
|
+
"compass.check.failed": "compass has no implementation yet; design phase",
|
|
96
|
+
"compass.check.overridden": "compass has no implementation yet; design phase",
|
|
97
|
+
"curator.finding.acknowledged": "curator check deferred; see plugins/curator/README.md",
|
|
98
|
+
"curator.finding.contradiction": "curator check deferred; see plugins/curator/README.md",
|
|
99
|
+
"curator.finding.redundant_pair": "curator check deferred; see plugins/curator/README.md",
|
|
100
|
+
"curator.finding.resolved": "curator check deferred; see plugins/curator/README.md",
|
|
101
|
+
"curator.finding.symbol_missing": "curator check deferred; see plugins/curator/README.md",
|
|
102
|
+
"curator.finding.unused_low_signal": "curator check deferred; see plugins/curator/README.md",
|
|
103
|
+
"curator.finding.url_unchecked": "curator check deferred; see plugins/curator/README.md",
|
|
104
|
+
"historian.config.warning": "no config-validation-warning branch exists in historian-config.sh; nothing in the plugin emits this type yet",
|
|
105
|
+
"historian.prune.complete": "historian/README.md lists retention-sweep pruning under Status as deferred to a follow-up landing",
|
|
106
|
+
"historian.purge.complete": "historian/README.md lists manual purge under Status as deferred to a follow-up landing",
|
|
107
|
+
"librarian.proposal.merged": "librarian-cli.sh only implements the accept/reject subcommands; the merge-into-X resolution from docs/design.md has no code path",
|
|
108
|
+
"librarian.proposal.superseded": "librarian-cli.sh only implements the accept/reject subcommands; the supersede-X resolution from docs/design.md has no code path",
|
|
109
|
+
"meridian.hint.delivered": "plugin lives in another repo",
|
|
110
|
+
"meridian.hint.generated": "plugin lives in another repo",
|
|
111
|
+
"meridian.lesson.curated": "plugin lives in another repo",
|
|
112
|
+
"meridian.outcome.recorded": "plugin lives in another repo",
|
|
113
|
+
"meridian.playbook.updated": "plugin lives in another repo",
|
|
114
|
+
"meridian.reliance.measured": "plugin lives in another repo",
|
|
115
|
+
"onlooker.session.summary": "emitted by the agent, not this repo",
|
|
116
|
+
"oracle.calibration.complete": "plugin lives in another repo",
|
|
117
|
+
"oracle.calibration.requested": "plugin lives in another repo",
|
|
118
|
+
"relay.handoff.captured": "plugin lives in another repo",
|
|
119
|
+
"relay.handoff.injected": "plugin lives in another repo",
|
|
120
|
+
"scribe.capture.complete": "no scribe_emit_event call for this type exists anywhere in plugins/scribe; only scribe.distill.complete and onlooker.artifact.ready are wired up",
|
|
121
|
+
"scribe.distill.complete": "the emit call in scripts/lib/scribe-distill.sh is real, but test/bats/scribe-extract.bats only drives the min_turns skip path, which returns before reaching it",
|
|
122
|
+
"sentinel.allowed": "plugin lives in another repo",
|
|
123
|
+
"sentinel.blocked": "plugin lives in another repo",
|
|
124
|
+
"sentinel.reviewed": "plugin lives in another repo",
|
|
125
|
+
"task.fail": "mapTaskHookInput in scripts/lib/onlooker-event.mjs only derives task.start/task.complete from TaskCreated/TaskCompleted; no branch distinguishes a failed task",
|
|
126
|
+
"tool.agent.complete": "the Agent/PostToolUse branch in onlooker-event.mjs is real, but test/bats/tool-history-tracker.bats only drives Read and failing-Bash tool inputs, never Agent",
|
|
127
|
+
"tool.file.edit": "the Edit branch in onlooker-event.mjs is real, but test/bats/tool-history-tracker.bats never drives an Edit tool input",
|
|
128
|
+
"tool.file.write": "the Write branch in onlooker-event.mjs is real, but test/bats/tool-history-tracker.bats never drives a Write tool input",
|
|
129
|
+
"tool.web.fetch": "the WebFetch branch in onlooker-event.mjs is real, but test/bats/tool-history-tracker.bats never drives a WebFetch tool input"
|
|
130
|
+
}
|
|
131
|
+
}
|
package/test/helpers/setup.bash
CHANGED
|
@@ -68,3 +68,25 @@ load_validate_path() {
|
|
|
68
68
|
"$ONLOOKER_COMPACT_TRACKERS_DIR" \
|
|
69
69
|
"$ONLOOKER_METRICS_DIR"
|
|
70
70
|
}
|
|
71
|
+
|
|
72
|
+
# Run a command that is expected to fail schema validation, without recording
|
|
73
|
+
# the deliberate rejection in the suite-wide emission report.
|
|
74
|
+
#
|
|
75
|
+
# The report exists so a payload that drifts from the schema turns CI red. A
|
|
76
|
+
# test that deliberately emits an invalid payload would otherwise write a
|
|
77
|
+
# valid:false line indistinguishable from real drift, making the gate
|
|
78
|
+
# permanently red from intentional tests. Unsetting the report directory for
|
|
79
|
+
# the duration keeps the negative test honest — it still asserts the emitter
|
|
80
|
+
# rejects — without polluting the gate.
|
|
81
|
+
#
|
|
82
|
+
# Sets $status and $output exactly as bats' `run` does.
|
|
83
|
+
#
|
|
84
|
+
# Usage: expect_emission_rejected <command> [args...]
|
|
85
|
+
expect_emission_rejected() {
|
|
86
|
+
local saved="${ONLOOKER_TEST_REPORT_DIR:-}"
|
|
87
|
+
unset ONLOOKER_TEST_REPORT_DIR
|
|
88
|
+
run "$@"
|
|
89
|
+
if [ -n "$saved" ]; then
|
|
90
|
+
export ONLOOKER_TEST_REPORT_DIR="$saved"
|
|
91
|
+
fi
|
|
92
|
+
}
|