@onlooker-community/ecosystem 0.24.0 → 0.25.0

package/test/bats/assayer-events.bats

@@ -0,0 +1,99 @@
+#!/usr/bin/env bats
+
+# Validates every emitted assayer.* event against @onlooker-community/schema.
+#
+# The assayer.* event types ship in @onlooker-community/schema; until the
+# installed version includes them, these tests skip rather than fail. Once the
+# ecosystem's schema dependency is bumped to a release that carries them, they
+# run for real. See plugins/assayer/README.md (Requirements).
+
+setup() {
+	source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
+	setup_test_env
+
+	PLUGIN_ROOT="${REPO_ROOT}/plugins/assayer"
+	export CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT"
+	export ONLOOKER_EVENTS_LOG="${ONLOOKER_DIR}/logs/onlooker-events.jsonl"
+	mkdir -p "$(dirname "$ONLOOKER_EVENTS_LOG")"
+
+	export _ONLOOKER_EVENT_JS="${REPO_ROOT}/scripts/lib/onlooker-event.mjs"
+	export CLAUDE_SESSION_ID="bats-session-$$"
+
+	# shellcheck disable=SC1091
+	source "${PLUGIN_ROOT}/scripts/lib/assayer-events.sh"
+}
+
+# Skip when the installed schema predates the assayer.* event types.
+_require_assayer_schema() {
+	if ! grep -q "assayer.audit.started" \
+		"${REPO_ROOT}/node_modules/@onlooker-community/schema/schemas/event.v1.json" 2>/dev/null; then
+		skip "installed @onlooker-community/schema has no assayer.* types yet"
+	fi
+}
+
+_validate_latest_event() {
+	local last
+	last=$(tail -n 1 "$ONLOOKER_EVENTS_LOG")
+	[ -n "$last" ] || return 1
+	printf '%s' "$last" | ONLOOKER_DIR="$ONLOOKER_DIR" \
+		node "${REPO_ROOT}/scripts/lib/onlooker-event.mjs" validate >/dev/null
+}
+
+# Valid 26-char Crockford Base32 ULID (no I, L, O, or U).
+AUDIT_ID="01J0000000000000000000AB34"
+
+@test "assayer.audit.started validates" {
+	_require_assayer_schema
+	local p
+	p=$(jq -n --arg a "$AUDIT_ID" '{audit_id: $a, claim_count: 3, trigger: "stop", command_count: 5}')
+	assayer_emit_event "assayer.audit.started" "$p"
+	run _validate_latest_event
+	[ "$status" -eq 0 ]
+}
+
+@test "assayer.claim.contradicted validates" {
+	_require_assayer_schema
+	local p
+	p=$(jq -n --arg a "$AUDIT_ID" '{
+		audit_id: $a,
+		claim: "I ran the tests and they all pass.",
+		claim_type: "tests_pass",
+		evidence_command: "npm test",
+		result_excerpt: "1 failed, 32 passed",
+		confidence: 0.9
+	}')
+	assayer_emit_event "assayer.claim.contradicted" "$p"
+	run _validate_latest_event
+	[ "$status" -eq 0 ]
+}
+
+@test "assayer.claim.unverified validates" {
+	_require_assayer_schema
+	local p
+	p=$(jq -n --arg a "$AUDIT_ID" '{audit_id: $a, claim: "The deploy is healthy.", claim_type: "generic", reason: "no_matching_command"}')
+	assayer_emit_event "assayer.claim.unverified" "$p"
+	run _validate_latest_event
+	[ "$status" -eq 0 ]
+}
+
+@test "assayer.audit.complete validates" {
+	_require_assayer_schema
+	local p
+	p=$(jq -n --arg a "$AUDIT_ID" '{
+		audit_id: $a, claim_count: 3, corroborated: 1, contradicted: 1,
+		unverified: 1, verdict: "contradictions_found", duration_ms: 4200
+	}')
+	assayer_emit_event "assayer.audit.complete" "$p"
+	run _validate_latest_event
+	[ "$status" -eq 0 ]
+}
+
+@test "emission fails on unknown event type" {
+	run assayer_emit_event "assayer.no.such.event" '{"audit_id":"x"}'
+	[ "$status" -ne 0 ]
+}
+
+@test "assayer_emit_event returns 1 when payload is empty" {
+	run assayer_emit_event "assayer.audit.started" ""
+	[ "$status" -ne 0 ]
+}

package/test/bats/assayer-extract.bats

@@ -0,0 +1,76 @@
+#!/usr/bin/env bats
+
+# Exercises claim parsing: assayer_parse_claims and the extraction prompt.
+
+setup() {
+	source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
+	setup_test_env
+	PLUGIN_ROOT="${REPO_ROOT}/plugins/assayer"
+	# shellcheck disable=SC1091
+	source "${PLUGIN_ROOT}/scripts/lib/assayer-extract.sh"
+}
+
+@test "parses a clean JSON array of claims" {
+	run assayer_parse_claims '[{"text":"tests pass","type":"tests_pass","command_keyword":"test","confidence":0.9}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq 'length')" -eq 1 ]
+	[ "$(printf '%s' "$output" | jq -r '.[0].type')" = "tests_pass" ]
+}
+
+@test "strips markdown fences" {
+	local raw
+	raw=$'```json\n[{"text":"build ok","type":"build_succeeds","command_keyword":"build","confidence":0.8}]\n```'
+	run assayer_parse_claims "$raw"
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq 'length')" -eq 1 ]
+}
+
+@test "drops malformed entries and entries without text" {
+	run assayer_parse_claims '[{"text":"ok","type":"generic","confidence":0.7},{"no_text":true},{"text":""}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq 'length')" -eq 1 ]
+}
+
+@test "coerces unknown type to generic" {
+	run assayer_parse_claims '[{"text":"thing","type":"made_up","command_keyword":"x","confidence":0.7}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.[0].type')" = "generic" ]
+}
+
+@test "defaults confidence when missing or non-numeric" {
+	run assayer_parse_claims '[{"text":"thing","type":"generic","command_keyword":"x"}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.[0].confidence')" = "0.6" ]
+}
+
+@test "lowercases command_keyword" {
+	run assayer_parse_claims '[{"text":"thing","type":"generic","command_keyword":"TEST","confidence":0.7}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.[0].command_keyword')" = "test" ]
+}
+
+@test "non-array input yields empty array" {
+	run assayer_parse_claims '{"text":"not an array"}'
+	[ "$status" -eq 0 ]
+	[ "$output" = "[]" ]
+}
+
+@test "garbage input yields empty array" {
+	run assayer_parse_claims 'I could not find any claims.'
+	[ "$status" -eq 0 ]
+	[ "$output" = "[]" ]
+}
+
+@test "empty input yields empty array" {
+	run assayer_parse_claims ""
+	[ "$status" -eq 0 ]
+	[ "$output" = "[]" ]
+}
+
+@test "extraction prompt includes the message and the JSON contract" {
+	run assayer_build_extraction_prompt "I ran the tests and they pass." 5
+	[ "$status" -eq 0 ]
+	[[ "$output" == *"I ran the tests and they pass."* ]]
+	[[ "$output" == *"TESTABLE SUCCESS CLAIM"* ]]
+	[[ "$output" == *"at most 5 claims"* ]]
+}

package/test/bats/assayer-project-key.bats

@@ -0,0 +1,58 @@
+#!/usr/bin/env bats
+
+# Exercises Assayer project-key derivation.
+
+setup() {
+	source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
+	setup_test_env
+	PLUGIN_ROOT="${REPO_ROOT}/plugins/assayer"
+	# shellcheck disable=SC1091
+	source "${PLUGIN_ROOT}/scripts/lib/assayer-project-key.sh"
+
+	REPO="${BATS_TEST_TMPDIR}/repo"
+	mkdir -p "$REPO"
+	git -C "$REPO" init -q
+	git -C "$REPO" config user.email test@example.com
+	git -C "$REPO" config user.name test
+	(cd "$REPO" && printf 'x\n' >f && git add f && git commit -q -m init)
+}
+
+@test "key is 12 hex chars for a repo with a remote" {
+	git -C "$REPO" remote add origin https://example.com/foo/bar.git
+	run assayer_project_key "$REPO"
+	[ "$status" -eq 0 ]
+	[[ "$output" =~ ^[0-9a-f]{12}$ ]]
+}
+
+@test "key is stable across calls" {
+	git -C "$REPO" remote add origin https://example.com/foo/bar.git
+	a=$(assayer_project_key "$REPO")
+	b=$(assayer_project_key "$REPO")
+	[ "$a" = "$b" ]
+}
+
+@test "remote-keyed differs from root-keyed" {
+	local with_remote without_remote
+	without_remote=$(assayer_project_key "$REPO")
+	git -C "$REPO" remote add origin https://example.com/foo/bar.git
+	with_remote=$(assayer_project_key "$REPO")
+	[ "$with_remote" != "$without_remote" ]
+}
+
+@test "different remotes yield different keys" {
+	git -C "$REPO" remote add origin https://example.com/foo/one.git
+	local one
+	one=$(assayer_project_key "$REPO")
+	git -C "$REPO" remote set-url origin https://example.com/foo/two.git
+	local two
+	two=$(assayer_project_key "$REPO")
+	[ "$one" != "$two" ]
+}
+
+@test "non-repo cwd yields empty key" {
+	local non_repo="${BATS_TEST_TMPDIR}/not-a-repo"
+	mkdir -p "$non_repo"
+	run assayer_project_key "$non_repo"
+	[ "$status" -eq 0 ]
+	[ -z "$output" ]
+}

package/test/bats/assayer-stop-hook.bats

@@ -0,0 +1,81 @@
+#!/usr/bin/env bats
+
+# Exercises the Assayer Stop hook's gating behavior. Does not invoke claude -p
+# (the hook bails before the extraction step when preconditions fail).
+# Verifies: disabled-by-default, no-git, recursion guard, no-transcript, and
+# stdout silence (advisory hook must never block Stop).
+
+setup() {
+	source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
+	setup_test_env
+
+	PLUGIN_ROOT="${REPO_ROOT}/plugins/assayer"
+	export CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT"
+	HOOK="${PLUGIN_ROOT}/scripts/hooks/assayer-stop.sh"
+
+	REPO="${BATS_TEST_TMPDIR}/repo"
+	mkdir -p "$REPO"
+	git -C "$REPO" init -q
+	git -C "$REPO" config user.email test@example.com
+	git -C "$REPO" config user.name test
+	(cd "$REPO" && printf 'initial\n' >README.md && git add README.md && git commit -q -m init)
+
+	TRANSCRIPT="${BATS_TEST_TMPDIR}/transcript.jsonl"
+	printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"text","text":"All tests pass."}]}}' >"$TRANSCRIPT"
+}
+
+_make_input() {
+	local cwd="${1:-$REPO}" sid="${2:-test-session}" transcript="${3:-$TRANSCRIPT}"
+	jq -n --arg cwd "$cwd" --arg sid "$sid" --arg tp "$transcript" \
+		'{cwd: $cwd, session_id: $sid, transcript_path: $tp}'
+}
+
+@test "exits 0 silently when assayer.enabled is false (default)" {
+	local input
+	input=$(_make_input)
+	run bash -c "printf '%s' '$input' | ONLOOKER_DIR='$ONLOOKER_DIR' '$HOOK'"
+	[ "$status" -eq 0 ]
+	[ -z "$output" ]
+}
+
+@test "exits 0 when cwd is not a git repo" {
+	local non_repo="${BATS_TEST_TMPDIR}/not-a-repo"
+	mkdir -p "$non_repo"
+	local input
+	input=$(_make_input "$non_repo")
+	run bash -c "printf '%s' '$input' | ONLOOKER_DIR='$ONLOOKER_DIR' '$HOOK'"
+	[ "$status" -eq 0 ]
+}
+
+@test "recursion guard: ASSAYER_NESTED=1 causes immediate exit 0" {
+	mkdir -p "${REPO}/.claude"
+	printf '%s\n' '{"assayer":{"enabled":true}}' >"${REPO}/.claude/settings.json"
+	local input
+	input=$(_make_input)
+	run bash -c "printf '%s' '$input' | ASSAYER_NESTED=1 ONLOOKER_DIR='$ONLOOKER_DIR' '$HOOK'"
+	[ "$status" -eq 0 ]
+	[ -z "$output" ]
+}
+
+@test "exits 0 when enabled but transcript is missing" {
+	mkdir -p "${REPO}/.claude"
+	printf '%s\n' '{"assayer":{"enabled":true}}' >"${REPO}/.claude/settings.json"
+	local input
+	input=$(_make_input "$REPO" "test-session" "${BATS_TEST_TMPDIR}/nope.jsonl")
+	run bash -c "printf '%s' '$input' | ONLOOKER_DIR='$ONLOOKER_DIR' '$HOOK'"
+	[ "$status" -eq 0 ]
+	[ -z "$output" ]
+}
+
+@test "exits 0 when enabled but final message is empty" {
+	mkdir -p "${REPO}/.claude"
+	printf '%s\n' '{"assayer":{"enabled":true}}' >"${REPO}/.claude/settings.json"
+	# Transcript with no assistant text turn.
+	local empty_transcript="${BATS_TEST_TMPDIR}/empty.jsonl"
+	printf '%s\n' '{"type":"user","message":{"content":[{"type":"text","text":"hi"}]}}' >"$empty_transcript"
+	local input
+	input=$(_make_input "$REPO" "test-session" "$empty_transcript")
+	run bash -c "printf '%s' '$input' | ONLOOKER_DIR='$ONLOOKER_DIR' '$HOOK'"
+	[ "$status" -eq 0 ]
+	[ -z "$output" ]
+}

package/test/bats/assayer-transcript.bats

@@ -0,0 +1,72 @@
+#!/usr/bin/env bats
+
+# Exercises the transcript reader against a synthetic JSONL transcript shaped
+# like a real Claude Code session log.
+
+setup() {
+	source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
+	setup_test_env
+	PLUGIN_ROOT="${REPO_ROOT}/plugins/assayer"
+	# shellcheck disable=SC1091
+	source "${PLUGIN_ROOT}/scripts/lib/assayer-transcript.sh"
+
+	TRANSCRIPT="${BATS_TEST_TMPDIR}/transcript.jsonl"
+	{
+		# An assistant turn that runs a passing build, then a failing test.
+		printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"text","text":"Running the build."},{"type":"tool_use","name":"Bash","id":"t1","input":{"command":"npm run build"}}]}}'
+		printf '%s\n' '{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t1","is_error":false,"content":"build ok"}]}}'
+		printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Bash","id":"t2","input":{"command":"npm test"}}]}}'
+		printf '%s\n' '{"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"t2","is_error":true,"content":"1 failed"}]}}'
+		# A non-Bash tool call should be ignored.
+		printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"tool_use","name":"Read","id":"t3","input":{"file_path":"x"}}]}}'
+		# Final assistant message with the claims.
+		printf '%s\n' '{"type":"assistant","message":{"content":[{"type":"text","text":"Done. The build passes and the tests are green."}]}}'
+	} >"$TRANSCRIPT"
+}
+
+@test "final assistant message returns the last text turn" {
+	run assayer_final_assistant_message "$TRANSCRIPT" 6000
+	[ "$status" -eq 0 ]
+	[ "$output" = "Done. The build passes and the tests are green." ]
+}
+
+@test "final assistant message truncates to max_chars" {
+	run assayer_final_assistant_message "$TRANSCRIPT" 10
+	[ "$status" -eq 0 ]
+	[ "${#output}" -eq 10 ]
+}
+
+@test "missing transcript yields empty final message" {
+	run assayer_final_assistant_message "${BATS_TEST_TMPDIR}/nope.jsonl" 6000
+	[ "$status" -eq 0 ]
+	[ -z "$output" ]
+}
+
+@test "collects Bash commands with their is_error status" {
+	run assayer_collect_commands "$TRANSCRIPT"
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq 'length')" -eq 2 ]
+	[ "$(printf '%s' "$output" | jq -r '.[0].command')" = "npm run build" ]
+	[ "$(printf '%s' "$output" | jq -r '.[0].is_error')" = "false" ]
+	[ "$(printf '%s' "$output" | jq -r '.[1].command')" = "npm test" ]
+	[ "$(printf '%s' "$output" | jq -r '.[1].is_error')" = "true" ]
+}
+
+@test "captures the failing command's output excerpt" {
+	run assayer_collect_commands "$TRANSCRIPT"
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.[1].excerpt')" = "1 failed" ]
+}
+
+@test "non-Bash tool calls are excluded" {
+	run assayer_collect_commands "$TRANSCRIPT"
+	[ "$status" -eq 0 ]
+	# Only the two Bash commands, never the Read call.
+	[ "$(printf '%s' "$output" | jq '[.[] | select(.command | contains("file"))] | length')" -eq 0 ]
+}
+
+@test "missing transcript yields empty command array" {
+	run assayer_collect_commands "${BATS_TEST_TMPDIR}/nope.jsonl"
+	[ "$status" -eq 0 ]
+	[ "$output" = "[]" ]
+}

package/test/bats/assayer-ulid.bats

@@ -0,0 +1,31 @@
+#!/usr/bin/env bats
+
+# Exercises the Assayer ULID generator.
+
+setup() {
+	source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
+	setup_test_env
+	PLUGIN_ROOT="${REPO_ROOT}/plugins/assayer"
+	# shellcheck disable=SC1091
+	source "${PLUGIN_ROOT}/scripts/lib/assayer-ulid.sh"
+}
+
+@test "ulid is 26 chars of Crockford Base32" {
+	run assayer_ulid
+	[ "$status" -eq 0 ]
+	[ "${#output}" -eq 26 ]
+	[[ "$output" =~ ^[0-9A-HJKMNP-TV-Z]{26}$ ]]
+}
+
+@test "ulids are unique across calls" {
+	a=$(assayer_ulid)
+	b=$(assayer_ulid)
+	[ "$a" != "$b" ]
+}
+
+@test "ulids are lexicographically time-ordered" {
+	a=$(assayer_ulid)
+	sleep 0.01
+	b=$(assayer_ulid)
+	[[ "$a" < "$b" || "$a" == "$b" ]]
+}

package/test/bats/assayer-verify.bats

@@ -0,0 +1,89 @@
+#!/usr/bin/env bats
+
+# Exercises the deterministic claim verifier: assayer_classify_claim and
+# assayer_audit_verdict. Pure logic — no LLM, no schema, no filesystem.
+
+setup() {
+	source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
+	setup_test_env
+	PLUGIN_ROOT="${REPO_ROOT}/plugins/assayer"
+	# shellcheck disable=SC1091
+	source "${PLUGIN_ROOT}/scripts/lib/assayer-verify.sh"
+}
+
+CMDS_FAILING_TEST='[{"command":"npm test","is_error":true,"excerpt":"1 failed, 32 passed"},{"command":"git status","is_error":false,"excerpt":""}]'
+
+@test "tests_pass claim is contradicted by a failing test command" {
+	run assayer_classify_claim '{"text":"tests pass","type":"tests_pass","command_keyword":"test","confidence":0.9}' "$CMDS_FAILING_TEST"
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.verdict')" = "contradicted" ]
+	[ "$(printf '%s' "$output" | jq -r '.evidence_command')" = "npm test" ]
+	[ "$(printf '%s' "$output" | jq -r '.excerpt')" = "1 failed, 32 passed" ]
+}
+
+@test "build_succeeds claim is corroborated by a passing build" {
+	run assayer_classify_claim '{"text":"build is green","type":"build_succeeds","command_keyword":"build","confidence":0.9}' \
+		'[{"command":"npm run build","is_error":false,"excerpt":"done"}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.verdict')" = "corroborated" ]
+}
+
+@test "claim with no matching command is unverified (no_matching_command)" {
+	run assayer_classify_claim '{"text":"lint clean","type":"lint_clean","command_keyword":"lint","confidence":0.9}' \
+		'[{"command":"npm test","is_error":false,"excerpt":""}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.verdict')" = "unverified" ]
+	[ "$(printf '%s' "$output" | jq -r '.reason')" = "no_matching_command" ]
+}
+
+@test "generic claim with no keyword is unverified (ambiguous)" {
+	run assayer_classify_claim '{"text":"deploy healthy","type":"generic","command_keyword":"","confidence":0.9}' \
+		"$CMDS_FAILING_TEST"
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.verdict')" = "unverified" ]
+	[ "$(printf '%s' "$output" | jq -r '.reason')" = "ambiguous" ]
+}
+
+@test "most recent matching command wins (fix-and-rerun)" {
+	# Failing test first, passing test after a fix — the later run is authoritative.
+	run assayer_classify_claim '{"text":"tests pass now","type":"tests_pass","command_keyword":"test","confidence":0.9}' \
+		'[{"command":"npm test","is_error":true,"excerpt":"fail"},{"command":"npm test","is_error":false,"excerpt":"pass"}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.verdict')" = "corroborated" ]
+}
+
+@test "types_check claim matches a tsc command" {
+	run assayer_classify_claim '{"text":"types check out","type":"types_check","command_keyword":"","confidence":0.9}' \
+		'[{"command":"npx tsc --noEmit","is_error":true,"excerpt":"TS2345"}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.verdict')" = "contradicted" ]
+}
+
+@test "command_keyword matches when type is generic" {
+	run assayer_classify_claim '{"text":"migration ran","type":"generic","command_keyword":"migrate","confidence":0.9}' \
+		'[{"command":"rails db:migrate","is_error":false,"excerpt":"migrated"}]'
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.verdict')" = "corroborated" ]
+}
+
+@test "empty claim defaults to unverified ambiguous" {
+	run assayer_classify_claim "" "$CMDS_FAILING_TEST"
+	[ "$status" -eq 0 ]
+	[ "$(printf '%s' "$output" | jq -r '.verdict')" = "unverified" ]
+}
+
+@test "audit verdict is contradictions_found when any contradiction" {
+	[ "$(assayer_audit_verdict 1 3 0)" = "contradictions_found" ]
+}
+
+@test "audit verdict is clean with corroborations and no contradictions" {
+	[ "$(assayer_audit_verdict 0 2 1)" = "clean" ]
+}
+
+@test "audit verdict is clean when only unverified claims" {
+	[ "$(assayer_audit_verdict 0 0 2)" = "clean" ]
+}
+
+@test "audit verdict is nothing_to_verify when all counts zero" {
+	[ "$(assayer_audit_verdict 0 0 0)" = "nothing_to_verify" ]
+}