cc-safe-setup 29.8.0 → 30.0.5

package/tests/test-fabricated-command-detector.sh

@@ -0,0 +1,296 @@
+#!/bin/bash
+# Tests for fabricated-command-detector.sh
+# Run: bash tests/test-fabricated-command-detector.sh
+set -uo pipefail
+
+HOOK="$(dirname "$0")/../examples/fabricated-command-detector.sh"
+PASS=0 FAIL=0
+
+# Helper: build a synthetic jsonl transcript with one assistant message and
+# zero-or-more Bash tool calls, then run the hook against it.
+# Args:
+#   $1 — description
+#   $2 — expected exit code
+#   $3 — assistant response text (markdown, may contain backtick tokens)
+#   $4 — newline-separated list of Bash commands actually invoked (may be empty)
+#   $5 — env vars to set for this run (optional)
+run_test() {
+    local desc="$1" expected_exit="$2" response="$3" invoked="$4" envvars="${5:-}"
+    local tmpdir transcript actual_exit input
+    tmpdir=$(mktemp -d)
+    transcript="${tmpdir}/transcript.jsonl"
+
+    # Build invoked tool-use events (one per command)
+    if [ -n "$invoked" ]; then
+        while IFS= read -r cmd; do
+            [ -z "$cmd" ] && continue
+            jq -nc --arg cmd "$cmd" '{
+                "role": "assistant",
+                "message": {
+                    "role": "assistant",
+                    "content": [
+                        {"type": "tool_use", "name": "Bash", "input": {"command": $cmd}}
+                    ]
+                }
+            }' >> "$transcript"
+        done <<< "$invoked"
+    fi
+
+    # Build the final assistant message
+    jq -nc --arg text "$response" '{
+        "role": "assistant",
+        "message": {
+            "role": "assistant",
+            "content": [
+                {"type": "text", "text": $text}
+            ]
+        }
+    }' >> "$transcript"
+
+    input=$(jq -nc --arg t "$transcript" '{"transcript_path": $t}')
+
+    if [ -n "$envvars" ]; then
+        actual_exit=$(echo "$input" | env $envvars bash "$HOOK" >/dev/null 2>/dev/null; echo $?)
+    else
+        echo "$input" | bash "$HOOK" >/dev/null 2>/dev/null
+        actual_exit=$?
+    fi
+
+    rm -rf "$tmpdir"
+
+    if [ "$actual_exit" -eq "$expected_exit" ]; then
+        echo "  PASS: $desc"
+        PASS=$((PASS + 1))
+    else
+        echo "  FAIL: $desc (expected exit $expected_exit, got $actual_exit)"
+        FAIL=$((FAIL + 1))
+    fi
+}
+
+# Helper for stderr-content checks: run the hook and grep stderr for a substring.
+expect_stderr_contains() {
+    local desc="$1" response="$2" invoked="$3" expect_substring="$4" envvars="${5:-}"
+    local tmpdir transcript stderr_out input
+    tmpdir=$(mktemp -d)
+    transcript="${tmpdir}/transcript.jsonl"
+
+    if [ -n "$invoked" ]; then
+        while IFS= read -r cmd; do
+            [ -z "$cmd" ] && continue
+            jq -nc --arg cmd "$cmd" '{
+                "role": "assistant",
+                "message": {"role": "assistant", "content": [{"type": "tool_use", "name": "Bash", "input": {"command": $cmd}}]}
+            }' >> "$transcript"
+        done <<< "$invoked"
+    fi
+
+    jq -nc --arg text "$response" '{
+        "role": "assistant",
+        "message": {"role": "assistant", "content": [{"type": "text", "text": $text}]}
+    }' >> "$transcript"
+
+    input=$(jq -nc --arg t "$transcript" '{"transcript_path": $t}')
+
+    if [ -n "$envvars" ]; then
+        stderr_out=$(echo "$input" | env $envvars bash "$HOOK" 2>&1 >/dev/null)
+    else
+        stderr_out=$(echo "$input" | bash "$HOOK" 2>&1 >/dev/null)
+    fi
+
+    rm -rf "$tmpdir"
+
+    if echo "$stderr_out" | grep -q -- "$expect_substring"; then
+        echo "  PASS: $desc"
+        PASS=$((PASS + 1))
+    else
+        echo "  FAIL: $desc (stderr did not contain '$expect_substring')"
+        echo "       got: $stderr_out"
+        FAIL=$((FAIL + 1))
+    fi
+}
+
+echo "Testing fabricated-command-detector.sh"
+echo "======================================"
+
+# --- BASELINE: empty / missing inputs do not crash ---
+run_test "empty input exits 0" 0 "" "" ""
+run_test "missing transcript path exits 0" 0 "" "" ""
+
+# --- CORE: fabrication detection ---
+# Case 1: response references ./exploit, operator actually ran ./chage_pwn root
+run_test "fabricated ./exploit not invoked → advisory exit 0" \
+    0 \
+    "I ran the reproduction: \`./exploit\` and observed the failure." \
+    "./chage_pwn root" \
+    ""
+
+# Case 2: response uses the actual invoked command → no warning
+run_test "verified ./chage_pwn appears → exit 0 silently" \
+    0 \
+    "I ran \`./chage_pwn root\` and observed the failure." \
+    "./chage_pwn root" \
+    ""
+
+# Case 3: response has no command-shaped tokens → exit 0
+run_test "no command tokens in response → exit 0" \
+    0 \
+    "The reproduction succeeded with expected output." \
+    "./chage_pwn root" \
+    ""
+
+# Case 4: response has multiple command tokens, mix of verified and fabricated
+run_test "mixed verified/fabricated → exit 0 advisory" \
+    0 \
+    "First I ran \`./chage_pwn root\`, then \`./exploit\` to confirm." \
+    "./chage_pwn root" \
+    ""
+
+# Case 5: git fabrication
+run_test "fabricated git command not invoked → exit 0 advisory" \
+    0 \
+    "I ran \`git rebase --onto main\` to update the branch." \
+    "git status" \
+    ""
+
+# Case 6: git verified
+run_test "verified git command → exit 0 silently" \
+    0 \
+    "I ran \`git status\` to check the tree." \
+    "git status" \
+    ""
+
+# --- BLOCKING MODE ---
+run_test "BLOCK=1 with fabrication exits 2" \
+    2 \
+    "I ran \`./fake_cmd\` to verify." \
+    "./real_cmd arg" \
+    "CC_FABRICATED_CMD_BLOCK=1"
+
+run_test "BLOCK=1 with verified command exits 0" \
+    0 \
+    "I ran \`./real_cmd arg\` to verify." \
+    "./real_cmd arg" \
+    "CC_FABRICATED_CMD_BLOCK=1"
+
+# --- DISABLE FLAG ---
+run_test "DISABLE=1 short-circuits with fabrication → exit 0" \
+    0 \
+    "I ran \`./fake_cmd\` to verify." \
+    "./real_cmd arg" \
+    "CC_FABRICATED_CMD_DISABLE=1"
+
+# --- ALLOW PATTERN ---
+run_test "ALLOW filter suppresses example commands" \
+    0 \
+    "Run \`./example.sh\` to set up." \
+    "" \
+    "CC_FABRICATED_CMD_ALLOW=^./example"
+
+# --- STDERR CONTENT CHECKS ---
+expect_stderr_contains \
+    "stderr names the unverified token" \
+    "I ran \`./exploit\` for the reproduction." \
+    "./chage_pwn root" \
+    "./exploit" \
+    ""
+
+expect_stderr_contains \
+    "stderr cites issue #60340" \
+    "I ran \`./exploit\` for the reproduction." \
+    "./chage_pwn root" \
+    "#60340" \
+    ""
+
+expect_stderr_contains \
+    "stderr says fabricated-command-detector header" \
+    "I ran \`./exploit\` for the reproduction." \
+    "./chage_pwn root" \
+    "fabricated-command-detector" \
+    ""
+
+# --- LANGUAGE / VOCAB COVERAGE ---
+run_test "bash <script> fabrication detected" \
+    0 \
+    "Run \`bash deploy.sh\` to ship." \
+    "" \
+    ""
+
+run_test "npm run fabrication detected" \
+    0 \
+    "Run \`npm run release\` to publish." \
+    "" \
+    ""
+
+run_test "python <script> fabrication detected" \
+    0 \
+    "Run \`python build.py\` to compile." \
+    "" \
+    ""
+
+run_test "node <script> fabrication detected" \
+    0 \
+    "Run \`node migrate.js\` to migrate." \
+    "" \
+    ""
+
+# --- EDGE CASES ---
+run_test "command in fenced code block (multi-line) is NOT scanned by inline-only regex" \
+    0 \
+    "Run this:\n\`\`\`\n./fake\n\`\`\`\nDone." \
+    "" \
+    ""
+
+run_test "backtick-fenced non-command text not flagged as fabrication" \
+    0 \
+    "Use the \`foo\` variable and \`bar\` setting." \
+    "" \
+    ""
+
+run_test "substring match: invoked command contains the token" \
+    0 \
+    "I ran \`git status\` to check." \
+    "git status --short" \
+    ""
+
+# --- ENVIRONMENT / CAP ---
+run_test "MAX=1 with two fabrications still scans (advisory)" \
+    0 \
+    "Run \`./fake1\` and \`./fake2\` next." \
+    "" \
+    "CC_FABRICATED_CMD_MAX=1"
+
+# --- LARGE INPUTS ---
+run_test "response with no inline backticks → exit 0" \
+    0 \
+    "Plain text with no backtick spans at all." \
+    "" \
+    ""
+
+run_test "empty Bash history with one fabrication → exit 0 advisory" \
+    0 \
+    "I ran \`./script.sh\` to set up." \
+    "" \
+    ""
+
+run_test "TRANSCRIPT missing file → exit 0" \
+    0 \
+    "" \
+    "" \
+    ""
+
+# --- INVOKED-CONFIRMATION: same prefix not enough ---
+run_test "prefix-only match does not count as invoked" \
+    0 \
+    "I ran \`./run_tests --all\` to verify." \
+    "./run_tests" \
+    ""
+
+# This is correctly an advisory case — `./run_tests --all` is a different
+# command from `./run_tests`. The substring-direction guarantees that the
+# response's token appears IN an invoked command, not vice versa.
+# If the operator invokes `./run_tests`, but the response says `./run_tests --all`,
+# the response's claim of running `--all` was not actually invoked.
+
+echo ""
+echo "Result: $PASS passed, $FAIL failed"
+[ "$FAIL" -eq 0 ]

package/tests/test-false-disk-full-guard.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+# Tests for false-disk-full-guard.sh
+HOOK="examples/false-disk-full-guard.sh"
+PASS=0 FAIL=0
+ac() { if echo "$2" | grep -q "$3"; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); echo "FAIL: $1 (expected '$3' in: $2)"; fi; }
+anc() { if ! echo "$2" | grep -q "$3"; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); echo "FAIL: $1 (unexpected '$3')"; fi; }
+ax() { if [ "$2" -eq "$3" ]; then PASS=$((PASS+1)); else FAIL=$((FAIL+1)); echo "FAIL: $1 (exit $2 != $3)"; fi; }
+
+# Test 1: fabricated message + a dir with space → advisory fires, exit 0
+MSG="Command output was lost: the temp filesystem at /tmp is full (0MB free). The child process's stdout/stderr writes failed with ENOSPC."
+OUT=$(printf '{"tool_name":"Bash","tool_response":"%s"}' "$MSG" | bash "$HOOK" 2>&1); RC=$?
+ax "fires exits 0" "$RC" 0
+ac "warns false-positive" "$OUT" "false-disk-full-guard"
+ac "tells not to rm/prune" "$OUT" "Do NOT run rm"
+
+# Test 2: object-shaped tool_response (stdout field) → still detected
+OUT=$(printf '{"tool_name":"Bash","tool_response":{"stdout":"%s","stderr":""}}' "$MSG" | bash "$HOOK" 2>&1)
+ac "object response detected" "$OUT" "false-disk-full-guard"
+
+# Test 3: normal command output → silent
+OUT=$(printf '{"tool_name":"Bash","tool_response":"hello world\nexit 0"}' | bash "$HOOK" 2>&1); RC=$?
+ax "normal exits 0" "$RC" 0
+anc "normal stays silent" "$OUT" "false-disk-full-guard"
+
+# Test 4: genuinely no-match empty payload → silent, exit 0
+OUT=$(printf '{}' | bash "$HOOK" 2>&1); RC=$?
+ax "empty exits 0" "$RC" 0
+anc "empty silent" "$OUT" "false-disk-full-guard"
+
+# Test 5: ENOSPC phrasing variant detected
+MSG2="writes failed with ENOSPC"
+OUT=$(printf '{"tool_name":"Bash","tool_response":"%s"}' "$MSG2" | bash "$HOOK" 2>&1)
+ac "ENOSPC variant detected" "$OUT" "false-disk-full-guard"
+
+echo "PASS: $PASS  FAIL: $FAIL"
+[ "$FAIL" -eq 0 ]

package/tests/test-feature-deprecation-detector.sh

@@ -0,0 +1,117 @@
+#!/bin/sh
+# Tests for examples/feature-deprecation-detector.sh
+
+set -u
+HOOK="$(dirname "$0")/../examples/feature-deprecation-detector.sh"
+PASS=0
+FAIL=0
+
+assert_no_output() {
+    local desc="$1"
+    local output="$2"
+    if [ -z "$output" ]; then
+        echo "  PASS: $desc"
+        PASS=$((PASS + 1))
+    else
+        echo "  FAIL: $desc — unexpected output: $output"
+        FAIL=$((FAIL + 1))
+    fi
+}
+
+assert_contains() {
+    local desc="$1"
+    local output="$2"
+    local needle="$3"
+    if echo "$output" | grep -q "$needle"; then
+        echo "  PASS: $desc"
+        PASS=$((PASS + 1))
+    else
+        echo "  FAIL: $desc — expected '$needle' in output: $output"
+        FAIL=$((FAIL + 1))
+    fi
+}
+
+# Test 1: Empty input → no output
+output=$(echo '{}' | bash "$HOOK" 2>&1)
+assert_no_output "empty input produces no output" "$output"
+
+# Test 2: Tool output with no deprecation signal → no output
+output=$(echo '{"tool_response":{"output":"Hello, World"}}' | bash "$HOOK" 2>&1)
+assert_no_output "non-deprecation output produces no output" "$output"
+
+# Test 3: Known removal /buddy → references issue #45596
+output=$(echo '{"tool_response":{"output":"Unknown skill: buddy"}}' | bash "$HOOK" 2>&1)
+assert_contains "/buddy detected with #45596 reference" "$output" "45596"
+
+# Test 4: Known removal /buddy → references release version
+output=$(echo '{"tool_response":{"output":"Unknown skill: buddy"}}' | bash "$HOOK" 2>&1)
+assert_contains "/buddy detected with v2.1.97 release info" "$output" "v2.1.97"
+
+# Test 5: Unknown skill not in known list → still surfaces possible deprecation
+output=$(echo '{"tool_response":{"output":"Unknown skill: foobar"}}' | bash "$HOOK" 2>&1)
+assert_contains "unknown skill foobar surfaces possible deprecation" "$output" "may have been deprecated"
+
+# Test 6: Unknown command pattern
+output=$(echo '{"tool_response":{"output":"Unknown command: /helper"}}' | bash "$HOOK" 2>&1)
+assert_contains "Unknown command pattern detected" "$output" "helper"
+
+# Test 7: Command not found pattern
+output=$(echo '{"tool_response":{"output":"Command not found: /xyz"}}' | bash "$HOOK" 2>&1)
+assert_contains "Command not found pattern detected" "$output" "xyz"
+
+# Test 8: Explicit "is deprecated" notice
+output=$(echo '{"tool_response":{"output":"The /old-cmd skill is deprecated"}}' | bash "$HOOK" 2>&1)
+assert_contains "is deprecated pattern detected" "$output" "Deprecation signal"
+
+# Test 9: Explicit "has been removed" notice
+output=$(echo '{"tool_response":{"output":"This feature has been removed in v2.1.99"}}' | bash "$HOOK" 2>&1)
+assert_contains "has been removed pattern detected" "$output" "Deprecation signal"
+
+# Test 10: CC_DEPRECATION_QUIET=1 suppresses all output
+output=$(echo '{"tool_response":{"output":"Unknown skill: buddy"}}' | CC_DEPRECATION_QUIET=1 bash "$HOOK" 2>&1)
+assert_no_output "QUIET=1 suppresses output" "$output"
+
+# Test 11: Custom known removals list
+TMP=$(mktemp -d /tmp/test-deprecation.XXXXXX)
+trap 'rm -rf "$TMP"' EXIT
+cat > "$TMP/custom.txt" << 'EOF'
+/custom|v2.1.150 (2026-05-20)|Custom feature for org-specific test|https://example.com/custom-removal
+EOF
+output=$(echo '{"tool_response":{"output":"Unknown skill: custom"}}' | CC_DEPRECATION_KNOWN_REMOVALS="$TMP/custom.txt" bash "$HOOK" 2>&1)
+assert_contains "custom known removal works" "$output" "Custom feature for org-specific test"
+
+# Test 12: tool_response.content alternative key
+output=$(echo '{"tool_response":{"content":"Unknown skill: buddy"}}' | bash "$HOOK" 2>&1)
+assert_contains "tool_response.content key works" "$output" "buddy"
+
+# Test 13: Output with no deprecation signal but contains "buddy" word
+output=$(echo '{"tool_response":{"output":"I am your buddy"}}' | bash "$HOOK" 2>&1)
+assert_no_output "false positive avoided (buddy mention)" "$output"
+
+# Test 14: Multiple deprecation signals — first one wins, no duplicate output
+output=$(echo '{"tool_response":{"output":"Unknown skill: buddy and feature has been removed"}}' | bash "$HOOK" 2>&1)
+# Should detect /buddy first (more specific match)
+assert_contains "first signal wins for multi-signal output" "$output" "45596"
+
+# Test 15: Exit code is 0 regardless of detection
+echo '{"tool_response":{"output":"Unknown skill: buddy"}}' | bash "$HOOK" > /dev/null 2>&1
+rc=$?
+if [ $rc -eq 0 ]; then
+    echo "  PASS: exit code 0 (advisory)"
+    PASS=$((PASS + 1))
+else
+    echo "  FAIL: exit code $rc, expected 0"
+    FAIL=$((FAIL + 1))
+fi
+
+# Test 16: Malformed JSON input → graceful exit
+output=$(echo 'not valid json' | bash "$HOOK" 2>&1)
+assert_no_output "malformed JSON exits gracefully" "$output"
+
+# Test 17: tool_response with empty output → no detection
+output=$(echo '{"tool_response":{"output":""}}' | bash "$HOOK" 2>&1)
+assert_no_output "empty tool_response.output produces no output" "$output"
+
+echo ""
+echo "Results: $PASS passed, $FAIL failed"
+[ $FAIL -eq 0 ] && exit 0 || exit 1