@onlooker-community/ecosystem 0.22.0 → 0.23.1

package/test/bats/historian-prompt-submit.bats

@@ -0,0 +1,236 @@
+#!/usr/bin/env bats
+#
+# Exercises the historian UserPromptSubmit retrieval pipeline end-to-end
+# against a synthetic ollama daemon (a fake `curl` binary on PATH that
+# returns predictable embeddings keyed on sentinel substrings in the
+# prompt). Indexing happens via the real SessionEnd hook against the
+# same stub, so the test exercises both halves of the embedder
+# integration.
+
+setup() {
+  source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
+  setup_test_env
+
+  PLUGIN_ROOT="${REPO_ROOT}/plugins/historian"
+  export CLAUDE_PLUGIN_ROOT="$PLUGIN_ROOT"
+  export ONLOOKER_ECOSYSTEM_ROOT="$REPO_ROOT"
+
+  PROJECT_REPO="${BATS_TEST_TMPDIR}/repo"
+  mkdir -p "$PROJECT_REPO"
+  git -C "$PROJECT_REPO" init -q
+  git -C "$PROJECT_REPO" config user.email t@example.com
+  git -C "$PROJECT_REPO" config user.name "Test"
+  git -C "$PROJECT_REPO" remote add origin git@github.com:org/historian-retrieval-test.git
+
+  # shellcheck disable=SC1091
+  source "${PLUGIN_ROOT}/scripts/lib/historian-project-key.sh"
+  PROJECT_KEY=$(historian_project_key "$PROJECT_REPO")
+  [ -n "$PROJECT_KEY" ]
+
+  HIST_DIR="${ONLOOKER_DIR}/historian/${PROJECT_KEY}"
+  ONLOOKER_EVENTS_LOG="${ONLOOKER_DIR}/logs/onlooker-events.jsonl"
+
+  STUB_BIN="${BATS_TEST_TMPDIR}/bin"
+  mkdir -p "$STUB_BIN"
+  cat > "${STUB_BIN}/curl" <<'STUB'
+#!/usr/bin/env bash
+# Mini curl stub for historian bats tests.
+# Parses just enough of the curl arg shape to find the URL and the -d
+# payload. Returns deterministic embeddings keyed on sentinel substrings
+# in the prompt.
+url=""
+payload=""
+prev=""
+for arg in "$@"; do
+  case "$prev" in
+    -d|--data|--data-raw)
+      payload="$arg"; prev=""; continue ;;
+    --max-time|-o|-H|--header)
+      prev=""; continue ;;
+  esac
+  case "$arg" in
+    -d|--data|--data-raw|--max-time|-o|-H|--header)
+      prev="$arg" ;;
+    -*)
+      ;;
+    *)
+      [[ -z "$url" ]] && url="$arg" ;;
+  esac
+done
+
+# An env var toggles the probe success so the same stub serves the
+# "embedder unavailable" test case.
+if [[ "${HISTORIAN_STUB_OLLAMA_AVAILABLE:-1}" == "0" ]]; then
+  exit 7
+fi
+
+if [[ "$url" == */api/tags ]]; then
+  exit 0
+fi
+
+if [[ "$url" == */api/embeddings ]]; then
+  prompt=$(printf '%s' "$payload" | jq -r '.prompt // ""' 2>/dev/null)
+  case "$prompt" in
+    *redash*) printf '{"embedding":[1,0,0]}' ;;
+    *kafka*)  printf '{"embedding":[0,1,0]}' ;;
+    *postgres*) printf '{"embedding":[0,0,1]}' ;;
+    *) printf '{"embedding":[0.5,0.5,0.5]}' ;;
+  esac
+  exit 0
+fi
+
+exit 1
+STUB
+  chmod +x "${STUB_BIN}/curl"
+  export PATH="${STUB_BIN}:${PATH}"
+
+  TRANSCRIPT="${BATS_TEST_TMPDIR}/transcript.jsonl"
+  SESSION_ID="sess-retrieval"
+
+  mkdir -p "${PROJECT_REPO}/.claude"
+  printf '%s\n' \
+    '{"historian":{"enabled":true,"indexing":{"min_transcript_chars_to_index":50,"chunk_target_chars":400,"chunk_overlap_chars":50},"retrieval":{"cooldown_seconds":60,"max_retrievals_per_session":5,"min_prompt_chars":40,"min_similarity":0.55,"max_age_days":365}}}' \
+    > "${PROJECT_REPO}/.claude/settings.json"
+
+  INDEX_HOOK="${PLUGIN_ROOT}/scripts/hooks/historian-session-end.sh"
+  RETRIEVE_HOOK="${PLUGIN_ROOT}/scripts/hooks/historian-prompt-submit.sh"
+}
+
+_index_input() {
+  local sid="${1:-$SESSION_ID}"
+  jq -cn --arg cwd "$PROJECT_REPO" --arg sid "$sid" --arg transcript "$TRANSCRIPT" \
+    '{cwd:$cwd, session_id:$sid, transcript_path:$transcript, hook_event_name:"SessionEnd"}'
+}
+
+_retrieve_input() {
+  local prompt="$1" sid="${2:-current}"
+  jq -cn --arg cwd "$PROJECT_REPO" --arg sid "$sid" --arg prompt "$prompt" \
+    '{cwd:$cwd, session_id:$sid, prompt:$prompt, hook_event_name:"UserPromptSubmit"}'
+}
+
+_append_text_turn() {
+  local role="$1" text="$2"
+  jq -cn --arg role "$role" --arg text "$text" \
+    '{role:$role, content:$text}' >> "$TRANSCRIPT"
+}
+
+_index_session() {
+  local sid="$1"
+  shift
+  : > "$TRANSCRIPT"
+  while [ $# -gt 0 ]; do
+    _append_text_turn "user" "$1"; shift
+    [ $# -gt 0 ] && { _append_text_turn "assistant" "$1"; shift; }
+  done
+  bash -c "printf '%s' '$(_index_input "$sid")' | '$INDEX_HOOK'" >/dev/null
+}
+
+@test "retrieval no-op when historian is disabled" {
+  rm -f "${PROJECT_REPO}/.claude/settings.json"
+  run bash -c "printf '%s' '$(_retrieve_input "a prompt long enough to clear the floor and trigger retrieval but historian is off")' | '$RETRIEVE_HOOK'"
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.hookSpecificOutput.additionalContext == ""' >/dev/null
+  [ ! -f "$ONLOOKER_EVENTS_LOG" ] || ! grep -q '"historian.retrieval' "$ONLOOKER_EVENTS_LOG"
+}
+
+@test "retrieval skipped when prompt is shorter than min_prompt_chars" {
+  run bash -c "printf '%s' '$(_retrieve_input "tiny")' | '$RETRIEVE_HOOK'"
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.hookSpecificOutput.additionalContext == ""' >/dev/null
+  grep '"event_type":"historian.retrieval.complete"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e '.payload.outcome == "skipped" and .payload.skip_reason == "short_prompt"' >/dev/null
+}
+
+@test "indexing embeds chunks when ollama is up" {
+  _index_session "$SESSION_ID" \
+    "We are debugging a redash dashboard problem with timezone offsets and saved query parameters this morning." \
+    "Sure — the latest version always passes UTC because of a chart migration we did last week."
+
+  local jsonl="${HIST_DIR}/sessions/${SESSION_ID}.jsonl"
+  [ -f "$jsonl" ]
+  jq -e '.embedding | type == "array" and length == 3' "$jsonl" >/dev/null
+}
+
+@test "retrieval surfaces a matching past chunk" {
+  # Index a past session containing a "redash" topic.
+  _index_session "past-1" \
+    "We are debugging a redash dashboard problem with timezone offsets and saved query parameters this morning." \
+    "Sure — the latest version always passes UTC because of a chart migration we did last week."
+
+  # New session, same project, query about redash → should match.
+  run bash -c "printf '%s' '$(_retrieve_input "Hitting another redash dashboard timezone issue on the same saved query parameters again today")' | '$RETRIEVE_HOOK'"
+  [ "$status" -eq 0 ]
+
+  local ctx
+  ctx=$(echo "$output" | jq -r '.hookSpecificOutput.additionalContext')
+  [[ "$ctx" == *"Historian: a past chunk looks similar"* ]]
+  [[ "$ctx" == *"redash"* ]]
+
+  grep '"event_type":"historian.retrieval.surfaced"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e '.payload.similarity >= 0.55 and .payload.source_session_id == "past-1"' >/dev/null
+}
+
+@test "retrieval returns empty when no chunk clears the similarity floor" {
+  # Past session about kafka — query about postgres falls below the
+  # 0.55 floor (the embedding vectors are orthogonal in the stub).
+  _index_session "past-2" \
+    "Investigating kafka consumer lag on the ingest pipeline today after the rebalance event yesterday." \
+    "Looks like the rebalance left a stale offset; manual reset cleared it."
+
+  run bash -c "printf '%s' '$(_retrieve_input "Working on a postgres migration plan today for our settings tables to add new columns safely")' | '$RETRIEVE_HOOK'"
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.hookSpecificOutput.additionalContext == ""' >/dev/null
+
+  grep '"event_type":"historian.retrieval.complete"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e '.payload.outcome == "empty"' >/dev/null
+}
+
+@test "retrieval skipped on cooldown" {
+  _index_session "past-3" \
+    "Yet another redash dashboard query that we had to fix the timezone on this morning to make the report run again." \
+    "ok"
+
+  # First retrieval surfaces something.
+  run bash -c "printf '%s' '$(_retrieve_input "redash dashboard timezone problem again on the saved query parameters this morning afternoon")' | '$RETRIEVE_HOOK'"
+  [ "$status" -eq 0 ]
+  grep -q '"event_type":"historian.retrieval.surfaced"' "$ONLOOKER_EVENTS_LOG"
+
+  rm -f "$ONLOOKER_EVENTS_LOG"
+
+  # Immediate second retrieval (same session) hits the cooldown gate
+  # (60s) and gets skipped without calling the embedder.
+  run bash -c "printf '%s' '$(_retrieve_input "redash dashboard timezone problem follow-up just a moment after the previous prompt cleared")' | '$RETRIEVE_HOOK'"
+  [ "$status" -eq 0 ]
+  grep '"event_type":"historian.retrieval.complete"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e '.payload.outcome == "skipped" and .payload.skip_reason == "cooldown"' >/dev/null
+}
+
+@test "retrieval skipped when the embedder is unreachable" {
+  _index_session "past-4" \
+    "Yet another redash dashboard query that we had to fix the timezone on this morning to make the report run again." \
+    "ok"
+
+  # Turn off the stub so the probe fails.
+  HISTORIAN_STUB_OLLAMA_AVAILABLE=0 \
+    run bash -c "printf '%s' '$(_retrieve_input "redash dashboard timezone problem long enough to clear the prompt floor for retrieval")' | '$RETRIEVE_HOOK'"
+  [ "$status" -eq 0 ]
+  echo "$output" | jq -e '.hookSpecificOutput.additionalContext == ""' >/dev/null
+
+  grep -q '"event_type":"historian.embedder.unavailable"' "$ONLOOKER_EVENTS_LOG"
+  grep '"event_type":"historian.retrieval.complete"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e '.payload.outcome == "skipped" and .payload.skip_reason == "embedder_unavailable"' >/dev/null
+}
+
+@test "retrieval excludes chunks from the current session id" {
+  # Index the same session id we'll then query from — should be excluded.
+  _index_session "current" \
+    "Working on a redash dashboard right now in this very session of the test framework that we are running." \
+    "ok"
+
+  run bash -c "printf '%s' '$(_retrieve_input "redash dashboard timezone trouble inside this very session of the test framework")' | '$RETRIEVE_HOOK'"
+  [ "$status" -eq 0 ]
+
+  echo "$output" | jq -e '.hookSpecificOutput.additionalContext == ""' >/dev/null
+  grep '"event_type":"historian.retrieval.complete"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e '.payload.outcome == "empty"' >/dev/null
+}

package/test/bats/memory-recall-tracker.bats

@@ -0,0 +1,189 @@
+#!/usr/bin/env bats
+#
+# Exercises the substrate-level memory.recalled emitter. The hook fires
+# on SessionStart and emits one canonical memory.recalled event per
+# typed memory file in the project's typed memory store at
+# ~/.claude/projects/<encoded>/memory/.
+#
+# Curator's usage tracker depends on this signal; without it,
+# zero-recall findings can't be generated. The tests below pin both the
+# happy path (correct count and provenance) and the skip cases (no git
+# context, no memory store, compact source).
+
+setup() {
+  source "${BATS_TEST_DIRNAME}/../helpers/setup.bash"
+  setup_test_env
+
+  export CLAUDE_PLUGIN_ROOT="$REPO_ROOT"
+
+  PROJECT_REPO="${BATS_TEST_TMPDIR}/repo"
+  mkdir -p "$PROJECT_REPO"
+  git -C "$PROJECT_REPO" init -q
+  git -C "$PROJECT_REPO" config user.email t@example.com
+  git -C "$PROJECT_REPO" config user.name "Test"
+  git -C "$PROJECT_REPO" remote add origin git@github.com:org/memory-recall-test.git
+
+  # Derive the encoded project dir under CLAUDE_HOME so the hook resolves
+  # via the path-encoding fallback (CLAUDE_PROJECT_ENCODED unset).
+  ABS_CWD=$(cd "$PROJECT_REPO" && pwd -P)
+  ENCODED=$(printf '%s' "$ABS_CWD" | sed -E 's#/#-#g')
+  MEM_DIR="${TEST_HOME}/.claude/projects/${ENCODED}/memory"
+  mkdir -p "$MEM_DIR"
+
+  ONLOOKER_EVENTS_LOG="${ONLOOKER_DIR}/logs/onlooker-events.jsonl"
+  HOOK="${REPO_ROOT}/scripts/hooks/memory-recall-tracker.sh"
+}
+
+_input() {
+  local source="${1:-startup}"
+  jq -cn --arg cwd "$PROJECT_REPO" --arg sid "sess-mem-test" --arg source "$source" \
+    '{cwd:$cwd, session_id:$sid, source:$source, hook_event_name:"SessionStart"}'
+}
+
+_seed_memory() {
+  local fname="$1" type="$2" name="${3:-$fname}"
+  printf -- '---\nname: %s\ndescription: test\ntype: %s\n---\n\nBody.\n' \
+    "$name" "$type" > "${MEM_DIR}/${fname}"
+}
+
+@test "memory-recall emits one event per typed memory file" {
+  _seed_memory "user_role.md" "user"
+  _seed_memory "feedback_no_summaries.md" "feedback"
+  _seed_memory "project_auth_rewrite.md" "project"
+  _seed_memory "reference_dashboards.md" "reference"
+
+  run bash -c "printf '%s' '$(_input)' | '$HOOK'"
+  [ "$status" -eq 0 ]
+
+  local count
+  count=$(grep -c '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG")
+  [ "$count" -eq 4 ]
+
+  # One event per memory_type, with the right filename.
+  grep '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e 'select(.payload.memory_type == "user" and .payload.memory_file == "user_role.md")' >/dev/null
+  grep '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e 'select(.payload.memory_type == "feedback")' >/dev/null
+  grep '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e 'select(.payload.memory_type == "project")' >/dev/null
+  grep '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e 'select(.payload.memory_type == "reference")' >/dev/null
+
+  # recall_position values are 0..N-1, distinct.
+  local positions
+  positions=$(grep '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -r '.payload.recall_position' | sort -n | paste -sd, -)
+  [ "$positions" = "0,1,2,3" ]
+}
+
+@test "memory-recall skips MEMORY.md itself" {
+  _seed_memory "feedback_one.md" "feedback"
+  printf '%s\n' '- [One](feedback_one.md) — one' > "${MEM_DIR}/MEMORY.md"
+
+  run bash -c "printf '%s' '$(_input)' | '$HOOK'"
+  [ "$status" -eq 0 ]
+
+  # MEMORY.md is not its own memory; should NOT appear as a memory_file.
+  ! grep '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e 'select(.payload.memory_file == "MEMORY.md")' >/dev/null
+  local count
+  count=$(grep -c '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG")
+  [ "$count" -eq 1 ]
+}
+
+@test "memory-recall skips memories without a recognized type" {
+  _seed_memory "feedback_valid.md" "feedback"
+  # Memory with an unrecognized type field — should be silently dropped.
+  printf -- '---\nname: weird\ntype: unknown\n---\n\nBody.\n' \
+    > "${MEM_DIR}/weird.md"
+  # Memory with no frontmatter at all — also dropped.
+  printf '%s\n' 'just a body, no metadata' > "${MEM_DIR}/raw.md"
+
+  run bash -c "printf '%s' '$(_input)' | '$HOOK'"
+  [ "$status" -eq 0 ]
+
+  local count
+  count=$(grep -c '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG")
+  [ "$count" -eq 1 ]
+  grep '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -e '.payload.memory_file == "feedback_valid.md"' >/dev/null
+}
+
+@test "memory-recall emits nothing when the memory store is empty" {
+  # MEM_DIR exists (created in setup) but contains no *.md files. The
+  # hook walks the glob, finds zero matches, and emits no events.
+  run bash -c "printf '%s' '$(_input)' | '$HOOK'"
+  [ "$status" -eq 0 ]
+  [ ! -f "$ONLOOKER_EVENTS_LOG" ] || ! grep -q '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG"
+}
+
+@test "memory-recall emits nothing when the memory store directory does not exist" {
+  # This is the genuinely-missing-directory branch — the dir check at
+  # the top of the hook short-circuits before any file walk.
+  rm -rf "$MEM_DIR"
+  [ ! -d "$MEM_DIR" ]
+
+  run bash -c "printf '%s' '$(_input)' | '$HOOK'"
+  [ "$status" -eq 0 ]
+  [ ! -f "$ONLOOKER_EVENTS_LOG" ] || ! grep -q '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG"
+}
+
+@test "memory-recall is a no-op when cwd is not a git repo" {
+  local non_git="${BATS_TEST_TMPDIR}/no-git"
+  mkdir -p "$non_git"
+  _seed_memory "user_x.md" "user"
+
+  local input
+  input=$(jq -cn --arg cwd "$non_git" --arg sid "s" --arg source "startup" \
+    '{cwd:$cwd, session_id:$sid, source:$source}')
+
+  run bash -c "printf '%s' '$input' | '$HOOK'"
+  [ "$status" -eq 0 ]
+  [ ! -f "$ONLOOKER_EVENTS_LOG" ] || ! grep -q '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG"
+}
+
+@test "memory-recall skips compact source to avoid double-counting" {
+  _seed_memory "user_x.md" "user"
+
+  run bash -c "printf '%s' '$(_input compact)' | '$HOOK'"
+  [ "$status" -eq 0 ]
+  [ ! -f "$ONLOOKER_EVENTS_LOG" ] || ! grep -q '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG"
+}
+
+@test "memory-recall payload carries the same project_key for two clones" {
+  _seed_memory "user_x.md" "user"
+
+  run bash -c "printf '%s' '$(_input)' | '$HOOK'"
+  [ "$status" -eq 0 ]
+
+  local key
+  key=$(grep '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -r '.payload.project_key' | head -1)
+
+  # Second clone of the same remote at a different path. The key should
+  # match (SHA256 of remote URL, path-independent).
+  local clone2="${BATS_TEST_TMPDIR}/clone2"
+  mkdir -p "$clone2"
+  git -C "$clone2" init -q
+  git -C "$clone2" remote add origin git@github.com:org/memory-recall-test.git
+
+  local ABS_CWD2 ENCODED2 MEM_DIR2
+  ABS_CWD2=$(cd "$clone2" && pwd -P)
+  ENCODED2=$(printf '%s' "$ABS_CWD2" | sed -E 's#/#-#g')
+  MEM_DIR2="${TEST_HOME}/.claude/projects/${ENCODED2}/memory"
+  mkdir -p "$MEM_DIR2"
+  printf -- '---\nname: x\ntype: user\n---\n\nBody.\n' > "${MEM_DIR2}/user_x.md"
+
+  rm -f "$ONLOOKER_EVENTS_LOG"
+  local input2
+  input2=$(jq -cn --arg cwd "$clone2" --arg sid "s" --arg source "startup" \
+    '{cwd:$cwd, session_id:$sid, source:$source}')
+  run bash -c "printf '%s' '$input2' | '$HOOK'"
+  [ "$status" -eq 0 ]
+
+  local key2
+  key2=$(grep '"event_type":"memory.recalled"' "$ONLOOKER_EVENTS_LOG" \
+    | jq -r '.payload.project_key' | head -1)
+
+  [ "$key" = "$key2" ]
+}