@onlooker-community/ecosystem 0.22.0 → 0.23.1

package/plugins/historian/scripts/lib/historian-retriever.sh

@@ -0,0 +1,191 @@
+#!/usr/bin/env bash
+# Similarity-search retriever for Historian.
+#
+# Given a query embedding and a project key, walks every JSONL chunk
+# record under ~/.onlooker/historian/<key>/sessions/, computes cosine
+# similarity between the query vector and each chunk's `embedding`
+# field, and returns the top-K candidates above a similarity floor.
+#
+# Chunks indexed before the embedder shipped don't have an `embedding`
+# field; the retriever silently skips them rather than treating them as
+# zero-similarity. They'll join the index after the next SessionEnd
+# indexing pass.
+
+# Aggregate every chunk record for the project. Returns a JSON array.
+historian_retriever_load_all_chunks() {
+	local key="$1"
+	[[ -z "$key" ]] && { echo '[]'; return 0; }
+
+	local dir
+	dir=$(historian_sessions_dir "$key")
+	[[ -d "$dir" ]] || { echo '[]'; return 0; }
+
+	# Walk every *.jsonl, emit one JSON array. Use python3 to avoid the
+	# `jq -s` quirks around very large inputs and to control the chunk
+	# shape (drop the embedding from filtering candidates but keep it
+	# for the math).
+	python3 - "$dir" <<'PY'
+import json, os, sys
+dir_path = sys.argv[1]
+out = []
+try:
+    for name in sorted(os.listdir(dir_path)):
+        if not name.endswith(".jsonl"):
+            continue
+        path = os.path.join(dir_path, name)
+        try:
+            with open(path, "r", encoding="utf-8", errors="replace") as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        rec = json.loads(line)
+                    except json.JSONDecodeError:
+                        continue
+                    out.append(rec)
+        except OSError:
+            continue
+except FileNotFoundError:
+    pass
+print(json.dumps(out))
+PY
+}
+
+# Compute top-K cosine-similarity matches against the query embedding.
+#
+# The chunks are streamed from disk one line at a time so memory and
+# argv stay bounded as the per-project store grows. Earlier versions
+# passed the full chunks array as an argv string, which would trip the
+# OS ARG_MAX limit somewhere around tens of thousands of chunks; this
+# form never holds more than one chunk in memory at a time.
+#
+# Usage: historian_retriever_search <sessions_dir>
+#                                   <query_embedding_json>
+#                                   <top_k> <min_similarity>
+#                                   <max_age_days> <current_session_id>
+#
+# Output: JSON array sorted by similarity descending, length <= top_k.
+# Each entry: {
+#   chunk_id, session_id, similarity, age_days, body_redacted,
+#   chunk_index, start_turn_index, end_turn_index, source
+# }
+historian_retriever_search() {
+	local sessions_dir="${1:-}"
+	local query="${2:-[]}"
+	local top_k="${3:-5}"
+	local min_sim="${4:-0.55}"
+	local max_age_days="${5:-180}"
+	local current_session="${6:-}"
+
+	if [[ -z "$sessions_dir" || ! -d "$sessions_dir" ]]; then
+		echo '[]'
+		return 0
+	fi
+
+	python3 - "$sessions_dir" "$top_k" "$min_sim" "$max_age_days" "$current_session" "$query" <<'PY'
+import datetime, json, math, os, sys
+
+sessions_dir = sys.argv[1]
+top_k = int(sys.argv[2])
+min_sim = float(sys.argv[3])
+max_age_days = int(sys.argv[4])
+current_session = sys.argv[5]
+query = json.loads(sys.argv[6] or "null")
+
+
+def cosine(a, b):
+    if not a or not b or len(a) != len(b):
+        return None
+    dot = 0.0
+    na = 0.0
+    nb = 0.0
+    for x, y in zip(a, b):
+        dot += x * y
+        na += x * x
+        nb += y * y
+    if na <= 0.0 or nb <= 0.0:
+        return None
+    return dot / (math.sqrt(na) * math.sqrt(nb))
+
+
+def parse_iso(s):
+    if not s:
+        return None
+    try:
+        return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%SZ").replace(
+            tzinfo=datetime.timezone.utc
+        )
+    except ValueError:
+        return None
+
+
+if not isinstance(query, list) or not query:
+    print("[]")
+    sys.exit(0)
+
+now = datetime.datetime.now(datetime.timezone.utc)
+scored = []
+
+
+def consider(chunk):
+    sid = chunk.get("session_id", "")
+    # Exclude chunks from the session that is currently asking for
+    # context; a session retrieving its own chunks is a degenerate case.
+    if current_session and sid == current_session:
+        return
+    embedding = chunk.get("embedding")
+    if not isinstance(embedding, list) or not embedding:
+        return
+    sim = cosine(query, embedding)
+    if sim is None or sim < min_sim:
+        return
+    created = parse_iso(chunk.get("created_at"))
+    if created is None:
+        age_days = -1
+    else:
+        age_days = (now - created).days
+        if max_age_days > 0 and age_days > max_age_days:
+            return
+    scored.append(
+        {
+            "chunk_id": chunk.get("chunk_id"),
+            "session_id": sid,
+            "similarity": round(sim, 4),
+            "age_days": age_days,
+            "body_redacted": chunk.get("body_redacted", ""),
+            "chunk_index": chunk.get("chunk_index"),
+            "start_turn_index": chunk.get("start_turn_index"),
+            "end_turn_index": chunk.get("end_turn_index"),
+            "source": chunk.get("source", "local"),
+        }
+    )
+
+
+try:
+    names = sorted(os.listdir(sessions_dir))
+except OSError:
+    names = []
+
+for name in names:
+    if not name.endswith(".jsonl"):
+        continue
+    path = os.path.join(sessions_dir, name)
+    try:
+        with open(path, "r", encoding="utf-8", errors="replace") as f:
+            for line in f:
+                line = line.strip()
+                if not line:
+                    continue
+                try:
+                    chunk = json.loads(line)
+                except json.JSONDecodeError:
+                    continue
+                consider(chunk)
+    except OSError:
+        continue
+
+scored.sort(key=lambda c: c["similarity"], reverse=True)
+print(json.dumps(scored[:top_k]))
+PY
+}

package/plugins/historian/scripts/lib/historian-storage.sh

@@ -108,3 +108,50 @@ historian_storage_reset_session() {
 	[[ -f "$path" ]] || return 0
 	: > "$path"
 }
+
+# ============================================================================
+# Retrieval watermarks (per-session, scoped to the project key)
+# ============================================================================
+
+# Path used to hold the per-session retrieval state (count + last_ms) so
+# the rate gate persists across UserPromptSubmit invocations within a
+# single session. We key on (project, session) so cross-session retrieval
+# limits don't leak. The state file uses `last_ms` — an epoch-millisecond
+# timestamp of the last retrieval the rate gate let through — and the
+# cooldown gate compares (now_ms - last_ms) against cooldown_seconds.
+historian_retrieval_state_path() {
+	local key="$1"
+	local session_id="$2"
+	local safe
+	safe=$(printf '%s' "$session_id" | tr -cd '[:alnum:]._-')
+	[[ -z "$safe" ]] && safe="unknown"
+	printf '%s/retrieval-state/%s.json' "$(historian_project_dir "$key")" "$safe"
+}
+
+# Read the JSON document at the watermark path. Returns {"count":0,
+# "last_ms":0} when the file is absent or unreadable.
+historian_retrieval_state_read() {
+	local key="$1"
+	local session_id="$2"
+	local path
+	path=$(historian_retrieval_state_path "$key" "$session_id")
+	if [[ -f "$path" ]]; then
+		jq -c '. // {count:0, last_ms:0}' "$path" 2>/dev/null \
+			|| printf '%s' '{"count":0,"last_ms":0}'
+	else
+		printf '%s' '{"count":0,"last_ms":0}'
+	fi
+}
+
+# Bump the count and update last_ms.
+historian_retrieval_state_write() {
+	local key="$1"
+	local session_id="$2"
+	local count="$3"
+	local last_ms="$4"
+	local path
+	path=$(historian_retrieval_state_path "$key" "$session_id")
+	mkdir -p "$(dirname "$path")" 2>/dev/null
+	jq -cn --argjson count "$count" --argjson last_ms "$last_ms" \
+		'{ count: $count, last_ms: $last_ms }' > "$path" 2>/dev/null
+}

package/plugins/scribe/.claude-plugin/plugin.json

@@ -1,6 +1,6 @@
 {
   "name": "scribe",
-  "version": "0.2.0",
+  "version": "0.2.1",
   "description": "Intent documentation from agent activity. Captures why changes were made — problem context, decisions, tradeoffs — and distills them into readable artifacts at session end.",
   "author": {
     "name": "Onlooker Community",
@@ -8,5 +8,7 @@
   },
   "homepage": "https://onlooker.dev",
   "license": "MIT",
-  "requires": ["ecosystem"]
+  "requires": [
+    "ecosystem"
+  ]
 }

package/plugins/scribe/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## [0.2.1](https://github.com/onlooker-community/ecosystem/compare/scribe-v0.2.0...scribe-v0.2.1) (2026-06-04)
+
+
+### Bug Fixes
+
+* **scribe:** mark hook scripts executable :relieved: ([#64](https://github.com/onlooker-community/ecosystem/issues/64)) ([05603e5](https://github.com/onlooker-community/ecosystem/commit/05603e56895c009c1435d1712592adbbc4c15e61))
+
 ## [0.2.0](https://github.com/onlooker-community/ecosystem/compare/scribe-v0.1.0...scribe-v0.2.0) (2026-06-01)
 
 

package/scripts/hooks/memory-recall-tracker.sh

@@ -0,0 +1,206 @@
+#!/usr/bin/env bash
+# Onlooker Memory Recall Tracker
+# Invoked by SessionStart (matcher: *) when a session boots, resumes, or
+# restarts after compaction. Emits one canonical `memory.recalled` event
+# per typed-memory file present at the project's per-checkout memory
+# store path. This approximates the substrate signal "these memories are
+# now in the model's context for the session about to begin".
+#
+# Curator's usage tracker (and any future plugin that reasons about how
+# often a memory is in scope) depends on this. The signal is coarse —
+# per-session-load rather than per-recall — but actionable in aggregate.
+#
+# Hook contract:
+#   - Always exits 0. Never blocks SessionStart.
+#   - No-ops when there is no project memory store, no git context, or
+#     when the source is `compact` (compaction is metadata-only; the
+#     same memories remain in scope, so re-emitting would double-count).
+
+set -uo pipefail # No -e: never block session startup
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=../lib/validate-path.sh
+source "$SCRIPT_DIR/../lib/validate-path.sh"
+# shellcheck source=../lib/onlooker-schema.sh
+source "$SCRIPT_DIR/../lib/onlooker-schema.sh"
+
+# Standard hook health instrumentation. hook_register sets up the timer;
+# hook_set_context exports _HOOK_SESSION_ID + _HOOK_EVENT_NAME so failures
+# attach to the right session in ~/.onlooker/logs/hook-health.jsonl;
+# hook_success / hook_failure close the health record.
+hook_register "memory-recall-tracker" "Memory Recall Tracker" "Emits memory.recalled per typed memory file present at SessionStart"
+
+INPUT=$(cat 2>/dev/null || true)
+hook_set_context "$INPUT" "SessionStart"
+
+CWD=$(printf '%s' "$INPUT" | jq -r '.cwd // ""' 2>/dev/null) || CWD=""
+SESSION_ID=$(printf '%s' "$INPUT" | jq -r '.session_id // ""' 2>/dev/null) || SESSION_ID=""
+SOURCE=$(printf '%s' "$INPUT" | jq -r '.source // "startup"' 2>/dev/null) || SOURCE="startup"
+[[ -z "$CWD" ]] && CWD="$(pwd)"
+[[ -z "$SESSION_ID" ]] && SESSION_ID="unknown"
+
+# Compaction reloads the session with the same memories still in scope.
+# Re-emitting on each compaction would inflate usage counts; skip.
+if [[ "$SOURCE" == "compact" ]]; then
+	hook_success
+	exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# Resolve project_key. Mirrors the SHA256-of-remote-URL + common-dir
+# fallback every memory plugin uses (see plugins/librarian/scripts/lib/
+# librarian-project-key.sh and friends): if there's no origin remote,
+# anchor the key on git --git-common-dir rather than --show-toplevel so
+# two worktrees of the same local-only repo share a key.
+# ---------------------------------------------------------------------------
+
+_memory_sha256_first12() {
+	local input="$1"
+	if command -v shasum >/dev/null 2>&1; then
+		printf '%s' "$input" | shasum -a 256 2>/dev/null | cut -c1-12
+	elif command -v sha256sum >/dev/null 2>&1; then
+		printf '%s' "$input" | sha256sum 2>/dev/null | cut -c1-12
+	else
+		return 1
+	fi
+}
+
+_memory_repo_root_via_common_dir() {
+	local cwd="$1"
+	local common_dir toplevel
+	common_dir=$(git -C "$cwd" rev-parse --git-common-dir 2>/dev/null) || return 0
+	# git-common-dir may be relative; resolve relative to cwd.
+	if [[ -n "$common_dir" && "$common_dir" != /* ]]; then
+		common_dir="$(cd "$cwd" && cd "$common_dir" 2>/dev/null && pwd -P)" || common_dir=""
+	fi
+	if [[ -n "$common_dir" && -d "$common_dir" ]]; then
+		# common_dir is typically the .git dir of the main repo; its
+		# parent is the canonical repo root (shared across worktrees).
+		toplevel="$(cd "$common_dir/.." 2>/dev/null && pwd -P)" || toplevel=""
+	fi
+	if [[ -z "$toplevel" ]]; then
+		toplevel=$(git -C "$cwd" rev-parse --show-toplevel 2>/dev/null || true)
+		[[ -n "$toplevel" ]] && toplevel="$(cd "$toplevel" 2>/dev/null && pwd -P)"
+	fi
+	printf '%s' "$toplevel"
+}
+
+PROJECT_KEY=""
+if git -C "$CWD" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+	REMOTE=$(git -C "$CWD" remote get-url origin 2>/dev/null || true)
+	if [[ -n "$REMOTE" ]]; then
+		PROJECT_KEY=$(_memory_sha256_first12 "remote:${REMOTE}")
+	else
+		ROOT=$(_memory_repo_root_via_common_dir "$CWD")
+		if [[ -n "$ROOT" ]]; then
+			PROJECT_KEY=$(_memory_sha256_first12 "root:${ROOT}")
+		fi
+	fi
+fi
+
+if [[ -z "$PROJECT_KEY" ]]; then
+	hook_success
+	exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# Resolve the per-project typed-memory store at
+# ~/.claude/projects/<encoded>/memory/. Claude Code encodes the project
+# path by replacing path separators with `-` and prepending a leading `-`.
+# Prefer $CLAUDE_PROJECT_ENCODED when the harness has populated it; fall
+# back to deriving from CWD.
+# ---------------------------------------------------------------------------
+
+ENCODED="${CLAUDE_PROJECT_ENCODED:-}"
+if [[ -z "$ENCODED" ]]; then
+	# Encode the absolute cwd: drop leading slash, swap remaining `/` for
+	# `-`, prepend the leading `-`.
+	ABS_CWD=$(cd "$CWD" 2>/dev/null && pwd -P) || ABS_CWD=""
+	if [[ -n "$ABS_CWD" ]]; then
+		ENCODED=$(printf '%s' "$ABS_CWD" | sed -E 's#/#-#g')
+	fi
+fi
+
+MEMORY_DIR="${CLAUDE_HOME}/projects/${ENCODED}/memory"
+if [[ -z "$ENCODED" || ! -d "$MEMORY_DIR" ]]; then
+	hook_success
+	exit 0
+fi
+
+# ---------------------------------------------------------------------------
+# Walk every *.md file (excluding MEMORY.md itself, which is the index, not
+# a memory). For each, parse the YAML frontmatter's `type` field. Skip
+# files whose type isn't one of the four valid enum values — emitting
+# anything else would fail schema validation and the event would be
+# silently dropped.
+# ---------------------------------------------------------------------------
+
+_extract_type() {
+	local path="$1"
+	[[ -f "$path" ]] || return 0
+	# Parse frontmatter type via awk + sed (no python dep, no yq dep).
+	awk '
+		NR == 1 && /^---/ { in_fm = 1; next }
+		in_fm && /^---/ { exit }
+		in_fm
+	' "$path" 2>/dev/null \
+		| sed -nE 's/^type:[[:space:]]*(.*)$/\1/p' \
+		| head -1 \
+		| tr -d '"' \
+		| tr -d "'"
+}
+
+position=0
+for file in "$MEMORY_DIR"/*.md; do
+	[[ -f "$file" ]] || continue
+	fname=$(basename "$file")
+	[[ "$fname" == "MEMORY.md" ]] && continue
+
+	memory_type=$(_extract_type "$file")
+	case "$memory_type" in
+		user|feedback|project|reference)
+			;;
+		*)
+			# Untyped or unknown-typed memories don't fit the schema's
+			# enum. Skip silently rather than tank schema validation.
+			continue
+			;;
+	esac
+
+	payload=$(jq -cn \
+		--arg project_key "$PROJECT_KEY" \
+		--arg memory_file "$fname" \
+		--arg memory_type "$memory_type" \
+		--argjson recall_position "$position" \
+		'{
+			project_key: $project_key,
+			memory_file: $memory_file,
+			memory_type: $memory_type,
+			recall_position: $recall_position
+		}')
+
+	# Use the canonical ecosystem plugin name (matches the
+	# `${ONLOOKER_PLUGIN_NAME:-onlooker}` default that scripts/lib/
+	# onlooker-emit.sh and onlooker-event.mjs both fall back to). Other
+	# substrate-level emissions land under "onlooker" too, so this stays
+	# consistent with the existing event stream.
+	local_plugin="${ONLOOKER_PLUGIN_NAME:-onlooker}"
+
+	params=$(jq -cn \
+		--arg plugin "$local_plugin" \
+		--arg sid "$SESSION_ID" \
+		--arg type "memory.recalled" \
+		--argjson payload "$payload" \
+		'{ plugin: $plugin, session_id: $sid, event_type: $type, payload: $payload }')
+
+	event_json=$(printf '%s' "$params" \
+		| ONLOOKER_DIR="$ONLOOKER_DIR" ONLOOKER_PLUGIN_NAME="$local_plugin" \
+		  node "$_ONLOOKER_EVENT_JS" emit 2>/dev/null) || event_json=""
+	[[ -z "$event_json" ]] && continue
+
+	onlooker_append_event "$event_json" || true
+	position=$((position + 1))
+done
+
+hook_success
+exit 0