@onlooker-community/ecosystem 0.21.0 → 0.22.0

package/plugins/historian/scripts/lib/historian-emit.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Event emission helpers for Historian.
+#
+# Thin wrapper around onlooker-event.mjs `emit` mode for historian.* events.
+# Fail-soft: returns 0 on success or when the substrate is unavailable.
+
+_historian_resolve_event_js() {
+	local script_dir plugin_root ecosystem_root candidate
+	script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+	plugin_root="$(cd "${script_dir}/../.." && pwd)"
+
+	ecosystem_root="${ONLOOKER_ECOSYSTEM_ROOT:-}"
+	if [[ -z "$ecosystem_root" ]]; then
+		candidate="$(cd "${plugin_root}/../.." 2>/dev/null && pwd)"
+		if [[ -f "${candidate}/scripts/lib/onlooker-event.mjs" ]]; then
+			ecosystem_root="$candidate"
+		fi
+	fi
+
+	if [[ -n "$ecosystem_root" ]]; then
+		printf '%s/scripts/lib/onlooker-event.mjs' "$ecosystem_root"
+	fi
+}
+
+_HISTORIAN_EVENT_JS="${_HISTORIAN_EVENT_JS:-$(_historian_resolve_event_js)}"
+
+# Emit a historian.* event. Fail-soft: returns 0 on any error.
+# Usage: historian_emit <event_type> <session_id> <payload_json>
+historian_emit() {
+	local event_type="${1:-}"
+	local session_id="${2:-}"
+	local payload="${3:-{\}}"
+
+	[[ -z "$event_type" || -z "$session_id" ]] && return 0
+	[[ -z "$_HISTORIAN_EVENT_JS" || ! -f "$_HISTORIAN_EVENT_JS" ]] && return 0
+	command -v node >/dev/null 2>&1 || return 0
+	[[ -z "${ONLOOKER_EVENTS_LOG:-}" ]] && return 0
+
+	local params event_json
+	params=$(jq -cn \
+		--arg plugin "historian" \
+		--arg session_id "$session_id" \
+		--arg event_type "$event_type" \
+		--argjson payload "$payload" \
+		'{
+			plugin: $plugin,
+			session_id: $session_id,
+			event_type: $event_type,
+			payload: $payload
+		}') || return 0
+
+	event_json=$(
+		ONLOOKER_DIR="${ONLOOKER_DIR:-$HOME/.onlooker}" \
+		ONLOOKER_PLUGIN_NAME="historian" \
+		printf '%s' "$params" | node "$_HISTORIAN_EVENT_JS" emit 2>/dev/null
+	) || return 0
+	[[ -z "$event_json" ]] && return 0
+
+	mkdir -p "$(dirname "$ONLOOKER_EVENTS_LOG")" 2>/dev/null
+	printf '%s\n' "$event_json" >> "$ONLOOKER_EVENTS_LOG" 2>/dev/null
+}

package/plugins/historian/scripts/lib/historian-project-key.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# Project key derivation for Historian.
+#
+# Historian stores chunk records under the ecosystem-wide 12-char hex
+# project key so state survives clone path changes and is shared across
+# worktrees / clones of the same repo.
+#
+# Resolution order:
+#   1. SHA256(`git remote get-url origin`) — preferred, machine-portable
+#   2. SHA256(realpath of `git rev-parse --show-toplevel`) — fallback for
+#      repos without an origin remote
+#
+# Returns the first 12 hex chars. Empty when not in a git repo at all.
+
+_historian_sha256_first12() {
+	local input="$1"
+	if command -v shasum >/dev/null 2>&1; then
+		printf '%s' "$input" | shasum -a 256 2>/dev/null | cut -c1-12
+	elif command -v sha256sum >/dev/null 2>&1; then
+		printf '%s' "$input" | sha256sum 2>/dev/null | cut -c1-12
+	else
+		return 1
+	fi
+}
+
+historian_project_remote_url() {
+	local cwd="${1:-}"
+	[[ -z "$cwd" || ! -d "$cwd" ]] && return 0
+	git -C "$cwd" remote get-url origin 2>/dev/null || true
+}
+
+historian_project_repo_root() {
+	local cwd="${1:-}"
+	[[ -z "$cwd" || ! -d "$cwd" ]] && return 0
+
+	if ! git -C "$cwd" rev-parse --is-inside-work-tree >/dev/null 2>&1; then
+		return 0
+	fi
+
+	local common_dir toplevel
+	common_dir=$(git -C "$cwd" rev-parse --git-common-dir 2>/dev/null) || return 0
+
+	if [[ -n "$common_dir" && "$common_dir" != /* ]]; then
+		common_dir="$(cd "$cwd" && cd "$common_dir" 2>/dev/null && pwd -P)" || common_dir=""
+	fi
+
+	if [[ -n "$common_dir" && -d "$common_dir" ]]; then
+		toplevel="$(cd "$common_dir/.." 2>/dev/null && pwd -P)" || toplevel=""
+	fi
+
+	if [[ -z "$toplevel" ]]; then
+		toplevel=$(git -C "$cwd" rev-parse --show-toplevel 2>/dev/null || true)
+		[[ -n "$toplevel" ]] && toplevel="$(cd "$toplevel" 2>/dev/null && pwd -P)"
+	fi
+
+	printf '%s' "$toplevel"
+}
+
+# Compute the project key for the given cwd. Prints the key or empty.
+# Usage: key=$(historian_project_key "$CWD")
+historian_project_key() {
+	local cwd="${1:-}"
+	[[ -z "$cwd" ]] && cwd="$(pwd)"
+
+	local remote
+	remote=$(historian_project_remote_url "$cwd")
+	if [[ -n "$remote" ]]; then
+		_historian_sha256_first12 "remote:$remote"
+		return 0
+	fi
+
+	local root
+	root=$(historian_project_repo_root "$cwd")
+	if [[ -n "$root" ]]; then
+		_historian_sha256_first12 "root:$root"
+		return 0
+	fi
+
+	return 0
+}

package/plugins/historian/scripts/lib/historian-sanitizer.sh

@@ -0,0 +1,123 @@
+#!/usr/bin/env bash
+# Sanitizer for Historian chunks.
+#
+# Three layers, in order:
+#   1. Secret-shaped substrings are redacted to "[REDACTED:secret]".
+#      Patterns cover AWS access keys, GitHub PATs, Anthropic API keys,
+#      bearer tokens, and KEY=value-style env assignments containing
+#      key/secret/token in the key name.
+#   2. `[historian:skip]` markers cause the entire chunk to be dropped.
+#   3. Path-deny: if the chunk references any path under
+#      `never_index_paths` (substring match against each entry), the
+#      chunk is dropped.
+#
+# Input: JSON array of chunk records from the chunker (each with `body`).
+# Output: JSON array of surviving chunk records, each with `body_redacted`
+#         (instead of `body`) and a `redaction_count`, plus a sibling
+#         array of `dropped` records keyed by reason.
+
+# Usage: historian_sanitizer_run <chunks_json> <never_index_paths_json>
+#                                 <redact_secret_patterns> <drop_skip_marker>
+#
+# The two boolean args honor the corresponding config knobs:
+#   redact_secret_patterns: false → skip the secret regex substitutions
+#                                    (chunk bodies copy through unchanged)
+#   drop_skip_marker: false → keep chunks even when they contain the
+#                              [historian:skip] marker
+#
+# Output: { "kept": [...], "dropped": [...] }
+historian_sanitizer_run() {
+	local chunks="${1:-[]}"
+	local never_index_paths="${2:-[]}"
+	local redact_secrets="${3:-true}"
+	local drop_skip="${4:-true}"
+
+	python3 - "$chunks" "$never_index_paths" "$redact_secrets" "$drop_skip" <<'PY'
+import json, re, sys
+
+chunks = json.loads(sys.argv[1] or "[]")
+deny_paths = json.loads(sys.argv[2] or "[]")
+redact_secrets = sys.argv[3] != "false"
+drop_skip = sys.argv[4] != "false"
+
+# Secret-shaped patterns. Conservative — false positives are acceptable;
+# false negatives are the failure mode we care about. Bearer matches
+# case-insensitively because the "Bearer" scheme is case-insensitive per
+# RFC 6750 and uppercase / lowercase variants occur in the wild.
+SECRET_PATTERNS = [
+    # AWS access keys (AKIA followed by 16 base32-ish chars).
+    re.compile(r"\bAKIA[0-9A-Z]{16}\b"),
+    # GitHub PATs.
+    re.compile(r"\bghp_[A-Za-z0-9]{20,}\b"),
+    re.compile(r"\bgho_[A-Za-z0-9]{20,}\b"),
+    re.compile(r"\bghs_[A-Za-z0-9]{20,}\b"),
+    re.compile(r"\bghu_[A-Za-z0-9]{20,}\b"),
+    re.compile(r"\bghr_[A-Za-z0-9]{20,}\b"),
+    # Anthropic API keys.
+    re.compile(r"\bsk-ant-[A-Za-z0-9_-]{20,}\b"),
+    # Bearer tokens in headers. Case-insensitive on the scheme name only.
+    re.compile(r"(?i:Bearer)\s+[A-Za-z0-9._\-+/=]{20,}"),
+    # KEY=value where KEY contains key/secret/token (case-insensitive).
+    # We redact only the value (everything after the first =).
+    re.compile(
+        r"\b([A-Z][A-Z0-9_]*(?:KEY|SECRET|TOKEN|PASSWORD|PASSWD)[A-Z0-9_]*)\s*=\s*\S+",
+        re.IGNORECASE,
+    ),
+]
+
+
+def sanitize(body):
+    count = 0
+    out = body
+    for pat in SECRET_PATTERNS[:-1]:
+        new = pat.sub("[REDACTED:secret]", out)
+        matches = pat.findall(out)
+        if matches:
+            count += len(matches)
+            out = new
+    # KEY=value form: preserve the key, redact the value.
+    last = SECRET_PATTERNS[-1]
+    matches = list(last.finditer(out))
+    if matches:
+        count += len(matches)
+
+        def repl(m):
+            key = m.group(1)
+            return f"{key}=[REDACTED:secret]"
+
+        out = last.sub(repl, out)
+    return out, count
+
+
+SKIP_MARKER = "[historian:skip]"
+
+
+kept = []
+dropped = []
+for chunk in chunks:
+    body = chunk.get("body", "")
+    if drop_skip and SKIP_MARKER in body:
+        dropped.append({
+            "chunk_index": chunk.get("chunk_index"),
+            "reason": "skip_marker",
+        })
+        continue
+    if deny_paths and any(p and p in body for p in deny_paths):
+        dropped.append({
+            "chunk_index": chunk.get("chunk_index"),
+            "reason": "never_index_path",
+        })
+        continue
+    if redact_secrets:
+        redacted, count = sanitize(body)
+    else:
+        redacted, count = body, 0
+    new_chunk = dict(chunk)
+    new_chunk.pop("body", None)
+    new_chunk["body_redacted"] = redacted
+    new_chunk["redaction_count"] = count
+    kept.append(new_chunk)
+
+print(json.dumps({"kept": kept, "dropped": dropped}))
+PY
+}

package/plugins/historian/scripts/lib/historian-storage.sh

@@ -0,0 +1,110 @@
+#!/usr/bin/env bash
+# Storage layout helpers for Historian.
+#
+# Layout (under $ONLOOKER_DIR/historian/<project-key>/):
+#   manifest.json              project metadata (remote_url, repo_root, last_seen_at)
+#   sessions/<session_id>.jsonl  append-only chunk records, one per line
+#
+# Chunk record shape:
+#   { chunk_id, session_id, chunk_index, start_turn_index, end_turn_index,
+#     body_redacted, body_chars, created_at, source, redaction_count }
+#
+# Append-only writes keep the indexing path simple and safe to re-run; if a
+# session is re-indexed (rare; SessionEnd should fire once), callers can
+# truncate the file before appending or accept duplicate chunk records.
+
+historian_storage_root() {
+	local base="${ONLOOKER_DIR:-$HOME/.onlooker}"
+	printf '%s/historian' "$base"
+}
+
+historian_project_dir() {
+	local key="$1"
+	printf '%s/%s' "$(historian_storage_root)" "$key"
+}
+
+historian_sessions_dir() {
+	local key="$1"
+	printf '%s/sessions' "$(historian_project_dir "$key")"
+}
+
+historian_session_file() {
+	local key="$1"
+	local session_id="$2"
+	# Sanitize session_id for filesystem use: strip anything outside
+	# [A-Za-z0-9._-]. session_id comes from the Claude Code hook payload
+	# and is normally a clean ULID-ish string, but guard against
+	# unexpected shapes.
+	local safe
+	safe=$(printf '%s' "$session_id" | tr -cd '[:alnum:]._-')
+	[[ -z "$safe" ]] && safe="unknown"
+	printf '%s/%s.jsonl' "$(historian_sessions_dir "$key")" "$safe"
+}
+
+historian_storage_init() {
+	local key="$1"
+	[[ -z "$key" ]] && return 1
+	local project_dir
+	project_dir=$(historian_project_dir "$key")
+	mkdir -p "$project_dir/sessions" 2>/dev/null
+}
+
+# Usage: historian_storage_write_manifest <key> <remote_url> <repo_root>
+historian_storage_write_manifest() {
+	local key="$1"
+	local remote_url="$2"
+	local repo_root="$3"
+	[[ -z "$key" ]] && return 1
+
+	historian_storage_init "$key" || return 1
+	local manifest_path now
+	manifest_path="$(historian_project_dir "$key")/manifest.json"
+	now=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+	jq -n \
+		--arg key "$key" \
+		--arg remote "$remote_url" \
+		--arg root "$repo_root" \
+		--arg now "$now" \
+		'{
+			project_key: $key,
+			remote_url: (if $remote == "" then null else $remote end),
+			repo_root: (if $root == "" then null else $root end),
+			last_seen_at: $now
+		}' > "$manifest_path" 2>/dev/null
+}
+
+# Append a single chunk record (one JSON line) to a session's file.
+# Usage: historian_storage_append_chunk <key> <session_id> <chunk_json>
+historian_storage_append_chunk() {
+	local key="$1"
+	local session_id="$2"
+	local chunk_json="$3"
+	[[ -z "$key" || -z "$session_id" || -z "$chunk_json" ]] && return 1
+
+	historian_storage_init "$key" || return 1
+	local path
+	path=$(historian_session_file "$key" "$session_id")
+	printf '%s\n' "$chunk_json" >> "$path" 2>/dev/null
+}
+
+# Count chunks for a session. Returns 0 when the file is absent.
+historian_storage_chunk_count() {
+	local key="$1"
+	local session_id="$2"
+	local path
+	path=$(historian_session_file "$key" "$session_id")
+	[[ -f "$path" ]] || { echo 0; return 0; }
+	wc -l < "$path" 2>/dev/null | tr -d ' '
+}
+
+# Reset (truncate) the chunk file for a session. Used when SessionEnd
+# re-runs against a transcript that was previously indexed.
+historian_storage_reset_session() {
+	local key="$1"
+	local session_id="$2"
+	local path
+	path=$(historian_session_file "$key" "$session_id")
+	[[ -f "$path" ]] || return 0
+	: > "$path"
+}

package/plugins/historian/scripts/lib/historian-transcript.sh

@@ -0,0 +1,83 @@
+#!/usr/bin/env bash
+# Transcript reading for Historian.
+#
+# Claude Code records each session's transcript as JSONL where each line
+# is an entry like { "role": "user"|"assistant"|"system", "content": "...",
+# ... }. Historian only embeds user + assistant turns — tool calls and tool
+# results are dropped at this stage so the chunked content stays
+# semantically focused on the conversation.
+
+# Load the transcript and emit a JSON array of normalized turn records:
+#   [
+#     { "turn_index": 0, "role": "user", "content": "..." },
+#     { "turn_index": 1, "role": "assistant", "content": "..." },
+#     ...
+#   ]
+#
+# Returns an empty array when the transcript is absent or unreadable.
+#
+# Usage: historian_transcript_load <transcript_path>
+historian_transcript_load() {
+	local path="${1:-}"
+	[[ -z "$path" || ! -f "$path" ]] && { echo '[]'; return 0; }
+
+	# Filter to user/assistant role entries with non-empty content, keep
+	# their original order (the JSONL is recorded chronologically), and
+	# attach a turn_index. Content may be a string OR an array of content
+	# blocks (Anthropic SDK shape); flatten array forms to text.
+	python3 - "$path" <<'PY'
+import json, sys
+
+path = sys.argv[1]
+out = []
+turn_index = 0
+try:
+    with open(path, "r", encoding="utf-8", errors="replace") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rec = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+            role = rec.get("role") or rec.get("type")
+            if role not in ("user", "assistant"):
+                continue
+            raw = rec.get("content", "")
+            if isinstance(raw, list):
+                # Anthropic content-blocks form. Concatenate the text-typed
+                # blocks; drop tool_use / tool_result entries here.
+                parts = []
+                for block in raw:
+                    if not isinstance(block, dict):
+                        continue
+                    if block.get("type") in (None, "text"):
+                        t = block.get("text") or ""
+                        if t:
+                            parts.append(t)
+                content = "\n\n".join(parts)
+            else:
+                content = str(raw)
+            content = content.strip()
+            if not content:
+                continue
+            out.append({
+                "turn_index": turn_index,
+                "role": role,
+                "content": content,
+            })
+            turn_index += 1
+except OSError:
+    pass
+
+print(json.dumps(out))
+PY
+}
+
+# Return the total content character count across normalized turns.
+# Usage: historian_transcript_char_count <turns_json>
+historian_transcript_char_count() {
+	local turns="${1:-[]}"
+	printf '%s' "$turns" | jq '[.[] | (.content | length)] | add // 0' 2>/dev/null
+}

package/plugins/historian/scripts/lib/historian-ulid.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Minimal ULID generator for Historian chunk IDs.
+#
+# Spec: https://github.com/ulid/spec — 48-bit timestamp + 80-bit randomness,
+# lexicographically sortable, Crockford Base32. Monotonicity within a single
+# millisecond is not required at SessionEnd cadence.
+
+_HISTORIAN_ULID_ALPHABET="0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+
+_historian_ulid_encode() {
+	local n="$1"
+	local len="$2"
+	local out=""
+	local i
+	for ((i = 0; i < len; i++)); do
+		out="${_HISTORIAN_ULID_ALPHABET:$((n % 32)):1}${out}"
+		n=$((n / 32))
+	done
+	printf '%s' "$out"
+}
+
+historian_ulid() {
+	local now_ms
+	if [[ "$(uname)" == "Darwin" ]]; then
+		now_ms=$(python3 -c 'import time; print(int(time.time() * 1000))' 2>/dev/null) \
+			|| now_ms=$(($(date +%s) * 1000))
+	else
+		now_ms=$(date +%s%3N 2>/dev/null) || now_ms=$(($(date +%s) * 1000))
+	fi
+
+	local rand_hi rand_lo
+	rand_hi=$((RANDOM * 32768 + RANDOM))
+	rand_lo=$((RANDOM * 32768 + RANDOM))
+	rand_hi=$(((rand_hi * 256 + RANDOM % 256) & ((1 << 40) - 1)))
+	rand_lo=$(((rand_lo * 256 + RANDOM % 256) & ((1 << 40) - 1)))
+
+	local ts_part hi_part lo_part
+	ts_part=$(_historian_ulid_encode "$now_ms" 10)
+	hi_part=$(_historian_ulid_encode "$rand_hi" 8)
+	lo_part=$(_historian_ulid_encode "$rand_lo" 8)
+
+	printf '%s%s%s' "$ts_part" "$hi_part" "$lo_part"
+}

package/release-please-config.json

@@ -190,6 +190,22 @@
           "jsonpath": "$.version"
         }
       ]
+    },
+    "plugins/historian": {
+      "changelog-path": "CHANGELOG.md",
+      "release-type": "simple",
+      "bump-minor-pre-major": true,
+      "bump-patch-for-minor-pre-major": false,
+      "component": "historian",
+      "draft": false,
+      "prerelease": false,
+      "extra-files": [
+        {
+          "type": "json",
+          "path": ".claude-plugin/plugin.json",
+          "jsonpath": "$.version"
+        }
+      ]
     }
   },
   "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json"