@onlooker-community/ecosystem 0.19.0 → 0.21.0

package/plugins/curator/scripts/lib/curator-storage.sh

@@ -0,0 +1,176 @@
+#!/usr/bin/env bash
+# Storage layout helpers for Curator.
+#
+# Layout (under $ONLOOKER_DIR/curator/<project-key>/):
+#   manifest.json              project metadata (remote_url, repo_root, last_seen_at)
+#   last_cheap_scan.json       watermark: when cheap-tier last ran
+#   last_llm_sweep.json        watermark: when LLM sweep last ran
+#   findings/<ulid>.json       one finding per file (open, acknowledged, or resolved)
+
+# ============================================================================
+# Path helpers
+# ============================================================================
+
+curator_storage_root() {
+	local base="${ONLOOKER_DIR:-$HOME/.onlooker}"
+	printf '%s/curator' "$base"
+}
+
+curator_project_dir() {
+	local key="$1"
+	printf '%s/%s' "$(curator_storage_root)" "$key"
+}
+
+curator_findings_dir() {
+	local key="$1"
+	printf '%s/findings' "$(curator_project_dir "$key")"
+}
+
+curator_storage_init() {
+	local key="$1"
+	[[ -z "$key" ]] && return 1
+	local project_dir
+	project_dir=$(curator_project_dir "$key")
+	mkdir -p "$project_dir/findings" 2>/dev/null
+}
+
+curator_storage_write_manifest() {
+	local key="$1"
+	local remote_url="$2"
+	local repo_root="$3"
+	[[ -z "$key" ]] && return 1
+
+	curator_storage_init "$key" || return 1
+	local manifest_path now
+	manifest_path="$(curator_project_dir "$key")/manifest.json"
+	now=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+
+	jq -n \
+		--arg key "$key" \
+		--arg remote "$remote_url" \
+		--arg root "$repo_root" \
+		--arg now "$now" \
+		'{
+			project_key: $key,
+			remote_url: (if $remote == "" then null else $remote end),
+			repo_root: (if $root == "" then null else $root end),
+			last_seen_at: $now
+		}' > "$manifest_path" 2>/dev/null
+}
+
+# ============================================================================
+# Watermarks
+# ============================================================================
+
+curator_last_cheap_scan_path() {
+	printf '%s/last_cheap_scan.json' "$(curator_project_dir "$1")"
+}
+
+curator_last_llm_sweep_path() {
+	printf '%s/last_llm_sweep.json' "$(curator_project_dir "$1")"
+}
+
+curator_storage_read_watermark() {
+	local path="$1"
+	[[ -f "$path" ]] || return 0
+	jq -r '.scanned_at // empty' "$path" 2>/dev/null
+}
+
+curator_storage_write_watermark() {
+	local path="$1"
+	[[ -z "$path" ]] && return 1
+	mkdir -p "$(dirname "$path")" 2>/dev/null
+	local now
+	now=$(date -u +"%Y-%m-%dT%H:%M:%SZ")
+	jq -n --arg t "$now" '{ scanned_at: $t }' > "$path" 2>/dev/null
+}
+
+# ============================================================================
+# Findings
+# ============================================================================
+
+# Write a finding to disk, keyed by ULID. Dedup is by deduped_hash so a
+# repeat scan that surfaces the same fact does not write a new finding.
+#
+# Usage: curator_storage_write_finding <key> <ulid> <json>
+curator_storage_write_finding() {
+	local key="$1"
+	local id="$2"
+	local json="$3"
+	[[ -z "$key" || -z "$id" || -z "$json" ]] && return 1
+
+	curator_storage_init "$key" || return 1
+	local path
+	path="$(curator_findings_dir "$key")/${id}.json"
+	printf '%s\n' "$json" > "$path" 2>/dev/null && printf '%s' "$path"
+}
+
+# Load all findings for a project key as a JSON array.
+curator_storage_load_findings() {
+	local key="$1"
+	[[ -z "$key" ]] && { echo '[]'; return 0; }
+	local dir
+	dir=$(curator_findings_dir "$key")
+	[[ -d "$dir" ]] || { echo '[]'; return 0; }
+
+	local file all='[]'
+	for file in "$dir"/*.json; do
+		[[ -f "$file" ]] || continue
+		local item
+		item=$(jq '.' "$file" 2>/dev/null) || continue
+		all=$(printf '%s' "$all" | jq --argjson item "$item" '. + [$item]')
+	done
+	printf '%s' "$all"
+}
+
+# Return 0 if a finding with the given dedup hash already exists (open).
+curator_storage_has_finding_with_hash() {
+	local key="$1"
+	local hash="$2"
+	[[ -z "$key" || -z "$hash" ]] && return 1
+	local existing
+	existing=$(curator_storage_load_findings "$key")
+	printf '%s' "$existing" | jq -e --arg h "$hash" '
+		any(.[]; (.deduped_hash // "") == $h and (.status // "open") == "open")
+	' >/dev/null 2>&1
+}
+
+curator_storage_count_open() {
+	local key="$1"
+	local all
+	all=$(curator_storage_load_findings "$key")
+	printf '%s' "$all" | jq '[.[] | select((.status // "open") == "open")] | length' 2>/dev/null
+}
+
+# Open-finding counts grouped by kind. Used by the surfacer to render a
+# pointer like "2 path-broken, 1 date-decayed".
+#
+# jq's group_by groups CONSECUTIVE matches, so the array must be sorted
+# by .kind first or the same kind can produce multiple groups (and the
+# downstream summary double-counts).
+curator_storage_open_counts_by_kind() {
+	local key="$1"
+	local all
+	all=$(curator_storage_load_findings "$key")
+	printf '%s' "$all" | jq -c '
+		[.[] | select((.status // "open") == "open")]
+		| sort_by(.kind)
+		| group_by(.kind)
+		| map({ kind: .[0].kind, count: length })
+		| sort_by(-.count)
+	'
+}
+
+# Hash a finding's identity-relevant fields. Two findings with the same
+# kind + memory_file + matched_phrase (where applicable) share a hash.
+# Plain shasum input — no expensive normalization needed.
+curator_finding_hash() {
+	local raw="$1"
+	if command -v shasum >/dev/null 2>&1; then
+		printf '%s' "$raw" | shasum -a 256 2>/dev/null | cut -c1-16
+	elif command -v sha256sum >/dev/null 2>&1; then
+		printf '%s' "$raw" | sha256sum 2>/dev/null | cut -c1-16
+	else
+		return 1
+	fi
+}

package/plugins/curator/scripts/lib/curator-ulid.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+# Minimal ULID generator for Curator finding IDs.
+#
+# Spec: https://github.com/ulid/spec — 48-bit timestamp + 80-bit randomness,
+# lexicographically sortable, Crockford Base32. Monotonicity within a single
+# millisecond is not required; findings are written infrequently.
+
+_CURATOR_ULID_ALPHABET="0123456789ABCDEFGHJKMNPQRSTVWXYZ"
+
+_curator_ulid_encode() {
+	local n="$1"
+	local len="$2"
+	local out=""
+	local i
+	for ((i = 0; i < len; i++)); do
+		out="${_CURATOR_ULID_ALPHABET:$((n % 32)):1}${out}"
+		n=$((n / 32))
+	done
+	printf '%s' "$out"
+}
+
+curator_ulid() {
+	local now_ms
+	if [[ "$(uname)" == "Darwin" ]]; then
+		now_ms=$(python3 -c 'import time; print(int(time.time() * 1000))' 2>/dev/null) \
+			|| now_ms=$(($(date +%s) * 1000))
+	else
+		now_ms=$(date +%s%3N 2>/dev/null) || now_ms=$(($(date +%s) * 1000))
+	fi
+
+	local rand_hi rand_lo
+	rand_hi=$((RANDOM * 32768 + RANDOM))
+	rand_lo=$((RANDOM * 32768 + RANDOM))
+	rand_hi=$(((rand_hi * 256 + RANDOM % 256) & ((1 << 40) - 1)))
+	rand_lo=$(((rand_lo * 256 + RANDOM % 256) & ((1 << 40) - 1)))
+
+	local ts_part hi_part lo_part
+	ts_part=$(_curator_ulid_encode "$now_ms" 10)
+	hi_part=$(_curator_ulid_encode "$rand_hi" 8)
+	lo_part=$(_curator_ulid_encode "$rand_lo" 8)
+
+	printf '%s%s%s' "$ts_part" "$hi_part" "$lo_part"
+}

package/plugins/historian/docs/adr/001-local-embeddings-only.md

@@ -0,0 +1,96 @@
+# ADR-001: Historian Uses Local Embeddings by Default
+
+- Status: Accepted
+- Date: 2026-06-02
+- Deciders: Meagan
+- Tags: historian, embeddings, privacy, data-egress, local-first
+
+## Context and Problem Statement
+
+Historian's central operation is embedding session transcripts. Two timeframes consume embeddings: indexing (once per session, in bulk) and retrieval (potentially every user prompt). The model choice — local vs. remote — drives privacy posture, cost shape, latency profile, retrieval quality, and dependency surface, all at once.
+
+Remote embedding APIs (Voyage, OpenAI, Cohere, Anthropic's voyage models) are easier to set up, produce higher-quality embeddings, and impose no local compute requirement. They also mean every session's transcript content — including paths, identifiers, code snippets, internal terminology, and anything the user said in conversation — is sent over the wire to a third party, in the moment-by-moment course of normal use. Even a well-intentioned user with a benign API key may not realize how much they are streaming out, because the operation is invisible.
+
+Local embedding models (`nomic-embed-text` via ollama, `bge-small` via fastembed) sacrifice some retrieval quality and impose a one-time local setup cost in exchange for keeping all transcript content on the user's machine. Modern small embedding models are good enough for the precedent-retrieval task historian is designed for.
+
+This ADR records why local embedding is the architectural baseline, and what the remote opt-in path requires.
+
+## Decision Drivers
+
+- **Transcript content is the most sensitive thing historian touches.** It contains code paths the user is actively working on, names of internal systems, mid-thought work product, mistakes, and side discussions. A user adopting historian for productivity gains should not, as a side effect, opt into per-prompt transcript egress.
+- **Per-prompt latency matters.** Retrieval runs on `UserPromptSubmit` — a hot path. A remote round-trip introduces ~100–400ms of latency every time. A local embedding model adds ~20–60ms. Cumulatively across a working day this is material.
+- **Per-prompt cost matters.** Even at $0.00002 per 1K tokens, retrieval that fires on every long-enough prompt is a non-trivial recurring cost. Local embedding has a one-time setup cost and is free thereafter.
+- **Retrieval-quality budget.** Historian's surface ("here is one past chunk that looks similar") is informational, not load-bearing. The model can ignore a marginal match. Top-1 retrieval against `nomic-embed-text` is good enough for the "have we seen this before?" signal. The marginal quality gain from a frontier embedding model does not unlock new capabilities here.
+- **Fail-soft requirement.** Plugins in the Onlooker ecosystem must not block sessions they were not invited to. A remote-by-default embedder means a failed network call is a failed retrieval — silently. A local embedder still works without a network.
+- **Opt-in pattern for sensitive plugins.** Compass's `data_egress` block makes egress an explicit configuration decision. Historian inherits this pattern: the user must affirm both `embedder.backend: "remote"` AND `data_egress.allow_remote_embedding: true` to send transcript content off-machine. Two independent affirmations.
+
+## Considered Options
+
+1. **Remote-by-default, with a local fallback.** Easier first-run UX. The user gets best-in-class retrieval immediately. Local fallback covers offline scenarios but most users never see it.
+2. **Local-by-default, with a remote opt-in.** Privacy-by-default. Higher first-run friction (the user must install or already have ollama). Remote opt-in is gated by an explicit egress affirmation.
+3. **No embeddings at all — keyword search only.** Cheapest, simplest, no dependency. Retrieval quality is poor for paraphrased prompts ("the flaky test" vs. "tests timing out intermittently"). Misses the semantic-recall use case that motivated historian.
+4. **Hybrid: local for embedding, remote for reranking.** Embed everything locally; for the top-N candidates, call a remote reranker to break ties. Reduces remote egress dramatically while keeping high retrieval quality. More complex; defers to a future iteration.
+
+## Decision
+
+We adopt **Option 2: local embedding (via ollama with `nomic-embed-text`) is the default backend. Remote embedding is opt-in and requires both `embedder.backend: "remote"` and `data_egress.allow_remote_embedding: true`.**
+
+Backends, in preference order:
+
+1. **ollama with `nomic-embed-text`.** Recommended default. 768-dim embeddings, ~110MB model, fully local, good cosine-similarity behavior on prose and code-mixed content. Historian's `/historian setup` skill walks the user through `ollama pull nomic-embed-text`.
+2. **fastembed via Python sidecar.** Fallback for users without ollama. ONNX-based; slower startup but no separate runtime daemon.
+3. **Remote.** Opt-in only. Requires two-key affirmation. When enabled, historian still stores all embeddings and chunk bodies locally — only the embedding *computation* leaves the machine.
+
+The two-key affirmation pattern (`embedder.backend: "remote"` AND `data_egress.allow_remote_embedding: true`) means a user who copies a config snippet that flips one key still gets a hard fail at startup rather than silent egress. The mismatch logs `historian.config.warning` with the specific message "remote embedding configured but egress not allowed."
+
+Option 1 is rejected because the user-experience benefit (no install step) is small relative to the privacy cost (silent transcript egress on every prompt). A user who genuinely wants remote-grade quality can flip the two-key opt-in.
+
+Option 3 is rejected because keyword retrieval misses the use cases historian is designed for. The motivating examples (flaky-test deja-vu, "I tried this approach already and it didn't work") all involve paraphrase across sessions.
+
+Option 4 (hybrid local + remote reranker) is deferred. It is appealing — most egress avoided, near-frontier retrieval quality — but it triples the configuration surface and requires both backends to be working. The simpler local-only default is the right starting point; hybrid is a natural future opt-in.
+
+## Consequences
+
+### Positive
+
+- Transcript content stays on the user's machine by default. No silent egress.
+- Per-prompt retrieval latency stays in the 30–100ms range with a warm ollama process. The hot path is not slowed by network.
+- Retrieval cost is zero after model download.
+- The plugin works offline.
+- The two-key egress affirmation forces an explicit decision at configuration time, matching compass's pattern. A user who wants remote retrieval can opt in cleanly; a user who doesn't is never accidentally signed up.
+- Embedding-storage is forward-compatible with future cross-project sharing (`source: "team:<id>"`) without inheriting "we already sent everything to a third party" as a starting condition.
+
+### Negative
+
+- First-run UX requires installing ollama (~150MB runtime) and pulling the model (~110MB). The `/historian setup` skill streamlines this but it is friction relative to "edit a config and go."
+- Retrieval quality is lower than frontier embedding models on edge cases (e.g., highly idiosyncratic terminology, very short prompts, code-heavy chunks). For most prose-shaped recall the gap is small.
+- Local embedding consumes ~200MB of resident memory while the ollama process runs. On constrained machines this is noticeable.
+- A user who wants remote retrieval has to flip two keys, not one. This is intentional but adds documentation surface.
+
+### Neutral
+
+- Ollama as the default runtime ties historian to a specific local-LLM ecosystem. Fastembed as a fallback hedges this dependency. A pure-Rust backend (e.g., direct ONNX) is plausible if ollama proves to be the wrong bet.
+- The decision to store chunk bodies locally alongside embeddings is independent of this ADR but mutually reinforcing — co-located chunk bodies mean retrieval results can render without any network round-trip.
+
+## Implementation Notes
+
+- `embedder.backend: "ollama"` is checked first. If ollama is not on PATH or the configured host is unreachable, historian falls through to fastembed automatically. If fastembed is also unavailable, historian emits `historian.embedder.unavailable` and disables both indexing and retrieval for the session — no partial behavior.
+- `embedder.backend: "remote"` does NOT auto-fall-through if the remote endpoint is unreachable. A misconfigured or down remote backend produces `historian.embedder.unavailable` and stops; this prevents silent fallback that might violate the user's expectation of where embeddings happen.
+- The two-key affirmation check runs at `SessionStart`. A configuration with `backend: "remote"` and `allow_remote_embedding: false` logs a warning and forces `backend: "ollama"` for the session. The user is told their config has a mismatch.
+- The `nomic-embed-text` cosine-similarity distribution on this corpus puts unrelated chunks around 0.30–0.45 and related chunks around 0.55–0.85. `min_similarity: 0.55` is the default floor. The `/historian calibrate` skill (mentioned as an open question in the design doc) is the per-project tuning surface.
+- Remote backends, if enabled, must respect the same chunk-body redaction pipeline as local backends. Redaction is applied before any network call. Verified by an emitted `historian.chunk.sanitized` event preceding the embed.
+- Re-embedding after a backend change is not automatic. The vector store has no `embedding_model` column today; mixing vectors from two different models silently degrades retrieval. A future migration adds the column and forces a re-index on backend change. Until then, the docs warn against changing `embedder.backend` after data is stored.
+
+## Validation
+
+- A test session with a transcript of ~10K characters should index in ≤2 seconds end-to-end on ollama with `nomic-embed-text` running locally on a typical macOS dev laptop.
+- A `UserPromptSubmit` retrieval against a vector store with 500 chunks should complete in under 100ms wall-clock.
+- A misconfigured `backend: "remote"` with `allow_remote_embedding: false` must produce `historian.config.warning` at SessionStart and operate as if `backend: "ollama"` for the session. No network call may be made.
+- A purge via `/historian purge all` must remove all chunks for the project key and leave no embeddings behind. Verified by `historian.purge.completed` followed by an empty `historian.stats` report.
+
+## References
+
+- Compass design — precedent for explicit `data_egress` configuration blocks (`plugins/compass/docs/design.md#data-egress`)
+- Compass `data_egress` discussion in the design doc — the "near-zero egress" mode template historian inherits
+- Memory architecture overview (`docs/memory-architecture.md`)
+- Historian design (`../design.md`)

package/plugins/historian/docs/design.md

@@ -0,0 +1,317 @@
+# Historian — Plugin Design
+
+**Plugin name:** `historian`
+**Tagline:** *Recalls past sessions when they matter.*
+**Status:** Design (pre-implementation)
+
+Historian is the episodic memory layer. At `SessionEnd`, it chunks and embeds the session transcript and stores the vectors locally. At `UserPromptSubmit`, it computes the prompt's embedding, retrieves the most similar past chunks (above a similarity threshold), and surfaces them as `additionalContext` — "you worked on something like this in session X." The goal is precedent recall, not distillation: where librarian preserves the *conclusion* of a past session as a typed memory, historian preserves the *verbatim shape* of the conversation so future sessions can see what was actually tried, said, and rejected.
+
+It sits in the [memory architecture](../../../docs/memory-architecture.md) parallel to librarian and curator. It operates on its own substrate (a local vector store) and does not write to the typed memory store. See [ADR-001](adr/001-local-embeddings-only.md) for the embeddings-locality decision — the design assumes a local embedding model and a local vector store.
+
+---
+
+## Failure Modes Historian Addresses
+
+**A — "We've solved this exact bug before."** A user hits a flaky test and asks the model to investigate. The model debugs from scratch. Three months ago, the same flake was investigated in this repo, the root cause was identified, and the fix landed. Historian surfaces the past session's relevant chunks so the model can short-circuit to known-good context.
+
+**B — "I tried X already and it didn't work."** A user begins exploring an approach; the model elaborates the approach in detail. Two weeks ago the same approach was tried and abandoned. Without historian, this is invisible — the typed memory store may have a `feedback` entry ("X doesn't work here") but the *why* is in the transcript. Historian retrieves the dead-end discussion, not just the conclusion.
+
+**C — "What was the rationale we settled on?"** A code shape exists because of a past discussion. The commit message has "use the cached path" but no rationale. The typed memory store has "use cached path for hot writes" but no nuance. The original session has 40 turns of weighing tradeoffs. Historian retrieves the rationale-bearing chunks.
+
+**D — "We were in a similar situation in the other repo."** Cross-repo recall — out of scope by default. Historian is per-project. Cross-project retrieval is an opt-in mode noted in [Open Questions](#open-questions).
+
+---
+
+## Architecture
+
+```
+SessionEnd hook fires
+        │
+        ▼
+┌──────────────────────┐
+│   Transcript Reader  │  reads full session transcript JSONL
+└─────────┬────────────┘
+          │
+          ▼
+┌──────────────────────┐
+│   Chunker            │  rolling-window chunks; preserves turn boundaries
+│                      │  default: 600-token chunks with 100-token overlap
+└─────────┬────────────┘
+          │
+          ▼
+┌──────────────────────┐
+│   Sanitizer          │  redacts patterns: API keys, tokens, .env content
+│                      │  drops chunks marked [historian:skip] by the user
+└─────────┬────────────┘
+          │
+          ▼
+┌──────────────────────┐
+│   Local Embedder     │  ollama (default model: nomic-embed-text)
+│                      │  fallback: fastembed via Python sidecar
+└─────────┬────────────┘
+          │
+          ▼
+┌──────────────────────┐
+│   Vector Store       │  sqlite-vec at
+│                      │  ~/.onlooker/historian/<project-key>/vectors.db
+└──────────────────────┘
+
+UserPromptSubmit hook fires
+        │
+        ▼
+┌──────────────────────┐
+│   Rate Gate          │  per-turn budget; cooldown after recent retrieval
+└─────────┬────────────┘
+          │
+          ▼
+┌──────────────────────┐
+│   Query Embedder     │  embed the current user prompt
+└─────────┬────────────┘
+          │
+          ▼
+┌──────────────────────┐
+│   ANN Lookup         │  top-K candidates by cosine; filter by min_similarity
+│                      │  filter by age (configurable max_age_days)
+└─────────┬────────────┘
+          │ ≥1 result
+          ▼
+┌──────────────────────┐
+│   Surfacer           │  emits additionalContext block with the top match
+│                      │  ("Similar past session 47d ago — excerpt + link")
+└──────────────────────┘
+```
+
+### Transcript Reader
+
+At `SessionEnd`, reads the full transcript from `transcript_path` in the hook payload (same field compass and tribunal use). Parses as JSONL. Filters to user and assistant messages only — tool calls and tool results are dropped at this stage to keep the embedded content semantically focused. The resulting message list is the input to the chunker.
+
+If the transcript is shorter than `min_transcript_chars_to_index` (default: 1200), historian skips indexing — the session is too short to plausibly produce a useful precedent.
+
+### Chunker
+
+Chunks the message list into overlapping windows:
+
+- **Chunk size:** `chunk_target_tokens` (default: 600). Measured via the local tokenizer used by the embedding model.
+- **Overlap:** `chunk_overlap_tokens` (default: 100). Ensures cross-chunk concepts aren't sliced apart.
+- **Turn-boundary respect:** Chunks never split mid-turn. The chunker accumulates turns until adding the next would exceed the target; then it emits a chunk and starts a new one. If a single turn exceeds the target, it is emitted as-is and the next chunk begins after it (no mid-turn split).
+- **Metadata per chunk:** `session_id`, `chunk_index`, `start_turn_id`, `end_turn_id`, `created_at`, `chunk_token_count`.
+
+### Sanitizer
+
+Before embedding, each chunk is scanned for:
+
+1. **Secret patterns.** AWS-style keys (`AKIA...`), bearer tokens (`Bearer ey...`), Anthropic API keys, GitHub PATs (`ghp_...`), `.env`-style assignments (`SECRET_KEY=...`). Matches are replaced with `[REDACTED:secret]`.
+2. **`[historian:skip]` markers.** A chunk containing the literal string `[historian:skip]` is dropped entirely. This is the in-band escape for users to mark sensitive turns.
+3. **Path-aware redaction.** When a chunk references a path in `historian.never_index_paths`, the chunk is dropped. This is the path-level escape for "this directory's discussions should never be indexed."
+
+Redacted chunks are still embedded (the surrounding content has value); dropped chunks are not. Both decisions are logged as `historian.chunk.sanitized` and `historian.chunk.dropped` events with reasons.
+
+### Local Embedder
+
+Embedding runs locally to keep transcript content off the wire and avoid per-prompt API cost. See [ADR-001](adr/001-local-embeddings-only.md) for the full reasoning.
+
+Backends, in preference order:
+
+1. **ollama with `nomic-embed-text`.** Ollama is the recommended runtime. The `nomic-embed-text` model (~110MB) produces 768-dim embeddings, works offline, and is cheap to install. Historian's setup skill walks users through `ollama pull nomic-embed-text` if needed.
+2. **fastembed via Python sidecar.** A Python subprocess hosting fastembed (ONNX-based). Slower startup, no external runtime dependency. Used when ollama is not on PATH.
+3. **Disabled.** If neither backend is available, historian emits `historian.embedder.unavailable` once per session and skips indexing. Retrieval also degrades to "no results."
+
+Historian does not call a remote embedding API by default. Users who want a remote embedding model (e.g., for cross-repo retrieval or higher quality) can opt in via `embedder.backend: "remote"`, which gates on `data_egress.allow_remote_embedding: true` to make the egress decision explicit.
+
+### Vector Store
+
+`sqlite-vec` at `~/.onlooker/historian/<project-key>/vectors.db`. Schema:
+
+```sql
+CREATE TABLE chunks (
+  chunk_id TEXT PRIMARY KEY,            -- ULID
+  session_id TEXT NOT NULL,
+  chunk_index INTEGER NOT NULL,
+  start_turn_id TEXT,
+  end_turn_id TEXT,
+  body_redacted TEXT NOT NULL,          -- post-sanitizer text
+  created_at TEXT NOT NULL,
+  source TEXT NOT NULL                  -- "local" today; future "team:<id>" etc.
+);
+CREATE VIRTUAL TABLE chunks_vec USING vec0(
+  embedding FLOAT[768]
+);
+CREATE INDEX idx_chunks_session ON chunks(session_id);
+CREATE INDEX idx_chunks_created ON chunks(created_at);
+```
+
+The chunk body (`body_redacted`) is stored alongside the embedding so retrieval can return the actual text without re-reading the transcript. Storage cost is bounded by retention: chunks older than `retention_days` (default: 365) are pruned on a daily-cap basis.
+
+### Rate Gate
+
+At `UserPromptSubmit`:
+
+1. Skip if `disabled` in settings.
+2. Skip if a retrieval has run in the last `cooldown_seconds` (default: 60) for this session. Avoids spamming retrieval on rapid-fire prompts.
+3. Skip if more than `max_retrievals_per_session` (default: 5) have run this session. A precedent recall is most useful in the first handful of turns; later turns are usually deep in the current work and don't benefit.
+4. Skip if the prompt is shorter than `min_prompt_chars` (default: 60). Short prompts ("ok", "next", "do it") have no semantic signal.
+
+Each skip emits `historian.retrieval.skipped` with the reason.
+
+### Query Embedder, ANN Lookup, and Surfacer
+
+For retrievals that pass the rate gate:
+
+1. Embed the prompt using the same backend as indexing.
+2. `vec0` cosine-similarity lookup, top `retrieval_top_k` (default: 5).
+3. Filter to candidates with `similarity >= min_similarity` (default: 0.55, calibrated against `nomic-embed-text` cosine distribution).
+4. Filter to candidates within `max_age_days` (default: 180). Older chunks are deprioritized rather than dropped — they appear with a "long ago" hint in the surfaced context.
+5. Filter out chunks from the current session (a session retrieving itself is a degenerate case).
+
+The surfacer emits `additionalContext` of the form:
+
+> Historian: a prompt 47 days ago looked similar. Excerpt (session 01J…):
+>
+> > [chunk text, truncated to 400 chars]
+>
+> Full session: `~/.onlooker/historian/<project-key>/sessions/<session_id>/transcript.json` (preserved on `historian.session.archive: true`; otherwise transcript reference only).
+
+Only the top result is surfaced inline. The skill `/historian recall <query>` lets the user inspect more candidates.
+
+---
+
+## Integration Points
+
+**Archivist.** Independent. Archivist preserves session-level distillations; historian preserves raw chunks. Same source (transcript) but different storage and different retrieval semantics.
+
+**Librarian.** Independent at runtime. A future enhancement: when librarian classifies an artifact as "session-only — don't promote," historian could weight its source chunks lower in retrieval (the user has already signaled they're not durable). Deferred.
+
+**Curator.** Independent. Curator audits the typed memory store; historian's substrate is the vector DB.
+
+**Ecosystem substrate.** Historian writes to its own sub-path under `~/.onlooker/` and emits events via `onlooker-event.mjs`. No new substrate dependencies.
+
+**Compass / Tribunal / Echo / Warden / Governor.** No interaction.
+
+---
+
+## Configuration (`config.json`)
+
+```json
+{
+  "plugin_name": "historian",
+  "storage_path": "${ONLOOKER_DIR:-$HOME/.onlooker}",
+  "historian": {
+    "enabled": false,
+    "indexing": {
+      "trigger": "SessionEnd",
+      "min_transcript_chars_to_index": 1200,
+      "chunk_target_tokens": 600,
+      "chunk_overlap_tokens": 100,
+      "retention_days": 365,
+      "prune_daily_cap_chunks": 5000
+    },
+    "embedder": {
+      "backend": "ollama",
+      "ollama": {
+        "model": "nomic-embed-text",
+        "host": "http://127.0.0.1:11434",
+        "request_timeout_seconds": 8
+      },
+      "fastembed": {
+        "model": "BAAI/bge-small-en-v1.5",
+        "sidecar_command": "python3 -m historian_fastembed"
+      },
+      "remote": {
+        "enabled": false,
+        "provider": "voyage",
+        "model": "voyage-3-lite",
+        "note": "Remote embedding sends transcript chunks to a third-party API. Requires data_egress.allow_remote_embedding: true to take effect."
+      }
+    },
+    "sanitization": {
+      "redact_secret_patterns": true,
+      "drop_skip_marker": true,
+      "never_index_paths": []
+    },
+    "retrieval": {
+      "trigger": "UserPromptSubmit",
+      "cooldown_seconds": 60,
+      "max_retrievals_per_session": 5,
+      "min_prompt_chars": 60,
+      "retrieval_top_k": 5,
+      "min_similarity": 0.55,
+      "max_age_days": 180
+    },
+    "surfacer": {
+      "surface_top_n": 1,
+      "excerpt_chars_max": 400,
+      "include_age_hint": true
+    },
+    "data_egress": {
+      "allow_remote_embedding": false,
+      "note": "When false, embedding stays local. When true, transcript chunks are sent to the configured remote provider for embedding only — chunks are not stored remotely."
+    },
+    "session_archive": {
+      "enabled": false,
+      "note": "When true, the full transcript at session end is copied to ~/.onlooker/historian/<key>/sessions/<session_id>/transcript.json so retrieval surfaces can link to the source. When false, only chunk bodies are retained."
+    }
+  }
+}
+```
+
+---
+
+## Events
+
+| Event | Trigger | Key payload fields |
+|---|---|---|
+| `historian.indexing.started` | SessionEnd indexing run begins | `session_id`, `transcript_chars` |
+| `historian.indexing.completed` | Run succeeds | `chunks_indexed`, `chunks_dropped`, `duration_ms` |
+| `historian.indexing.skipped` | Indexing skipped | `reason: too_short\|embedder_unavailable\|disabled` |
+| `historian.chunk.sanitized` | Secret patterns redacted in a chunk | `chunk_id`, `redaction_count` |
+| `historian.chunk.dropped` | Chunk dropped entirely | `reason: skip_marker\|never_index_path` |
+| `historian.embedder.unavailable` | Backend unreachable | `backend`, `error_summary` |
+| `historian.retrieval.started` | UserPromptSubmit retrieval begins | `prompt_chars` |
+| `historian.retrieval.skipped` | Skipped by rate gate | `reason: cooldown\|budget\|short_prompt\|disabled` |
+| `historian.retrieval.empty` | No candidates above similarity floor | `top_similarity`, `min_similarity` |
+| `historian.retrieval.surfaced` | A precedent was surfaced as additionalContext | `chunk_id`, `similarity`, `age_days` |
+| `historian.prune.completed` | Daily retention prune ran | `chunks_pruned`, `chunks_remaining` |
+| `historian.purge.completed` | User-triggered purge ran | `scope: session\|date_range\|all`, `chunks_purged` |
+
+---
+
+## Skills
+
+**`/historian setup`** — checks for ollama on PATH, offers to install (or run the equivalent for fastembed), pulls the embedding model, and writes a confirmation to the storage dir.
+
+**`/historian recall <query>`** — runs an ad-hoc retrieval against the current project's vector store. Returns the top K matches with similarity scores and full chunk bodies. Useful for "remind me what I tried last time" queries that don't naturally surface during a session.
+
+**`/historian purge`** — interactive purge with three scopes: `session <id>` (remove all chunks from one session), `before <date>` (remove all chunks older than a date), `all` (full reset for this project). Always requires explicit user confirmation.
+
+**`/historian stats`** — reports vector store size, chunk count, oldest chunk date, embedding model, last index run, last retrieval, retrieval-hit rate over last 30 days.
+
+---
+
+## Open Questions
+
+1. **Cross-project retrieval.** A precedent in repo A may be relevant in repo B (e.g., the user solved a similar Postgres deadlock in two different services). Historian is per-project today. A `team:<id>` source mode could allow shared vector stores, but it introduces multi-user privacy concerns and a new substrate dependency. Deferred.
+
+2. **Retrieval-hit calibration.** `min_similarity = 0.55` is a guess based on `nomic-embed-text`'s typical cosine distribution. A `/historian calibrate` skill could label a small set of past prompt-chunk pairs as relevant/irrelevant and tune the threshold per-project.
+
+3. **Index-time vs. retrieval-time redaction.** Redaction at index time is permanent and safe. Retrieval-time redaction would let users tune redaction rules without re-indexing. The asymmetry: a secret indexed today can't be redacted later without a re-embed pass. The design picks index-time redaction as the default (irreversibility is the safer error) and leaves a re-embed path for the rare case of needing to update rules.
+
+4. **Chunk overlap policy on long single turns.** A 4000-token assistant turn becomes a single chunk (no mid-turn split). The next chunk begins after it, losing the trailing-context overlap. Acceptable today; a "soft split" mode that breaks at paragraph boundaries within long turns is a future option.
+
+5. **Session-archive storage cost.** With `session_archive: true`, the full transcript is preserved per session — typically tens of MB per active project per month. The retention cap applies to chunks, not archived transcripts; a separate `session_archive_retention_days` setting may be needed.
+
+6. **Embedding-model versioning.** Re-running indexing with a different embedding model produces vectors in a different geometric space. The vector store has no concept of model version today. Adding `embedding_model` and `embedding_dim` columns and filtering retrieval by model match is straightforward but not yet decided.
+
+7. **Coordination with `~/.onlooker/logs/onlooker-events.jsonl`.** The JSONL log itself contains a rich, structured record of past sessions. Historian could index event payloads (decisions, dead-ends, findings) alongside or instead of raw transcripts. The case against: the transcript already contains the conversation those events summarize; indexing both is duplication.
+
+8. **Interaction with compaction.** A long session that compacts mid-flow has a truncated transcript at `SessionEnd`. Historian only sees the post-compaction tail. If pre-compaction content was important, it lives only in archivist. Whether historian should also index archivist artifacts (as a complement to transcript chunks) is a deferred design question.
+
+---
+
+## Non-Goals
+
+- Does not call remote embedding APIs by default — local embedding is the architectural baseline (see [ADR-001](adr/001-local-embeddings-only.md)).
+- Does not write to the typed memory store — that is librarian's job.
+- Does not distill or summarize past sessions — preservation of verbatim shape is the point.
+- Does not perform cross-project retrieval by default.
+- Does not block any tool call — surfacer is informational only.
+- Does not retain transcripts beyond chunk bodies unless `session_archive: true` is explicitly enabled.

package/plugins/librarian/.claude-plugin/plugin.json

@@ -0,0 +1,14 @@
+{
+  "name": "librarian",
+  "version": "0.1.0",
+  "description": "Consolidation layer between archivist's per-session artifacts and the user's durable typed memory store. Detects which session decisions, dead-ends, and open questions deserve to live across sessions, classifies them into the user/feedback/project/reference types, and queues them as proposals for explicit confirmation. Auto-promotion is opt-in. Builds on the Onlooker ecosystem plugin.",
+  "author": {
+    "name": "Onlooker Community",
+    "url": "https://onlooker.dev"
+  },
+  "homepage": "https://onlooker.dev",
+  "repository": "https://github.com/onlooker-community/ecosystem",
+  "license": "MIT",
+  "skills": [],
+  "agents": []
+}