create-egregore 0.20.2 → 0.21.0-runtime-mvp.2

package/runtime/pi/bin/search.sh

@@ -1,68 +1,44 @@
 #!/usr/bin/env bash
-# Hybrid search over shared memory — powered by qmd (BM25 + vector + rerank, fully local).
+# Compatibility shell for Egregore's Local Retriever.
 # Usage:
-#   bash bin/search.sh query "text" [-n N] [--fast|--semantic] [--all]
+#   bash bin/search.sh query "text" [-n N] [--fast|--semantic] [--recent-days N] [--min-evidence N] [--compact] [--context-packet] [--enrich] [--all] [--new-search]
 #   bash bin/search.sh reindex [--embed]
+#   bash bin/search.sh install
 #   bash bin/search.sh status
+#   bash bin/search.sh start
+#   bash bin/search.sh stop
+#   bash bin/search.sh open memory/path/to/source.md
 #
-# The memory repo is registered as a qmd collection named "{slug}-memory".
-# Every call is scoped to that collection with -c, so other qmd collections
-# on the machine are never read or modified.
-#
-# Modes:
-#   (default)   qmd query   — hybrid BM25 + vector + rerank (needs local models, ~2GB one-time download)
-#   --fast      qmd search  — BM25 keyword only, instant, no models
-#   --semantic  qmd vsearch — vector similarity only (needs embeddings: reindex --embed)
-# Hybrid/semantic fall back to BM25 automatically if models or embeddings are missing.
+# The shell owns the established user-facing output. Retrieval infrastructure
+# is isolated behind the Egregore Retriever adapter.
+
+set -u
 
 SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
-export NODE_NO_WARNINGS=1
-# Do NOT set CI=1 here — it disables qmd's LLM ops (query expansion, rerank).
+MEM_PATH="$(readlink -f "$SCRIPT_DIR/memory" 2>/dev/null || true)"
 
-# qmd's progress spinner can leak into stdout when it misdetects a TTY;
-# strip ANSI escapes and spinner frames from anything we return.
-_clean() {
-  sed -E $'s/\x1b\\[[0-9;?]*[A-Za-z]//g' | grep -v 'Gathering information' || true
+_runtime() {
+  EGREGORE_ROOT="$SCRIPT_DIR" \
+    PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}" \
+    python3 -m egregore_runtime.harness_cli "$@"
 }
 
-SLUG=$(jq -r '.slug // "egregore"' "$SCRIPT_DIR/egregore.json" 2>/dev/null || echo "egregore")
-COLLECTION="${SLUG}-memory"
-MEM_PATH="$(readlink -f "$SCRIPT_DIR/memory" 2>/dev/null || true)"
-
-# qmd 2.5.x may render --full-path results as absolute `**file:**` paths,
-# while older output uses qmd:// URIs. Normalize both to the canonical
-# memory/... form users can open from an Egregore checkout, and parse both
-# forms when counting hits or attaching graph state.
-_normalize_file_paths() {
-  local line absolute_prefix uri_prefix
-  absolute_prefix="**file:** \`$MEM_PATH/"
-  uri_prefix="**file:** \`qmd://${COLLECTION}/"
-  while IFS= read -r line || [ -n "$line" ]; do
-    if [[ "$line" == "$absolute_prefix"* ]]; then
-      printf '**file:** `memory/%s\n' "${line#"$absolute_prefix"}"
-    elif [[ "$line" == "$uri_prefix"* ]]; then
-      printf '**file:** `memory/%s\n' "${line#"$uri_prefix"}"
-    else
-      printf '%s\n' "$line"
-    fi
-  done
+# Explicit index provisioning/rebuild remains a maintenance compatibility
+# surface. Normal harness query/open/lifecycle traffic never enters this
+# implementation-specific bridge.
+_maintenance_runtime() {
+  EGREGORE_ROOT="$SCRIPT_DIR" \
+    PYTHONPATH="$SCRIPT_DIR${PYTHONPATH:+:$PYTHONPATH}" \
+    python3 -m egregore_runtime.adapters.qmd_cli "$@"
 }
 
 _result_paths() {
-  local line path canonical_prefix absolute_prefix uri_prefix
+  local line path canonical_prefix
   canonical_prefix="**file:** \`memory/"
-  absolute_prefix="**file:** \`$MEM_PATH/"
-  uri_prefix="**file:** \`qmd://${COLLECTION}/"
   while IFS= read -r line || [ -n "$line" ]; do
     path=""
-    if [[ "$line" == "$canonical_prefix"* ]]; then
-      path="${line#"$canonical_prefix"}"
-    elif [[ "$line" == "$absolute_prefix"* ]]; then
-      path="${line#"$absolute_prefix"}"
-    elif [[ "$line" == "$uri_prefix"* ]]; then
-      path="${line#"$uri_prefix"}"
-    elif [[ "$line" == *"qmd://${COLLECTION}/"* ]]; then
-      path="${line#*"qmd://${COLLECTION}/"}"
+    if [[ "$line" == *"$canonical_prefix"* ]]; then
+      path="${line#*"$canonical_prefix"}"
     fi
     path="${path%%\`*}"
     case "$path" in
@@ -71,81 +47,48 @@ _result_paths() {
   done | sort -u
 }
 
-QMD_VERSION="2.5.3"  # pinned — bump deliberately, never float @latest
-
-_qmd() {
-  if command -v qmd >/dev/null 2>&1; then
-    qmd "$@"
-  else
-    npx -y "@tobilu/qmd@${QMD_VERSION}" "$@"
+# Legacy explicit graph annotation remains available while the graph is moved
+# behind its projection boundary. It is intentionally never selected by the
+# default Local retrieval path.
+#
+# An instance whose Runtime MVP upgrade has been activated retrieves through
+# Runtime/QMD exclusively: even the explicit --enrich flag makes zero graph
+# calls there. The dormant graph configuration is preserved untouched for
+# rollback only.
+_upgrade_instance_key() {
+  # Mirrors egregore_runtime.upgrade.instance_key: stable org identity
+  # composed with the main installation path. No org_id → fail closed.
+  local org main gitdir
+  org=$(jq -r '.org_id // empty' "$SCRIPT_DIR/egregore.json" 2>/dev/null)
+  [ -n "$org" ] || return 1
+  main="$SCRIPT_DIR"
+  if [ -f "$SCRIPT_DIR/.git" ]; then
+    gitdir=$(sed -n 's/^gitdir: //p' "$SCRIPT_DIR/.git" 2>/dev/null)
+    case "$gitdir" in
+      */.git/worktrees/*) main="${gitdir%/.git/worktrees/*}" ;;
+    esac
   fi
+  main=$(cd "$main" 2>/dev/null && pwd -P) || return 1
+  printf '%s|%s' "$org" "$main" | shasum -a 256 2>/dev/null | cut -c1-16
 }
 
-# --- Weight ladder ----------------------------------------------------------
-# Three tiers, chosen by what is already on disk — no mode ever triggers a
-# model download in a session's critical path:
-#   keyword   0 MB    BM25, works at install, forever
-#   semantic  ~334 MB embed model, fetched silently by the background warm
-#   hybrid    ~2.2 GB rerank + expansion — org opt-in (features.search_hybrid),
-#                     prefetched by the background warm, never inline
-MODELS_DIR="${XDG_CACHE_HOME:-$HOME/.cache}/qmd/models"
-_have_embed()  { ls "$MODELS_DIR"/*embedding* >/dev/null 2>&1; }
-_have_hybrid() { ls "$MODELS_DIR"/*reranker* >/dev/null 2>&1 && ls "$MODELS_DIR"/*expansion* >/dev/null 2>&1; }
-_hybrid_opted() { [ "$(jq -r '.features.search_hybrid // false' "$SCRIPT_DIR/egregore.json" 2>/dev/null)" = "true" ]; }
-
-_ensure() {
-  if [ -z "$MEM_PATH" ] || [ ! -d "$MEM_PATH" ]; then
-    echo "memory/ not found — run bin/search.sh from an Egregore instance with a memory symlink" >&2
-    exit 1
-  fi
-  if ! _qmd collection list 2>/dev/null | grep -q "^${COLLECTION} "; then
-    _qmd collection add "$MEM_PATH" --name "$COLLECTION" >/dev/null 2>&1 || true
-  fi
-  # Incremental re-index so new handoffs/decisions are searchable immediately.
-  # NEVER swallow a failure silently: a dead path in ANY registered collection
-  # (e.g. a deleted side-project dir) crashes `qmd update` for every collection,
-  # and the index quietly stops absorbing new documents. Caught in dogfooding
-  # 2026-07-07 — results looked fine while missing three days of handoffs.
-  if ! _qmd update >/dev/null 2>&1; then
-    echo "(search: index update FAILED — results may be missing recent documents. A registered qmd collection likely points at a deleted path: run 'qmd collection list', remove the dead one, retry.)" >&2
-  fi
-  _embed_bg
+_upgraded_runtime_active() {
+  local key record
+  key=$(_upgrade_instance_key) || return 1
+  [ -n "$key" ] || return 1
+  record="${EGREGORE_UPGRADE_ROOT:-$HOME/.egregore/runtime/upgrade}/${key}/active.json"
+  [ -f "$record" ] || return 1
+  [ "$(jq -r '.retrieval // empty' "$record" 2>/dev/null)" = "runtime-qmd" ]
 }
 
-# Keep vector embeddings warm without anyone running a manual step: fire a
-# detached, lock-guarded `qmd embed` on every query. Nothing pending → exits
-# in ~1s. First run on a machine → downloads models + embeds in the
-# background while queries fall back to keyword mode.
-_embed_bg() {
-  [ "${EGREGORE_SEARCH_NO_WARM:-0}" = "1" ] && return 0
-  local lock="${TMPDIR:-/tmp}/egregore-qmd-embed-${COLLECTION}.lock"
-  if [ -f "$lock" ]; then
-    # A crashed embed shouldn't block warming forever — expire the lock after 60 min.
-    if [ -n "$(find "$lock" -mmin +60 2>/dev/null)" ]; then rm -f "$lock"; else return 0; fi
-  fi
-  touch "$lock"
-  if ! _have_embed; then
-    echo "(first run: building the semantic index in the background — searches use keyword mode until it finishes)" >&2
-  fi
-  # Background fetch policy: `qmd embed` pulls only the ~334MB embed model
-  # (the semantic tier). The ~1.9GB hybrid pack is prefetched here ONLY when
-  # the org opted in — a session never waits on either.
-  ( ( _qmd embed >/dev/null 2>&1
-      if _hybrid_opted && ! _have_hybrid; then
-        _qmd query "warmup" -c "$COLLECTION" -n 1 >/dev/null 2>&1
-      fi
-      rm -f "$lock" ) & ) >/dev/null 2>&1
-}
-
-# Annotate results with live graph state (connected mode): each hit that the
-# graph knows (Session by filePath = handoffs; Artifact by filePath = knowledge)
-# gets one "↳ graph:" line — search finds the artifact, the graph tells you its
-# state, in a single call. Fails soft everywhere: local mode / graph offline /
-# unknown paths simply produce no annotations.
 _enrich() {
   local results paths params out
   results=$(cat)
   echo "$results"
+  if _upgraded_runtime_active; then
+    echo "↳ graph retrieval: not used by this Runtime version"
+    return 0
+  fi
   paths=$(printf '%s\n' "$results" | _result_paths)
   [ -z "$paths" ] && return 0
   params=$(printf '%s\n' "$paths" | jq -R . | jq -sc '{paths: .}')
@@ -168,76 +111,31 @@ _enrich() {
   done
 }
 
-cmd_query() {
-  local mode="query" n=6 enrich=auto all=()
-  local -a words=()
-  while [ $# -gt 0 ]; do
-    case "$1" in
-      --fast) mode="search" ;;
-      --semantic) mode="vsearch" ;;
-      --enrich) enrich=1 ;;
-      --no-enrich) enrich=0 ;;
-      --all) all=(--all) ;;
-      -n) n="$2"; shift ;;
-      *) words+=("$1") ;;
-    esac
-    shift
-  done
-  # Default mode rides the weight ladder: use the heaviest tier whose models
-  # are already local. Explicit flags always win; nothing downloads inline.
-  if [ "$mode" = "query" ] && ! _have_hybrid; then
-    if _have_embed; then mode="vsearch"; else mode="search"; fi
-  fi
-  # Enrichment is managed, not a user choice: on wherever a graph exists
-  # (connected mode), silently off everywhere else. Flags only override.
-  if [ "$enrich" = "auto" ]; then
-    if [ "$(jq -r '.mode // "connected"' "$SCRIPT_DIR/egregore.json" 2>/dev/null)" != "local" ] && \
-       [ -n "$(jq -r '.api_url // empty' "$SCRIPT_DIR/egregore.json" 2>/dev/null)" ]; then
-      enrich=1
-    else
-      enrich=0
-    fi
-  fi
-  local q="${words[*]}"
-  if [ -z "$q" ]; then
-    echo "usage: bash bin/search.sh query \"text\" [-n N] [--fast|--semantic] [--all]" >&2
-    exit 1
+_banner_and_emit() {
+  local label="$1" t0="$2" emit="$3" out="$4" hide_results="${5:-0}"
+  local hits dur product surface plural="" gap actual required
+  if [[ "$out" == EGREGORE_OBSERVE_CONTEXT_REUSED* ]]; then
+    printf '⌕ Egregore · using context already compiled for this prompt · 0s\n\n'
+    [ "$hide_results" = 1 ] || printf '%s\n' "$out" | tail -n +2
+    return 0
   fi
-  _ensure
-  local emit="cat"
-  [ "$enrich" = 1 ] && emit="_enrich"
-  local t0 out label
-  t0=$(date +%s)
-  if [ "$mode" != "search" ]; then
-    out=$(_qmd "$mode" "$q" -c "$COLLECTION" -n "$n" --format md --full-path "${all[@]}" 2>/dev/null | _clean | _normalize_file_paths)
-    if [ -n "$out" ] && [ "$out" != "No results found." ]; then
-      [ "$mode" = "query" ] && label="hybrid" || label="semantic"
-      _banner_and_emit "$label" "$t0" "$emit" "$q" "$out"
-      return 0
-    fi
-    echo "_(hybrid/semantic returned nothing — showing keyword results. If this persists, run: bash bin/search.sh reindex --embed)_"
-    echo ""
+  gap=$(printf '%s\n' "$out" | head -n 1)
+  if [[ "$gap" == EGREGORE_EVIDENCE_GAP* ]]; then
+    actual="${gap#*actual=}"
+    actual="${actual%% *}"
+    required="${gap#*required=}"
+    required="${required%% *}"
+    dur=$(( $(date +%s) - t0 ))
+    printf '⌕ Egregore · searching your organization’s memory · %s · evidence gap · %s/%s sources · %ss\n\n' \
+      "$label" "$actual" "$required" "$dur"
+    [ "$hide_results" = 1 ] || printf '%s\n' "$out" | tail -n +2
+    return 0
   fi
-  out=$(_qmd search "$q" -c "$COLLECTION" -n "$n" --format md --full-path "${all[@]}" | _clean | _normalize_file_paths)
-  _banner_and_emit "keyword" "$t0" "$emit" "$q" "$out"
-}
-
-# One visible product-attribution line per search — the tell that Egregore's
-# organizational retrieval layer (not an improvised agent grep loop) answered.
-# It names the graph only when enrichment actually ran, then reports mode,
-# hit count, and latency. Colored when stdout is a terminal; plain glyph in
-# harness transcripts. Also appends to ~/.egregore/search.log (local only,
-# never uploaded) so dogfooding can measure search-first rate and latency.
-_banner_and_emit() {
-  local label="$1" t0="$2" emit="$3" q="$4" out="$5"
-  local hits dur modef product surface plural=""
   hits=$(printf '%s\n' "$out" | _result_paths | wc -l | tr -d ' ')
   dur=$(( $(date +%s) - t0 ))
-  modef="$label"
   product="Egregore"
   surface="searching your organization’s memory"
   if [ "$emit" = "_enrich" ] && [ "$hits" -gt 0 ]; then
-    modef="${label}+graph"
     product="Egregore Connect"
     surface="${surface} and relationships"
   fi
@@ -248,35 +146,173 @@ _banner_and_emit() {
   else
     printf '%s\n\n' "$line"
   fi
-  printf '%s\n' "$out" | $emit
-  mkdir -p "$HOME/.egregore" 2>/dev/null
-  echo "$(date '+%Y-%m-%d %H:%M:%S') mode=${modef} hits=${hits} dur=${dur}s q=\"$(printf '%.80s' "$q")\"" >> "$HOME/.egregore/search.log" 2>/dev/null || true
+  [ "$hide_results" = 1 ] || printf '%s\n' "$out" | "$emit"
 }
 
-cmd_reindex() {
-  _ensure
-  _qmd update
-  if [ "${1:-}" = "--embed" ]; then
-    _qmd embed
-    # Hybrid pack prefetch only when the org opted in (features.search_hybrid).
-    if _hybrid_opted && ! _have_hybrid; then
-      _qmd query "warmup" -c "$COLLECTION" -n 1 >/dev/null 2>&1 || true
-    fi
+_context_packet_path() {
+  local session context_dir
+  # Same precedence as the Python runtime's session identity: an explicit
+  # EGREGORE_SESSION_ID wins, the session file is the fallback.
+  session="${EGREGORE_SESSION_ID:-}"
+  [ -n "$session" ] || session=$(cat "$SCRIPT_DIR/.egregore-session-id" 2>/dev/null || true)
+  session=$(printf '%s' "$session" | tr -cd 'A-Za-z0-9_.-')
+  [ -n "$session" ] || return 1
+  context_dir="${TMPDIR:-/tmp}/egregore-retrieval-context"
+  mkdir -p "$context_dir" 2>/dev/null || return 1
+  chmod 700 "$context_dir" 2>/dev/null || true
+  printf '%s/%s.context\n' "$context_dir" "$session"
+}
+
+cmd_query() {
+  local mode="lex+vec" label="hybrid" n=6 enrich=0 all=0 new_search=0 recent_days="" min_evidence=1 compact=0 context_packet=0
+  local -a words=()
+  while [ $# -gt 0 ]; do
+    case "$1" in
+      --fast) mode="lex"; label="keyword" ;;
+      --semantic) mode="vec"; label="semantic" ;;
+      --enrich) enrich=1 ;;
+      --no-enrich) enrich=0 ;;
+      --all) all=1 ;;
+      --new-search) new_search=1 ;;
+      --recent-days)
+        [ $# -ge 2 ] || { echo "search: --recent-days requires a positive integer" >&2; return 1; }
+        recent_days="$2"
+        shift
+        ;;
+      --min-evidence)
+        [ $# -ge 2 ] || { echo "search: --min-evidence requires a positive integer" >&2; return 1; }
+        min_evidence="$2"
+        shift
+        ;;
+      --compact) compact=1 ;;
+      --context-packet) context_packet=1 ;;
+      -n)
+        [ $# -ge 2 ] || { echo "search: -n requires a positive integer" >&2; return 1; }
+        n="$2"
+        shift
+        ;;
+      *) words+=("$1") ;;
+    esac
+    shift
+  done
+  local q="${words[*]}"
+  if [ -z "$q" ]; then
+    echo "usage: bash bin/search.sh query \"text\" [-n N] [--fast|--semantic] [--recent-days N] [--min-evidence N] [--compact] [--context-packet] [--all] [--new-search]" >&2
+    return 1
+  fi
+  [[ "$n" =~ ^[1-9][0-9]*$ ]] || { echo "search: -n requires a positive integer" >&2; return 1; }
+  [ -z "$recent_days" ] || [[ "$recent_days" =~ ^[1-9][0-9]*$ ]] || {
+    echo "search: --recent-days requires a positive integer" >&2
+    return 1
+  }
+  [[ "$min_evidence" =~ ^[1-9][0-9]*$ ]] || {
+    echo "search: --min-evidence requires a positive integer" >&2
+    return 1
+  }
+  [ "$all" = 1 ] && n=10000
+
+  local -a args=(query --task "$q" --mode "$mode" --limit "$n")
+  [ "$new_search" = 1 ] && args+=(--new-search)
+  [ -n "$recent_days" ] && args+=(--recent-days "$recent_days")
+  args+=(--min-evidence "$min_evidence")
+  [ "$compact" = 1 ] && args+=(--compact)
+  case "$mode" in
+    lex) args+=(--lex "$q") ;;
+    vec) args+=(--vec "$q") ;;
+    lex+vec) args+=(--lex "$q" --vec "$q") ;;
+  esac
+
+  local t0 out emit="cat" query_rc
+  t0=$(date +%s)
+  out=$(_runtime "${args[@]}" 2>&1)
+  query_rc=$?
+  if [ "$query_rc" -ne 0 ]; then
+    {
+      echo "⚠ Egregore Runtime retrieval failed."
+      printf '%s\n' "$out" | tail -3 | sed 's/^/  /'
+      echo "This is a Runtime failure — do not fall back to manual scanning (rg/grep/ls over memory), a global qmd, npx, or graph."
+      echo "Diagnose with: bash bin/search.sh readiness · report with /issue · stop here."
+    } >&2
+    return "$query_rc"
   fi
+  [ "$enrich" = 1 ] && emit="_enrich"
+  if [ "$context_packet" = 1 ]; then
+    local packet
+    packet=$(_context_packet_path) || {
+      echo "search: context packet requires an active Egregore session" >&2
+      return 1
+    }
+    umask 077
+    printf '%s\n' "$out" > "$packet"
+    _banner_and_emit "$label" "$t0" "$emit" "$out" 1
+  else
+    _banner_and_emit "$label" "$t0" "$emit" "$out"
+  fi
+}
+
+cmd_reindex() {
+  local -a args=(update)
+  [ "${1:-}" = "--embed" ] && args+=(--embed)
+  _maintenance_runtime "${args[@]}"
 }
 
 cmd_status() {
-  _qmd status 2>/dev/null
+  _runtime status
   echo ""
-  echo "collection: $COLLECTION → $MEM_PATH"
+  echo "canonical memory: $MEM_PATH"
 }
 
 case "${1:-}" in
-  query)   shift; cmd_query "$@" ;;
+  query) shift; cmd_query "$@" ;;
   reindex) shift; cmd_reindex "$@" ;;
-  status)  shift; cmd_status ;;
+  install) shift; _maintenance_runtime install "$@" ;;
+  status) shift; cmd_status ;;
+  readiness) shift; _runtime readiness "$@" ;;
+  handoffs)
+    shift
+    if ! _runtime handoffs "$@"; then
+      {
+        echo "⚠ Egregore Runtime lifecycle lookup failed."
+        echo "This is a Runtime failure — do not fall back to manual scanning (ls/rg/grep over memory), a semantic re-query, or graph."
+        echo "Diagnose with: bash bin/search.sh readiness · report with /issue · stop here."
+      } >&2
+      exit 1
+    fi
+    ;;
+  start) shift; _runtime start "$@" ;;
+  stop) shift; _runtime stop "$@" ;;
+  open)
+    shift
+    OPEN_PACKET=0; OPEN_PATH=""
+    for _arg in "$@"; do
+      case "$_arg" in
+        --context-packet) OPEN_PACKET=1 ;;
+        *) OPEN_PATH="$_arg" ;;
+      esac
+    done
+    if [ "$OPEN_PACKET" = 1 ]; then
+      # Claude: full documents attach privately through the PostToolUse
+      # packet instead of splashing raw markdown into the transcript —
+      # the same contract query --context-packet already follows.
+      packet=$(_context_packet_path) || {
+        echo "search: context packet requires an active Egregore session" >&2
+        exit 1
+      }
+      umask 077
+      if _runtime open "$OPEN_PATH" > "$packet" 2>"$packet.err"; then
+        rm -f "$packet.err"
+        echo "↳ opened $OPEN_PATH · content attached privately"
+      else
+        cat "$packet.err" >&2 2>/dev/null
+        rm -f "$packet" "$packet.err"
+        exit 1
+      fi
+    else
+      _runtime open "$OPEN_PATH"
+    fi
+    ;;
   *)
-    echo "usage: bash bin/search.sh {query|reindex|status}" >&2
+    echo "usage: bash bin/search.sh {query|reindex|install|start|stop|status|readiness|open}" >&2
     exit 1
     ;;
 esac