create-egregore 0.19.32 → 0.20.0

package/runtime/prime/bin/graph-wal.sh

@@ -0,0 +1,229 @@
+#!/usr/bin/env bash
+# Graph Write-Ahead Log — resilient graph writes with local buffer.
+# Mirrors bin/telemetry.sh architecture.
+#
+# Subcommands:
+#   append <cypher> <params_json>   Append entry to local JSONL buffer (no network)
+#   drain                           Execute pending entries via graph-batch.sh
+#   status                          Show pending count + file size as JSON
+#
+# Storage: ~/.egregore/graph-wal.jsonl
+# Buffer guard: 2MB max, auto-truncate to last 200 entries if exceeded.
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+
+if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
+  echo "Usage: graph-wal.sh <command>"
+  echo ""
+  echo "Graph Write-Ahead Log — resilient graph writes with local buffer."
+  echo ""
+  echo "Commands:"
+  echo "  append <cypher> <params>  Append entry to WAL (no network)"
+  echo "  drain                     Execute pending entries via graph-batch"
+  echo "  status                    Show pending count + file size (JSON)"
+  exit 0
+fi
+
+WAL_DIR="$HOME/.egregore"
+# WAL file is per-instance to prevent cross-org data leakage on multi-instance machines
+PROJ_HASH=$(echo -n "$SCRIPT_DIR" | md5 2>/dev/null || echo -n "$SCRIPT_DIR" | md5sum 2>/dev/null | cut -d' ' -f1)
+WAL_FILE="$WAL_DIR/graph-wal-${PROJ_HASH}.jsonl"
+MAX_BUFFER_BYTES=2097152  # 2MB
+MAX_ENTRIES_AFTER_TRUNCATE=200
+LOCK_DIR="$WAL_DIR/graph-wal-${PROJ_HASH}.lock"
+LOCK_ATTEMPTS="${GRAPH_WAL_LOCK_ATTEMPTS:-50}"
+
+# --- Cross-platform file locking via mkdir (atomic on all POSIX systems) ---
+_lock() {
+  local attempts=0
+  while ! mkdir "$LOCK_DIR" 2>/dev/null; do
+    attempts=$((attempts + 1))
+    if [ "$attempts" -gt "$LOCK_ATTEMPTS" ]; then
+      # Stale lock — check if holder PID is alive
+      local holder_pid
+      holder_pid=$(cat "$LOCK_DIR/pid" 2>/dev/null || echo "")
+      if [ -n "$holder_pid" ] && ! kill -0 "$holder_pid" 2>/dev/null; then
+        rm -rf "$LOCK_DIR" 2>/dev/null
+        continue
+      fi
+      echo "graph-wal: lock timeout after ${LOCK_ATTEMPTS} attempt(s)" >&2
+      return 1
+    fi
+    sleep 0.1
+  done
+  echo $$ > "$LOCK_DIR/pid" 2>/dev/null
+}
+
+_unlock() {
+  rm -rf "$LOCK_DIR" 2>/dev/null
+}
+
+trap '_unlock' EXIT
+
+# --- Resolve session ID (same pattern as telemetry.sh) ---
+_resolve_session_id() {
+  if [ -n "${EGREGORE_SESSION_ID:-}" ]; then
+    echo "$EGREGORE_SESSION_ID"
+    return
+  fi
+  local proj_hash
+  proj_hash=$(echo -n "$SCRIPT_DIR" | md5 2>/dev/null || echo -n "$SCRIPT_DIR" | md5sum 2>/dev/null | cut -d' ' -f1)
+  local sid_file="$WAL_DIR/session-${proj_hash}.id"
+  if [ -f "$sid_file" ]; then
+    cat "$sid_file" 2>/dev/null || echo "unknown"
+  else
+    echo "unknown"
+  fi
+}
+
+# --- append: O(1) local write, no network ---
+cmd_append() {
+  local cypher="${1:?Usage: graph-wal.sh append <cypher> <params_json>}"
+  local params="${2:-"{}"}"
+
+  mkdir -p "$WAL_DIR"
+
+  local ts
+  ts=$(date -u +%Y-%m-%dT%H:%M:%SZ)
+  local sid
+  sid=$(_resolve_session_id)
+
+  # Build JSON line
+  local line
+  line=$(jq -n -c \
+    --arg ts "$ts" \
+    --arg sid "$sid" \
+    --arg cypher "$cypher" \
+    --argjson params "$params" \
+    '{ts: $ts, sid: $sid, cypher: $cypher, params: $params}')
+
+  # Append under lock — if lock fails, append anyway (durability > mutual exclusion)
+  local _locked=true
+  _lock || _locked=false
+  echo "$line" >> "$WAL_FILE"
+
+  # Buffer guard: truncate to last N entries if file exceeds max size
+  # Only safe under lock — tail+mv is not atomic and races with concurrent writers
+  if [ "$_locked" = "true" ] && [ -f "$WAL_FILE" ]; then
+    local size
+    size=$(wc -c < "$WAL_FILE" 2>/dev/null | tr -d ' ')
+    if [ "$size" -gt "$MAX_BUFFER_BYTES" ] 2>/dev/null; then
+      local total_entries
+      total_entries=$(wc -l < "$WAL_FILE" 2>/dev/null | tr -d ' ')
+      local dropped=$((total_entries - MAX_ENTRIES_AFTER_TRUNCATE))
+      if [ "$dropped" -gt 0 ]; then
+        echo "graph-wal: buffer exceeded ${MAX_BUFFER_BYTES} bytes. Dropping $dropped oldest entries (keeping $MAX_ENTRIES_AFTER_TRUNCATE)." >&2
+      fi
+      local tmp="$WAL_FILE.truncate.$$"
+      tail -n "$MAX_ENTRIES_AFTER_TRUNCATE" "$WAL_FILE" > "$tmp" 2>/dev/null \
+        && mv "$tmp" "$WAL_FILE" \
+        || rm -f "$tmp"
+    fi
+  fi
+  [ "$_locked" = "true" ] && _unlock
+}
+
+# --- drain: execute pending entries via graph-batch.sh ---
+cmd_drain() {
+  if [ ! -f "$WAL_FILE" ] || [ ! -s "$WAL_FILE" ]; then
+    return 0
+  fi
+
+  # Atomic rotate under lock: move to temp file so new appends don't conflict
+  _lock || return 0
+  local drain_file="$WAL_FILE.drain.$$"
+  mv "$WAL_FILE" "$drain_file" 2>/dev/null || { _unlock; return 0; }
+  _unlock
+
+  local total
+  total=$(wc -l < "$drain_file" 2>/dev/null | tr -d ' ')
+  local batch_size=20
+  local offset=0
+  local all_ok=true
+
+  while [ "$offset" -lt "$total" ]; do
+    # Extract batch of entries (offset+1 through offset+batch_size)
+    local batch_lines
+    batch_lines=$(tail -n "+$((offset + 1))" "$drain_file" | head -n "$batch_size")
+
+    # Build batch JSON array: [{statement, parameters}, ...]
+    local batch_json
+    batch_json=$(echo "$batch_lines" | jq -s '[.[] | {statement: .cypher, parameters: .params}]' 2>/dev/null)
+
+    if [ -z "$batch_json" ] || [ "$batch_json" = "[]" ]; then
+      offset=$((offset + batch_size))
+      continue
+    fi
+
+    # Execute via graph-batch.sh
+    if bash "$SCRIPT_DIR/bin/graph-batch.sh" "$batch_json" >/dev/null 2>&1; then
+      offset=$((offset + batch_size))
+    else
+      all_ok=false
+      break
+    fi
+  done
+
+  if [ "$all_ok" = "true" ]; then
+    # All batches succeeded — remove drain file
+    rm -f "$drain_file"
+  else
+    # Failure — merge unprocessed entries back
+    # Extract remaining entries (from offset+1 to end)
+    local remaining
+    remaining=$(tail -n "+$((offset + 1))" "$drain_file" 2>/dev/null)
+
+    if [ -n "$remaining" ]; then
+      _lock || { echo "$remaining" > "$WAL_FILE" 2>/dev/null; rm -f "$drain_file"; return 1; }
+      if [ -f "$WAL_FILE" ]; then
+        # New entries were written while draining; prepend remaining
+        local merge_file="$WAL_FILE.merge.$$"
+        { echo "$remaining"; cat "$WAL_FILE"; } > "$merge_file" 2>/dev/null \
+          && mv "$merge_file" "$WAL_FILE" \
+          && rm -f "$drain_file" \
+          || rm -f "$merge_file"
+      else
+        echo "$remaining" > "$WAL_FILE"
+        rm -f "$drain_file"
+      fi
+      _unlock
+    else
+      rm -f "$drain_file"
+    fi
+  fi
+}
+
+# --- status: pending count + file size as JSON ---
+cmd_status() {
+  if [ -f "$WAL_FILE" ] && [ -s "$WAL_FILE" ]; then
+    local count size
+    count=$(wc -l < "$WAL_FILE" 2>/dev/null | tr -d ' ')
+    size=$(wc -c < "$WAL_FILE" 2>/dev/null | tr -d ' ')
+    jq -n --argjson pending "$count" --argjson bytes "$size" '{pending: $pending, bytes: $bytes}'
+  else
+    echo '{"pending":0,"bytes":0}'
+  fi
+}
+
+# --- Main dispatch ---
+case "${1:-status}" in
+  append)
+    shift
+    cmd_append "$@"
+    ;;
+  drain)
+    cmd_drain
+    ;;
+  status)
+    cmd_status
+    ;;
+  *)
+    echo "Usage: graph-wal.sh <command>"
+    echo ""
+    echo "Commands:"
+    echo "  append <cypher> <params>  Append entry to WAL (no network)"
+    echo "  drain                     Execute pending entries"
+    echo "  status                    Show pending count + file size"
+    ;;
+esac

package/runtime/prime/bin/graph.sh

@@ -0,0 +1,223 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+CONFIG="$SCRIPT_DIR/egregore.json"
+
+if [ ! -f "$CONFIG" ]; then
+  echo "Error: egregore.json not found. Run onboarding first." >&2
+  exit 1
+fi
+
+# --- Local mode gate: bail immediately, no .env sourcing, no network ---
+_MODE=$(jq -r '.mode // "connected"' "$CONFIG" 2>/dev/null)
+if [ "$_MODE" = "local" ]; then
+  case "${1:-}" in
+    query)  echo '{"results":[]}';;
+    schema) echo '{}';;
+    test)   echo '{"status":"offline","reason":"local_mode"}';;
+    *)      echo '{"results":[]}';;
+  esac
+  exit 0
+fi
+
+# Load specific variables from .env if it exists (safe extraction, no arbitrary code execution).
+# A worktree missing its .env symlink resolves through worktree-links.sh, which
+# self-heals the link and falls back to the main checkout's .env.
+_ENV_FILE="$SCRIPT_DIR/.env"
+if [ ! -f "$_ENV_FILE" ] && [ -f "$SCRIPT_DIR/bin/lib/worktree-links.sh" ]; then
+  # shellcheck source=bin/lib/worktree-links.sh
+  source "$SCRIPT_DIR/bin/lib/worktree-links.sh" 2>/dev/null || true
+  _ENV_FILE="$(egregore_env_file "$SCRIPT_DIR" 2>/dev/null || true)"
+fi
+if [ -n "$_ENV_FILE" ] && [ -f "$_ENV_FILE" ]; then
+  EGREGORE_API_URL="${EGREGORE_API_URL:-$(grep '^EGREGORE_API_URL=' "$_ENV_FILE" 2>/dev/null | cut -d'=' -f2- || true)}"
+  EGREGORE_API_KEY="${EGREGORE_API_KEY:-$(grep '^EGREGORE_API_KEY=' "$_ENV_FILE" 2>/dev/null | cut -d'=' -f2- || true)}"
+fi
+
+# api_url comes from egregore.json (committed, non-secret)
+# api_key comes from .env only (EGREGORE_API_KEY) — never from egregore.json
+API_URL="${EGREGORE_API_URL:-$(jq -r '.api_url // empty' "$CONFIG")}"
+API_KEY="${EGREGORE_API_KEY:-}"
+
+# --- Millisecond timer (macOS + Linux compatible) ---
+_millis() {
+  if command -v gdate &>/dev/null; then
+    gdate +%s%3N
+  else
+    # macOS date doesn't support %N — use python3 fallback
+    python3 -c "import time; print(int(time.time() * 1000))" 2>/dev/null || echo "0"
+  fi
+}
+
+# --- Optional read-through cache (opt-in via EGREGORE_GRAPH_CACHE_TTL) ---
+# Read-only queries can be served from a local cache that bin/attendant.sh
+# keeps warm in the background, so callers on the launch path (session-start
+# greeting) pay zero network round-trips when warm. TTL in seconds; unset/0
+# disables. Mutating queries are never cached.
+_CACHE_TTL="${EGREGORE_GRAPH_CACHE_TTL:-0}"
+# Key the cache by the MAIN checkout so worktree sessions hit the same warm
+# cache the attendant maintains (its replay uses the same derivation).
+# Worktrees have .git as a FILE pointing into the main repo's .git/worktrees/.
+# api_url + slug are part of the key so entries never cross tenants when the
+# same checkout is repointed at a different org/API (cache entries are keyed
+# only by query+params below, so the directory must carry the tenant).
+_CACHE_ROOT="$SCRIPT_DIR"
+if [ -f "$SCRIPT_DIR/.git" ]; then
+  _WT_GITDIR=$(sed 's/^gitdir: //' "$SCRIPT_DIR/.git" 2>/dev/null || true)
+  if [ -n "$_WT_GITDIR" ]; then
+    _CACHE_ROOT=$(cd "$_WT_GITDIR/../../.." 2>/dev/null && pwd || echo "$SCRIPT_DIR")
+  fi
+fi
+# The scope is a non-secret fingerprint of the EFFECTIVE endpoint + credential
+# (already resolved above: process env → .env → committed api_url). Committed
+# config alone is not enough — an EGREGORE_API_URL/KEY override or a swapped
+# .env can point this checkout at a different tenant than egregore.json names,
+# and the entry key below carries only query+params. cksum output is a CRC,
+# not a reversible or usable credential.
+_CACHE_SCOPE=$(echo -n "${API_URL:-}|${API_KEY:-}" | cksum | cut -d' ' -f1)
+_CACHE_DIR="$HOME/.egregore/graph-cache/$(echo -n "${_CACHE_ROOT}|${_CACHE_SCOPE}" | cksum | cut -d' ' -f1)"
+
+_cache_key() {
+  echo -n "$1|${2:-}" | cksum | cut -d' ' -f1
+}
+
+_is_read_query() {
+  ! echo "$1" | grep -qiE '\b(MERGE|CREATE|SET|DELETE|DETACH|REMOVE)\b'
+}
+
+if [ -n "$API_URL" ] && [ -n "$API_KEY" ]; then
+  # === API MODE: Call Egregore API gateway ===
+
+  run_query() {
+    local cypher="$1"
+    local params
+    if [ -n "${2:-}" ]; then
+      params="$2"
+    else
+      params="{}"
+    fi
+
+    local cache_file=""
+    if [ "$_CACHE_TTL" != "0" ] && _is_read_query "$cypher"; then
+      cache_file="$_CACHE_DIR/$(_cache_key "$cypher" "$params").json"
+      if [ -f "$cache_file" ]; then
+        local now cts
+        now=$(date +%s)
+        cts=$(jq -r '.ts // 0' "$cache_file" 2>/dev/null || echo 0)
+        if [ $(( now - cts )) -lt "$_CACHE_TTL" ]; then
+          jq -c '.result' "$cache_file" 2>/dev/null && return 0
+        fi
+      fi
+    fi
+
+    local body
+    body=$(jq -n --arg stmt "$cypher" --argjson params "$params" \
+      '{statement: $stmt, parameters: $params}')
+
+    local start_ms
+    start_ms=$(_millis)
+
+    local response
+    response=$(curl -s -X POST "${API_URL}/api/graph/query" \
+      -H "Authorization: Bearer $API_KEY" \
+      -H "Content-Type: application/json" \
+      -d "$body" \
+      --max-time 30)
+
+    local end_ms
+    end_ms=$(_millis)
+    local latency_ms=$(( end_ms - start_ms ))
+
+    # Check for HTTP-level errors
+    if echo "$response" | jq -e '.detail' >/dev/null 2>&1; then
+      echo "API error:" >&2
+      echo "$response" | jq '.detail' >&2
+      bash "$SCRIPT_DIR/bin/telemetry.sh" emit "error" \
+        "$(jq -n --arg source "graph" --arg error_code "api_error" --argjson latency "$latency_ms" \
+          '{source: $source, error_code: $error_code, latency_ms: $latency}')" 2>/dev/null &
+      exit 1
+    fi
+
+    # Emit latency telemetry (fire-and-forget)
+    bash "$SCRIPT_DIR/bin/telemetry.sh" emit "graph_query" \
+      "$(jq -n --argjson latency "$latency_ms" '{latency_ms: $latency}')" 2>/dev/null &
+
+    # Store for the attendant to keep warm (query+params travel with the result)
+    if [ -n "$cache_file" ]; then
+      mkdir -p "$_CACHE_DIR" 2>/dev/null
+      jq -n --arg q "$cypher" --argjson p "$params" --argjson ts "$(date +%s)" --argjson r "$response" \
+        '{query: $q, params: $p, ts: $ts, result: $r}' > "$cache_file" 2>/dev/null || true
+    fi
+
+    echo "$response"
+  }
+
+  get_schema() {
+    local response
+    response=$(curl -s -X GET "${API_URL}/api/graph/schema" \
+      -H "Authorization: Bearer $API_KEY" \
+      --max-time 30)
+    echo "$response" | jq .
+  }
+
+  test_connection() {
+    local response
+    response=$(curl -s -X GET "${API_URL}/api/graph/test" \
+      -H "Authorization: Bearer $API_KEY" \
+      --max-time 10)
+
+    if echo "$response" | jq -e '.status == "ok"' >/dev/null 2>&1; then
+      echo "Connected to Egregore API at $API_URL"
+    else
+      echo "Failed to connect to Egregore API at $API_URL" >&2
+      echo "$response" >&2
+      exit 1
+    fi
+  }
+
+elif [ -n "$API_URL" ]; then
+  # === CONNECTED CONFIG, MISSING KEY ===
+  # api_url is configured, so the org expects a live graph, but no API key was
+  # found. Faking '{"results":[]}' here made a missing credential (e.g. a
+  # worktree without its .env symlink) indistinguishable from an empty graph —
+  # fail loudly instead. `test` keeps its offline JSON contract: dashboards,
+  # render-card, and startup-check consume it.
+  _missing_credentials() {
+    {
+      echo "Error: api_url is configured but EGREGORE_API_KEY is empty (looked in: ${_ENV_FILE:-$SCRIPT_DIR/.env})."
+      echo "  If this is a worktree, its .env symlink may be missing. Run /env to configure."
+    } >&2
+    exit 1
+  }
+  run_query() { _missing_credentials; }
+  get_schema() { _missing_credentials; }
+  test_connection() { echo '{"status":"offline","reason":"no_api_key"}'; }
+
+else
+  # === OFFLINE MODE: no api_url configured (OSS/unconfigured) ===
+  run_query() { echo '{"results":[]}'; }
+  get_schema() { echo '{}'; }
+  test_connection() { echo '{"status":"offline","reason":"no_api_key"}'; }
+fi
+
+case "${1:-help}" in
+  query)
+    shift
+    run_query "$@"
+    ;;
+  schema)
+    get_schema
+    ;;
+  test)
+    test_connection
+    ;;
+  help|*)
+    echo "Usage: graph.sh <command>"
+    echo ""
+    echo "Commands:"
+    echo "  query <cypher> [params_json]  Run a Cypher query"
+    echo "  schema                        Show graph schema"
+    echo "  test                          Test connection"
+    ;;
+esac

package/runtime/prime/bin/handoff-pr-backfill.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+# handoff-pr-backfill.sh — fill in PR numbers in a handoff file's ## Repo State
+# table, then re-commit + push the memory repo. Detached fire-and-forget from
+# /handoff's hot path so user-visible latency doesn't include N × `gh pr list`.
+#
+# Usage:
+#   handoff-pr-backfill.sh <abs-handoff-file> <topic>
+#
+# Reads the ## Repo State table, looks up each (repo, branch)'s open PR via
+# `gh pr list`, rewrites the PR column from `—` to `#N`, commits the memory
+# repo, pushes to main. Silent on all failures — this is cosmetic.
+#
+# Detached usage (the skill / orchestrator call it this way):
+#   ( bash bin/handoff-pr-backfill.sh "$ABS_FILE" "$TOPIC" >/dev/null 2>&1 & ) >/dev/null 2>&1
+
+set -uo pipefail
+
+ABS_FILE="${1:-}"
+TOPIC="${2:-handoff}"
+
+[ -n "$ABS_FILE" ] && [ -f "$ABS_FILE" ] || exit 0
+
+SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+CONFIG="$SCRIPT_DIR/egregore.json"
+
+command -v gh >/dev/null 2>&1 || exit 0
+command -v jq >/dev/null 2>&1 || exit 0
+
+GITHUB_ORG=$(jq -r '.github_org // empty' "$CONFIG" 2>/dev/null)
+[ -n "$GITHUB_ORG" ] || exit 0
+
+# Extract the ## Repo State table rows: format is `| repo | branch | — | base |`.
+# Skip header and separator lines (first two rows of the markdown table).
+ROWS=$(awk '
+  /^## Repo State/ { in_section=1; next }
+  in_section && /^## / { in_section=0 }
+  in_section && /^\|/ && !/^\|[-\s]/ && !/^\| Repo/ { print }
+' "$ABS_FILE" 2>/dev/null)
+
+[ -n "$ROWS" ] || exit 0
+
+UPDATED=0
+while IFS= read -r row; do
+  [ -z "$row" ] && continue
+  # Parse: | repo | branch | pr | base |
+  repo=$(echo "$row" | awk -F'|' '{gsub(/^ +| +$/,"",$2); print $2}')
+  branch=$(echo "$row" | awk -F'|' '{gsub(/^ +| +$/,"",$3); print $3}')
+  pr_col=$(echo "$row" | awk -F'|' '{gsub(/^ +| +$/,"",$4); print $4}')
+
+  # Only backfill `—` cells (leave `#N` alone)
+  [ "$pr_col" = "—" ] || continue
+  [ -n "$repo" ] && [ -n "$branch" ] || continue
+
+  pr_num=$(gh pr list --repo "$GITHUB_ORG/$repo" --head "$branch" --json number --jq '.[0].number // empty' 2>/dev/null || echo "")
+  [ -n "$pr_num" ] || continue
+
+  # Rewrite the `—` in this branch's row to `#N`. Match by branch name to
+  # avoid cross-repo collisions when repos share branch names.
+  awk -v branch="$branch" -v pr="#$pr_num" '
+    BEGIN { in_section=0 }
+    /^## Repo State/ { in_section=1; print; next }
+    in_section && /^## / { in_section=0 }
+    in_section && $0 ~ "\\| " branch " " && /—/ {
+      sub(/—/, pr)
+      print; next
+    }
+    { print }
+  ' "$ABS_FILE" > "$ABS_FILE.tmp" && mv "$ABS_FILE.tmp" "$ABS_FILE"
+  UPDATED=1
+done <<< "$ROWS"
+
+[ "$UPDATED" = "1" ] || exit 0
+
+# Re-commit + push the memory repo with the backfilled PR numbers.
+MEMORY_DIR="$SCRIPT_DIR/memory"
+if [ -d "$MEMORY_DIR" ]; then
+  (
+    cd "$MEMORY_DIR" || exit 0
+    git add -A >/dev/null 2>&1
+    git commit -m "Backfill PR numbers for handoff: $TOPIC" >/dev/null 2>&1 || exit 0
+    git pull --rebase origin main --quiet >/dev/null 2>&1
+    git push origin main --quiet >/dev/null 2>&1
+  ) || true
+fi
+
+exit 0