mishkan-harness 0.1.0

package/payload/mishkan/scripts/ensure-curated-box.sh

@@ -0,0 +1,66 @@
+#!/usr/bin/env bash
+# MISHKAN — idempotently ensure the CURATED box (global singleton) is up + seeded.
+# Called by /mishkan-init. The curated box is shared across ALL projects, so this
+# NEVER recreates or reseeds an already-running, already-populated box — it only
+# fills what's missing. Safe to run repeatedly.
+#
+# Does: ensure .env.curated exists (generated from the example, reusing the work
+# stack's LLM key + a fresh Neo4j password) -> ensure curated_db -> bring the box
+# up if down -> seed only if the curated graph is empty. See decision D-007.
+set -euo pipefail
+
+COGNEE_DIR="${HOME}/.claude/mishkan/cognee"
+cd "$COGNEE_DIR" || { echo "cognee dir not found: $COGNEE_DIR" >&2; exit 1; }
+
+# 0. work stack owns the shared network + Ollama + Postgres — it must be up first.
+if ! docker ps --format '{{.Names}}' | grep -qx mishkan-cognee-pg; then
+  echo "work stack not running — bring it up first (docker-compose.yml), then re-run." >&2
+  exit 1
+fi
+
+# 1. ensure .env.curated (SOPS-manage the real file; this is the bootstrap fallback).
+if [ ! -f .env.curated ]; then
+  [ -f .env.curated.example ] || { echo ".env.curated.example missing" >&2; exit 1; }
+  echo "generating .env.curated (reusing work LLM key; fresh local passwords)..."
+  LLMKEY="$(grep '^LLM_API_KEY=' .env | cut -d= -f2-)"
+  PGPW="$(grep '^DB_PASSWORD=' .env | cut -d= -f2-)"
+  umask 077
+  sed -e "s|^LLM_API_KEY=.*|LLM_API_KEY=${LLMKEY}|" \
+      -e "s|^GRAPH_DATABASE_PASSWORD=.*|GRAPH_DATABASE_PASSWORD=$(openssl rand -hex 16)|" \
+      -e "s|^DB_PASSWORD=.*|DB_PASSWORD=${PGPW}|" \
+      -e "s|^DEFAULT_USER_PASSWORD=.*|DEFAULT_USER_PASSWORD=$(openssl rand -hex 20)|" \
+      .env.curated.example > .env.curated
+  chmod 600 .env.curated
+fi
+
+# 2. ensure the isolated curated_db in the shared Postgres.
+if ! docker exec mishkan-cognee-pg psql -U cognee -d cognee_db -tAc \
+     "SELECT 1 FROM pg_database WHERE datname='curated_db'" | grep -q 1; then
+  docker exec mishkan-cognee-pg psql -U cognee -d cognee_db -c \
+    "CREATE DATABASE curated_db OWNER cognee;"
+fi
+
+# 3. bring the curated box up if it's not already running.
+if ! docker ps --format '{{.Names}}' | grep -qx mishkan-curated-mcp; then
+  docker compose --env-file .env.curated -f docker-compose.curated.yml up -d
+fi
+
+# 4. wait for curated-mcp health (bounded).
+echo "waiting for mishkan-curated-mcp..."
+for _ in $(seq 1 40); do
+  [ "$(docker inspect -f '{{.State.Health.Status}}' mishkan-curated-mcp 2>/dev/null)" = "healthy" ] && break
+  sleep 5
+done
+
+# 5. seed ONLY if the curated graph is empty (never reseed a populated singleton).
+N4PW="$(grep '^GRAPH_DATABASE_PASSWORD=' .env.curated | cut -d= -f2-)"
+COUNT="$(docker exec mishkan-curated-neo4j cypher-shell -u neo4j -p "$N4PW" --format plain \
+         "MATCH (n:CuratedResource) RETURN count(n);" 2>/dev/null | tail -1 | tr -dc '0-9')"
+if [ "${COUNT:-0}" = "0" ]; then
+  echo "curated graph empty — seeding..."
+  "${HOME}/.claude/mishkan/scripts/seed-curated-library.sh"
+else
+  echo "curated library already populated (${COUNT} nodes) — leaving as-is."
+fi
+
+echo "curated box ready on :${CURATED_MCP_PORT:-7730}"

package/payload/mishkan/scripts/mishkan-ingest.sh

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+# MISHKAN — selectively ingest docs into the project's work cognee store.
+# Default: nothing enters memory unless tagged (`mishkan: ingest` frontmatter)
+# or explicitly listed — preventing PII bleed and oversized-doc errors.
+# Runs cognee.add -> cognify -> memify (extraction + enrichment).
+#
+#   mishkan-ingest.sh --tagged-only                # walk ./docs/ for tagged
+#   mishkan-ingest.sh docs/SECURITY.md docs/ROADMAP.md
+#   mishkan-ingest.sh --dataset=research docs/research.md
+set -euo pipefail
+
+TAGGED_ONLY=false
+DATASET="$(basename "$PWD")"
+PATHS=()
+CONTAINER="${COGNEE_CONTAINER:-mishkan-cognee-mcp}"
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --tagged-only) TAGGED_ONLY=true; shift ;;
+    --dataset=*) DATASET="${1#--dataset=}"; shift ;;
+    -h|--help) sed -n '2,12p' "$0"; exit 0 ;;
+    *) PATHS+=("$1"); shift ;;
+  esac
+done
+
+[ ${#PATHS[@]} -eq 0 ] && PATHS=("./docs")
+
+docker ps --format '{{.Names}}' | grep -qx "$CONTAINER" \
+  || { echo "work cognee container '$CONTAINER' is not running" >&2; exit 1; }
+
+# Collect candidate files (md only).
+FILES=()
+for p in "${PATHS[@]}"; do
+  if [ -d "$p" ]; then
+    while IFS= read -r f; do FILES+=("$f"); done < <(find "$p" -type f -name "*.md")
+  elif [ -f "$p" ]; then
+    FILES+=("$p")
+  else
+    echo "warn: skipping (not found): $p" >&2
+  fi
+done
+
+# Filter to tagged docs if requested. A doc is tagged when its YAML frontmatter
+# (the block between the first two `---` lines) contains `mishkan: ingest`.
+if $TAGGED_ONLY; then
+  KEPT=()
+  for f in "${FILES[@]}"; do
+    if awk '
+      BEGIN{infm=0;ok=0}
+      NR==1 && $0=="---"{infm=1;next}
+      infm && $0=="---"{exit}
+      infm && /^[[:space:]]*mishkan:[[:space:]]*ingest[[:space:]]*$/{ok=1;exit}
+      NR>50 && !infm{exit}
+      END{exit !ok}
+    ' "$f"; then KEPT+=("$f"); fi
+  done
+  FILES=("${KEPT[@]}")
+fi
+
+if [ ${#FILES[@]} -eq 0 ]; then
+  echo "no docs selected (tagged_only=$TAGGED_ONLY, paths='${PATHS[*]}')"
+  exit 0
+fi
+
+echo "ingesting ${#FILES[@]} file(s) into dataset '${DATASET}':"
+printf '  %s\n' "${FILES[@]}"
+
+# Stage files into the container, then run add -> cognify -> memify.
+docker exec "$CONTAINER" sh -c 'rm -rf /home/cognee/ingest_buf && mkdir -p /home/cognee/ingest_buf'
+for f in "${FILES[@]}"; do
+  docker cp "$f" "${CONTAINER}:/home/cognee/ingest_buf/$(basename "$f")"
+done
+
+PY_SCRIPT="$(mktemp)"
+cat > "$PY_SCRIPT" <<'PY'
+import asyncio, glob, sys, cognee
+DATASET = sys.argv[1]
+FILES = sorted(glob.glob("/home/cognee/ingest_buf/*"))
+async def m():
+    if not FILES:
+        print(">> no files"); return
+    await cognee.add(FILES, dataset_name=DATASET)
+    print(f">> added {len(FILES)} file(s) -> {DATASET}", flush=True)
+    await cognee.cognify(datasets=[DATASET])
+    print(">> cognified", flush=True)
+    await cognee.memify(dataset=DATASET)
+    print(">> memified", flush=True)
+asyncio.run(m())
+PY
+docker cp "$PY_SCRIPT" "${CONTAINER}:/home/cognee/_mishkan_ingest.py"
+rm -f "$PY_SCRIPT"
+docker exec -i -w /app/cognee-mcp "$CONTAINER" uv run python -u /home/cognee/_mishkan_ingest.py "$DATASET"

package/payload/mishkan/scripts/observability-aggregate.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# MISHKAN — observability aggregation.
+# Reads logs/*.jsonl (raw PostToolUse events), computes per-tool and per-outcome
+# counts plus per-session activity, and writes a summary the improvement layer
+# reads. Triggered by /sprint-close.
+#
+# Raw events carry tool/outcome/session/timestamp; agent/team/sprint/token fields
+# are enriched here where derivable and left null otherwise (hook payload limit).
+set -euo pipefail
+
+MISHKAN="${HOME}/.claude/mishkan"
+LOG_DIR="${MISHKAN}/logs"
+TS="$(date -u +%Y%m%dT%H%M%SZ)"
+OUT="${LOG_DIR}/aggregate-${TS}.json"
+
+command -v python3 >/dev/null 2>&1 || { echo "python3 required" >&2; exit 1; }
+mkdir -p "$LOG_DIR"
+
+python3 - "$LOG_DIR" "$OUT" <<'PY'
+import sys, json, glob, os
+from collections import Counter, defaultdict
+log_dir, out_path = sys.argv[1], sys.argv[2]
+tool_counts = Counter()
+outcome_counts = Counter()
+per_session = defaultdict(int)
+total = 0
+for f in glob.glob(os.path.join(log_dir, "*.jsonl")):
+    if os.path.basename(f).startswith(("aggregate-", "milestones")):
+        continue
+    for line in open(f):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            e = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        total += 1
+        for t in e.get("tool_calls", []):
+            tool_counts[t] += 1
+        outcome_counts[e.get("outcome", "unknown")] += 1
+        per_session[e.get("session", "unknown")] += 1
+summary = {
+    "generated": os.path.basename(out_path),
+    "total_events": total,
+    "tool_calls": dict(tool_counts.most_common()),
+    "outcomes": dict(outcome_counts),
+    "sessions": dict(per_session),
+    "note": "agent/team/sprint/token enrichment pending Cognee-side join; raw hook payload lacks them.",
+}
+json.dump(summary, open(out_path, "w"), indent=2)
+print(f"aggregated {total} events -> {out_path}")
+print("top tools:", dict(tool_counts.most_common(5)))
+PY
+
+echo "Summary written. Push to Cognee as Sprint/Agent metric nodes once the"
+echo "container is up (see config/improvement-queries.md for the queries it feeds)."

package/payload/mishkan/scripts/seed-curated-library.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# MISHKAN — seed the curated library into Cognee.
+# Reads config/curated-library.yaml, emits a JSONL of CuratedResource entries,
+# then ingests them as typed Team + CuratedResource nodes via the structured
+# low-level pipeline (ingest-curated.py) run INSIDE the cognee-mcp container.
+# No LLM extraction — embeddings only (use local Ollama embeddings to avoid
+# cloud rate walls on bulk ingest).
+#
+# One-time bootstrap. The ingest PRUNES the graph first for a clean, reproducible
+# seed — run before real session knowledge accumulates.
+set -euo pipefail
+
+MISHKAN="${HOME}/.claude/mishkan"
+LIB="${MISHKAN}/config/curated-library.yaml"
+OUT="${MISHKAN}/cognee/curated-resources.jsonl"
+INGEST="${MISHKAN}/cognee/ingest-curated.py"
+CONTAINER="${COGNEE_CONTAINER:-mishkan-curated-mcp}"
+CTR_JSONL="/home/cognee/curated-resources.jsonl"
+CTR_INGEST="/home/cognee/ingest-curated.py"
+
+command -v python3 >/dev/null 2>&1 || { echo "python3 required" >&2; exit 1; }
+[ -f "$LIB" ] || { echo "curated library not found: $LIB" >&2; exit 1; }
+mkdir -p "${MISHKAN}/cognee"
+
+# Convert YAML → JSONL of CuratedResource nodes (ontology type=CuratedResource).
+python3 - "$LIB" "$OUT" <<'PY'
+import sys, json
+try:
+    import yaml
+except ImportError:
+    sys.exit("pyyaml required: pip install pyyaml")
+lib_path, out_path = sys.argv[1], sys.argv[2]
+data = yaml.safe_load(open(lib_path))
+n = 0
+with open(out_path, "w") as f:
+    for team, items in (data or {}).items():
+        for it in items:
+            node = {
+                "type": "CuratedResource",
+                "team": team,
+                "name": it["name"],
+                "url": it["url"],
+                "problem_class": it.get("problem_class", ""),
+                "source_tier": "curated",
+            }
+            f.write(json.dumps(node) + "\n")
+            n += 1
+print(f"wrote {n} CuratedResource nodes -> {out_path}")
+PY
+
+# Ingest into Cognee via the structured low-level pipeline inside the container.
+if docker ps --format '{{.Names}}' 2>/dev/null | grep -qx "${CONTAINER}"; then
+  echo "Ingesting into ${CONTAINER} (structured Team + CuratedResource nodes)..."
+  [ -f "$INGEST" ] || { echo "ingest script missing: $INGEST" >&2; exit 1; }
+  docker cp "$OUT" "${CONTAINER}:${CTR_JSONL}"
+  docker cp "$INGEST" "${CONTAINER}:${CTR_INGEST}"
+  docker exec -i -w /app/cognee-mcp "${CONTAINER}" uv run python -u "${CTR_INGEST}"
+  echo "Seed complete. Verify: MATCH (r:CuratedResource) RETURN r.team, count(*);"
+else
+  echo "Cognee container '${CONTAINER}' not running. JSONL is ready at ${OUT};"
+  echo "bring the stack up and re-run this script (or set COGNEE_CONTAINER)."
+fi

package/payload/mishkan/scripts/sync-profile.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+# MISHKAN — profile propagation (mechanical layer).
+# Copies the canonical engineer profile to the runtime path every reference uses,
+# and audits the harness for references + drift. The semantic re-derivation of
+# digests drawn from the profile is Seraiah's job, not this script's.
+#
+# Usage:
+#   sync-profile.sh            # copy canonical -> runtime, then report references
+#   sync-profile.sh --check    # report references + drift only (no copy)
+#
+# Canonical source resolution:
+#   1. $MISHKAN_PROFILE_SRC if set
+#   2. <repo>/docs/engineer/profile.md  (when run from a MISHKAN checkout)
+#   3. the existing runtime copy (treated as source if no repo canonical found)
+set -uo pipefail
+
+MISHKAN="${HOME}/.claude/mishkan"
+RUNTIME="${MISHKAN}/profile.md"
+CHECK_ONLY=0
+[ "${1:-}" = "--check" ] && CHECK_ONLY=1
+
+resolve_src() {
+  if [ -n "${MISHKAN_PROFILE_SRC:-}" ] && [ -f "$MISHKAN_PROFILE_SRC" ]; then
+    echo "$MISHKAN_PROFILE_SRC"; return
+  fi
+  # walk up from cwd looking for docs/engineer/profile.md
+  local d; d="$(pwd)"
+  while [ "$d" != "/" ]; do
+    [ -f "$d/docs/engineer/profile.md" ] && { echo "$d/docs/engineer/profile.md"; return; }
+    d="$(dirname "$d")"
+  done
+  [ -f "$RUNTIME" ] && { echo "$RUNTIME"; return; }
+  echo ""; return
+}
+
+SRC="$(resolve_src)"
+if [ -z "$SRC" ]; then
+  echo "sync-profile: no canonical profile found (set MISHKAN_PROFILE_SRC or run from a MISHKAN checkout)." >&2
+  exit 1
+fi
+
+if [ "$CHECK_ONLY" -eq 0 ]; then
+  if [ "$SRC" != "$RUNTIME" ]; then
+    mkdir -p "$MISHKAN"
+    cp "$SRC" "$RUNTIME"
+    echo "synced: $SRC -> $RUNTIME"
+  else
+    echo "canonical == runtime ($RUNTIME); nothing to copy."
+  fi
+fi
+
+echo "--- references to the engineer profile across the harness ---"
+# Files that reference the runtime path (exclude this script and the runtime file itself).
+grep -rl "profile.md" "$MISHKAN" "${HOME}/.claude/CLAUDE.md" --exclude=sync-profile.sh 2>/dev/null \
+  | grep -v "/profile.md$" | sed "s#${HOME}#~#g" || echo "  (no path references found)"
+
+# Stale check excludes this script (its grep pattern contains the legacy string by design).
+STALE="$(grep -rl "Y4NN_profile" "$MISHKAN" "${HOME}/.claude/CLAUDE.md" --exclude=sync-profile.sh 2>/dev/null || true)"
+if [ -n "$STALE" ]; then
+  echo "--- DRIFT: stale 'Y4NN_profile' references (should be 'profile.md') ---"
+  echo "$STALE" | sed "s#${HOME}#~#g"
+  echo ">> Ask Seraiah to update these and re-derive any digests."
+else
+  echo "no stale references. (Semantic digests in CLAUDE.md are Seraiah's to re-derive if the profile changed materially.)"
+fi

package/payload/mishkan/scripts/validate-research-log.sh

@@ -0,0 +1,108 @@
+#!/usr/bin/env bash
+# validate-research-log.sh — enforce the research-log.schema.json contract.
+#
+# Usage:
+#   validate-research-log.sh <path-to-research-log.json>
+#
+# Exit codes:
+#   0  → valid
+#   1  → invalid (schema violation or missing required field)
+#   2  → environment problem (jq/ajv missing, schema not found, file unreadable)
+#
+# Why this exists: Baruch is the terminal stage of the research pipeline. Its
+# output is the durable record of every research run. A free-text reporter
+# eventually drifts — a schema-validated one cannot. This script is the
+# enforcement layer around payload/mishkan/templates/research-log.schema.json.
+
+set -euo pipefail
+
+LOG_PATH="${1:-}"
+if [[ -z "$LOG_PATH" ]]; then
+  echo "usage: $(basename "$0") <path-to-research-log.json>" >&2
+  exit 2
+fi
+if [[ ! -r "$LOG_PATH" ]]; then
+  echo "error: cannot read $LOG_PATH" >&2
+  exit 2
+fi
+
+SCHEMA_CANDIDATES=(
+  "${MISHKAN_HOME:-$HOME/.claude/mishkan}/templates/research-log.schema.json"
+  "$(dirname "$0")/../templates/research-log.schema.json"
+)
+SCHEMA_PATH=""
+for c in "${SCHEMA_CANDIDATES[@]}"; do
+  if [[ -r "$c" ]]; then SCHEMA_PATH="$c"; break; fi
+done
+if [[ -z "$SCHEMA_PATH" ]]; then
+  echo "error: research-log.schema.json not found in any known location" >&2
+  printf '  searched: %s\n' "${SCHEMA_CANDIDATES[@]}" >&2
+  exit 2
+fi
+
+# Layer 1 — JSON well-formedness (fast, always available).
+if ! command -v jq >/dev/null 2>&1; then
+  echo "error: jq is required" >&2
+  exit 2
+fi
+if ! jq -e . "$LOG_PATH" >/dev/null 2>&1; then
+  echo "invalid: $LOG_PATH is not valid JSON" >&2
+  exit 1
+fi
+
+# Layer 2 — Required-field check (works without ajv; mirrors schema 'required').
+REQUIRED=(agent team sprint trigger query_intent tools_invoked
+          research_output_summary applied_to_task outcome
+          knowledge_graph_write curated_library_match)
+missing=()
+for f in "${REQUIRED[@]}"; do
+  if ! jq -e --arg k "$f" 'has($k)' "$LOG_PATH" >/dev/null 2>&1; then
+    missing+=("$f")
+  fi
+done
+if (( ${#missing[@]} > 0 )); then
+  echo "invalid: missing required fields: ${missing[*]}" >&2
+  exit 1
+fi
+
+# Enum checks (cheap, semantic).
+trigger=$(jq -r '.trigger' "$LOG_PATH")
+case "$trigger" in
+  faced_problem|requested) ;;
+  *) echo "invalid: trigger must be one of faced_problem|requested (got '$trigger')" >&2; exit 1 ;;
+esac
+
+outcome=$(jq -r '.outcome' "$LOG_PATH")
+case "$outcome" in
+  resolved|partial|blocked) ;;
+  *) echo "invalid: outcome must be one of resolved|partial|blocked (got '$outcome')" >&2; exit 1 ;;
+esac
+
+sprint=$(jq -r '.sprint' "$LOG_PATH")
+if [[ ! "$sprint" =~ ^S[0-9]+$ ]]; then
+  echo "invalid: sprint must match ^S[0-9]+\$ (got '$sprint')" >&2
+  exit 1
+fi
+
+# Layer 3 — Full JSON Schema validation when ajv is available.
+# This catches additionalProperties, type mismatches, and format violations
+# that the layer-2 fast checks do not.
+if command -v ajv >/dev/null 2>&1; then
+  if ! ajv validate -s "$SCHEMA_PATH" -d "$LOG_PATH" --strict=false >/dev/null 2>&1; then
+    echo "invalid: schema validation failed (ajv):" >&2
+    ajv validate -s "$SCHEMA_PATH" -d "$LOG_PATH" --strict=false >&2 || true
+    exit 1
+  fi
+elif command -v check-jsonschema >/dev/null 2>&1; then
+  if ! check-jsonschema --schemafile "$SCHEMA_PATH" "$LOG_PATH" >/dev/null 2>&1; then
+    echo "invalid: schema validation failed (check-jsonschema):" >&2
+    check-jsonschema --schemafile "$SCHEMA_PATH" "$LOG_PATH" >&2 || true
+    exit 1
+  fi
+else
+  # No JSON-Schema validator available; layers 1+2 still ran.
+  echo "warn: ajv/check-jsonschema not installed; ran fast checks only" >&2
+fi
+
+echo "valid: $LOG_PATH"
+exit 0