@kontourai/flow-agents 0.1.2 → 0.3.0

package/evals/acceptance/test_knowledge_kit_live.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+# test_knowledge_kit_live.sh — Acceptance: Knowledge Kit S5 live example
+#
+# Gated on:
+#   1. ollama binary at /run/current-system/sw/bin/ollama
+#   2. qwen3:1.7b model pulled (checked via ollama list)
+#   3. Python venv with strands-agents[ollama] at /tmp/strands-py-live/venv
+#
+# Skips cleanly if any gate is absent (matching other harness conventions).
+# Starts ollama serve, runs the live example, asserts evidence, stops ollama.
+#
+# Assertions:
+#   A1. Script exits 0 (overall PASS printed)
+#   A2. <workspace>/.telemetry/full.jsonl exists and contains tool.invoke + tool.result
+#   A3. <workspace>/.flow-agents/.telemetry/full.jsonl exists and contains
+#       session.start, tool.invoke, tool.result (FlowAgentsHooks events)
+#   A4. No new .telemetry directory created in the workspace's parent directory
+#       by this script (pre-existing parent-dir .telemetry is not counted)
+#   A5. At least 1 compiled record in <workspace>/.knowledge-store/records/
+#   A6. Compiled record has provenance source_ids referencing raw records
+#
+set -uo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+OLLAMA_BIN="/run/current-system/sw/bin/ollama"
+VENV_PYTHON="/tmp/strands-py-live/venv/bin/python3"
+EXAMPLE_SCRIPT="$ROOT_DIR/integrations/strands/examples/knowledge_kit_live.py"
+
+pass=0
+fail=0
+skip=0
+OLLAMA_STARTED=0
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+_skip() { echo "  ○ $1"; skip=$((skip + 1)); }
+
+cleanup() {
+  if [[ "$OLLAMA_STARTED" -eq 1 ]]; then
+    pkill -f "ollama serve" 2>/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+echo "=== Acceptance: Knowledge Kit S5 Live Example ==="
+echo ""
+
+# ── Gate checks ─────────────────────────────────────────────────────────────
+if [[ ! -x "$OLLAMA_BIN" ]]; then
+  _skip "ollama binary not found at $OLLAMA_BIN"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 0
+fi
+
+if [[ ! -x "$VENV_PYTHON" ]]; then
+  _skip "Python venv not found at $VENV_PYTHON — run: python3 -m venv /tmp/strands-py-live/venv && /tmp/strands-py-live/venv/bin/pip install 'strands-agents[ollama]'"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 0
+fi
+
+_pass "Gate: ollama binary present"
+_pass "Gate: Python venv with strands-agents present"
+echo ""
+
+# ── Start ollama serve ───────────────────────────────────────────────────────
+echo "--- Starting ollama serve ---"
+"$OLLAMA_BIN" serve > /tmp/ollama-knowledge-kit-live.log 2>&1 &
+OLLAMA_STARTED=1
+
+# Wait for server to be ready (up to 15 seconds)
+for i in {1..15}; do
+  if curl -s localhost:11434/v1/models >/dev/null 2>&1; then
+    _pass "ollama serve ready (${i}s)"
+    break
+  fi
+  if [[ "$i" -eq 15 ]]; then
+    _fail "ollama serve did not start within 15 seconds"
+    echo ""
+    echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+    exit 1
+  fi
+  sleep 1
+done
+
+# Model gate AFTER server start: ollama list errors when no server is running,
+# which previously misreported a pulled model as missing (skip-path bug).
+if ! "$OLLAMA_BIN" list 2>/dev/null | grep -q "qwen3:1.7b"; then
+  _skip "qwen3:1.7b model not pulled — run: ollama pull qwen3:1.7b"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 0
+fi
+_pass "Gate: qwen3:1.7b model pulled"
+echo ""
+
+# ── Run the example ──────────────────────────────────────────────────────────
+echo "--- Running knowledge_kit_live.py ---"
+EXAMPLE_OUTPUT="$(mktemp /tmp/knowledge-kit-live-output.XXXXXX)"
+
+FLOW_AGENTS_ROOT="$ROOT_DIR" \
+  "$VENV_PYTHON" "$EXAMPLE_SCRIPT" 2>&1 | tee "$EXAMPLE_OUTPUT"
+EXAMPLE_EXIT="${PIPESTATUS[0]}"
+
+echo ""
+
+# ── Assert A1: script exits 0 ─────────────────────────────────────────────
+if [[ "$EXAMPLE_EXIT" -eq 0 ]]; then
+  _pass "A1: example script exits 0"
+else
+  _fail "A1: example script exited $EXAMPLE_EXIT"
+fi
+
+# Extract workspace path from script output
+WORKSPACE="$(grep "^Workspace: " "$EXAMPLE_OUTPUT" | head -1 | sed 's/^Workspace: //')"
+if [[ -z "$WORKSPACE" ]]; then
+  _fail "Could not extract workspace path from script output"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 1
+fi
+
+echo "  Workspace: $WORKSPACE"
+KIT_TELEMETRY="$WORKSPACE/.telemetry/full.jsonl"
+SESSION_TELEMETRY="$WORKSPACE/.flow-agents/.telemetry/full.jsonl"
+STORE_RECORDS="$WORKSPACE/.knowledge-store/records"
+
+# ── Assert A2: kit telemetry contains tool.invoke + tool.result ───────────
+if [[ -f "$KIT_TELEMETRY" ]] && \
+  node -e "
+const fs = require('fs');
+const lines = fs.readFileSync('$KIT_TELEMETRY', 'utf8').trim().split('\n').filter(Boolean);
+const types = lines.map(l => { try { return JSON.parse(l).event_type; } catch(e) { return ''; } });
+const required = ['tool.invoke', 'tool.result'];
+const missing = required.filter(t => !types.includes(t));
+if (missing.length > 0) { process.stderr.write('missing: ' + missing.join(', ') + '\n'); process.exit(1); }
+" 2>/dev/null; then
+  _pass "A2: kit telemetry contains tool.invoke + tool.result gate events"
+else
+  _fail "A2: kit telemetry missing or lacks required event types (tool.invoke, tool.result)"
+fi
+
+# ── Assert A3: session telemetry contains session.start, tool.invoke, tool.result ─
+if [[ -f "$SESSION_TELEMETRY" ]] && \
+  node -e "
+const fs = require('fs');
+const lines = fs.readFileSync('$SESSION_TELEMETRY', 'utf8').trim().split('\n').filter(Boolean);
+const types = lines.map(l => { try { return JSON.parse(l).event_type; } catch(e) { return ''; } });
+const required = ['session.start', 'tool.invoke', 'tool.result'];
+const missing = required.filter(t => !types.includes(t));
+if (missing.length > 0) { process.stderr.write('missing: ' + missing.join(', ') + '\n'); process.exit(1); }
+" 2>/dev/null; then
+  _pass "A3: session telemetry contains session.start, tool.invoke, tool.result"
+else
+  _fail "A3: session telemetry missing or lacks required FlowAgentsHooks events"
+fi
+
+# ── Assert A4: workspace telemetry does not leak to parent ────────────────
+# This assertion checks that telemetry written during this test run does not
+# appear in the parent directory. We verify that the workspace telemetry is
+# contained within WORKSPACE, not in its parent.
+# (Pre-existing .telemetry in the system temp dir is not counted as a leak.)
+PARENT_TELEMETRY="$(dirname "$WORKSPACE")/.telemetry"
+if [[ -d "$PARENT_TELEMETRY" ]]; then
+  # Only fail if the directory was modified during our test (mtime within last 60s)
+  PARENT_MTIME="$(find "$PARENT_TELEMETRY" -newer "$EXAMPLE_OUTPUT" -name "*.jsonl" 2>/dev/null | wc -l | tr -d ' ')"
+  if [[ "$PARENT_MTIME" -gt 0 ]]; then
+    _fail "A4: telemetry leaked — new .jsonl files written to workspace parent directory during this test"
+  else
+    _pass "A4: workspace telemetry contained within workspace (pre-existing parent .telemetry not modified by this test)"
+  fi
+else
+  _pass "A4: no .telemetry in workspace parent directory"
+fi
+
+# ── Assert A5: at least 1 compiled record exists ─────────────────────────
+COMPILED_COUNT=0
+if [[ -d "$STORE_RECORDS" ]]; then
+  COMPILED_COUNT=$(grep -rl "type: compiled" "$STORE_RECORDS"/*.md 2>/dev/null | wc -l | tr -d ' ')
+fi
+if [[ "$COMPILED_COUNT" -ge 1 ]]; then
+  _pass "A5: compiled record found in store ($COMPILED_COUNT)"
+else
+  _fail "A5: no compiled records found in $STORE_RECORDS"
+fi
+
+# ── Assert A6: compiled record has provenance source_ids ─────────────────
+PROVENANCE_OK=0
+if [[ -d "$STORE_RECORDS" ]]; then
+  for compiled_md in "$STORE_RECORDS"/*.md; do
+    [[ -f "$compiled_md" ]] || continue
+    if grep -q "type: compiled" "$compiled_md" && grep -q "source_ids:" "$compiled_md"; then
+      # Verify at least 2 raw ids are referenced
+      SOURCE_COUNT=$(grep -c "^  - " "$compiled_md" 2>/dev/null || echo 0)
+      if [[ "$SOURCE_COUNT" -ge 2 ]]; then
+        PROVENANCE_OK=1
+        break
+      fi
+    fi
+  done
+fi
+if [[ "$PROVENANCE_OK" -eq 1 ]]; then
+  _pass "A6: compiled record has provenance source_ids with resolving raw refs"
+else
+  _fail "A6: compiled record missing source_ids or insufficient provenance refs"
+fi
+
+# ── Cleanup temp files ───────────────────────────────────────────────────
+rm -f "$EXAMPLE_OUTPUT"
+if [[ -d "$WORKSPACE" ]]; then
+  rm -rf "$WORKSPACE"
+fi
+
+echo ""
+echo "==========================="
+total=$((pass + fail))
+echo "Results: ${pass}/${total} passed, ${fail} failed, ${skip} skipped"
+[[ "$fail" -gt 0 ]] && exit 1
+exit 0

package/evals/acceptance/test_opencode_harness.sh

@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT_DIR/evals/lib/node.sh"
+TMP_WORK=""
+pass=0
+fail=0
+skip=0
+
+cleanup() {
+  [[ -n "$TMP_WORK" ]] && rm -rf "$TMP_WORK"
+}
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+_skip() { echo "  ○ $1"; skip=$((skip + 1)); }
+
+wait_for_telemetry() {
+  local file="$1"
+  local i=0
+  while [[ $i -lt 150 ]]; do
+    [[ -s "$file" ]] && return 0
+    sleep 0.1
+    i=$((i + 1))
+  done
+  return 1
+}
+
+echo "=== Harness Acceptance: opencode ==="
+echo ""
+
+if ! command -v opencode >/dev/null 2>&1; then
+  _skip "opencode CLI not installed"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 0
+fi
+
+cd "$ROOT_DIR"
+flow_agents_node scripts/build-universal-bundles.js >/dev/null
+
+TMP_WORK="$(mktemp -d /tmp/opencode-acceptance-work.XXXXXX)"
+(cd dist/opencode && bash install.sh "$TMP_WORK") >/dev/null
+
+echo "--- Plugin Load + Telemetry ---"
+cd "$TMP_WORK"
+rm -rf .telemetry
+
+MODEL_ARGS=()
+if [[ -n "${FLOW_AGENTS_ACCEPT_OPENCODE_MODEL:-}" ]]; then
+  MODEL_ARGS=(-m "$FLOW_AGENTS_ACCEPT_OPENCODE_MODEL")
+fi
+
+# Models sometimes answer without calling the tool (nondeterminism), which
+# would void the tool.invoke/tool.result assertions — force the tool call
+# and retry once if no tool events landed.
+ACCEPT_PROMPT="You MUST call the read tool before replying — answering from memory is a failure. Read the first 5 lines of README.md with the read tool, then reply: done"
+run_output=""
+provider_error=0
+for _attempt in 1 2; do
+  run_output="$(opencode run "${MODEL_ARGS[@]}" "$ACCEPT_PROMPT" 2>&1 || true)"
+  if echo "$run_output" | grep -qi "error"; then
+    provider_error=1
+    break
+  fi
+  provider_error=0
+  for _i in $(seq 1 50); do
+    [[ -s "$TMP_WORK/.telemetry/full.jsonl" ]] && grep -q '"tool.invoke"' "$TMP_WORK/.telemetry/full.jsonl" 2>/dev/null && break
+    sleep 0.3
+  done
+  grep -q '"tool.invoke"' "$TMP_WORK/.telemetry/full.jsonl" 2>/dev/null && break
+done
+
+LATEST_LOG="$(ls -t ~/.local/share/opencode/log/*.log 2>/dev/null | head -1 || true)"
+if [[ -n "$LATEST_LOG" ]] && grep -q "plugins/flow-agents.js loading plugin" "$LATEST_LOG" 2>/dev/null; then
+  _pass "opencode log confirms flow-agents plugin loaded"
+else
+  _fail "opencode log did not confirm flow-agents plugin loaded"
+fi
+
+telemetry_file="$TMP_WORK/.telemetry/full.jsonl"
+if [[ "$provider_error" -eq 1 ]]; then
+  _skip "opencode telemetry assertions skipped (provider/auth error)"
+  _skip "opencode telemetry tool events skipped (provider/auth error)"
+else
+  if wait_for_telemetry "$telemetry_file"; then
+    _pass "opencode telemetry log was written"
+  else
+    _fail "opencode telemetry log was not written"
+  fi
+
+  if [[ -f "$telemetry_file" ]] && \
+    node -e "
+const fs = require('fs');
+const lines = fs.readFileSync('$telemetry_file', 'utf8').trim().split('\n');
+const types = lines.map(l => { try { return JSON.parse(l).event_type; } catch(e) { return ''; } });
+const hasInvoke = types.some(t => t === 'tool.invoke');
+const hasResult = types.some(t => t === 'tool.result');
+process.exit(hasInvoke && hasResult ? 0 : 1);
+" 2>/dev/null; then
+    _pass "opencode telemetry contains tool.invoke and tool.result events"
+  else
+    _fail "opencode telemetry missing tool.invoke or tool.result events"
+  fi
+fi
+
+PARENT_TELEMETRY="$(dirname "$TMP_WORK")/.telemetry"
+if [[ -d "$PARENT_TELEMETRY" ]]; then
+  _fail "opencode wrote .telemetry to workspace parent directory"
+else
+  _pass "no .telemetry leak to workspace parent directory"
+fi
+
+echo ""
+echo "==========================="
+total=$((pass + fail))
+echo "Results: ${pass}/${total} passed, ${fail} failed, ${skip} skipped"
+[[ "$fail" -gt 0 ]] && exit 1
+exit 0

package/evals/acceptance/test_pi_harness.sh

@@ -0,0 +1,113 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT_DIR/evals/lib/node.sh"
+TMP_WORK=""
+pass=0
+fail=0
+skip=0
+
+cleanup() {
+  [[ -n "$TMP_WORK" ]] && rm -rf "$TMP_WORK"
+}
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+_skip() { echo "  ○ $1"; skip=$((skip + 1)); }
+
+wait_for_telemetry() {
+  local file="$1"
+  local i=0
+  while [[ $i -lt 150 ]]; do
+    [[ -s "$file" ]] && return 0
+    sleep 0.1
+    i=$((i + 1))
+  done
+  return 1
+}
+
+echo "=== Harness Acceptance: pi ==="
+echo ""
+
+if ! command -v pi >/dev/null 2>&1; then
+  _skip "pi CLI not installed"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 0
+fi
+
+cd "$ROOT_DIR"
+flow_agents_node scripts/build-universal-bundles.js >/dev/null
+
+TMP_WORK="$(mktemp -d /tmp/pi-acceptance-work.XXXXXX)"
+(cd dist/pi && bash install.sh "$TMP_WORK") >/dev/null
+
+echo "--- Telemetry ---"
+cd "$TMP_WORK"
+rm -rf .telemetry
+
+run_output="$(pi --approve -p \
+  "Use your read tool to read the first 5 lines of README.md, then reply: done" 2>&1 || true)"
+provider_error=0
+if echo "$run_output" | grep -qi "error"; then
+  provider_error=1
+fi
+
+telemetry_file="$TMP_WORK/.telemetry/full.jsonl"
+if [[ "$provider_error" -eq 1 ]]; then
+  _skip "pi telemetry assertions skipped (provider/auth error)"
+  _skip "pi telemetry event types skipped (provider/auth error)"
+  _skip "pi telemetry session events skipped (provider/auth error)"
+else
+  if wait_for_telemetry "$telemetry_file"; then
+    _pass "pi telemetry log was written"
+  else
+    _fail "pi telemetry log was not written"
+  fi
+
+  if [[ -f "$telemetry_file" ]] && \
+    node -e "
+const fs = require('fs');
+const lines = fs.readFileSync('$telemetry_file', 'utf8').trim().split('\n');
+const types = lines.map(l => { try { return JSON.parse(l).event_type; } catch(e) { return ''; } });
+const required = ['session.start', 'tool.invoke', 'tool.result', 'session.end'];
+const missing = required.filter(t => !types.includes(t));
+if (missing.length > 0) { process.stderr.write('missing: ' + missing.join(', ') + '\n'); process.exit(1); }
+process.exit(0);
+" 2>/dev/null; then
+    _pass "pi telemetry contains session.start, tool.invoke, tool.result, session.end"
+  else
+    _fail "pi telemetry missing one or more required event types (session.start, tool.invoke, tool.result, session.end)"
+  fi
+
+  # Assert session.start appears exactly once (guards against before_agent_start double-emit).
+  if [[ -f "$telemetry_file" ]] && \
+    node -e "
+const fs = require('fs');
+const lines = fs.readFileSync('$telemetry_file', 'utf8').trim().split('\n');
+const types = lines.map(l => { try { return JSON.parse(l).event_type; } catch(e) { return ''; } });
+const count = types.filter(t => t === 'session.start').length;
+if (count !== 1) { process.stderr.write('session.start count=' + count + ' (expected exactly 1)\n'); process.exit(1); }
+process.exit(0);
+" 2>/dev/null; then
+    _pass "pi telemetry: session.start appears exactly once (no double-emit)"
+  else
+    _fail "pi telemetry: session.start count is not 1 (double-emit or missing)"
+  fi
+fi
+
+PARENT_TELEMETRY="$(dirname "$TMP_WORK")/.telemetry"
+if [[ -d "$PARENT_TELEMETRY" ]]; then
+  _fail "pi wrote .telemetry to workspace parent directory"
+else
+  _pass "no .telemetry leak to workspace parent directory"
+fi
+
+echo ""
+echo "==========================="
+total=$((pass + fail))
+echo "Results: ${pass}/${total} passed, ${fail} failed, ${skip} skipped"
+[[ "$fail" -gt 0 ]] && exit 1
+exit 0