@kontourai/flow-agents 0.2.0 → 0.3.0

package/evals/acceptance/test_knowledge_kit_live.sh

@@ -0,0 +1,221 @@
+#!/usr/bin/env bash
+# test_knowledge_kit_live.sh — Acceptance: Knowledge Kit S5 live example
+#
+# Gated on:
+#   1. ollama binary at /run/current-system/sw/bin/ollama
+#   2. qwen3:1.7b model pulled (checked via ollama list)
+#   3. Python venv with strands-agents[ollama] at /tmp/strands-py-live/venv
+#
+# Skips cleanly if any gate is absent (matching other harness conventions).
+# Starts ollama serve, runs the live example, asserts evidence, stops ollama.
+#
+# Assertions:
+#   A1. Script exits 0 (overall PASS printed)
+#   A2. <workspace>/.telemetry/full.jsonl exists and contains tool.invoke + tool.result
+#   A3. <workspace>/.flow-agents/.telemetry/full.jsonl exists and contains
+#       session.start, tool.invoke, tool.result (FlowAgentsHooks events)
+#   A4. No new .telemetry directory created in the workspace's parent directory
+#       by this script (pre-existing parent-dir .telemetry is not counted)
+#   A5. At least 1 compiled record in <workspace>/.knowledge-store/records/
+#   A6. Compiled record has provenance source_ids referencing raw records
+#
+set -uo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+
+OLLAMA_BIN="/run/current-system/sw/bin/ollama"
+VENV_PYTHON="/tmp/strands-py-live/venv/bin/python3"
+EXAMPLE_SCRIPT="$ROOT_DIR/integrations/strands/examples/knowledge_kit_live.py"
+
+pass=0
+fail=0
+skip=0
+OLLAMA_STARTED=0
+
+_pass() { echo "  ✓ $1"; pass=$((pass + 1)); }
+_fail() { echo "  ✗ $1"; fail=$((fail + 1)); }
+_skip() { echo "  ○ $1"; skip=$((skip + 1)); }
+
+cleanup() {
+  if [[ "$OLLAMA_STARTED" -eq 1 ]]; then
+    pkill -f "ollama serve" 2>/dev/null || true
+  fi
+}
+trap cleanup EXIT
+
+echo "=== Acceptance: Knowledge Kit S5 Live Example ==="
+echo ""
+
+# ── Gate checks ─────────────────────────────────────────────────────────────
+if [[ ! -x "$OLLAMA_BIN" ]]; then
+  _skip "ollama binary not found at $OLLAMA_BIN"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 0
+fi
+
+if [[ ! -x "$VENV_PYTHON" ]]; then
+  _skip "Python venv not found at $VENV_PYTHON — run: python3 -m venv /tmp/strands-py-live/venv && /tmp/strands-py-live/venv/bin/pip install 'strands-agents[ollama]'"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 0
+fi
+
+_pass "Gate: ollama binary present"
+_pass "Gate: Python venv with strands-agents present"
+echo ""
+
+# ── Start ollama serve ───────────────────────────────────────────────────────
+echo "--- Starting ollama serve ---"
+"$OLLAMA_BIN" serve > /tmp/ollama-knowledge-kit-live.log 2>&1 &
+OLLAMA_STARTED=1
+
+# Wait for server to be ready (up to 15 seconds)
+for i in {1..15}; do
+  if curl -s localhost:11434/v1/models >/dev/null 2>&1; then
+    _pass "ollama serve ready (${i}s)"
+    break
+  fi
+  if [[ "$i" -eq 15 ]]; then
+    _fail "ollama serve did not start within 15 seconds"
+    echo ""
+    echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+    exit 1
+  fi
+  sleep 1
+done
+
+# Model gate AFTER server start: ollama list errors when no server is running,
+# which previously misreported a pulled model as missing (skip-path bug).
+if ! "$OLLAMA_BIN" list 2>/dev/null | grep -q "qwen3:1.7b"; then
+  _skip "qwen3:1.7b model not pulled — run: ollama pull qwen3:1.7b"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 0
+fi
+_pass "Gate: qwen3:1.7b model pulled"
+echo ""
+
+# ── Run the example ──────────────────────────────────────────────────────────
+echo "--- Running knowledge_kit_live.py ---"
+EXAMPLE_OUTPUT="$(mktemp /tmp/knowledge-kit-live-output.XXXXXX)"
+
+FLOW_AGENTS_ROOT="$ROOT_DIR" \
+  "$VENV_PYTHON" "$EXAMPLE_SCRIPT" 2>&1 | tee "$EXAMPLE_OUTPUT"
+EXAMPLE_EXIT="${PIPESTATUS[0]}"
+
+echo ""
+
+# ── Assert A1: script exits 0 ─────────────────────────────────────────────
+if [[ "$EXAMPLE_EXIT" -eq 0 ]]; then
+  _pass "A1: example script exits 0"
+else
+  _fail "A1: example script exited $EXAMPLE_EXIT"
+fi
+
+# Extract workspace path from script output
+WORKSPACE="$(grep "^Workspace: " "$EXAMPLE_OUTPUT" | head -1 | sed 's/^Workspace: //')"
+if [[ -z "$WORKSPACE" ]]; then
+  _fail "Could not extract workspace path from script output"
+  echo ""
+  echo "Results: ${pass}/$((pass + fail)) passed, ${fail} failed, ${skip} skipped"
+  exit 1
+fi
+
+echo "  Workspace: $WORKSPACE"
+KIT_TELEMETRY="$WORKSPACE/.telemetry/full.jsonl"
+SESSION_TELEMETRY="$WORKSPACE/.flow-agents/.telemetry/full.jsonl"
+STORE_RECORDS="$WORKSPACE/.knowledge-store/records"
+
+# ── Assert A2: kit telemetry contains tool.invoke + tool.result ───────────
+if [[ -f "$KIT_TELEMETRY" ]] && \
+  node -e "
+const fs = require('fs');
+const lines = fs.readFileSync('$KIT_TELEMETRY', 'utf8').trim().split('\n').filter(Boolean);
+const types = lines.map(l => { try { return JSON.parse(l).event_type; } catch(e) { return ''; } });
+const required = ['tool.invoke', 'tool.result'];
+const missing = required.filter(t => !types.includes(t));
+if (missing.length > 0) { process.stderr.write('missing: ' + missing.join(', ') + '\n'); process.exit(1); }
+" 2>/dev/null; then
+  _pass "A2: kit telemetry contains tool.invoke + tool.result gate events"
+else
+  _fail "A2: kit telemetry missing or lacks required event types (tool.invoke, tool.result)"
+fi
+
+# ── Assert A3: session telemetry contains session.start, tool.invoke, tool.result ─
+if [[ -f "$SESSION_TELEMETRY" ]] && \
+  node -e "
+const fs = require('fs');
+const lines = fs.readFileSync('$SESSION_TELEMETRY', 'utf8').trim().split('\n').filter(Boolean);
+const types = lines.map(l => { try { return JSON.parse(l).event_type; } catch(e) { return ''; } });
+const required = ['session.start', 'tool.invoke', 'tool.result'];
+const missing = required.filter(t => !types.includes(t));
+if (missing.length > 0) { process.stderr.write('missing: ' + missing.join(', ') + '\n'); process.exit(1); }
+" 2>/dev/null; then
+  _pass "A3: session telemetry contains session.start, tool.invoke, tool.result"
+else
+  _fail "A3: session telemetry missing or lacks required FlowAgentsHooks events"
+fi
+
+# ── Assert A4: workspace telemetry does not leak to parent ────────────────
+# This assertion checks that telemetry written during this test run does not
+# appear in the parent directory. We verify that the workspace telemetry is
+# contained within WORKSPACE, not in its parent.
+# (Pre-existing .telemetry in the system temp dir is not counted as a leak.)
+PARENT_TELEMETRY="$(dirname "$WORKSPACE")/.telemetry"
+if [[ -d "$PARENT_TELEMETRY" ]]; then
+  # Only fail if the directory was modified during our test (mtime within last 60s)
+  PARENT_MTIME="$(find "$PARENT_TELEMETRY" -newer "$EXAMPLE_OUTPUT" -name "*.jsonl" 2>/dev/null | wc -l | tr -d ' ')"
+  if [[ "$PARENT_MTIME" -gt 0 ]]; then
+    _fail "A4: telemetry leaked — new .jsonl files written to workspace parent directory during this test"
+  else
+    _pass "A4: workspace telemetry contained within workspace (pre-existing parent .telemetry not modified by this test)"
+  fi
+else
+  _pass "A4: no .telemetry in workspace parent directory"
+fi
+
+# ── Assert A5: at least 1 compiled record exists ─────────────────────────
+COMPILED_COUNT=0
+if [[ -d "$STORE_RECORDS" ]]; then
+  COMPILED_COUNT=$(grep -rl "type: compiled" "$STORE_RECORDS"/*.md 2>/dev/null | wc -l | tr -d ' ')
+fi
+if [[ "$COMPILED_COUNT" -ge 1 ]]; then
+  _pass "A5: compiled record found in store ($COMPILED_COUNT)"
+else
+  _fail "A5: no compiled records found in $STORE_RECORDS"
+fi
+
+# ── Assert A6: compiled record has provenance source_ids ─────────────────
+PROVENANCE_OK=0
+if [[ -d "$STORE_RECORDS" ]]; then
+  for compiled_md in "$STORE_RECORDS"/*.md; do
+    [[ -f "$compiled_md" ]] || continue
+    if grep -q "type: compiled" "$compiled_md" && grep -q "source_ids:" "$compiled_md"; then
+      # Verify at least 2 raw ids are referenced
+      SOURCE_COUNT=$(grep -c "^  - " "$compiled_md" 2>/dev/null || echo 0)
+      if [[ "$SOURCE_COUNT" -ge 2 ]]; then
+        PROVENANCE_OK=1
+        break
+      fi
+    fi
+  done
+fi
+if [[ "$PROVENANCE_OK" -eq 1 ]]; then
+  _pass "A6: compiled record has provenance source_ids with resolving raw refs"
+else
+  _fail "A6: compiled record missing source_ids or insufficient provenance refs"
+fi
+
+# ── Cleanup temp files ───────────────────────────────────────────────────
+rm -f "$EXAMPLE_OUTPUT"
+if [[ -d "$WORKSPACE" ]]; then
+  rm -rf "$WORKSPACE"
+fi
+
+echo ""
+echo "==========================="
+total=$((pass + fail))
+echo "Results: ${pass}/${total} passed, ${fail} failed, ${skip} skipped"
+[[ "$fail" -gt 0 ]] && exit 1
+exit 0

package/evals/acceptance/test_pi_harness.sh

@@ -81,6 +81,21 @@ process.exit(0);
   else
     _fail "pi telemetry missing one or more required event types (session.start, tool.invoke, tool.result, session.end)"
   fi
+
+  # Assert session.start appears exactly once (guards against before_agent_start double-emit).
+  if [[ -f "$telemetry_file" ]] && \
+    node -e "
+const fs = require('fs');
+const lines = fs.readFileSync('$telemetry_file', 'utf8').trim().split('\n');
+const types = lines.map(l => { try { return JSON.parse(l).event_type; } catch(e) { return ''; } });
+const count = types.filter(t => t === 'session.start').length;
+if (count !== 1) { process.stderr.write('session.start count=' + count + ' (expected exactly 1)\n'); process.exit(1); }
+process.exit(0);
+" 2>/dev/null; then
+    _pass "pi telemetry: session.start appears exactly once (no double-emit)"
+  else
+    _fail "pi telemetry: session.start count is not 1 (double-emit or missing)"
+  fi
 fi
 
 PARENT_TELEMETRY="$(dirname "$TMP_WORK")/.telemetry"

package/evals/integration/test_runtime_adapter_activation.sh

@@ -86,16 +86,128 @@ elif node - "$UNKNOWN_OUT" <<'NODE'
 const fs = require("node:fs");
 const data = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
 if (!data.available_adapters?.includes("codex-local")) throw new Error("available adapters missing codex-local");
+if (!data.available_adapters?.includes("strands-local")) throw new Error("available adapters missing strands-local");
 if (!data.errors?.length) throw new Error("unknown adapter did not report errors");
 console.log("ok");
 NODE
 then
-  pass "unknown adapter reports available adapters"
+  pass "unknown adapter reports available adapters (codex-local and strands-local)"
 else
   fail "unknown adapter diagnostics missing"
   sed -n '1,120p' "$UNKNOWN_OUT"
 fi
 
+# -------------------------------------------------------------------------
+# strands-local adapter activation (Issue #32 AC1)
+# -------------------------------------------------------------------------
+
+echo ""
+echo "=== strands-local Adapter Activation Checks (Issue #32 AC1) ==="
+
+STRANDS_DEST="$TMP_DIR/strands-dest"
+STRANDS_OUT="$TMP_DIR/strands-activation.json"
+mkdir -p "$STRANDS_DEST"
+
+# Use the builder kit (stable fixture) — activate for strands-local from the repo source root
+if flow_agents_node "$CLI" activate --dest "$STRANDS_DEST" --source-root "$ROOT" --adapter strands-local --format json >"$STRANDS_OUT" 2>&1; then
+  pass "strands-local activation succeeds"
+else
+  fail "strands-local activation failed"
+  sed -n '1,220p' "$STRANDS_OUT"
+fi
+
+if node - "$STRANDS_OUT" "$STRANDS_DEST" "$ROOT/kits/catalog.json" <<'NODE'
+const fs = require("node:fs");
+const path = require("node:path");
+const data = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
+const dest = process.argv[3];
+const catalog = process.argv[4];
+
+// Verify selected_adapter
+if (data.selected_adapter !== "strands-local") throw new Error(`expected strands-local, got: ${data.selected_adapter}`);
+if (JSON.stringify(data.supported_asset_classes) !== JSON.stringify(["flows"])) throw new Error(`unexpected supported_asset_classes: ${JSON.stringify(data.supported_asset_classes)}`);
+
+// Verify builder kit flows are generated (builder kit is in catalog.json)
+const ids = new Set(data.generated_runtime_files.map((item) => item.asset_id));
+for (const expected of ["builder.shape", "builder.build", "strands-local.activation"]) {
+  if (!ids.has(expected)) throw new Error(`missing generated asset: ${expected}`);
+}
+
+// Verify generated runtime files actually exist on disk
+for (const item of data.generated_runtime_files) {
+  if (item.asset_class === "activation-manifest") continue;
+  const generatedPath = path.join(dest, item.path);
+  if (!fs.existsSync(generatedPath)) throw new Error(`generated file missing: ${generatedPath}`);
+  // Verify runtime files are under .flow-agents/runtime/strands/flows/
+  if (!item.path.includes(".flow-agents/runtime/strands/flows/")) {
+    throw new Error(`generated path not under strands runtime dir: ${item.path}`);
+  }
+}
+
+// Verify activation.json written at strands runtime dir
+const manifestPath = path.join(dest, ".flow-agents/runtime/strands/activation.json");
+if (!fs.existsSync(manifestPath)) throw new Error("strands runtime activation.json missing");
+const manifest = JSON.parse(fs.readFileSync(manifestPath, "utf8"));
+if (manifest.adapter !== "strands-local") throw new Error(`activation.json adapter mismatch: ${manifest.adapter}`);
+if (!Array.isArray(manifest.skipped_assets)) throw new Error("activation.json missing skipped_assets array");
+
+// Verify skipped_assets have expected fields (parity with codex-local)
+for (const item of manifest.skipped_assets) {
+  for (const key of ["asset_class", "path", "kit_id", "asset_id", "reason"]) {
+    if (!(key in item)) throw new Error(`skipped asset missing ${key}: ${JSON.stringify(item)}`);
+  }
+  if (!item.reason.includes("diagnostic-only")) throw new Error(`unexpected skip reason: ${item.reason}`);
+}
+
+// Non-flow asset classes should appear in skipped_assets
+const skippedClasses = new Set(manifest.skipped_assets.map((item) => item.asset_class));
+// builder kit has flows only; skipped_assets check requires a kit with non-flow assets,
+// which the codex-local path already validates via mixed-runtime-kit above.
+// Here we just confirm the field structure is present.
+if (!Array.isArray(data.skipped_assets)) throw new Error("result skipped_assets is not an array");
+
+// Catalog not mutated
+if (path.resolve(catalog) === path.resolve(path.join(dest, ".flow-agents/runtime/strands/activation.json"))) {
+  throw new Error("activation generated over kits/catalog.json");
+}
+
+console.log("ok");
+NODE
+then
+  pass "strands-local: runtime flow files, activation.json, and skipped_assets present with correct structure"
+else
+  fail "strands-local: activation diagnostics incomplete or incorrect"
+  sed -n '1,220p' "$STRANDS_OUT"
+fi
+
+# Verify codex-local activation is still intact (AC3 — existing tests still pass)
+if flow_agents_node "$CLI" activate --dest "$STRANDS_DEST" --source-root "$ROOT" --format json >"$TMP_DIR/codex-after-strands.json" 2>&1; then
+  pass "codex-local still activates after strands-local has run"
+else
+  fail "codex-local activation failed after strands-local activation"
+  sed -n '1,220p' "$TMP_DIR/codex-after-strands.json"
+fi
+
+if node - "$TMP_DIR/codex-after-strands.json" "$STRANDS_DEST" <<'NODE'
+const fs = require("node:fs");
+const path = require("node:path");
+const data = JSON.parse(fs.readFileSync(process.argv[2], "utf8"));
+const dest = process.argv[3];
+if (data.selected_adapter !== "codex-local") throw new Error(`expected codex-local, got: ${data.selected_adapter}`);
+const manifestPath = path.join(dest, ".flow-agents/runtime/codex/activation.json");
+if (!fs.existsSync(manifestPath)) throw new Error("codex activation.json still not present");
+// Strands runtime dir must also still exist
+const strandsManifestPath = path.join(dest, ".flow-agents/runtime/strands/activation.json");
+if (!fs.existsSync(strandsManifestPath)) throw new Error("strands activation.json was removed by codex-local run");
+console.log("ok");
+NODE
+then
+  pass "codex-local and strands-local runtime dirs co-exist independently (AC3)"
+else
+  fail "co-existence check failed"
+  sed -n '1,220p' "$TMP_DIR/codex-after-strands.json"
+fi
+
 echo ""
 if [[ "$errors" -eq 0 ]]; then
   echo "Runtime adapter activation checks passed."