@kontourai/flow-agents 0.1.1 → 0.1.2

package/evals/integration/test_utterance_check.sh

@@ -0,0 +1,271 @@
+#!/usr/bin/env bash
+# test_utterance_check.sh — Survey utterance check hook and CLI adapter coverage
+set -uo pipefail
+
+ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+source "$ROOT/evals/lib/node.sh"
+
+TMPDIR_EVAL="$(mktemp -d)"
+errors=0
+
+cleanup() {
+  rm -rf "$TMPDIR_EVAL"
+}
+trap cleanup EXIT
+
+_pass() { echo "  ✓ $1"; }
+_fail() { echo "  ✗ $1"; errors=$((errors + 1)); }
+
+echo "=== Utterance Check Hook and CLI Adapter ==="
+
+HOOK="$ROOT/scripts/hooks/utterance-check.js"
+RUN_HOOK="$ROOT/scripts/hooks/run-hook.js"
+
+# ---------------------------------------------------------------------------
+# Hook: pass-through when disabled (default)
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- hook: disabled by default ---"
+
+INPUT_JSON='{"hook_event_name":"PostToolUse","tool_response":"The coverage is 92% and all tests pass."}'
+
+if node "$HOOK" >"$TMPDIR_EVAL/disabled.out" 2>"$TMPDIR_EVAL/disabled.err" <<< "$INPUT_JSON"; then
+  if grep -qF '"hook_event_name"' "$TMPDIR_EVAL/disabled.out"; then
+    _pass "utterance check hook passes through when FLOW_AGENTS_UTTERANCE_CHECK_ENABLED is unset"
+  else
+    _fail "utterance check hook pass-through output was not the raw input"
+  fi
+else
+  _fail "utterance check hook should exit 0 when disabled"
+fi
+
+# ---------------------------------------------------------------------------
+# Hook: pass-through with empty input
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- hook: empty input ---"
+
+if FLOW_AGENTS_UTTERANCE_CHECK_ENABLED=true node "$HOOK" >"$TMPDIR_EVAL/empty.out" 2>"$TMPDIR_EVAL/empty.err" <<< '{}'; then
+  _pass "utterance check hook passes through when no utterance text is present"
+else
+  _fail "utterance check hook should exit 0 on empty input"
+fi
+
+# ---------------------------------------------------------------------------
+# Hook: pass-through when CLI is not built yet
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- hook: missing CLI gracefully fails open ---"
+
+if FLOW_AGENTS_UTTERANCE_CHECK_ENABLED=true \
+   node "$HOOK" >"$TMPDIR_EVAL/nocli.out" 2>"$TMPDIR_EVAL/nocli.err" <<JSON
+{"hook_event_name":"PostToolUse","tool_response":"Some agent text."}
+JSON
+then
+  # Either built CLI path worked, or hook failed open (exit 0)
+  _pass "utterance check hook fails open when CLI or survey is not available"
+else
+  _fail "utterance check hook should not block when CLI is unavailable"
+fi
+
+# ---------------------------------------------------------------------------
+# Hook: respects SA_DISABLED_HOOKS through run-hook.js
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- hook: run-hook.js respects SA_DISABLED_HOOKS ---"
+
+HOOK_INPUT='{"hook_event_name":"PostToolUse","tool_response":"text"}'
+
+if SA_DISABLED_HOOKS=post:utterance-check \
+   node "$RUN_HOOK" post:utterance-check utterance-check.js standard,strict \
+   >"$TMPDIR_EVAL/disabled-runner.out" 2>"$TMPDIR_EVAL/disabled-runner.err" <<< "$HOOK_INPUT"
+then
+  if cmp -s "$TMPDIR_EVAL/disabled-runner.out" <(printf '%s
+' "$HOOK_INPUT"); then
+    _pass "run-hook.js passes input through when hook id is in SA_DISABLED_HOOKS"
+  else
+    _fail "run-hook.js disabled hook output did not match raw input"
+  fi
+else
+  _fail "run-hook.js with disabled hook should exit 0"
+fi
+
+# ---------------------------------------------------------------------------
+# CLI: build and test --not-configured
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- cli: not-configured output ---"
+
+# Build the TypeScript source if needed
+if [[ ! -f "$ROOT/build/src/cli.js" ]]; then
+  echo "  (building TypeScript source...)"
+  if ! (cd "$ROOT" && npm run build --silent 2>"$TMPDIR_EVAL/build.err"); then
+    _fail "TypeScript build failed: $(cat "$TMPDIR_EVAL/build.err" | head -5)"
+    errors=$((errors + 1))
+    echo ""
+    echo "Utterance check integration tests failed: $errors issue(s)."
+    exit 1
+  fi
+fi
+
+if node "$ROOT/build/src/cli.js" utterance-check check --not-configured \
+   >"$TMPDIR_EVAL/not-configured.out" 2>"$TMPDIR_EVAL/not-configured.err"
+then
+  if node -e '
+    const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
+    if (r.status !== "not_configured") process.exit(1);
+    if (!Array.isArray(r.statements)) process.exit(2);
+    if (typeof r.summary !== "string") process.exit(3);
+  ' "$TMPDIR_EVAL/not-configured.out"; then
+    _pass "CLI outputs not_configured JSON when --not-configured is set"
+  else
+    _fail "CLI not-configured output did not match expected shape"
+  fi
+else
+  _fail "CLI should exit 0 with --not-configured"
+fi
+
+# ---------------------------------------------------------------------------
+# CLI: --help exits 0 and prints usage
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- cli: help output ---"
+
+if node "$ROOT/build/src/cli.js" utterance-check --help \
+   >"$TMPDIR_EVAL/help.out" 2>"$TMPDIR_EVAL/help.err"; then
+  if grep -q 'utterance-check check' "$TMPDIR_EVAL/help.err"; then
+    _pass "CLI --help prints usage"
+  else
+    _fail "CLI --help did not print expected usage text"
+  fi
+else
+  _fail "CLI --help should exit 0"
+fi
+
+# ---------------------------------------------------------------------------
+# CLI: missing --utterance exits non-zero
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- cli: missing utterance flag ---"
+
+if node "$ROOT/build/src/cli.js" utterance-check check \
+   >"$TMPDIR_EVAL/no-utterance.out" 2>"$TMPDIR_EVAL/no-utterance.err"
+then
+  _fail "CLI check without --utterance should exit non-zero"
+else
+  _pass "CLI check without --utterance exits non-zero (usage error)"
+fi
+
+# ---------------------------------------------------------------------------
+# CLI: survey not installed → not_configured output, exits 1
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- cli: @kontourai/survey not installed ---"
+
+# Run with a NODE_PATH that does not include any survey package, so the
+# dynamic import fails. node's module resolution will not find @kontourai/survey
+# from this test since it is not installed in flow-agents/node_modules.
+if node "$ROOT/build/src/cli.js" utterance-check check \
+   --utterance "The test coverage is 92%." \
+   >"$TMPDIR_EVAL/no-survey.out" 2>"$TMPDIR_EVAL/no-survey.err"
+then
+  # survey might be installed; check for not_configured or ok status
+  status_val=$(node -e 'console.log(JSON.parse(require("fs").readFileSync(process.argv[1],"utf8")).status)' \
+    "$TMPDIR_EVAL/no-survey.out" 2>/dev/null || echo "parse-error")
+  if [[ "$status_val" == "ok" || "$status_val" == "not_configured" ]]; then
+    _pass "CLI utterance check produces valid report (status: $status_val)"
+  else
+    _fail "CLI utterance check output had unexpected status: $status_val"
+  fi
+else
+  exit_code=$?
+  # Exit 1 means not_configured (survey not installed) — expected in CI
+  if [[ "$exit_code" -eq 1 ]]; then
+    if node -e '
+      const r = JSON.parse(require("fs").readFileSync(process.argv[1], "utf8"));
+      if (r.status !== "not_configured") process.exit(1);
+    ' "$TMPDIR_EVAL/no-survey.out" 2>/dev/null; then
+      _pass "CLI outputs not_configured when @kontourai/survey is not installed"
+    else
+      _fail "CLI exit 1 but output was not not_configured JSON"
+    fi
+  else
+    _fail "CLI should exit 0 or 1, got exit code: $exit_code"
+  fi
+fi
+
+# ---------------------------------------------------------------------------
+# CLI: utterance check registers as a valid flow-agents command
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- cli: command registration ---"
+
+if node "$ROOT/build/src/cli.js" commands 2>/dev/null | grep -q 'utterance-check'; then
+  _pass "utterance-check is registered as a flow-agents CLI command"
+else
+  _fail "utterance-check is not registered in flow-agents CLI commands"
+fi
+
+# ---------------------------------------------------------------------------
+# Hook: module.exports shape
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- hook: module.exports contract ---"
+
+if node -e '
+  const h = require(process.argv[1]);
+  if (typeof h.run !== "function") { console.error("run missing"); process.exit(1); }
+  if (typeof h.extractUtteranceText !== "function") { console.error("extractUtteranceText missing"); process.exit(2); }
+  if (typeof h.findPackageRoot !== "function") { console.error("findPackageRoot missing"); process.exit(3); }
+' "$HOOK"; then
+  _pass "utterance-check hook exports run, extractUtteranceText, findPackageRoot"
+else
+  _fail "utterance-check hook module.exports is missing expected functions"
+fi
+
+# ---------------------------------------------------------------------------
+# Hook: extractUtteranceText extracts from PostToolUse and Stop events
+# ---------------------------------------------------------------------------
+
+echo ""
+echo "--- hook: extractUtteranceText ---"
+
+if node -e '
+  const { extractUtteranceText } = require(process.argv[1]);
+  const postToolUse = { hook_event_name: "PostToolUse", tool_response: "The answer is 42." };
+  const text = extractUtteranceText(postToolUse);
+  if (text !== "The answer is 42.") { console.error("PostToolUse extract failed:", text); process.exit(1); }
+  const stopWithContent = { hook_event_name: "Stop", content: [{ type: "text", text: "Done!" }] };
+  const text2 = extractUtteranceText(stopWithContent);
+  if (text2 !== "Done!") { console.error("Stop content extract failed:", text2); process.exit(2); }
+  const emptyEvent = { hook_event_name: "PostToolUse" };
+  const text3 = extractUtteranceText(emptyEvent);
+  if (text3 !== null) { console.error("Empty event should return null, got:", text3); process.exit(3); }
+' "$HOOK"; then
+  _pass "extractUtteranceText handles PostToolUse, Stop content, and empty events"
+else
+  _fail "extractUtteranceText behavior was unexpected"
+fi
+
+# ---------------------------------------------------------------------------
+# Summary
+# ---------------------------------------------------------------------------
+
+echo ""
+if [[ "$errors" -eq 0 ]]; then
+  echo "Utterance check integration tests passed."
+  exit 0
+fi
+
+echo "Utterance check integration tests failed: $errors issue(s)."
+exit 1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@kontourai/flow-agents",
-  "version": "0.1.1",
+  "version": "0.1.2",
   "description": "Flow Agents — a Kontour product that applies Flow and Veritas discipline inside the agent tools you already use: Claude Code, Codex, Kiro, and GitHub Actions.",
   "keywords": [
     "agents",

package/scripts/README.md

@@ -59,6 +59,7 @@ renamed, or changes category, update the table and the validator together.
 | `report-only-guard.js` | policy hook | `evals/integration/test_hook_category_behaviors.sh` | Protects report-only specialist roles from production edits. |
 | `stop-format-typecheck.js` | policy hook | `evals/integration/test_hook_category_behaviors.sh` | Runs stop-time format/typecheck feedback. |
 | `stop-goal-fit.js` | policy hook | `evals/integration/test_goal_fit_hook.sh` | Warns when a workflow is about to stop short of Goal Fit. |
+| `utterance-check.js` | policy hook | `evals/integration/test_utterance_check.sh` | Optionally checks agent utterances for evidence coverage using @kontourai/survey (disabled by default; opt-in via FLOW_AGENTS_UTTERANCE_CHECK_ENABLED). |
 | `workflow-steering.js` | policy hook | `evals/integration/test_workflow_steering_hook.sh` | Provides workflow guidance from current artifact state. |
 | `pre-commit-quality.js` | repo guardrail hook | `evals/integration/test_hook_category_behaviors.sh` | Supports repository Git hook checks, not installed runtime hooks. |
 | `desktop-notify.sh` | local notification helper | `evals/integration/test_hook_category_behaviors.sh` | Optional local desktop notification helper. |

package/scripts/hooks/utterance-check.js

@@ -0,0 +1,225 @@
+#!/usr/bin/env node
+/**
+ * Utterance Check Hook — ADR 0003 §9 / survey integration.
+ *
+ * Optionally inspects agent output text for evidence coverage using
+ * @kontourai/survey's surveyAgentUtterance. Injects badge guidance into the
+ * agent context when concerning statements (unsupported/disputed/rejected) are
+ * found.
+ *
+ * Disabled by default. Enable with:
+ *   FLOW_AGENTS_UTTERANCE_CHECK_ENABLED=true
+ *
+ * Hook category: PostToolUse / Stop (non-blocking, always exits 0).
+ *
+ * Strict mode (blocks Stop when concerning badges present):
+ *   FLOW_AGENTS_UTTERANCE_CHECK_STRICT=true
+ *
+ * Bundle path (optional trust bundle for claim resolution):
+ *   FLOW_AGENTS_UTTERANCE_CHECK_BUNDLE_PATH=/path/to/bundle.json
+ *
+ * Agent ID (for provenance):
+ *   FLOW_AGENTS_UTTERANCE_CHECK_AGENT_ID=my-agent
+ */
+
+'use strict';
+
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+
+const MAX_STDIN = 1024 * 1024;
+const CLI_TIMEOUT_MS = 30000;
+// Maximum utterance text to pass to the CLI to keep stdin under MAX_STDIN.
+const MAX_UTTERANCE_CHARS = 8192;
+
+// ---------------------------------------------------------------------------
+// Helpers
+// ---------------------------------------------------------------------------
+
+function parseJson(raw) {
+  try { return JSON.parse(raw || '{}'); } catch { return {}; }
+}
+
+/**
+ * Walk up from startDir to find the flow-agents package root.
+ * Identified by having both package.json and build/src/cli.js present.
+ */
+function findPackageRoot(startDir) {
+  let dir = path.resolve(startDir || __dirname);
+  for (let depth = 0; depth < 10; depth++) {
+    if (
+      fs.existsSync(path.join(dir, 'package.json')) &&
+      fs.existsSync(path.join(dir, 'build', 'src', 'cli.js'))
+    ) {
+      return dir;
+    }
+    const parent = path.dirname(dir);
+    if (parent === dir) break;
+    dir = parent;
+  }
+  // Fallback: assume hooks dir is scripts/hooks/, so root is two levels up.
+  return path.resolve(__dirname, '..', '..');
+}
+
+/**
+ * Extract agent utterance text from the hook event input.
+ * - PostToolUse: tool_response or tool_output field
+ * - Stop: stop_reason or agent_message field (varies by harness)
+ * - Fallback: returns null (hook passes through)
+ */
+function extractUtteranceText(input) {
+  if (!input || typeof input !== 'object') return null;
+
+  // PostToolUse: agent output is in tool_response or tool_output
+  const resp = input.tool_response || input.tool_output;
+  if (typeof resp === 'string' && resp.trim()) return resp.trim();
+
+  // Stop: some harnesses expose agent message content
+  const agentMsg = input.agent_message || input.message;
+  if (typeof agentMsg === 'string' && agentMsg.trim()) return agentMsg.trim();
+
+  // Stop with content array (Claude Code format)
+  if (Array.isArray(input.content)) {
+    const texts = input.content
+      .filter(b => b && b.type === 'text' && typeof b.text === 'string')
+      .map(b => String(b.text).trim())
+      .filter(Boolean);
+    if (texts.length > 0) return texts.join('\n\n');
+  }
+
+  return null;
+}
+
+function safeOneLineExcerpt(text, max = 120) {
+  const s = String(text || '').replace(/\s+/g, ' ').trim();
+  return s.length > max ? `${s.slice(0, max - 3)}...` : s;
+}
+
+// ---------------------------------------------------------------------------
+// Hook runner
+// ---------------------------------------------------------------------------
+
+function run(rawInput) {
+  const enabled = String(process.env.FLOW_AGENTS_UTTERANCE_CHECK_ENABLED || '').toLowerCase() === 'true';
+  if (!enabled) return rawInput;
+
+  let input;
+  try { input = JSON.parse(rawInput || '{}'); } catch { return rawInput; }
+
+  const utteranceText = extractUtteranceText(input);
+  if (!utteranceText) return rawInput;
+
+  // Truncate very long utterances before passing to CLI.
+  const utterance = utteranceText.length > MAX_UTTERANCE_CHARS
+    ? utteranceText.slice(0, MAX_UTTERANCE_CHARS)
+    : utteranceText;
+
+  const packageRoot = findPackageRoot(__dirname);
+  const cliPath = path.join(packageRoot, 'build', 'src', 'cli.js');
+
+  if (!fs.existsSync(cliPath)) {
+    process.stderr.write(
+      `[UtteranceCheck] CLI not found at ${cliPath}. Run npm run build in the flow-agents checkout.\n`
+    );
+    return rawInput;
+  }
+
+  const cliArgs = ['utterance-check', 'check', '--utterance', utterance];
+
+  const bundlePath = process.env.FLOW_AGENTS_UTTERANCE_CHECK_BUNDLE_PATH;
+  if (bundlePath) cliArgs.push('--bundle-path', bundlePath);
+
+  const agentId = process.env.FLOW_AGENTS_UTTERANCE_CHECK_AGENT_ID || 'flow-agents-hook';
+  cliArgs.push('--agent-id', agentId);
+
+  const strict = String(process.env.FLOW_AGENTS_UTTERANCE_CHECK_STRICT || '').toLowerCase() === 'true';
+  if (strict) cliArgs.push('--strict');
+
+  const result = spawnSync(process.execPath, [cliPath, ...cliArgs], {
+    encoding: 'utf8',
+    timeout: CLI_TIMEOUT_MS,
+    cwd: packageRoot,
+    env: { ...process.env },
+  });
+
+  if (result.error || result.signal || result.status === null) {
+    const detail = result.error
+      ? result.error.message
+      : result.signal
+        ? `signal ${result.signal}`
+        : 'missing exit status';
+    process.stderr.write(`[UtteranceCheck] CLI execution failed (failing open): ${detail}\n`);
+    return rawInput;
+  }
+
+  // Parse the JSON report from stdout.
+  let report = null;
+  try { report = JSON.parse(String(result.stdout || '').trim()); } catch { /* pass through */ }
+
+  if (result.stderr) {
+    process.stderr.write(String(result.stderr));
+  }
+
+  if (!report) return rawInput;
+
+  // If survey is not configured, pass through silently.
+  if (report.status === 'not_configured') return rawInput;
+
+  // Build guidance text from the badge report.
+  const statements = Array.isArray(report.statements) ? report.statements : [];
+  const concerning = statements.filter(s =>
+    s && (s.badge === 'unsupported' || s.badge === 'disputed' || s.badge === 'rejected')
+  );
+
+  if (concerning.length === 0) return rawInput;
+
+  const lines = [
+    `UTTERANCE CHECK: ${concerning.length} statement(s) in this response lack evidence coverage.`,
+    `Summary: ${report.summary || 'unknown'}`,
+    ...concerning.slice(0, 4).map(s =>
+      `  - [${s.badge}] "${safeOneLineExcerpt(s.excerpt)}"`
+    ),
+    'Evidence note: unsupported = no matching claim in the trust bundle; disputed = conflicting evidence; rejected = claim was rejected.',
+    'Cite sources, note gaps, or run survey-utterance-check to record coverage.',
+  ];
+
+  // For PostToolUse: append guidance to the raw input as additional context.
+  // For Stop: report to stderr (non-blocking warning) unless strict mode.
+  const event = input.hook_event_name || '';
+  const guidance = '\n\n---\n' + lines.join('\n') + '\n---';
+
+  if (strict && result.status === 2) {
+    return {
+      stdout: rawInput,
+      stderr: lines.join('\n'),
+      exitCode: 2,
+    };
+  }
+
+  if (event === 'PostToolUse') {
+    return rawInput + guidance;
+  }
+
+  process.stderr.write(`[UtteranceCheck] ${lines.join('\n')}\n`);
+  return rawInput;
+}
+
+if (require.main === module) {
+  let data = '';
+  process.stdin.setEncoding('utf8');
+  process.stdin.on('data', chunk => {
+    if (data.length < MAX_STDIN) data += chunk.substring(0, MAX_STDIN - data.length);
+  });
+  process.stdin.on('end', () => {
+    const output = run(data);
+    if (output && typeof output === 'object') {
+      if (output.stderr) process.stderr.write(output.stderr.endsWith('\n') ? output.stderr : `${output.stderr}\n`);
+      process.stdout.write(String(output.stdout ?? data));
+      process.exit(Number.isInteger(output.exitCode) ? output.exitCode : 0);
+    }
+    process.stdout.write(String(output));
+  });
+}
+
+module.exports = { run, extractUtteranceText, findPackageRoot };

package/skills/idea-to-backlog/SKILL.md

@@ -102,7 +102,7 @@ Gate: the opportunity is worth shaping, or it is parked/rejected.
 
 ### 4. Explore Options
 
-Use `knowledge-search`, `search-first`, `explore`, or `crowdsource` when context is missing.
+Use `search-first` or `explore` when context is missing.
 
 Decide the path: