@aiassesstech/mighty-mark 0.5.4 → 0.5.6

package/CHANGELOG.md

@@ -5,6 +5,19 @@ All notable changes to `@aiassesstech/mighty-mark` will be documented in this fi
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.5.5] — 2026-03-08
+
+**Fleet-bus diagnostics script + updated VPS smoke test.**
+
+### Added
+- `fleet-bus-diagnostics.sh` — command-line fleet-bus health checker (cards, audit chain, message flow, gateway init, live ping test).
+- Supports `--live` (gateway ping test), `--verbose` (detailed event listing).
+
+### Fixed
+- Updated `vps-smoke-test.mjs` to use current `enforceRouting(FleetMessage)` API (was using outdated 3-arg signature).
+- Added all 6 agent cards to smoke test (was only checking 4).
+- Fixed `AuditTrailWriter` usage to use `.record()` instead of `.write()`.
+
 ## [0.5.4] — 2026-03-07
 
 **Documentation sync — updated check counts, rules, guides, and deployment playbook.**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aiassesstech/mighty-mark",
-  "version": "0.5.4",
+  "version": "0.5.6",
   "description": "System Health Sentinel for AI Assess Tech Fleet — autonomous monitoring, watchdog recovery, and fleet infrastructure oversight.",
   "type": "module",
   "main": "dist/index.js",

package/src/scripts/fleet-bus-diagnostics.sh

@@ -0,0 +1,454 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+##############################################################################
+# Fleet-Bus Diagnostics — Command-Line Health & Connectivity Checks
+#
+# Run from the VPS:
+#   bash /opt/mighty-mark/fleet-bus-diagnostics.sh
+#   bash /opt/mighty-mark/fleet-bus-diagnostics.sh --live    # includes live ping test
+#   bash /opt/mighty-mark/fleet-bus-diagnostics.sh --verbose # show audit event details
+#
+# What it checks:
+#   1. Agent cards on disk (structure, staleness)
+#   2. Audit trail integrity (SHA-256 hash chain per agent)
+#   3. Recent message flow (who sent what to whom)
+#   4. Gateway fleet-bus initialization logs
+#   5. (--live) Send a fleet/ping via gateway and check for response
+##############################################################################
+
+OPENCLAW_HOME="${OPENCLAW_HOME:-$HOME/.openclaw}"
+FLEET_BUS_DIR="$OPENCLAW_HOME/workspace/fleet-bus"
+CARDS_DIR="$FLEET_BUS_DIR/cards"
+AUDIT_DIR="$FLEET_BUS_DIR/audit"
+AGENTS=("jessie" "grillo" "noah" "nole" "sam" "mighty-mark")
+LIVE_TEST=false
+VERBOSE=false
+
+for arg in "$@"; do
+  case "$arg" in
+    --live) LIVE_TEST=true ;;
+    --verbose) VERBOSE=true ;;
+    --help|-h)
+      echo "Usage: $0 [--live] [--verbose]"
+      echo "  --live     Send a live fleet/ping through the gateway"
+      echo "  --verbose  Show detailed audit event data"
+      exit 0
+      ;;
+  esac
+done
+
+PASS=0
+WARN=0
+FAIL=0
+
+check() {
+  local label=$1 status=$2 detail=$3
+  case "$status" in
+    PASS) echo "  ✅ $label — $detail"; PASS=$((PASS + 1)) ;;
+    WARN) echo "  ⚠️  $label — $detail"; WARN=$((WARN + 1)) ;;
+    FAIL) echo "  ❌ $label — $detail"; FAIL=$((FAIL + 1)) ;;
+  esac
+}
+
+echo ""
+echo "╔══════════════════════════════════════════════╗"
+echo "║  Fleet-Bus Diagnostics                      ║"
+echo "╚══════════════════════════════════════════════╝"
+echo ""
+echo "  Fleet-bus dir: $FLEET_BUS_DIR"
+echo "  Timestamp:     $(date -u +%Y-%m-%dT%H:%M:%SZ)"
+echo ""
+
+###############################################################################
+# 1. Fleet-Bus Directory Structure
+###############################################################################
+echo "=== 1. Directory Structure ==="
+
+if [ -d "$FLEET_BUS_DIR" ]; then
+  check "Fleet-bus directory" "PASS" "$FLEET_BUS_DIR exists"
+else
+  check "Fleet-bus directory" "FAIL" "$FLEET_BUS_DIR not found"
+  echo ""
+  echo "  Fleet-bus directory missing. Is the gateway running?"
+  echo "  Try: systemctl restart openclaw-gateway"
+  exit 1
+fi
+
+if [ -d "$CARDS_DIR" ]; then
+  CARD_COUNT=$(ls -1 "$CARDS_DIR"/*.json 2>/dev/null | wc -l | tr -d ' ')
+  check "Cards directory" "PASS" "$CARDS_DIR — $CARD_COUNT card file(s)"
+else
+  check "Cards directory" "FAIL" "$CARDS_DIR not found"
+fi
+
+if [ -d "$AUDIT_DIR" ]; then
+  AUDIT_COUNT=$(ls -1 "$AUDIT_DIR"/*.audit.jsonl 2>/dev/null | wc -l | tr -d ' ')
+  check "Audit directory" "PASS" "$AUDIT_DIR — $AUDIT_COUNT audit file(s)"
+else
+  check "Audit directory" "WARN" "$AUDIT_DIR not found — no audit trail yet"
+fi
+echo ""
+
+###############################################################################
+# 2. Agent Cards
+###############################################################################
+echo "=== 2. Agent Cards ==="
+
+NOW_EPOCH=$(date +%s)
+
+for AGENT in "${AGENTS[@]}"; do
+  CARD_FILE="$CARDS_DIR/${AGENT}.json"
+  if [ ! -f "$CARD_FILE" ]; then
+    check "Card: $AGENT" "FAIL" "not found at $CARD_FILE"
+    continue
+  fi
+
+  CARD_DATA=$(python3 -c "
+import json, sys
+with open('$CARD_FILE') as f:
+    c = json.load(f)
+role = c.get('role', '?')
+status = c.get('status', '?')
+auth = c.get('authLevel', '?')
+mode = c.get('communicationMode', '?')
+last_seen = c.get('lastSeen', '')
+methods = len(c.get('supportedMethods', []))
+print(f'{role}|{status}|{auth}|{mode}|{last_seen}|{methods}')
+" 2>/dev/null || echo "ERROR")
+
+  if [ "$CARD_DATA" = "ERROR" ]; then
+    check "Card: $AGENT" "FAIL" "could not parse JSON"
+    continue
+  fi
+
+  IFS='|' read -r ROLE STATUS AUTH MODE LAST_SEEN METHODS <<< "$CARD_DATA"
+
+  STALE=""
+  if [ -n "$LAST_SEEN" ]; then
+    SEEN_EPOCH=$(date -d "$LAST_SEEN" +%s 2>/dev/null || date -j -f "%Y-%m-%dT%H:%M:%S" "${LAST_SEEN%%.*}" +%s 2>/dev/null || echo "0")
+    if [ "$SEEN_EPOCH" -gt 0 ]; then
+      AGE_HOURS=$(( (NOW_EPOCH - SEEN_EPOCH) / 3600 ))
+      if [ "$AGE_HOURS" -gt 48 ]; then
+        STALE=" ⚠️ stale (${AGE_HOURS}h ago)"
+      fi
+    fi
+  fi
+
+  check "Card: $AGENT" "PASS" "role=$ROLE status=$STATUS auth=$AUTH mode=$MODE methods=$METHODS${STALE}"
+done
+echo ""
+
+###############################################################################
+# 3. Audit Trail Integrity
+###############################################################################
+echo "=== 3. Audit Trail ==="
+
+TOTAL_EVENTS=0
+declare -A AGENT_EVENTS
+
+for AGENT in "${AGENTS[@]}"; do
+  AUDIT_FILE="$AUDIT_DIR/${AGENT}.audit.jsonl"
+  if [ ! -f "$AUDIT_FILE" ]; then
+    check "Audit: $AGENT" "WARN" "no audit file yet"
+    AGENT_EVENTS[$AGENT]=0
+    continue
+  fi
+
+  EVENT_COUNT=$(wc -l < "$AUDIT_FILE" | tr -d ' ')
+  TOTAL_EVENTS=$((TOTAL_EVENTS + EVENT_COUNT))
+  AGENT_EVENTS[$AGENT]=$EVENT_COUNT
+
+  CHAIN_FILE="$AUDIT_DIR/${AGENT}.chain.json"
+  if [ -f "$CHAIN_FILE" ]; then
+    CHAIN_COUNT=$(python3 -c "
+import json
+with open('$CHAIN_FILE') as f:
+    c = json.load(f)
+print(c.get('eventCount', 0))
+" 2>/dev/null || echo "?")
+    if [ "$CHAIN_COUNT" = "$EVENT_COUNT" ]; then
+      check "Audit: $AGENT" "PASS" "$EVENT_COUNT events, chain state consistent"
+    else
+      check "Audit: $AGENT" "WARN" "$EVENT_COUNT events in JSONL but chain says $CHAIN_COUNT"
+    fi
+  else
+    check "Audit: $AGENT" "WARN" "$EVENT_COUNT events but no chain state file"
+  fi
+done
+
+echo ""
+echo "  Total audit events across fleet: $TOTAL_EVENTS"
+echo ""
+
+###############################################################################
+# 4. Recent Message Flow
+###############################################################################
+echo "=== 4. Message Flow (Last 50 Events) ==="
+
+HAS_MESSAGES=false
+
+for AGENT in "${AGENTS[@]}"; do
+  AUDIT_FILE="$AUDIT_DIR/${AGENT}.audit.jsonl"
+  [ ! -f "$AUDIT_FILE" ] && continue
+  [ "${AGENT_EVENTS[$AGENT]}" = "0" ] && continue
+
+  FLOW=$(python3 -c "
+import json, sys
+
+events = []
+with open('$AUDIT_FILE') as f:
+    for line in f:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            events.append(json.loads(line))
+        except:
+            pass
+
+recent = events[-50:] if len(events) > 50 else events
+
+sent = []
+received = []
+violations = []
+errors = []
+
+for e in recent:
+    et = e.get('eventType', '')
+    msg = e.get('message', {}) or {}
+    ts = e.get('timestamp', '?')[:19]
+    outcome = e.get('outcome', '?')
+    fr = msg.get('from', '?')
+    to = msg.get('to', '?')
+    method = msg.get('method', '?')
+
+    if et == 'message_sent':
+        sent.append(f'    {ts}  {fr} → {to}  [{method}]  outcome={outcome}')
+    elif et == 'message_received':
+        received.append(f'    {ts}  {fr} → {to}  [{method}]  outcome={outcome}')
+    elif et == 'routing_violation':
+        violations.append(f'    {ts}  {fr} → {to}  [{method}]  reason={e.get(\"routingDecision\", \"?\")}'[:120])
+    elif outcome in ('error', 'failed'):
+        errors.append(f'    {ts}  {et}  outcome={outcome}  details={e.get(\"details\", \"?\")}'[:120])
+
+verbose = '$VERBOSE' == 'true'
+if sent:
+    print(f'  📤 Sent ({len(sent)}):')
+    for s in (sent if verbose else sent[-5:]):
+        print(s)
+    if not verbose and len(sent) > 5:
+        print(f'    ... and {len(sent) - 5} more (use --verbose)')
+if received:
+    print(f'  📥 Received ({len(received)}):')
+    for r in (received if verbose else received[-5:]):
+        print(r)
+    if not verbose and len(received) > 5:
+        print(f'    ... and {len(received) - 5} more (use --verbose)')
+if violations:
+    print(f'  🚫 Routing Violations ({len(violations)}):')
+    for v in violations:
+        print(v)
+if errors:
+    print(f'  💥 Errors ({len(errors)}):')
+    for e in errors:
+        print(e)
+
+if sent or received:
+    sys.exit(0)
+elif violations or errors:
+    sys.exit(2)
+else:
+    sys.exit(1)
+" 2>/dev/null)
+
+  EXIT_CODE=$?
+
+  if [ $EXIT_CODE -eq 0 ]; then
+    HAS_MESSAGES=true
+    echo "  ── $AGENT ──"
+    echo "$FLOW"
+    echo ""
+  elif [ $EXIT_CODE -eq 2 ]; then
+    HAS_MESSAGES=true
+    echo "  ── $AGENT (issues detected) ──"
+    echo "$FLOW"
+    echo ""
+  fi
+done
+
+if ! $HAS_MESSAGES; then
+  echo "  No fleet messages found in any agent's audit trail."
+  echo "  This means agents haven't communicated via fleet-bus yet."
+  echo ""
+  echo "  To trigger a test, ask Jessie to delegate a task to Sam:"
+  echo '    "Jessie, ask Sam to write a hello world script"'
+  echo ""
+fi
+
+###############################################################################
+# 5. Gateway Fleet-Bus Initialization
+###############################################################################
+echo "=== 5. Gateway Initialization ==="
+
+if command -v journalctl &>/dev/null; then
+  INIT_LOGS=$(journalctl -u openclaw-gateway --since "24 hours ago" --no-pager 2>/dev/null | grep -i 'fleet-bus.*initialized' || true)
+
+  if [ -n "$INIT_LOGS" ]; then
+    INIT_COUNT=$(echo "$INIT_LOGS" | wc -l | tr -d ' ')
+    check "Fleet-bus init logs" "PASS" "$INIT_COUNT initialization(s) in last 24h"
+
+    INITIALIZED_AGENTS=""
+    for AGENT in "${AGENTS[@]}"; do
+      if echo "$INIT_LOGS" | grep -q "fleet-bus:${AGENT}"; then
+        INITIALIZED_AGENTS="$INITIALIZED_AGENTS $AGENT"
+      fi
+    done
+
+    MISSING_AGENTS=""
+    for AGENT in "${AGENTS[@]}"; do
+      if ! echo "$INITIALIZED_AGENTS" | grep -q "$AGENT"; then
+        MISSING_AGENTS="$MISSING_AGENTS $AGENT"
+      fi
+    done
+
+    if [ -n "$MISSING_AGENTS" ]; then
+      check "Agent init coverage" "WARN" "not seen in logs:$MISSING_AGENTS"
+    else
+      check "Agent init coverage" "PASS" "all 6 agents initialized"
+    fi
+
+    if $VERBOSE; then
+      echo ""
+      echo "  Recent init logs:"
+      echo "$INIT_LOGS" | tail -12 | while read -r line; do
+        echo "    $line"
+      done
+    fi
+  else
+    check "Fleet-bus init logs" "WARN" "no fleet-bus initialization in last 24h"
+  fi
+else
+  check "Fleet-bus init logs" "WARN" "journalctl not available — cannot check gateway logs"
+fi
+echo ""
+
+###############################################################################
+# 6. Fleet-Bus Transport Test (requires --live)
+###############################################################################
+if $LIVE_TEST; then
+  echo "=== 6. Live Transport Test (fleet/ping via callGateway) ==="
+
+  LIVE_SCRIPT=$(mktemp /tmp/fleet-live-ping-XXXXXX.mjs)
+  cat > "$LIVE_SCRIPT" << 'NODESCRIPT'
+import { existsSync } from 'node:fs';
+import { dirname, join } from 'node:path';
+import { randomBytes } from 'node:crypto';
+
+async function resolveCallGateway() {
+  const mainScript = process.argv[1] ?? '';
+  const home = process.env.HOME ?? '/root';
+  const packageNames = ['clawdbot', 'openclaw'];
+  const globalRoots = [
+    '/usr/lib/node_modules',
+    '/usr/local/lib/node_modules',
+    join(home, '.local', 'lib', 'node_modules'),
+  ];
+  const paths = globalRoots.flatMap(root =>
+    packageNames.map(pkg => join(root, pkg, 'dist', 'gateway', 'call.js'))
+  );
+  for (const p of paths) {
+    if (!existsSync(p)) continue;
+    try {
+      const mod = await import(`file://${p}`);
+      if (typeof mod.callGateway === 'function') return mod.callGateway;
+    } catch {}
+  }
+  return null;
+}
+
+try {
+  const callGateway = await resolveCallGateway();
+  if (!callGateway) {
+    console.log('RESULT:FAIL:callGateway not resolved — gateway/call.js not found');
+    process.exit(0);
+  }
+
+  const msg = {
+    id: `fm-diag-${Date.now()}-${randomBytes(4).toString('hex')}`,
+    from: 'mighty-mark',
+    to: 'jessie',
+    method: 'fleet/ping',
+    params: { source: 'diagnostics', timestamp: new Date().toISOString() },
+    timestamp: new Date().toISOString(),
+    nonce: randomBytes(8).toString('hex'),
+    version: '1.0',
+  };
+
+  const serialized = '%%FLEET%%' + JSON.stringify(msg);
+  const start = Date.now();
+
+  const result = await callGateway({
+    method: 'agent',
+    params: {
+      agentId: 'jessie',
+      message: serialized,
+      deliver: false,
+      timeoutSeconds: 15,
+      label: 'fleet:fleet/ping',
+    },
+    timeoutMs: 20000,
+    mode: 'backend',
+    clientName: 'fleet-bus-diagnostics',
+    clientDisplayName: 'Fleet Diagnostics',
+  });
+
+  const duration = Date.now() - start;
+  const hasResponse = result && (typeof result === 'object' || typeof result === 'string');
+
+  if (hasResponse) {
+    const summary = typeof result === 'string' ? result.slice(0, 100) : JSON.stringify(result).slice(0, 100);
+    console.log(`RESULT:PASS:fleet/ping delivered to jessie (${duration}ms) — response: ${summary}`);
+  } else {
+    console.log(`RESULT:PASS:fleet/ping accepted by gateway (${duration}ms) — fire-and-forget`);
+  }
+} catch (err) {
+  console.log(`RESULT:FAIL:${err.message?.slice(0, 150) || 'unknown error'}`);
+}
+process.exit(0);
+NODESCRIPT
+
+  LIVE_RESULT=$(node "$LIVE_SCRIPT" 2>/dev/null || echo "RESULT:FAIL:node execution failed")
+  rm -f "$LIVE_SCRIPT"
+
+  LIVE_STATUS=$(echo "$LIVE_RESULT" | grep '^RESULT:' | head -1 | cut -d: -f2)
+  LIVE_DETAIL=$(echo "$LIVE_RESULT" | grep '^RESULT:' | head -1 | cut -d: -f3-)
+
+  if [ "$LIVE_STATUS" = "PASS" ]; then
+    check "Live ping (Mark → Jessie)" "PASS" "$LIVE_DETAIL"
+  elif [ "$LIVE_STATUS" = "FAIL" ]; then
+    check "Live ping (Mark → Jessie)" "FAIL" "$LIVE_DETAIL"
+  else
+    check "Live ping (Mark → Jessie)" "WARN" "unexpected result: $LIVE_RESULT"
+  fi
+  echo ""
+fi
+
+###############################################################################
+# Summary
+###############################################################################
+echo "══════════════════════════════════════════════"
+echo "  Results: $PASS passed, $WARN warnings, $FAIL failed"
+echo "══════════════════════════════════════════════"
+echo ""
+
+if [ $FAIL -gt 0 ]; then
+  echo "  ⚡ Fleet-bus has $FAIL failing checks — investigate above."
+  exit 1
+elif [ $WARN -gt 0 ]; then
+  echo "  ⚠️  $WARN warnings — review above for details."
+  exit 0
+else
+  echo "  🎯 Fleet-bus is healthy."
+  exit 0
+fi