@dotsetlabs/dotclaw 2.4.0 → 2.6.0

package/scripts/release-slo-check.js

@@ -0,0 +1,214 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import { execFileSync } from 'node:child_process';
+
+const DEFAULT_SLOS = {
+  min_records: 20,
+  min_success_rate: 0.95,
+  max_error_rate: 0.05,
+  max_empty_success_rate: 0.01,
+  min_tool_success_rate: 0.95,
+  max_p95_latency_ms: 120000,
+  max_error_class_rate: {
+    auth: 0.02,
+    rate_limit: 0.1,
+    timeout: 0.1,
+    context_overflow: 0.05,
+    unknown: 0.1
+  }
+};
+
+function parseArgs(argv) {
+  const args = {
+    days: 7,
+    dir: '',
+    input: '',
+    thresholds: '',
+    enforce: false
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--days' && i + 1 < argv.length) {
+      const value = Number(argv[i + 1]);
+      if (Number.isFinite(value) && value > 0) args.days = Math.floor(value);
+      i += 1;
+      continue;
+    }
+    if (arg === '--dir' && i + 1 < argv.length) {
+      args.dir = argv[i + 1];
+      i += 1;
+      continue;
+    }
+    if (arg === '--input' && i + 1 < argv.length) {
+      args.input = argv[i + 1];
+      i += 1;
+      continue;
+    }
+    if (arg === '--thresholds' && i + 1 < argv.length) {
+      args.thresholds = argv[i + 1];
+      i += 1;
+      continue;
+    }
+    if (arg === '--enforce') {
+      args.enforce = true;
+    }
+  }
+  return args;
+}
+
+function readJson(filePath) {
+  return JSON.parse(fs.readFileSync(filePath, 'utf-8'));
+}
+
+function classifyErrorMessage(message) {
+  const lower = String(message || '').toLowerCase();
+  if (/invalid.?api.?key|unauthorized|forbidden|payment|required|insufficient.?credit|\b401\b|\b402\b|\b403\b/.test(lower)) {
+    return 'auth';
+  }
+  if (/rate.?limit|too many requests|\b429\b/.test(lower)) {
+    return 'rate_limit';
+  }
+  if (/timeout|timed out|deadline|econnreset|econnrefused|enotfound|eai_again/.test(lower)) {
+    return 'timeout';
+  }
+  if (/context.?length|maximum.?context|too many tokens|token.?limit/.test(lower)) {
+    return 'context_overflow';
+  }
+  return 'unknown';
+}
+
+function mergeThresholds(base, overrides) {
+  if (!overrides || typeof overrides !== 'object') return base;
+  const merged = JSON.parse(JSON.stringify(base));
+  for (const [key, value] of Object.entries(overrides)) {
+    if (value && typeof value === 'object' && !Array.isArray(value) && merged[key] && typeof merged[key] === 'object') {
+      merged[key] = mergeThresholds(merged[key], value);
+    } else {
+      merged[key] = value;
+    }
+  }
+  return merged;
+}
+
+function computeEmptySuccessRate(report) {
+  const success = Number(report.records_success || 0);
+  const empty = Number(report.empty_success_responses || 0);
+  if (!Number.isFinite(success) || success <= 0) return null;
+  if (!Number.isFinite(empty) || empty < 0) return null;
+  return empty / success;
+}
+
+export function evaluateReleaseSlo(report, thresholds = DEFAULT_SLOS) {
+  const failures = [];
+  const checks = [];
+
+  const recordsTotal = Number(report.records_total || 0);
+  const successRate = Number(report.success_rate || 0);
+  const errorRate = recordsTotal > 0 ? Number(report.records_error || 0) / recordsTotal : 0;
+  const emptySuccessRate = computeEmptySuccessRate(report) ?? 0;
+  const toolSuccessRate = Number(report?.tool_calls?.success_rate ?? 0);
+  const p95Latency = Number(report?.latency_ms?.p95 ?? 0);
+
+  checks.push({ name: 'records_total', actual: recordsTotal, threshold: thresholds.min_records, comparator: '>=' });
+  if (recordsTotal < thresholds.min_records) {
+    failures.push(`records_total ${recordsTotal} below ${thresholds.min_records}`);
+  }
+
+  checks.push({ name: 'success_rate', actual: successRate, threshold: thresholds.min_success_rate, comparator: '>=' });
+  if (successRate < thresholds.min_success_rate) {
+    failures.push(`success_rate ${successRate} below ${thresholds.min_success_rate}`);
+  }
+
+  checks.push({ name: 'error_rate', actual: Number(errorRate.toFixed(4)), threshold: thresholds.max_error_rate, comparator: '<=' });
+  if (errorRate > thresholds.max_error_rate) {
+    failures.push(`error_rate ${errorRate.toFixed(4)} above ${thresholds.max_error_rate}`);
+  }
+
+  checks.push({ name: 'empty_success_rate', actual: Number(emptySuccessRate.toFixed(4)), threshold: thresholds.max_empty_success_rate, comparator: '<=' });
+  if (emptySuccessRate > thresholds.max_empty_success_rate) {
+    failures.push(`empty_success_rate ${emptySuccessRate.toFixed(4)} above ${thresholds.max_empty_success_rate}`);
+  }
+
+  checks.push({ name: 'tool_success_rate', actual: toolSuccessRate, threshold: thresholds.min_tool_success_rate, comparator: '>=' });
+  if (toolSuccessRate < thresholds.min_tool_success_rate) {
+    failures.push(`tool_success_rate ${toolSuccessRate} below ${thresholds.min_tool_success_rate}`);
+  }
+
+  checks.push({ name: 'latency_p95_ms', actual: p95Latency, threshold: thresholds.max_p95_latency_ms, comparator: '<=' });
+  if (p95Latency > thresholds.max_p95_latency_ms) {
+    failures.push(`latency_p95_ms ${p95Latency} above ${thresholds.max_p95_latency_ms}`);
+  }
+
+  const classCounts = {
+    auth: 0,
+    rate_limit: 0,
+    timeout: 0,
+    context_overflow: 0,
+    unknown: 0
+  };
+  const topErrors = Array.isArray(report.top_errors) ? report.top_errors : [];
+  for (const entry of topErrors) {
+    const errorClass = classifyErrorMessage(entry.key);
+    classCounts[errorClass] += Number(entry.count || 0);
+  }
+
+  for (const [errorClass, maxRate] of Object.entries(thresholds.max_error_class_rate || {})) {
+    const rate = recordsTotal > 0 ? classCounts[errorClass] / recordsTotal : 0;
+    checks.push({
+      name: `error_class_${errorClass}_rate`,
+      actual: Number(rate.toFixed(4)),
+      threshold: maxRate,
+      comparator: '<='
+    });
+    if (rate > maxRate) {
+      failures.push(`error_class_${errorClass}_rate ${rate.toFixed(4)} above ${maxRate}`);
+    }
+  }
+
+  return {
+    checks,
+    failures,
+    passed: failures.length === 0
+  };
+}
+
+function buildBaselineReport(args) {
+  if (args.input) {
+    return readJson(args.input);
+  }
+  const baselineScript = path.join(process.cwd(), 'scripts', 'benchmark-baseline.js');
+  const baselineArgs = [baselineScript, '--days', String(args.days)];
+  if (args.dir) {
+    baselineArgs.push('--dir', args.dir);
+  }
+  const stdout = execFileSync(process.execPath, baselineArgs, { encoding: 'utf-8' });
+  return JSON.parse(stdout);
+}
+
+function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const report = buildBaselineReport(args);
+  const thresholds = args.thresholds
+    ? mergeThresholds(DEFAULT_SLOS, readJson(args.thresholds))
+    : DEFAULT_SLOS;
+  const evaluation = evaluateReleaseSlo(report, thresholds);
+  const output = {
+    source: args.input || 'benchmark-baseline',
+    thresholds,
+    report,
+    checks: evaluation.checks,
+    failures: evaluation.failures,
+    passed: evaluation.passed
+  };
+
+  console.log(JSON.stringify(output, null, 2));
+  if (args.enforce && !evaluation.passed) {
+    process.exitCode = 1;
+  }
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  main();
+}

package/scripts/run-live-canary.js

@@ -0,0 +1,339 @@
+#!/usr/bin/env node
+
+import path from 'node:path';
+
+import { DATA_DIR } from '../dist/config.js';
+import { initDatabase } from '../dist/db.js';
+import { loadJson } from '../dist/utils.js';
+import {
+  AgentExecutionError,
+  createTraceBase,
+  executeAgentRun,
+  recordAgentTelemetry,
+} from '../dist/agent-execution.js';
+import { routeRequest } from '../dist/request-router.js';
+import { writeTrace } from '../dist/trace-writer.js';
+import { percentile } from './benchmark-baseline.js';
+
+function parseArgs(argv) {
+  const args = {
+    rounds: 8,
+    toolAllow: ['Read', 'Write', 'Edit', 'Glob', 'Grep', 'Bash'],
+    chatJid: '',
+    groupFolder: 'main',
+    userId: 'canary-live-user',
+    userName: 'Canary',
+    source: 'live-canary',
+    reasoningEffort: 'low',
+    maxToolSteps: 40,
+    timeoutMs: 180_000,
+    promptPrefix: '[CANARY:LIVE]'
+  };
+
+  for (let i = 0; i < argv.length; i += 1) {
+    const arg = argv[i];
+    if (arg === '--rounds' && i + 1 < argv.length) {
+      const value = Number(argv[i + 1]);
+      if (Number.isFinite(value) && value > 0) args.rounds = Math.floor(value);
+      i += 1;
+      continue;
+    }
+    if (arg === '--chat-jid' && i + 1 < argv.length) {
+      args.chatJid = argv[i + 1];
+      i += 1;
+      continue;
+    }
+    if (arg === '--group-folder' && i + 1 < argv.length) {
+      args.groupFolder = argv[i + 1];
+      i += 1;
+      continue;
+    }
+    if (arg === '--max-tool-steps' && i + 1 < argv.length) {
+      const value = Number(argv[i + 1]);
+      if (Number.isFinite(value) && value > 0) args.maxToolSteps = Math.floor(value);
+      i += 1;
+      continue;
+    }
+    if (arg === '--timeout-ms' && i + 1 < argv.length) {
+      const value = Number(argv[i + 1]);
+      if (Number.isFinite(value) && value > 0) args.timeoutMs = Math.floor(value);
+      i += 1;
+      continue;
+    }
+    if (arg === '--reasoning-effort' && i + 1 < argv.length) {
+      const value = String(argv[i + 1]).trim().toLowerCase();
+      if (value === 'off' || value === 'low' || value === 'medium' || value === 'high') {
+        args.reasoningEffort = value;
+      }
+      i += 1;
+      continue;
+    }
+    if (arg === '--prompt-prefix' && i + 1 < argv.length) {
+      args.promptPrefix = String(argv[i + 1] || '').trim() || args.promptPrefix;
+      i += 1;
+      continue;
+    }
+    if (arg === '--source' && i + 1 < argv.length) {
+      args.source = String(argv[i + 1] || '').trim() || args.source;
+      i += 1;
+      continue;
+    }
+    if (arg === '--tool-allow' && i + 1 < argv.length) {
+      const tools = String(argv[i + 1] || '')
+        .split(',')
+        .map((item) => item.trim())
+        .filter(Boolean);
+      if (tools.length > 0) args.toolAllow = tools;
+      i += 1;
+    }
+  }
+
+  return args;
+}
+
+function loadRegisteredGroups() {
+  const filePath = path.join(DATA_DIR, 'registered_groups.json');
+  return loadJson(filePath, {});
+}
+
+function resolveChatAndGroup(args) {
+  const groups = loadRegisteredGroups();
+  const entries = Object.entries(groups);
+  if (!entries.length) {
+    throw new Error(`No registered groups found in ${path.join(DATA_DIR, 'registered_groups.json')}`);
+  }
+
+  if (args.chatJid) {
+    const group = groups[args.chatJid];
+    if (!group || typeof group !== 'object') {
+      throw new Error(`Chat not registered: ${args.chatJid}`);
+    }
+    return { chatJid: args.chatJid, group };
+  }
+
+  const matching = entries.find(([, group]) => group?.folder === args.groupFolder);
+  if (matching) return { chatJid: matching[0], group: matching[1] };
+  return { chatJid: entries[0][0], group: entries[0][1] };
+}
+
+function buildRoundPrompts(prefix, round, stamp) {
+  const canaryFile = `inbox/live-canary-${stamp}-r${String(round).padStart(2, '0')}.txt`;
+  return [
+    `${prefix} [SCENARIO:tool_heavy] Round ${round}: Create file "${canaryFile}" with 3 lines: alpha-${round}, beta-${round}, gamma-${round}. Then read it back and return a 1-sentence summary with exact filename.`,
+    `${prefix} [SCENARIO:memory] Round ${round}: From this same conversation session, what exact filename did you just create and what was line 2? Answer in one concise sentence.`,
+    `${prefix} [SCENARIO:tool_heavy] Round ${round}: List the 5 newest files under inbox/, read the newest one, and return exactly 2 bullet points with key details.`,
+  ];
+}
+
+function summarizeResults(executions) {
+  const rows = executions.length;
+  const successRows = executions.filter((item) => item.output?.status === 'success');
+  const errorRows = executions.filter((item) => item.output?.status === 'error' || item.error);
+  const emptySuccess = successRows.filter((item) => {
+    const text = typeof item.output?.result === 'string' ? item.output.result.trim() : '';
+    return !text;
+  }).length;
+  const latencies = successRows
+    .map((item) => Number(item.output?.latency_ms))
+    .filter((value) => Number.isFinite(value) && value >= 0);
+  const toolCalls = successRows.flatMap((item) => Array.isArray(item.output?.tool_calls) ? item.output.tool_calls : []);
+  const failedToolCalls = toolCalls.filter((call) => !call?.ok).length;
+  const errorCounts = new Map();
+  for (const item of errorRows) {
+    const key = String(item.error || item.output?.error || 'unknown error').trim() || 'unknown error';
+    errorCounts.set(key, (errorCounts.get(key) || 0) + 1);
+  }
+  const topErrors = Array.from(errorCounts.entries())
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, 8)
+    .map(([error, count]) => ({ error, count }));
+
+  const memoryCandidates = executions.filter((item) => /\[(?:scenario:)?memory(?:_carryover)?\]/i.test(item.prompt));
+  const memoryPassed = memoryCandidates.filter((item) => {
+    const text = typeof item.output?.result === 'string' ? item.output.result.trim() : '';
+    return item.output?.status === 'success' && text.length > 0;
+  }).length;
+  const toolHeavyCandidates = executions.filter((item) => /\[(?:scenario:)?tool_heavy\]/i.test(item.prompt));
+  const toolHeavyPassed = toolHeavyCandidates.filter((item) => {
+    if (item.output?.status !== 'success') return false;
+    const calls = Array.isArray(item.output?.tool_calls) ? item.output.tool_calls : [];
+    if (calls.length < 2) return false;
+    const failed = calls.filter((call) => !call?.ok).length;
+    return failed <= Math.floor(calls.length * 0.2);
+  }).length;
+
+  return {
+    rows_total: rows,
+    rows_success: successRows.length,
+    rows_error: errorRows.length,
+    success_rate: rows > 0 ? Number((successRows.length / rows).toFixed(4)) : null,
+    empty_success_rate: successRows.length > 0 ? Number((emptySuccess / successRows.length).toFixed(4)) : null,
+    latency_ms: {
+      p50: percentile(latencies, 50),
+      p90: percentile(latencies, 90),
+      p95: percentile(latencies, 95),
+      p99: percentile(latencies, 99),
+    },
+    tool_calls: {
+      total: toolCalls.length,
+      failed: failedToolCalls,
+      success_rate: toolCalls.length > 0
+        ? Number(((toolCalls.length - failedToolCalls) / toolCalls.length).toFixed(4))
+        : null,
+    },
+    top_errors: topErrors,
+    scenarios: {
+      memory_carryover: {
+        candidates: memoryCandidates.length,
+        passed: memoryPassed,
+        pass_rate: memoryCandidates.length > 0 ? Number((memoryPassed / memoryCandidates.length).toFixed(4)) : null,
+      },
+      tool_heavy: {
+        candidates: toolHeavyCandidates.length,
+        passed: toolHeavyPassed,
+        pass_rate: toolHeavyCandidates.length > 0 ? Number((toolHeavyPassed / toolHeavyCandidates.length).toFixed(4)) : null,
+      }
+    }
+  };
+}
+
+async function runOne(params) {
+  const traceBase = createTraceBase({
+    chatId: params.chatJid,
+    groupFolder: params.group.folder,
+    userId: params.userId,
+    inputText: params.prompt,
+    source: params.source
+  });
+
+  let output = null;
+  let context = null;
+  let errorMessage = null;
+
+  try {
+    const execution = await executeAgentRun({
+      group: params.group,
+      prompt: params.prompt,
+      chatJid: params.chatJid,
+      userId: params.userId,
+      userName: params.userName,
+      recallQuery: params.prompt,
+      recallMaxResults: params.routing.recallMaxResults,
+      recallMaxTokens: params.routing.recallMaxTokens,
+      sessionId: params.sessionId,
+      persistSession: true,
+      useGroupLock: true,
+      useSemaphore: true,
+      modelFallbacks: params.routing.fallbacks,
+      reasoningEffort: params.reasoningEffort,
+      modelMaxOutputTokens: params.routing.maxOutputTokens || undefined,
+      maxToolSteps: params.maxToolSteps,
+      lane: 'maintenance',
+      toolAllow: params.toolAllow,
+      timeoutMs: params.timeoutMs
+    });
+    output = execution.output;
+    context = execution.context;
+    if (output.status === 'error') {
+      errorMessage = output.error || 'Unknown error';
+    }
+    return {
+      output,
+      context,
+      errorMessage,
+      nextSessionId: output?.newSessionId || params.sessionId
+    };
+  } catch (err) {
+    if (err instanceof AgentExecutionError) {
+      context = err.context;
+      errorMessage = err.message;
+    } else {
+      errorMessage = err instanceof Error ? err.message : String(err);
+    }
+    return {
+      output,
+      context,
+      errorMessage,
+      nextSessionId: params.sessionId
+    };
+  } finally {
+    if (context) {
+      recordAgentTelemetry({
+        traceBase,
+        output,
+        context,
+        metricsSource: 'live_canary',
+        toolAuditSource: 'heartbeat',
+        errorMessage: errorMessage || undefined
+      });
+    } else {
+      writeTrace({
+        ...traceBase,
+        output_text: output?.result ?? null,
+        model_id: output?.model || 'unknown',
+        memory_recall: [],
+        error_code: errorMessage || undefined,
+        source: params.source
+      });
+    }
+  }
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  const startedAt = new Date().toISOString();
+  initDatabase();
+  const resolved = resolveChatAndGroup(args);
+  const routing = routeRequest();
+  const stamp = new Date().toISOString().replace(/[:.]/g, '-');
+
+  const executions = [];
+  let sessionId;
+
+  for (let round = 1; round <= args.rounds; round += 1) {
+    const prompts = buildRoundPrompts(args.promptPrefix, round, stamp);
+    for (const prompt of prompts) {
+      const result = await runOne({
+        chatJid: resolved.chatJid,
+        group: resolved.group,
+        prompt,
+        userId: args.userId,
+        userName: args.userName,
+        source: args.source,
+        routing,
+        sessionId,
+        reasoningEffort: args.reasoningEffort,
+        maxToolSteps: args.maxToolSteps,
+        timeoutMs: args.timeoutMs,
+        toolAllow: args.toolAllow
+      });
+      sessionId = result.nextSessionId;
+      executions.push({
+        round,
+        prompt,
+        output: result.output,
+        error: result.errorMessage
+      });
+    }
+  }
+
+  const summary = summarizeResults(executions);
+  const output = {
+    generated_at: new Date().toISOString(),
+    started_at: startedAt,
+    chat_jid: resolved.chatJid,
+    group_folder: resolved.group.folder,
+    source: args.source,
+    rounds: args.rounds,
+    prompts_executed: executions.length,
+    model: routing.model,
+    fallbacks: routing.fallbacks,
+    metrics: summary
+  };
+  console.log(JSON.stringify(output, null, 2));
+}
+
+main().catch((err) => {
+  console.error(err instanceof Error ? err.message : String(err));
+  process.exitCode = 1;
+});