ai-engineering-loop 2.10.0 → 2.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,64 @@
1
+ 'use strict';
2
+
3
+ const fs = require('fs');
4
+ const path = require('path');
5
+ const { spawnSync } = require('child_process');
6
+ const { destructiveFindings } = require('./guard.js');
7
+ const { redactHomePaths, redactSecrets } = require('./safe-context.js');
8
+
9
+ const FILE = 'safety-fact.json';
10
+ const LEVELS = {
11
+ 1: 'stated',
12
+ 2: 'pointed at the line',
13
+ 3: 'walked the failure path',
14
+ 4: 'ran real code',
15
+ 5: 'reproduced in the running app'
16
+ };
17
+
18
+ const fail = (message, code = 'INVALID_USAGE') => Object.assign(new Error(message), { code });
19
+ const factPath = (rootDir, runId) => path.join(rootDir, '.ai-engineering-loop', 'runs', runId, FILE);
20
+
21
+ function recordSafetyFact(rootDir, runId, { fact, command = null, live = false, level = null, timeoutMs = 120000, now = new Date() }) {
22
+ const text = String(fact || '').trim();
23
+ if (!text) throw fail('--fact is required: the one fact this change is safe because of');
24
+ let record;
25
+ if (command) {
26
+ if (destructiveFindings(command).length) throw fail('The proof command looks destructive; prove the fact with a read-only script or test');
27
+ const result = spawnSync(command, { cwd: rootDir, encoding: 'utf8', timeout: timeoutMs, shell: true });
28
+ if (result.status !== 0) {
29
+ throw fail(`The proof command did not pass (exit ${result.status ?? 'none'}), so the fact is not proven. Fix the change or the fact, then record again.`, 'SAFETY_FACT_NOT_PROVEN');
30
+ }
31
+ record = {
32
+ fact: text,
33
+ level: live ? 5 : 4,
34
+ command: redactHomePaths(redactSecrets(command).text),
35
+ exitCode: 0,
36
+ outputTail: redactHomePaths(redactSecrets(String(result.stdout || '')).text).slice(-1000)
37
+ };
38
+ } else {
39
+ const claimed = level == null ? 1 : Number(level);
40
+ if (![1, 2, 3].includes(claimed)) throw fail('--level without --cmd must be 1, 2 or 3; levels 4 and 5 need a proof command that passes');
41
+ if (live) throw fail('--live needs --cmd: level 5 is a passing proof run against the running app');
42
+ record = { fact: text, level: claimed };
43
+ }
44
+ const value = { schemaVersion: 1, runId, ...record, levelName: LEVELS[record.level], recordedAt: now.toISOString() };
45
+ fs.writeFileSync(factPath(rootDir, runId), `${JSON.stringify(value, null, 2)}\n`, { mode: 0o600 });
46
+ return value;
47
+ }
48
+
49
+ function loadSafetyFact(rootDir, runId) {
50
+ try {
51
+ return JSON.parse(fs.readFileSync(factPath(rootDir, runId), 'utf8'));
52
+ } catch {
53
+ return null;
54
+ }
55
+ }
56
+
57
+ function safetyFactProblem(rootDir, run, { minimum = 4 } = {}) {
58
+ const value = loadSafetyFact(rootDir, run.runId);
59
+ if (!value) return 'no safety fact recorded';
60
+ if (value.level < minimum) return `safety fact is level ${value.level} (${LEVELS[value.level]}); level ${minimum} needs a proof command that passes`;
61
+ return null;
62
+ }
63
+
64
+ module.exports = { LEVELS, loadSafetyFact, recordSafetyFact, safetyFactProblem };
package/lib/stats.js CHANGED
@@ -73,7 +73,10 @@ function computeStats(rootDir, { last = null } = {}) {
73
73
  }
74
74
  const labeled = Object.entries(tokensByRole).filter(([role]) => role !== 'unlabeled').reduce((sum, [, n]) => sum + n, 0);
75
75
  const checking = ['reviewer', 'judge', 'verification'].reduce((sum, role) => sum + (tokensByRole[role] || 0), 0);
76
+ const { runInterventions, summarizeInterventions } = require('./interventions.js');
77
+ const interventions = summarizeInterventions(runs.map((runId) => runInterventions(path.join(runsDir(rootDir), runId))));
76
78
  return {
79
+ interventions,
77
80
  revertRate: reverts.rate,
78
81
  reverted: reverts.reverted,
79
82
  delivered: reverts.delivered,
@@ -100,6 +103,12 @@ function formatStats(stats) {
100
103
  ];
101
104
  lines.push(`Revert rate: ${stats.reverted.length}/${stats.delivered} delivered Run(s) (${Math.round(stats.revertRate * 1000) / 10}%)${stats.reverted.length ? `: ${stats.reverted.map((item) => item.runId).join(', ')}` : ''}`);
102
105
  if (stats.verificationTokenShare != null) lines.push(`Verification token share: ${Math.round(stats.verificationTokenShare * 100)}% (reviewer + judge + verification of role-labelled usage)`);
106
+ if (stats.interventions?.runs) {
107
+ const i = stats.interventions;
108
+ lines.push(`Human interventions per Run: ${i.perRun} (${i.answers} answered questions, ${i.escalations} escalations, ${i.outcomes} problems found after delivery)`);
109
+ const ran = i.proofs.reproduced + i.proofs.not_reproduced;
110
+ if (ran) lines.push(`Finding proofs: ${i.proofs.reproduced}/${ran} reproduced (${Math.round(i.reproducedRate * 100)}%); ${i.proofs.not_reproduced} blocking finding(s) did not reproduce and stopped blocking`);
111
+ }
103
112
  if (stats.bottleneck) lines.push(`Bottleneck: ${stats.bottleneck}. Fix that harness layer first.`);
104
113
  for (const [tier, bucket] of Object.entries(stats.tiers)) {
105
114
  lines.push(`- ${tier}: ${bucket.runs} run(s), false-victory rate ${Math.round(bucket.falseVictoryRate * 100)}%`);
@@ -160,6 +160,28 @@ function startupRiskFiles(rootDir, run) {
160
160
  return [...new Set(extractChangedPaths(diff))].filter((file) => STARTUP_RISK.test(file));
161
161
  }
162
162
 
163
+ function formalCoveredFiles(rootDir, run) {
164
+ let text = '';
165
+ try {
166
+ text = fs.readFileSync(path.join(rootDir, '.ai-engineering-loop', 'verification.md'), 'utf8');
167
+ } catch {
168
+ return [];
169
+ }
170
+ const match = text.match(/\*\*formal_covers\*\*:\s*(.+)/i);
171
+ if (!match) return [];
172
+ const globs = match[1].replace(/`/g, '').split(',').map((glob) => glob.trim()).filter(Boolean);
173
+ const artifact = run?.artifacts?.diff;
174
+ if (!artifact?.path || !globs.length) return [];
175
+ let diff = '';
176
+ try {
177
+ diff = fs.readFileSync(path.join(rootDir, artifact.path), 'utf8');
178
+ } catch {
179
+ return [];
180
+ }
181
+ const { matchesGlob } = require('./impact-map.js');
182
+ return [...new Set(extractChangedPaths(diff))].filter((file) => globs.some((glob) => matchesGlob(file, glob)));
183
+ }
184
+
163
185
  function readJson(filePath) {
164
186
  try {
165
187
  return JSON.parse(fs.readFileSync(filePath, 'utf8'));
@@ -193,6 +215,7 @@ function finalVerificationStatus(rootDir, run) {
193
215
  }
194
216
 
195
217
  module.exports = {
218
+ formalCoveredFiles,
196
219
  VERIFICATION_CATALOG,
197
220
  keyOfCommand,
198
221
  DEFAULT_PLAN_KEYS,
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "ai-engineering-loop",
3
- "version": "2.10.0",
3
+ "version": "2.11.0",
4
4
  "description": "A visual workspace that helps AI coding agents plan, build, review, and ship reliable software.",
5
5
  "main": "bin/ai-engineering-loop.js",
6
6
  "bin": {
@@ -53,6 +53,10 @@
53
53
  "minLength": 1
54
54
  }
55
55
  },
56
+ "measure": {
57
+ "description": "Optional (C-33) for performance ACs: Judge PASS needs `measurement record` with at least 3 runs, a limiter, and spread smaller than the effect.",
58
+ "type": "boolean"
59
+ },
56
60
  "rubric": {
57
61
  "description": "Optional (C-09) for subjective ACs: criteria the Devil's Advocate scores with evidence in ledger.rubricScores. Judge PASS needs every criterion scored pass:true.",
58
62
  "type": "array",