@njeybe/nativ 2.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/LICENSE +21 -0
- package/README.md +334 -0
- package/bin/cli.js +31 -0
- package/dist/commands/bench.d.ts +8 -0
- package/dist/commands/bench.js +52 -0
- package/dist/commands/db.d.ts +22 -0
- package/dist/commands/db.js +415 -0
- package/dist/commands/doctor.d.ts +38 -0
- package/dist/commands/doctor.js +169 -0
- package/dist/commands/hook.d.ts +31 -0
- package/dist/commands/hook.js +105 -0
- package/dist/commands/init.d.ts +3 -0
- package/dist/commands/init.js +144 -0
- package/dist/commands/mcp.d.ts +5 -0
- package/dist/commands/mcp.js +16 -0
- package/dist/commands/setup.d.ts +17 -0
- package/dist/commands/setup.js +51 -0
- package/dist/commands/status.d.ts +5 -0
- package/dist/commands/status.js +75 -0
- package/dist/commands/task.d.ts +83 -0
- package/dist/commands/task.js +807 -0
- package/dist/commands/test-gen.d.ts +14 -0
- package/dist/commands/test-gen.js +52 -0
- package/dist/commands/triage.d.ts +70 -0
- package/dist/commands/triage.js +343 -0
- package/dist/commands/update.d.ts +14 -0
- package/dist/commands/update.js +112 -0
- package/dist/commands/validate.d.ts +1 -0
- package/dist/commands/validate.js +89 -0
- package/dist/commands/verify.d.ts +8 -0
- package/dist/commands/verify.js +65 -0
- package/dist/commands/worktree.d.ts +47 -0
- package/dist/commands/worktree.js +396 -0
- package/dist/core/benchmark.d.ts +57 -0
- package/dist/core/benchmark.js +772 -0
- package/dist/core/enforcement.d.ts +80 -0
- package/dist/core/enforcement.js +271 -0
- package/dist/core/legacy-templates.d.ts +24 -0
- package/dist/core/legacy-templates.js +60 -0
- package/dist/core/lock-manager.d.ts +54 -0
- package/dist/core/lock-manager.js +260 -0
- package/dist/core/root-resolver.d.ts +32 -0
- package/dist/core/root-resolver.js +149 -0
- package/dist/core/setup-assets.d.ts +123 -0
- package/dist/core/setup-assets.js +438 -0
- package/dist/core/telemetry.d.ts +156 -0
- package/dist/core/telemetry.js +416 -0
- package/dist/core/test-generator-types.d.ts +227 -0
- package/dist/core/test-generator-types.js +179 -0
- package/dist/core/test-generator.d.ts +34 -0
- package/dist/core/test-generator.js +1690 -0
- package/dist/core/tier1-liaison.d.ts +139 -0
- package/dist/core/tier1-liaison.js +320 -0
- package/dist/core/verifier.d.ts +52 -0
- package/dist/core/verifier.js +249 -0
- package/dist/core/version.d.ts +5 -0
- package/dist/core/version.js +14 -0
- package/dist/core/worktree-merge.d.ts +30 -0
- package/dist/core/worktree-merge.js +98 -0
- package/dist/db/data-engine.d.ts +46 -0
- package/dist/db/data-engine.js +802 -0
- package/dist/db/diff.d.ts +39 -0
- package/dist/db/diff.js +420 -0
- package/dist/db/env-parser.d.ts +86 -0
- package/dist/db/env-parser.js +549 -0
- package/dist/db/introspector.d.ts +21 -0
- package/dist/db/introspector.js +835 -0
- package/dist/db/types.d.ts +217 -0
- package/dist/db/types.js +9 -0
- package/dist/governor/circuit-breaker.d.ts +100 -0
- package/dist/governor/circuit-breaker.js +231 -0
- package/dist/governor/evaluator.d.ts +67 -0
- package/dist/governor/evaluator.js +851 -0
- package/dist/governor/index.d.ts +6 -0
- package/dist/governor/index.js +6 -0
- package/dist/governor/rules/api-rules.d.ts +2 -0
- package/dist/governor/rules/api-rules.js +140 -0
- package/dist/governor/rules/db-rules.d.ts +2 -0
- package/dist/governor/rules/db-rules.js +180 -0
- package/dist/governor/store.d.ts +33 -0
- package/dist/governor/store.js +84 -0
- package/dist/governor/types.d.ts +64 -0
- package/dist/governor/types.js +1 -0
- package/dist/index.d.ts +3 -0
- package/dist/index.js +358 -0
- package/dist/mcp/server.d.ts +3 -0
- package/dist/mcp/server.js +309 -0
- package/dist/providers/claude-api.d.ts +5 -0
- package/dist/providers/claude-api.js +75 -0
- package/dist/providers/claude-cli.d.ts +13 -0
- package/dist/providers/claude-cli.js +134 -0
- package/dist/providers/gemini.d.ts +8 -0
- package/dist/providers/gemini.js +78 -0
- package/dist/providers/index.d.ts +5 -0
- package/dist/providers/index.js +5 -0
- package/dist/providers/registry.d.ts +38 -0
- package/dist/providers/registry.js +121 -0
- package/dist/providers/types.d.ts +74 -0
- package/dist/providers/types.js +25 -0
- package/dist/runner/agent-supervisor.d.ts +363 -0
- package/dist/runner/agent-supervisor.js +1672 -0
- package/dist/scanner/context-builder.d.ts +2 -0
- package/dist/scanner/context-builder.js +29 -0
- package/dist/scanner/detector.d.ts +2 -0
- package/dist/scanner/detector.js +760 -0
- package/dist/scanner/types.d.ts +80 -0
- package/dist/scanner/types.js +50 -0
- package/dist/server/studio-server.d.ts +39 -0
- package/dist/server/studio-server.js +1931 -0
- package/dist/server/studio-ui.d.ts +15 -0
- package/dist/server/studio-ui.js +5321 -0
- package/package.json +70 -0
- package/plugin/.claude-plugin/plugin.json +18 -0
- package/plugin/.mcp.json +10 -0
- package/plugin/agents/architect.md +28 -0
- package/plugin/agents/verifier.md +28 -0
- package/plugin/agents/worker.md +26 -0
- package/plugin/hooks/hooks.json +28 -0
- package/plugin/skills/nativ/SKILL.md +75 -0
- package/templates/.claudeignore +45 -0
- package/templates/AGENTS.md +67 -0
- package/templates/CLAUDE.md +32 -0
- package/templates/GEMINI.md +66 -0
- package/templates/claude-agents/architect.md +28 -0
- package/templates/claude-agents/verifier.md +28 -0
- package/templates/claude-agents/worker.md +26 -0
- package/templates/dot-ai/api_contracts.json +66 -0
- package/templates/dot-ai/context.md +38 -0
- package/templates/dot-ai/db_schema.json +39 -0
- package/templates/dot-ai/master_plan.json +28 -0
- package/templates/dot-ai/subagents/backend.md +11 -0
- package/templates/dot-ai/subagents/database.md +10 -0
- package/templates/dot-ai/subagents/db-migration.md +11 -0
- package/templates/dot-ai/subagents/devops-agent.md +11 -0
- package/templates/dot-ai/subagents/flutter-developer.md +11 -0
- package/templates/dot-ai/subagents/frontend.md +126 -0
- package/templates/dot-ai/subagents/qa-tester.md +13 -0
- package/templates/dot-ai/subagents/security-auditor.md +11 -0
- package/templates/dot-ai/ui_specs.md +78 -0
|
@@ -0,0 +1,416 @@
|
|
|
1
|
+
import fs from 'node:fs';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import pc from 'picocolors';
|
|
4
|
+
import { withFileLock } from './lock-manager.js';
|
|
5
|
+
/** Oldest entries are dropped past this, so a chatty agent cannot grow telemetry.json without bound. */
|
|
6
|
+
export const MAX_ROLE_VIOLATIONS = 200;
|
|
7
|
+
/** Model whose rates price heuristic estimates; matches the native engine's default model. */
|
|
8
|
+
export const DEFAULT_TELEMETRY_MODEL = 'claude-opus-5-5';
|
|
9
|
+
/** List price of DEFAULT_TELEMETRY_MODEL; also the fallback for models missing from MODEL_PRICING. */
|
|
10
|
+
export const INPUT_TOKEN_COST_PER_M = 4.0; // $4.00 / 1M tokens
|
|
11
|
+
export const OUTPUT_TOKEN_COST_PER_M = 20.0; // $20.00 / 1M tokens
|
|
12
|
+
function standardPricing(inputPerM, outputPerM, cacheReadPerM = inputPerM * 0.1) {
|
|
13
|
+
return { inputPerM, outputPerM, cacheWritePerM: inputPerM * 1.25, cacheReadPerM };
|
|
14
|
+
}
|
|
15
|
+
/** First-party Claude API list prices in USD per 1M tokens. */
|
|
16
|
+
export const MODEL_PRICING = {
|
|
17
|
+
'claude-fable-5-1': standardPricing(10, 50, 0.25),
|
|
18
|
+
'claude-fable-5': standardPricing(10, 50),
|
|
19
|
+
'claude-opus-5-5': standardPricing(4, 20, 0.2),
|
|
20
|
+
'claude-opus-5': standardPricing(5, 25),
|
|
21
|
+
'claude-opus-4-8': standardPricing(5, 25),
|
|
22
|
+
'claude-opus-4-7': standardPricing(5, 25),
|
|
23
|
+
'claude-opus-4-6': standardPricing(5, 25),
|
|
24
|
+
'claude-sonnet-5': standardPricing(2, 10),
|
|
25
|
+
'claude-sonnet-4-6': standardPricing(3, 15),
|
|
26
|
+
'claude-haiku-4-5': standardPricing(1, 5),
|
|
27
|
+
};
|
|
28
|
+
/** Unknown models fall back to the heuristic estimate rates so spend is never silently zero. */
|
|
29
|
+
const FALLBACK_PRICING = standardPricing(INPUT_TOKEN_COST_PER_M, OUTPUT_TOKEN_COST_PER_M);
|
|
30
|
+
/** True when MODEL_PRICING knows the model (exactly or as a dated / suffixed variant). */
|
|
31
|
+
export function isPricedModel(model) {
|
|
32
|
+
return typeof model === 'string' && Object.keys(MODEL_PRICING).some((id) => model === id || model.startsWith(`${id}-`));
|
|
33
|
+
}
|
|
34
|
+
/** Resolves pricing by exact ID, then by longest known prefix (e.g. dated or `-fast` variants). */
|
|
35
|
+
export function resolveModelPricing(model) {
|
|
36
|
+
const exact = MODEL_PRICING[model];
|
|
37
|
+
if (exact)
|
|
38
|
+
return exact;
|
|
39
|
+
const prefix = Object.keys(MODEL_PRICING)
|
|
40
|
+
.filter((id) => model.startsWith(id))
|
|
41
|
+
.sort((a, b) => b.length - a.length)[0];
|
|
42
|
+
return prefix ? MODEL_PRICING[prefix] : FALLBACK_PRICING;
|
|
43
|
+
}
|
|
44
|
+
/** Prices one usage slice with the model's input, output, cache-write and cache-read rates. */
|
|
45
|
+
export function computeActualCostUsd(model, usage) {
|
|
46
|
+
const p = resolveModelPricing(model);
|
|
47
|
+
const cost = (usage.inputTokens * p.inputPerM +
|
|
48
|
+
usage.outputTokens * p.outputPerM +
|
|
49
|
+
usage.cacheCreationTokens * p.cacheWritePerM +
|
|
50
|
+
usage.cacheReadTokens * p.cacheReadPerM) /
|
|
51
|
+
1_000_000;
|
|
52
|
+
return Math.round(cost * 1_000_000) / 1_000_000;
|
|
53
|
+
}
|
|
54
|
+
export function emptyActualUsage(model) {
|
|
55
|
+
return {
|
|
56
|
+
model,
|
|
57
|
+
turns: 0,
|
|
58
|
+
inputTokens: 0,
|
|
59
|
+
outputTokens: 0,
|
|
60
|
+
cacheCreationTokens: 0,
|
|
61
|
+
cacheReadTokens: 0,
|
|
62
|
+
thinkingTokens: 0,
|
|
63
|
+
costUsd: 0,
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/** Sums two usage records; the result keeps `a.model` unless `a` is empty. */
|
|
67
|
+
export function mergeActualUsage(a, b) {
|
|
68
|
+
return {
|
|
69
|
+
model: a.turns > 0 ? a.model : b.model,
|
|
70
|
+
turns: a.turns + b.turns,
|
|
71
|
+
inputTokens: a.inputTokens + b.inputTokens,
|
|
72
|
+
outputTokens: a.outputTokens + b.outputTokens,
|
|
73
|
+
cacheCreationTokens: a.cacheCreationTokens + b.cacheCreationTokens,
|
|
74
|
+
cacheReadTokens: a.cacheReadTokens + b.cacheReadTokens,
|
|
75
|
+
thinkingTokens: a.thinkingTokens + b.thinkingTokens,
|
|
76
|
+
costUsd: Math.round((a.costUsd + b.costUsd) * 1_000_000) / 1_000_000,
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
export function cacheHitRate(usage) {
|
|
80
|
+
const promptTokens = usage.inputTokens + usage.cacheCreationTokens + usage.cacheReadTokens;
|
|
81
|
+
return promptTokens > 0 ? Math.round((usage.cacheReadTokens / promptTokens) * 10000) / 10000 : 0;
|
|
82
|
+
}
|
|
83
|
+
/** Prices estimated input/output tokens at a model's list rates. */
|
|
84
|
+
export function priceEstimate(model, inputEstimated, outputEstimated) {
|
|
85
|
+
const p = resolveModelPricing(model);
|
|
86
|
+
const cost = (inputEstimated / 1_000_000) * p.inputPerM + (outputEstimated / 1_000_000) * p.outputPerM;
|
|
87
|
+
return Math.round(cost * 10000) / 10000;
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Calculates estimated tokens and cost based on contract slice, target files, and diff heuristics.
|
|
91
|
+
*/
|
|
92
|
+
export function estimateTokens(targetDir, task, model = DEFAULT_TELEMETRY_MODEL) {
|
|
93
|
+
let fileBytes = 0;
|
|
94
|
+
if (Array.isArray(task.targetFiles)) {
|
|
95
|
+
for (const relFile of task.targetFiles) {
|
|
96
|
+
const fullPath = path.resolve(targetDir, relFile);
|
|
97
|
+
if (fs.existsSync(fullPath)) {
|
|
98
|
+
try {
|
|
99
|
+
const stat = fs.statSync(fullPath);
|
|
100
|
+
if (stat.isFile()) {
|
|
101
|
+
fileBytes += stat.size;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
// ignore unreadable files
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
// Base prompt context (contracts + role guide + system prompt) ~ 3,500 tokens
|
|
111
|
+
const baseContextTokens = 3500;
|
|
112
|
+
const fileTokens = Math.round(fileBytes / 4);
|
|
113
|
+
const inputEstimated = baseContextTokens + fileTokens;
|
|
114
|
+
// Output estimation: diff or file modification approximation
|
|
115
|
+
let outputEstimated = 350; // minimum baseline generation
|
|
116
|
+
if (fileBytes > 0) {
|
|
117
|
+
outputEstimated = Math.max(350, Math.round(fileBytes / 6));
|
|
118
|
+
}
|
|
119
|
+
return {
|
|
120
|
+
inputEstimated,
|
|
121
|
+
outputEstimated,
|
|
122
|
+
totalEstimated: inputEstimated + outputEstimated,
|
|
123
|
+
costUsdEstimated: priceEstimate(model, inputEstimated, outputEstimated),
|
|
124
|
+
};
|
|
125
|
+
}
|
|
126
|
+
/**
|
|
127
|
+
* Loads project telemetry from .ai/telemetry.json or returns a fresh initialized state.
|
|
128
|
+
*/
|
|
129
|
+
export function loadTelemetry(telemetryPath, projectName = 'project') {
|
|
130
|
+
if (fs.existsSync(telemetryPath)) {
|
|
131
|
+
try {
|
|
132
|
+
const raw = fs.readFileSync(telemetryPath, 'utf8');
|
|
133
|
+
const parsed = JSON.parse(raw);
|
|
134
|
+
if (parsed && typeof parsed === 'object' && Array.isArray(parsed.tasks)) {
|
|
135
|
+
// Files written before the current model table name retired models (e.g. claude-3-7-sonnet);
|
|
136
|
+
// estimates are priced at the current default instead.
|
|
137
|
+
if (!isPricedModel(parsed.modelTierDefault))
|
|
138
|
+
parsed.modelTierDefault = DEFAULT_TELEMETRY_MODEL;
|
|
139
|
+
return parsed;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
catch {
|
|
143
|
+
// Fall through to initial structure
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return {
|
|
147
|
+
$schema: 'http://json-schema.org/draft-07/schema#',
|
|
148
|
+
version: '1.0.0',
|
|
149
|
+
projectName,
|
|
150
|
+
lastUpdated: new Date().toISOString(),
|
|
151
|
+
modelTierDefault: DEFAULT_TELEMETRY_MODEL,
|
|
152
|
+
summary: {
|
|
153
|
+
totalTasksCompleted: 0,
|
|
154
|
+
totalDurationMs: 0,
|
|
155
|
+
estimatedTotalTokens: 0,
|
|
156
|
+
estimatedTotalCostUsd: 0,
|
|
157
|
+
verificationPassRate: 1.0,
|
|
158
|
+
totalVerificationsRun: 0,
|
|
159
|
+
totalVerificationsPassed: 0,
|
|
160
|
+
circuitBreakerTrips: 0,
|
|
161
|
+
},
|
|
162
|
+
tasks: [],
|
|
163
|
+
};
|
|
164
|
+
}
|
|
165
|
+
/**
|
|
166
|
+
* Atomically writes telemetry to disk with Windows EPERM resilience.
|
|
167
|
+
*/
|
|
168
|
+
export function saveTelemetry(telemetryPath, telemetry) {
|
|
169
|
+
telemetry.lastUpdated = new Date().toISOString();
|
|
170
|
+
const dir = path.dirname(telemetryPath);
|
|
171
|
+
if (!fs.existsSync(dir)) {
|
|
172
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
173
|
+
}
|
|
174
|
+
const tmpPath = `${telemetryPath}.${process.pid}.${Date.now()}.${Math.random().toString(36).slice(2, 8)}.tmp`;
|
|
175
|
+
try {
|
|
176
|
+
fs.writeFileSync(tmpPath, JSON.stringify(telemetry, null, 2) + '\n', 'utf8');
|
|
177
|
+
let renamed = false;
|
|
178
|
+
for (let attempt = 0; attempt < 10; attempt++) {
|
|
179
|
+
try {
|
|
180
|
+
fs.renameSync(tmpPath, telemetryPath);
|
|
181
|
+
renamed = true;
|
|
182
|
+
break;
|
|
183
|
+
}
|
|
184
|
+
catch (err) {
|
|
185
|
+
if (err.code === 'EPERM' || err.code === 'EBUSY') {
|
|
186
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);
|
|
187
|
+
}
|
|
188
|
+
else {
|
|
189
|
+
throw err;
|
|
190
|
+
}
|
|
191
|
+
}
|
|
192
|
+
}
|
|
193
|
+
if (!renamed) {
|
|
194
|
+
fs.copyFileSync(tmpPath, telemetryPath);
|
|
195
|
+
try {
|
|
196
|
+
fs.unlinkSync(tmpPath);
|
|
197
|
+
}
|
|
198
|
+
catch {
|
|
199
|
+
// Best effort
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return true;
|
|
203
|
+
}
|
|
204
|
+
catch {
|
|
205
|
+
fs.rmSync(tmpPath, { force: true });
|
|
206
|
+
return false;
|
|
207
|
+
}
|
|
208
|
+
}
|
|
209
|
+
/**
|
|
210
|
+
* Recomputes aggregate summary metrics across all recorded tasks.
|
|
211
|
+
*/
|
|
212
|
+
export function recomputeTelemetrySummary(telemetry) {
|
|
213
|
+
const completedTasks = telemetry.tasks.filter((t) => t.status === 'completed');
|
|
214
|
+
telemetry.summary.totalTasksCompleted = completedTasks.length;
|
|
215
|
+
let totalDuration = 0;
|
|
216
|
+
let totalTokens = 0;
|
|
217
|
+
let totalCost = 0;
|
|
218
|
+
for (const t of completedTasks) {
|
|
219
|
+
totalDuration += t.durationMs || 0;
|
|
220
|
+
if (t.tokens) {
|
|
221
|
+
// Token estimates are kept as recorded; only the rate is current, so older records are re-priced.
|
|
222
|
+
t.tokens.costUsdEstimated = priceEstimate(telemetry.modelTierDefault, t.tokens.inputEstimated || 0, t.tokens.outputEstimated || 0);
|
|
223
|
+
totalTokens += t.tokens.totalEstimated || 0;
|
|
224
|
+
totalCost += t.tokens.costUsdEstimated;
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
telemetry.summary.totalDurationMs = totalDuration;
|
|
228
|
+
telemetry.summary.estimatedTotalTokens = totalTokens;
|
|
229
|
+
telemetry.summary.estimatedTotalCostUsd = Math.round(totalCost * 10000) / 10000;
|
|
230
|
+
let actual = emptyActualUsage(telemetry.modelTierDefault);
|
|
231
|
+
for (const t of telemetry.tasks) {
|
|
232
|
+
if (t.actualUsage)
|
|
233
|
+
actual = mergeActualUsage(actual, t.actualUsage);
|
|
234
|
+
}
|
|
235
|
+
telemetry.summary.actualSpendUsd = Math.round(actual.costUsd * 10000) / 10000;
|
|
236
|
+
telemetry.summary.actualInputTokens = actual.inputTokens;
|
|
237
|
+
telemetry.summary.actualOutputTokens = actual.outputTokens;
|
|
238
|
+
telemetry.summary.cacheReadTokens = actual.cacheReadTokens;
|
|
239
|
+
telemetry.summary.cacheCreationTokens = actual.cacheCreationTokens;
|
|
240
|
+
telemetry.summary.thinkingTokens = actual.thinkingTokens;
|
|
241
|
+
telemetry.summary.cacheHitRate = cacheHitRate(actual);
|
|
242
|
+
if (telemetry.summary.totalVerificationsRun > 0) {
|
|
243
|
+
telemetry.summary.verificationPassRate =
|
|
244
|
+
Math.round((telemetry.summary.totalVerificationsPassed / telemetry.summary.totalVerificationsRun) * 100) / 100;
|
|
245
|
+
}
|
|
246
|
+
else {
|
|
247
|
+
telemetry.summary.verificationPassRate = 1.0;
|
|
248
|
+
}
|
|
249
|
+
}
|
|
250
|
+
/**
|
|
251
|
+
* Records the start of an agent task in .ai/telemetry.json.
|
|
252
|
+
*/
|
|
253
|
+
export async function recordTaskStart(targetDir, task) {
|
|
254
|
+
const telemetryPath = path.join(targetDir, '.ai', 'telemetry.json');
|
|
255
|
+
await withFileLock(telemetryPath, () => {
|
|
256
|
+
const telemetry = loadTelemetry(telemetryPath, path.basename(targetDir));
|
|
257
|
+
const existing = telemetry.tasks.find((t) => t.taskId === task.id);
|
|
258
|
+
if (existing) {
|
|
259
|
+
existing.startedAt = new Date().toISOString();
|
|
260
|
+
existing.status = 'in_progress';
|
|
261
|
+
}
|
|
262
|
+
else {
|
|
263
|
+
telemetry.tasks.push({
|
|
264
|
+
taskId: task.id,
|
|
265
|
+
title: task.title,
|
|
266
|
+
assignedSubagent: task.assignedSubagent,
|
|
267
|
+
startedAt: new Date().toISOString(),
|
|
268
|
+
durationMs: 0,
|
|
269
|
+
status: 'in_progress',
|
|
270
|
+
notes: task.notes || '',
|
|
271
|
+
});
|
|
272
|
+
}
|
|
273
|
+
saveTelemetry(telemetryPath, telemetry);
|
|
274
|
+
});
|
|
275
|
+
}
|
|
276
|
+
/**
|
|
277
|
+
* Records completion of an agent task, tracking duration, verification results, tokens, and cost.
|
|
278
|
+
*/
|
|
279
|
+
export async function recordTaskComplete(targetDir, task, verificationResult, notes) {
|
|
280
|
+
const telemetryPath = path.join(targetDir, '.ai', 'telemetry.json');
|
|
281
|
+
return withFileLock(telemetryPath, () => {
|
|
282
|
+
const telemetry = loadTelemetry(telemetryPath, path.basename(targetDir));
|
|
283
|
+
let record = telemetry.tasks.find((t) => t.taskId === task.id);
|
|
284
|
+
const now = new Date();
|
|
285
|
+
const completedAt = now.toISOString();
|
|
286
|
+
if (!record) {
|
|
287
|
+
record = {
|
|
288
|
+
taskId: task.id,
|
|
289
|
+
title: task.title,
|
|
290
|
+
assignedSubagent: task.assignedSubagent,
|
|
291
|
+
startedAt: completedAt,
|
|
292
|
+
durationMs: 0,
|
|
293
|
+
status: 'completed',
|
|
294
|
+
};
|
|
295
|
+
telemetry.tasks.push(record);
|
|
296
|
+
}
|
|
297
|
+
const startTime = new Date(record.startedAt || completedAt).getTime();
|
|
298
|
+
record.completedAt = completedAt;
|
|
299
|
+
record.durationMs = Math.max(0, now.getTime() - startTime);
|
|
300
|
+
record.status = 'completed';
|
|
301
|
+
if (notes)
|
|
302
|
+
record.notes = notes;
|
|
303
|
+
if (verificationResult) {
|
|
304
|
+
record.verification = {
|
|
305
|
+
command: verificationResult.command,
|
|
306
|
+
durationMs: verificationResult.durationMs,
|
|
307
|
+
exitCode: verificationResult.exitCode,
|
|
308
|
+
skipped: verificationResult.skipped === true,
|
|
309
|
+
};
|
|
310
|
+
if (!verificationResult.skipped) {
|
|
311
|
+
telemetry.summary.totalVerificationsRun++;
|
|
312
|
+
if (verificationResult.success) {
|
|
313
|
+
telemetry.summary.totalVerificationsPassed++;
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
record.tokens = estimateTokens(targetDir, task, telemetry.modelTierDefault);
|
|
318
|
+
recomputeTelemetrySummary(telemetry);
|
|
319
|
+
saveTelemetry(telemetryPath, telemetry);
|
|
320
|
+
return record;
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
/**
|
|
324
|
+
* Folds one native run's API-reported usage into the task's grounded telemetry.
|
|
325
|
+
* Creates the task record when the run was dispatched without `nativ task start`.
|
|
326
|
+
*/
|
|
327
|
+
export async function recordRunnerUsage(targetDir, task, usage) {
|
|
328
|
+
const telemetryPath = path.join(targetDir, '.ai', 'telemetry.json');
|
|
329
|
+
return withFileLock(telemetryPath, () => {
|
|
330
|
+
const telemetry = loadTelemetry(telemetryPath, path.basename(targetDir));
|
|
331
|
+
let record = telemetry.tasks.find((t) => t.taskId === task.id);
|
|
332
|
+
if (!record) {
|
|
333
|
+
record = {
|
|
334
|
+
taskId: task.id,
|
|
335
|
+
title: task.title,
|
|
336
|
+
assignedSubagent: task.assignedSubagent,
|
|
337
|
+
startedAt: new Date().toISOString(),
|
|
338
|
+
durationMs: 0,
|
|
339
|
+
status: 'in_progress',
|
|
340
|
+
};
|
|
341
|
+
telemetry.tasks.push(record);
|
|
342
|
+
}
|
|
343
|
+
record.actualUsage = record.actualUsage ? mergeActualUsage(record.actualUsage, usage) : { ...usage };
|
|
344
|
+
recomputeTelemetrySummary(telemetry);
|
|
345
|
+
saveTelemetry(telemetryPath, telemetry);
|
|
346
|
+
return record;
|
|
347
|
+
});
|
|
348
|
+
}
|
|
349
|
+
/**
|
|
350
|
+
* Logs a role-enforcement violation to .ai/telemetry.json. Best effort: it never throws, because the
|
|
351
|
+
* hook that calls it must not fail an agent's tool call over bookkeeping.
|
|
352
|
+
*/
|
|
353
|
+
export async function recordRoleViolation(targetDir, violation) {
|
|
354
|
+
try {
|
|
355
|
+
const telemetryPath = path.join(targetDir, '.ai', 'telemetry.json');
|
|
356
|
+
await withFileLock(telemetryPath, () => {
|
|
357
|
+
const telemetry = loadTelemetry(telemetryPath, path.basename(targetDir));
|
|
358
|
+
const violations = telemetry.violations ?? [];
|
|
359
|
+
violations.push({ at: new Date().toISOString(), ...violation });
|
|
360
|
+
telemetry.violations = violations.slice(-MAX_ROLE_VIOLATIONS);
|
|
361
|
+
telemetry.summary.roleViolations = (telemetry.summary.roleViolations ?? 0) + 1;
|
|
362
|
+
saveTelemetry(telemetryPath, telemetry);
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
catch {
|
|
366
|
+
// Bookkeeping only.
|
|
367
|
+
}
|
|
368
|
+
}
|
|
369
|
+
/**
|
|
370
|
+
* Increments circuit breaker trips counter in telemetry.
|
|
371
|
+
*/
|
|
372
|
+
export async function recordCircuitBreakerTrip(targetDir, taskId) {
|
|
373
|
+
const telemetryPath = path.join(targetDir, '.ai', 'telemetry.json');
|
|
374
|
+
await withFileLock(telemetryPath, () => {
|
|
375
|
+
const telemetry = loadTelemetry(telemetryPath, path.basename(targetDir));
|
|
376
|
+
telemetry.summary.circuitBreakerTrips++;
|
|
377
|
+
const record = telemetry.tasks.find((t) => t.taskId === taskId);
|
|
378
|
+
if (record) {
|
|
379
|
+
record.status = 'blocked';
|
|
380
|
+
}
|
|
381
|
+
saveTelemetry(telemetryPath, telemetry);
|
|
382
|
+
});
|
|
383
|
+
}
|
|
384
|
+
/**
|
|
385
|
+
* Returns formatted telemetry dashboard summary lines for CLI display.
|
|
386
|
+
*/
|
|
387
|
+
export function formatTelemetrySummary(telemetry, plan) {
|
|
388
|
+
const { summary } = telemetry;
|
|
389
|
+
const minutes = (summary.totalDurationMs / 60000).toFixed(1);
|
|
390
|
+
const passPct = Math.round(summary.verificationPassRate * 100);
|
|
391
|
+
// Tasks completed before telemetry existed (or with --no-verify from older builds) have no record.
|
|
392
|
+
const untracked = plan ? Math.max(0, plan.completedTasks - summary.totalTasksCompleted) : 0;
|
|
393
|
+
const lines = [
|
|
394
|
+
pc.bold(pc.cyan('\n⚡ Execution & Cost Telemetry')),
|
|
395
|
+
pc.dim(' Model Tier Benchmark: ') + pc.white(telemetry.modelTierDefault),
|
|
396
|
+
pc.dim(' Tasks Completed: ') +
|
|
397
|
+
pc.green(`${summary.totalTasksCompleted}`) +
|
|
398
|
+
(untracked ? pc.dim(` measured (+${untracked} completed without telemetry)`) : ''),
|
|
399
|
+
pc.dim(' Total Time Active: ') + pc.white(`${minutes} min (${summary.totalDurationMs}ms)`),
|
|
400
|
+
pc.dim(' Estimated Tokens: ') + pc.yellow(`${summary.estimatedTotalTokens.toLocaleString()}`),
|
|
401
|
+
pc.dim(' Estimated Cost: ') + pc.green(`$${summary.estimatedTotalCostUsd.toFixed(4)} USD`),
|
|
402
|
+
...(summary.actualSpendUsd
|
|
403
|
+
? [
|
|
404
|
+
pc.dim(' Actual Spend: ') +
|
|
405
|
+
pc.green(`$${summary.actualSpendUsd.toFixed(4)} USD`) +
|
|
406
|
+
pc.dim(` (cache hit ${Math.round((summary.cacheHitRate ?? 0) * 100)}%)`),
|
|
407
|
+
]
|
|
408
|
+
: []),
|
|
409
|
+
pc.dim(' Verification Pass: ') +
|
|
410
|
+
(passPct >= 80 ? pc.green(`${passPct}%`) : pc.red(`${passPct}%`)) +
|
|
411
|
+
pc.dim(` (${summary.totalVerificationsPassed}/${summary.totalVerificationsRun} runs)`),
|
|
412
|
+
pc.dim(' Circuit Breaker Trips:') +
|
|
413
|
+
(summary.circuitBreakerTrips > 0 ? pc.red(` ${summary.circuitBreakerTrips}`) : pc.dim(' 0')),
|
|
414
|
+
];
|
|
415
|
+
return lines.join('\n');
|
|
416
|
+
}
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Contract-to-test generator types (nativ test gen).
|
|
3
|
+
* Result/request shapes conform to POST /api/tests/generate in .ai/api_contracts.json.
|
|
4
|
+
* Endpoint suites are built from .ai/api_contracts.json, database integrity suites from .ai/db_schema.json.
|
|
5
|
+
*/
|
|
6
|
+
import type { DatabaseEngine, TableSchema } from '../db/types.js';
|
|
7
|
+
export type TestFramework = 'vitest' | 'jest' | 'node:test' | 'pytest' | 'go';
|
|
8
|
+
export declare const TEST_FRAMEWORKS: readonly TestFramework[];
|
|
9
|
+
export declare function isTestFramework(value: string): value is TestFramework;
|
|
10
|
+
export type TestLanguage = 'typescript' | 'javascript' | 'python' | 'go';
|
|
11
|
+
/** Kinds of generated suite; serialized as `suiteType` on each generated file. */
|
|
12
|
+
export type TestSuiteType = 'api-contract' | 'db-integrity' | 'support';
|
|
13
|
+
export declare const DEFAULT_TEST_OUTPUT_DIR = "tests/contract";
|
|
14
|
+
export declare const DEFAULT_TEST_FRAMEWORK: TestFramework;
|
|
15
|
+
export declare const DEFAULT_TEST_BASE_URL = "http://localhost:3000";
|
|
16
|
+
/** POST /api/tests/generate request body. */
|
|
17
|
+
export interface TestGenerateRequest {
|
|
18
|
+
outputDir?: string;
|
|
19
|
+
framework?: TestFramework;
|
|
20
|
+
baseUrl?: string;
|
|
21
|
+
dryRun?: boolean;
|
|
22
|
+
}
|
|
23
|
+
/** Options accepted by the generator (CLI flags, studio endpoint, MCP tool). */
|
|
24
|
+
export interface TestGenOptions extends TestGenerateRequest {
|
|
25
|
+
/** Project root containing the .ai/ directory. */
|
|
26
|
+
targetDir: string;
|
|
27
|
+
}
|
|
28
|
+
/** TestGenOptions with every default applied. */
|
|
29
|
+
export interface ResolvedTestGenOptions {
|
|
30
|
+
targetDir: string;
|
|
31
|
+
outputDir: string;
|
|
32
|
+
framework: TestFramework;
|
|
33
|
+
baseUrl: string;
|
|
34
|
+
dryRun: boolean;
|
|
35
|
+
}
|
|
36
|
+
export interface GeneratedTestFile {
|
|
37
|
+
/** Path relative to outputDir, forward slashes. */
|
|
38
|
+
relativePath: string;
|
|
39
|
+
content: string;
|
|
40
|
+
suiteType: TestSuiteType;
|
|
41
|
+
}
|
|
42
|
+
/** POST /api/tests/generate response body. */
|
|
43
|
+
export interface TestGenResult {
|
|
44
|
+
projectName: string;
|
|
45
|
+
framework: TestFramework;
|
|
46
|
+
outputDir: string;
|
|
47
|
+
files: GeneratedTestFile[];
|
|
48
|
+
endpointsCovered: number;
|
|
49
|
+
tablesCovered: number;
|
|
50
|
+
durationMs: number;
|
|
51
|
+
}
|
|
52
|
+
export type HttpMethod = 'GET' | 'POST' | 'PUT' | 'PATCH' | 'DELETE';
|
|
53
|
+
/**
|
|
54
|
+
* A field in a contract body is either a nested object of fields or a descriptor string
|
|
55
|
+
* such as "string ('dev' | 'prod')", "integer (optional, default 1)" or "array of TableSchema".
|
|
56
|
+
*/
|
|
57
|
+
export type ContractFieldValue = string | null | {
|
|
58
|
+
[field: string]: ContractFieldValue;
|
|
59
|
+
};
|
|
60
|
+
export interface ApiEndpointRequest {
|
|
61
|
+
headers?: Record<string, string>;
|
|
62
|
+
queryParams?: Record<string, string>;
|
|
63
|
+
pathParams?: Record<string, string>;
|
|
64
|
+
body?: ContractFieldValue;
|
|
65
|
+
}
|
|
66
|
+
export interface ApiEndpointResponse {
|
|
67
|
+
description?: string;
|
|
68
|
+
body?: ContractFieldValue;
|
|
69
|
+
}
|
|
70
|
+
export interface ApiEndpointContract {
|
|
71
|
+
id: string;
|
|
72
|
+
path: string;
|
|
73
|
+
method: HttpMethod;
|
|
74
|
+
description?: string;
|
|
75
|
+
auth: boolean;
|
|
76
|
+
request: ApiEndpointRequest;
|
|
77
|
+
/** Keyed by HTTP status code ("200", "400", ...). */
|
|
78
|
+
responses: Record<string, ApiEndpointResponse>;
|
|
79
|
+
}
|
|
80
|
+
export interface ApiContractsDocument {
|
|
81
|
+
version?: string;
|
|
82
|
+
projectName: string;
|
|
83
|
+
baseUrl?: string;
|
|
84
|
+
authScheme?: string;
|
|
85
|
+
endpoints: ApiEndpointContract[];
|
|
86
|
+
}
|
|
87
|
+
export interface DbSchemaDocument {
|
|
88
|
+
version?: string;
|
|
89
|
+
projectName: string;
|
|
90
|
+
engine: DatabaseEngine;
|
|
91
|
+
tables: TableSchema[];
|
|
92
|
+
}
|
|
93
|
+
export type PrimitiveKind = 'string' | 'number' | 'integer' | 'boolean' | 'null' | 'any';
|
|
94
|
+
interface SchemaNodeBase {
|
|
95
|
+
/** Field may be absent ("optional", "required if ..."). */
|
|
96
|
+
optional?: boolean;
|
|
97
|
+
/** Condition from "required if <expr>" descriptors, kept verbatim. */
|
|
98
|
+
requiredIf?: string;
|
|
99
|
+
/** Free-text parenthetical from the descriptor, e.g. "default 1" or "e.g. 'MONGO_URI'". */
|
|
100
|
+
note?: string;
|
|
101
|
+
}
|
|
102
|
+
export interface PrimitiveNode extends SchemaNodeBase {
|
|
103
|
+
kind: 'primitive';
|
|
104
|
+
type: PrimitiveKind;
|
|
105
|
+
}
|
|
106
|
+
export interface EnumNode extends SchemaNodeBase {
|
|
107
|
+
kind: 'enum';
|
|
108
|
+
base: 'string' | 'number';
|
|
109
|
+
values: Array<string | number>;
|
|
110
|
+
}
|
|
111
|
+
export interface ObjectNode extends SchemaNodeBase {
|
|
112
|
+
kind: 'object';
|
|
113
|
+
/** Empty with `open: true` for untyped "object" descriptors. */
|
|
114
|
+
properties: Record<string, SchemaNode>;
|
|
115
|
+
open: boolean;
|
|
116
|
+
}
|
|
117
|
+
export interface ArrayNode extends SchemaNodeBase {
|
|
118
|
+
kind: 'array';
|
|
119
|
+
items: SchemaNode;
|
|
120
|
+
}
|
|
121
|
+
export interface UnionNode extends SchemaNodeBase {
|
|
122
|
+
kind: 'union';
|
|
123
|
+
variants: SchemaNode[];
|
|
124
|
+
}
|
|
125
|
+
/** Named model such as TableSchema or TableDiff; checked structurally as an object. */
|
|
126
|
+
export interface RefNode extends SchemaNodeBase {
|
|
127
|
+
kind: 'ref';
|
|
128
|
+
name: string;
|
|
129
|
+
}
|
|
130
|
+
export type SchemaNode = PrimitiveNode | EnumNode | ObjectNode | ArrayNode | UnionNode | RefNode;
|
|
131
|
+
/**
|
|
132
|
+
* Parses a contract descriptor string into a SchemaNode.
|
|
133
|
+
* Examples: "boolean", "string | null", "integer (optional, default 25, max 100)",
|
|
134
|
+
* "string ('dev' | 'prod')", "boolean (required if env === 'prod')", "array of TableSchema",
|
|
135
|
+
* "array of { key: string, targetEnv: 'dev' | 'prod' }".
|
|
136
|
+
*/
|
|
137
|
+
export declare function parseFieldDescriptor(descriptor: string): SchemaNode;
|
|
138
|
+
/** Builds a SchemaNode from a contract body value (nested objects, descriptor strings, or null). */
|
|
139
|
+
export declare function buildSchemaNode(value: ContractFieldValue | undefined): SchemaNode;
|
|
140
|
+
/** True when a schema node accepts null (nullable union or explicit null). */
|
|
141
|
+
export declare function isNullableNode(node: SchemaNode): boolean;
|
|
142
|
+
export type TestCaseKind = 'route' | 'auth-required' | 'invalid-payload' | 'response-schema' | 'table-exists' | 'column' | 'primary-key' | 'nullability' | 'unique-constraint';
|
|
143
|
+
export interface TestRequestSpec {
|
|
144
|
+
method: HttpMethod;
|
|
145
|
+
/** Contract path with `:param` placeholders filled from pathParams. */
|
|
146
|
+
path: string;
|
|
147
|
+
query?: Record<string, string | number | boolean>;
|
|
148
|
+
headers?: Record<string, string>;
|
|
149
|
+
body?: unknown;
|
|
150
|
+
}
|
|
151
|
+
export type TestAssertion = {
|
|
152
|
+
kind: 'status';
|
|
153
|
+
expected: number | number[];
|
|
154
|
+
} | {
|
|
155
|
+
kind: 'response-shape';
|
|
156
|
+
schema: SchemaNode;
|
|
157
|
+
} | {
|
|
158
|
+
kind: 'table-exists';
|
|
159
|
+
table: string;
|
|
160
|
+
} | {
|
|
161
|
+
kind: 'column-exists';
|
|
162
|
+
table: string;
|
|
163
|
+
column: string;
|
|
164
|
+
type: string;
|
|
165
|
+
} | {
|
|
166
|
+
kind: 'primary-key';
|
|
167
|
+
table: string;
|
|
168
|
+
columns: string[];
|
|
169
|
+
} | {
|
|
170
|
+
kind: 'nullability';
|
|
171
|
+
table: string;
|
|
172
|
+
column: string;
|
|
173
|
+
nullable: boolean;
|
|
174
|
+
} | {
|
|
175
|
+
kind: 'unique-constraint';
|
|
176
|
+
table: string;
|
|
177
|
+
index: string;
|
|
178
|
+
columns: string[];
|
|
179
|
+
};
|
|
180
|
+
export interface TestCaseSpec {
|
|
181
|
+
/** Stable id, e.g. "db-status:route". */
|
|
182
|
+
id: string;
|
|
183
|
+
name: string;
|
|
184
|
+
kind: TestCaseKind;
|
|
185
|
+
/** Present for endpoint cases; absent for database integrity cases. */
|
|
186
|
+
request?: TestRequestSpec;
|
|
187
|
+
assertions: TestAssertion[];
|
|
188
|
+
}
|
|
189
|
+
export type TestSuiteSource = {
|
|
190
|
+
kind: 'endpoint';
|
|
191
|
+
endpointId: string;
|
|
192
|
+
method: HttpMethod;
|
|
193
|
+
path: string;
|
|
194
|
+
} | {
|
|
195
|
+
kind: 'table';
|
|
196
|
+
table: string;
|
|
197
|
+
entityType: 'table' | 'collection';
|
|
198
|
+
};
|
|
199
|
+
export interface TestSuiteSpec {
|
|
200
|
+
id: string;
|
|
201
|
+
name: string;
|
|
202
|
+
suiteType: TestSuiteType;
|
|
203
|
+
source: TestSuiteSource;
|
|
204
|
+
cases: TestCaseSpec[];
|
|
205
|
+
}
|
|
206
|
+
export interface TestRenderContext {
|
|
207
|
+
projectName: string;
|
|
208
|
+
baseUrl: string;
|
|
209
|
+
/** Engine from .ai/db_schema.json; null when no schema contract exists. */
|
|
210
|
+
engine: DatabaseEngine | null;
|
|
211
|
+
/** Relative import path from a suite file to the generated support file, when one is emitted. */
|
|
212
|
+
supportImportPath?: string;
|
|
213
|
+
}
|
|
214
|
+
/** Renders framework-agnostic suites into source files for one test framework. */
|
|
215
|
+
export interface TestFrameworkAdapter {
|
|
216
|
+
framework: TestFramework;
|
|
217
|
+
language: TestLanguage;
|
|
218
|
+
/** Extension including the dot, e.g. ".test.ts" or "_test.go". */
|
|
219
|
+
fileExtension: string;
|
|
220
|
+
/** Relative path (under outputDir) for a suite's file. */
|
|
221
|
+
suiteFileName(suite: TestSuiteSpec): string;
|
|
222
|
+
renderSuite(suite: TestSuiteSpec, ctx: TestRenderContext): string;
|
|
223
|
+
/** Shared helpers (HTTP client, shape matcher, DB connection stub) emitted once per run. */
|
|
224
|
+
renderSupportFiles?(ctx: TestRenderContext): GeneratedTestFile[];
|
|
225
|
+
}
|
|
226
|
+
export type TestFrameworkAdapterRegistry = Partial<Record<TestFramework, TestFrameworkAdapter>>;
|
|
227
|
+
export {};
|