@dotsetlabs/bellwether 2.1.0 → 2.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +35 -0
- package/README.md +48 -31
- package/dist/cli/commands/check.js +49 -6
- package/dist/cli/commands/dashboard.d.ts +3 -0
- package/dist/cli/commands/dashboard.js +69 -0
- package/dist/cli/commands/discover.js +24 -2
- package/dist/cli/commands/explore.js +49 -6
- package/dist/cli/commands/watch.js +12 -1
- package/dist/cli/index.js +27 -34
- package/dist/cli/utils/headers.d.ts +12 -0
- package/dist/cli/utils/headers.js +63 -0
- package/dist/config/defaults.d.ts +2 -0
- package/dist/config/defaults.js +2 -0
- package/dist/config/template.js +12 -0
- package/dist/config/validator.d.ts +38 -18
- package/dist/config/validator.js +10 -0
- package/dist/constants/core.d.ts +4 -2
- package/dist/constants/core.js +13 -2
- package/dist/dashboard/index.d.ts +3 -0
- package/dist/dashboard/index.js +6 -0
- package/dist/dashboard/runtime/artifact-index.d.ts +45 -0
- package/dist/dashboard/runtime/artifact-index.js +238 -0
- package/dist/dashboard/runtime/command-profiles.d.ts +764 -0
- package/dist/dashboard/runtime/command-profiles.js +691 -0
- package/dist/dashboard/runtime/config-service.d.ts +21 -0
- package/dist/dashboard/runtime/config-service.js +73 -0
- package/dist/dashboard/runtime/job-runner.d.ts +26 -0
- package/dist/dashboard/runtime/job-runner.js +292 -0
- package/dist/dashboard/security/input-validation.d.ts +3 -0
- package/dist/dashboard/security/input-validation.js +27 -0
- package/dist/dashboard/security/localhost-guard.d.ts +5 -0
- package/dist/dashboard/security/localhost-guard.js +52 -0
- package/dist/dashboard/server.d.ts +14 -0
- package/dist/dashboard/server.js +293 -0
- package/dist/dashboard/types.d.ts +55 -0
- package/dist/dashboard/types.js +2 -0
- package/dist/dashboard/ui.d.ts +2 -0
- package/dist/dashboard/ui.js +2264 -0
- package/dist/discovery/discovery.js +20 -1
- package/dist/discovery/types.d.ts +1 -1
- package/dist/docs/contract.js +7 -1
- package/dist/errors/retry.js +15 -1
- package/dist/errors/types.d.ts +10 -0
- package/dist/errors/types.js +28 -0
- package/dist/logging/logger.js +5 -2
- package/dist/transport/env-filter.d.ts +6 -0
- package/dist/transport/env-filter.js +76 -0
- package/dist/transport/http-transport.js +10 -0
- package/dist/transport/mcp-client.d.ts +16 -9
- package/dist/transport/mcp-client.js +119 -88
- package/dist/transport/sse-transport.js +19 -0
- package/dist/version.js +2 -2
- package/package.json +5 -15
- package/man/bellwether.1 +0 -204
- package/man/bellwether.1.md +0 -148
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import { existsSync, readFileSync, statSync } from 'fs';
|
|
2
|
+
import { isAbsolute, join, resolve } from 'path';
|
|
3
|
+
import { getGoldenStorePath } from '../../baseline/golden-output.js';
|
|
4
|
+
import { PATHS, CONTRACT_TESTING } from '../../constants.js';
|
|
5
|
+
import { loadConfig } from '../../config/loader.js';
|
|
6
|
+
import { CONFIG_DEFAULTS } from '../../config/defaults.js';
|
|
7
|
+
import { getConfigDocument } from './config-service.js';
|
|
8
|
+
function resolvePathFromCwd(cwd, value) {
|
|
9
|
+
if (isAbsolute(value)) {
|
|
10
|
+
return value;
|
|
11
|
+
}
|
|
12
|
+
return resolve(cwd, value);
|
|
13
|
+
}
|
|
14
|
+
function resolvePathFromOutput(outputDir, value) {
|
|
15
|
+
if (isAbsolute(value)) {
|
|
16
|
+
return value;
|
|
17
|
+
}
|
|
18
|
+
return resolve(outputDir, value);
|
|
19
|
+
}
|
|
20
|
+
function resolveComparePath(cwd, outputDir, comparePath) {
|
|
21
|
+
if (isAbsolute(comparePath)) {
|
|
22
|
+
return comparePath;
|
|
23
|
+
}
|
|
24
|
+
const outputCandidate = resolve(outputDir, comparePath);
|
|
25
|
+
const cwdCandidate = resolve(cwd, comparePath);
|
|
26
|
+
if (existsSync(outputCandidate) || !existsSync(cwdCandidate)) {
|
|
27
|
+
return outputCandidate;
|
|
28
|
+
}
|
|
29
|
+
return cwdCandidate;
|
|
30
|
+
}
|
|
31
|
+
function loadConfigOrDefaults(cwd, configPath) {
|
|
32
|
+
try {
|
|
33
|
+
const configDocument = getConfigDocument(cwd, configPath);
|
|
34
|
+
if (!configDocument.exists) {
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
return loadConfig(configDocument.absolutePath);
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
function resolveContractDefinitionPath(config, outputDir) {
|
|
44
|
+
const explicitContractPath = config?.contract.path;
|
|
45
|
+
if (explicitContractPath && explicitContractPath.trim()) {
|
|
46
|
+
return resolvePathFromOutput(outputDir, explicitContractPath);
|
|
47
|
+
}
|
|
48
|
+
const candidates = CONTRACT_TESTING.CONTRACT_FILENAMES.map((filename) => resolvePathFromOutput(outputDir, filename));
|
|
49
|
+
const existingCandidate = candidates.find((candidate) => existsSync(candidate));
|
|
50
|
+
if (existingCandidate) {
|
|
51
|
+
return existingCandidate;
|
|
52
|
+
}
|
|
53
|
+
return candidates[0] ?? resolvePathFromOutput(outputDir, CONTRACT_TESTING.CONTRACT_FILENAMES[0]);
|
|
54
|
+
}
|
|
55
|
+
function buildArtifactContext(cwd, configPath) {
|
|
56
|
+
const config = loadConfigOrDefaults(cwd, configPath);
|
|
57
|
+
const outputDir = resolvePathFromCwd(cwd, config?.output.dir ?? CONFIG_DEFAULTS.output.dir);
|
|
58
|
+
const docsDir = resolvePathFromCwd(cwd, config?.output.docsDir ?? CONFIG_DEFAULTS.output.docsDir);
|
|
59
|
+
const files = config?.output.files ?? CONFIG_DEFAULTS.output.files;
|
|
60
|
+
const baselinePathValue = config?.baseline.path ?? PATHS.DEFAULT_BASELINE_FILE;
|
|
61
|
+
const baselineSaveValue = config?.baseline.savePath ?? baselinePathValue;
|
|
62
|
+
const baselineCompareValue = config?.baseline.comparePath ?? baselinePathValue;
|
|
63
|
+
return {
|
|
64
|
+
cwd,
|
|
65
|
+
outputDir,
|
|
66
|
+
docsDir,
|
|
67
|
+
baselinePath: resolvePathFromOutput(outputDir, baselinePathValue),
|
|
68
|
+
baselineSavePath: resolvePathFromOutput(outputDir, baselineSaveValue),
|
|
69
|
+
baselineComparePath: resolveComparePath(cwd, outputDir, baselineCompareValue),
|
|
70
|
+
contractPath: resolveContractDefinitionPath(config, outputDir),
|
|
71
|
+
goldenStorePath: getGoldenStorePath(outputDir),
|
|
72
|
+
files: {
|
|
73
|
+
checkReport: files.checkReport,
|
|
74
|
+
exploreReport: files.exploreReport,
|
|
75
|
+
contractDoc: files.contractDoc,
|
|
76
|
+
agentsDoc: files.agentsDoc,
|
|
77
|
+
},
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
function toArtifactSummary(id, label, kind, path) {
|
|
81
|
+
const exists = existsSync(path);
|
|
82
|
+
if (!exists) {
|
|
83
|
+
return {
|
|
84
|
+
id,
|
|
85
|
+
label,
|
|
86
|
+
kind,
|
|
87
|
+
path,
|
|
88
|
+
exists: false,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
91
|
+
const stats = statSync(path);
|
|
92
|
+
return {
|
|
93
|
+
id,
|
|
94
|
+
label,
|
|
95
|
+
kind,
|
|
96
|
+
path,
|
|
97
|
+
exists: true,
|
|
98
|
+
sizeBytes: stats.size,
|
|
99
|
+
updatedAt: stats.mtime.toISOString(),
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
export function listArtifacts(cwd, configPath) {
|
|
103
|
+
const context = buildArtifactContext(cwd, configPath);
|
|
104
|
+
const summaries = [
|
|
105
|
+
toArtifactSummary('check-report', 'Check JSON Report', 'json', join(context.outputDir, context.files.checkReport)),
|
|
106
|
+
toArtifactSummary('explore-report', 'Explore JSON Report', 'json', join(context.outputDir, context.files.exploreReport)),
|
|
107
|
+
toArtifactSummary('contract-doc', 'CONTRACT.md', 'markdown', join(context.docsDir, context.files.contractDoc)),
|
|
108
|
+
toArtifactSummary('agents-doc', 'AGENTS.md', 'markdown', join(context.docsDir, context.files.agentsDoc)),
|
|
109
|
+
toArtifactSummary('contract-definition', 'Contract Definition', 'yaml', context.contractPath),
|
|
110
|
+
toArtifactSummary('golden-store', 'Golden Output Store', 'json', context.goldenStorePath),
|
|
111
|
+
toArtifactSummary('baseline-save', 'Baseline Save Path', 'json', context.baselineSavePath),
|
|
112
|
+
toArtifactSummary('baseline-compare', 'Baseline Compare Path', 'json', context.baselineComparePath),
|
|
113
|
+
toArtifactSummary('baseline-default', 'Baseline Default Path', 'json', context.baselinePath),
|
|
114
|
+
];
|
|
115
|
+
return summaries;
|
|
116
|
+
}
|
|
117
|
+
export function readArtifact(cwd, artifactId, configPath) {
|
|
118
|
+
const artifact = listArtifacts(cwd, configPath).find((entry) => entry.id === artifactId);
|
|
119
|
+
if (!artifact) {
|
|
120
|
+
throw new Error(`Unknown artifact: ${artifactId}`);
|
|
121
|
+
}
|
|
122
|
+
if (!artifact.exists) {
|
|
123
|
+
throw new Error(`Artifact does not exist: ${artifact.path}`);
|
|
124
|
+
}
|
|
125
|
+
const raw = readFileSync(artifact.path, 'utf-8');
|
|
126
|
+
if (artifact.kind !== 'json') {
|
|
127
|
+
return { artifact, raw };
|
|
128
|
+
}
|
|
129
|
+
try {
|
|
130
|
+
const parsedJson = JSON.parse(raw);
|
|
131
|
+
return { artifact, raw, parsedJson };
|
|
132
|
+
}
|
|
133
|
+
catch (error) {
|
|
134
|
+
return {
|
|
135
|
+
artifact,
|
|
136
|
+
raw,
|
|
137
|
+
parseError: error instanceof Error ? error.message : String(error),
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
function asRecord(value) {
|
|
142
|
+
if (!value || typeof value !== 'object' || Array.isArray(value)) {
|
|
143
|
+
return null;
|
|
144
|
+
}
|
|
145
|
+
return value;
|
|
146
|
+
}
|
|
147
|
+
function getArrayLength(value) {
|
|
148
|
+
if (!Array.isArray(value)) {
|
|
149
|
+
return undefined;
|
|
150
|
+
}
|
|
151
|
+
return value.length;
|
|
152
|
+
}
|
|
153
|
+
export function getArtifactOverview(cwd, configPath) {
|
|
154
|
+
const artifacts = listArtifacts(cwd, configPath);
|
|
155
|
+
const checkArtifact = artifacts.find((artifact) => artifact.id === 'check-report');
|
|
156
|
+
const exploreArtifact = artifacts.find((artifact) => artifact.id === 'explore-report');
|
|
157
|
+
const goldenArtifact = artifacts.find((artifact) => artifact.id === 'golden-store');
|
|
158
|
+
const contractArtifact = artifacts.find((artifact) => artifact.id === 'contract-definition');
|
|
159
|
+
const check = checkArtifact
|
|
160
|
+
? summarizeInterviewArtifact(cwd, checkArtifact, configPath)
|
|
161
|
+
: { available: false, path: '' };
|
|
162
|
+
const explore = exploreArtifact
|
|
163
|
+
? summarizeInterviewArtifact(cwd, exploreArtifact, configPath)
|
|
164
|
+
: { available: false, path: '' };
|
|
165
|
+
const golden = goldenArtifact
|
|
166
|
+
? summarizeGoldenArtifact(cwd, goldenArtifact, configPath)
|
|
167
|
+
: { available: false, path: '' };
|
|
168
|
+
return {
|
|
169
|
+
check,
|
|
170
|
+
explore,
|
|
171
|
+
golden,
|
|
172
|
+
contract: {
|
|
173
|
+
available: Boolean(contractArtifact?.exists),
|
|
174
|
+
path: contractArtifact?.path ?? '',
|
|
175
|
+
},
|
|
176
|
+
};
|
|
177
|
+
}
|
|
178
|
+
function summarizeInterviewArtifact(cwd, artifact, configPath) {
|
|
179
|
+
if (!artifact.exists) {
|
|
180
|
+
return {
|
|
181
|
+
available: false,
|
|
182
|
+
path: artifact.path,
|
|
183
|
+
};
|
|
184
|
+
}
|
|
185
|
+
try {
|
|
186
|
+
const report = readArtifact(cwd, artifact.id, configPath);
|
|
187
|
+
const parsed = asRecord(report.parsedJson);
|
|
188
|
+
if (!parsed) {
|
|
189
|
+
return {
|
|
190
|
+
available: true,
|
|
191
|
+
path: artifact.path,
|
|
192
|
+
};
|
|
193
|
+
}
|
|
194
|
+
const discovery = asRecord(parsed.discovery);
|
|
195
|
+
const metadata = asRecord(parsed.metadata);
|
|
196
|
+
return {
|
|
197
|
+
available: true,
|
|
198
|
+
path: artifact.path,
|
|
199
|
+
summary: typeof parsed.summary === 'string' ? parsed.summary : undefined,
|
|
200
|
+
toolCount: getArrayLength(discovery?.tools),
|
|
201
|
+
promptCount: getArrayLength(discovery?.prompts),
|
|
202
|
+
resourceCount: getArrayLength(discovery?.resources),
|
|
203
|
+
durationMs: typeof metadata?.durationMs === 'number' ? metadata.durationMs : undefined,
|
|
204
|
+
errorCount: typeof metadata?.errorCount === 'number' ? metadata.errorCount : undefined,
|
|
205
|
+
};
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
return {
|
|
209
|
+
available: true,
|
|
210
|
+
path: artifact.path,
|
|
211
|
+
};
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
function summarizeGoldenArtifact(cwd, artifact, configPath) {
|
|
215
|
+
if (!artifact.exists) {
|
|
216
|
+
return {
|
|
217
|
+
available: false,
|
|
218
|
+
path: artifact.path,
|
|
219
|
+
};
|
|
220
|
+
}
|
|
221
|
+
try {
|
|
222
|
+
const report = readArtifact(cwd, artifact.id, configPath);
|
|
223
|
+
const parsed = asRecord(report.parsedJson);
|
|
224
|
+
return {
|
|
225
|
+
available: true,
|
|
226
|
+
path: artifact.path,
|
|
227
|
+
outputCount: getArrayLength(parsed?.outputs),
|
|
228
|
+
lastUpdated: typeof parsed?.lastUpdated === 'string' ? parsed.lastUpdated : undefined,
|
|
229
|
+
};
|
|
230
|
+
}
|
|
231
|
+
catch {
|
|
232
|
+
return {
|
|
233
|
+
available: true,
|
|
234
|
+
path: artifact.path,
|
|
235
|
+
};
|
|
236
|
+
}
|
|
237
|
+
}
|
|
238
|
+
//# sourceMappingURL=artifact-index.js.map
|