@specmarket/cli 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-DLEMNRTH.js → chunk-OTXWWFAO.js} +24 -2
- package/dist/chunk-OTXWWFAO.js.map +1 -0
- package/dist/{config-OAU6SJLC.js → config-5JMI3YAR.js} +2 -2
- package/dist/index.js +1283 -389
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
- package/src/commands/init.test.ts +162 -23
- package/src/commands/init.ts +349 -17
- package/src/commands/issues.test.ts +8 -3
- package/src/commands/issues.ts +2 -9
- package/src/commands/login.ts +2 -6
- package/src/commands/publish.test.ts +14 -1
- package/src/commands/publish.ts +1 -0
- package/src/commands/run.test.ts +206 -0
- package/src/commands/run.ts +63 -3
- package/src/commands/validate.test.ts +83 -6
- package/src/commands/validate.ts +96 -114
- package/src/lib/format-detection.test.ts +4 -4
- package/src/lib/format-detection.ts +3 -3
- package/src/lib/meta-instructions.test.ts +340 -0
- package/src/lib/meta-instructions.ts +562 -0
- package/src/lib/ralph-loop.test.ts +404 -0
- package/src/lib/ralph-loop.ts +475 -98
- package/src/lib/telemetry.ts +5 -0
- package/dist/chunk-DLEMNRTH.js.map +0 -1
- /package/dist/{config-OAU6SJLC.js.map → config-5JMI3YAR.js.map} +0 -0
|
@@ -172,14 +172,13 @@ describe('handleIssuesList', () => {
|
|
|
172
172
|
});
|
|
173
173
|
});
|
|
174
174
|
|
|
175
|
-
it('
|
|
175
|
+
it('passes label filter to server as labels array', async () => {
|
|
176
176
|
mockQuery.mockImplementation((fn: string) => {
|
|
177
177
|
if (fn === 'specs.get') return MOCK_SPEC;
|
|
178
178
|
if (fn === 'issues.list') {
|
|
179
179
|
return {
|
|
180
180
|
page: [
|
|
181
|
-
{ ...MOCK_ISSUE, labels: ['
|
|
182
|
-
{ ...MOCK_ISSUE, number: 2, labels: ['enhancement'], title: 'Add feature' },
|
|
181
|
+
{ ...MOCK_ISSUE, labels: ['enhancement'], title: 'Add feature' },
|
|
183
182
|
],
|
|
184
183
|
isDone: true,
|
|
185
184
|
continueCursor: null,
|
|
@@ -189,6 +188,12 @@ describe('handleIssuesList', () => {
|
|
|
189
188
|
|
|
190
189
|
await handleIssuesList('@alice/my-spec', { label: 'enhancement' });
|
|
191
190
|
|
|
191
|
+
expect(mockQuery).toHaveBeenCalledWith('issues.list', {
|
|
192
|
+
specId: 'spec123',
|
|
193
|
+
status: 'open',
|
|
194
|
+
labels: ['enhancement'],
|
|
195
|
+
paginationOpts: { numItems: 50, cursor: null },
|
|
196
|
+
});
|
|
192
197
|
expect(consoleSpy).toHaveBeenCalledWith(
|
|
193
198
|
expect.stringContaining('1 issue(s)')
|
|
194
199
|
);
|
package/src/commands/issues.ts
CHANGED
|
@@ -75,20 +75,13 @@ export async function handleIssuesList(
|
|
|
75
75
|
const result = await client.query(api.issues.list, {
|
|
76
76
|
specId: spec._id,
|
|
77
77
|
status: statusFilter,
|
|
78
|
+
labels: opts.label ? [opts.label] : undefined,
|
|
78
79
|
paginationOpts: { numItems: 50, cursor: null },
|
|
79
80
|
});
|
|
80
81
|
|
|
81
82
|
spinner.stop();
|
|
82
83
|
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
// Client-side label filter (backend doesn't support it directly)
|
|
86
|
-
if (opts.label) {
|
|
87
|
-
const label = opts.label.toLowerCase();
|
|
88
|
-
issues = issues.filter((i: any) =>
|
|
89
|
-
i.labels.some((l: string) => l.toLowerCase() === label)
|
|
90
|
-
);
|
|
91
|
-
}
|
|
84
|
+
const issues = result.page;
|
|
92
85
|
|
|
93
86
|
if (issues.length === 0) {
|
|
94
87
|
const statusLabel = statusFilter ?? 'any';
|
package/src/commands/login.ts
CHANGED
|
@@ -1,10 +1,9 @@
|
|
|
1
1
|
import { Command } from 'commander';
|
|
2
2
|
import chalk from 'chalk';
|
|
3
3
|
import ora from 'ora';
|
|
4
|
-
import { EXIT_CODES } from '@specmarket/shared';
|
|
4
|
+
import { EXIT_CODES, TOKEN_EXPIRY_MS, DEFAULT_WEB_URL } from '@specmarket/shared';
|
|
5
5
|
import { saveCredentials, loadCredentials } from '../lib/auth.js';
|
|
6
6
|
import { getConvexClient } from '../lib/convex-client.js';
|
|
7
|
-
import { TOKEN_EXPIRY_MS } from '@specmarket/shared';
|
|
8
7
|
import type { Credentials } from '@specmarket/shared';
|
|
9
8
|
import createDebug from 'debug';
|
|
10
9
|
|
|
@@ -101,10 +100,7 @@ async function handleTokenLogin(token: string): Promise<void> {
|
|
|
101
100
|
* 6. On expiry/timeout: show error
|
|
102
101
|
*/
|
|
103
102
|
async function handleDeviceCodeLogin(): Promise<void> {
|
|
104
|
-
const
|
|
105
|
-
const baseUrl = config.convexUrl ?? process.env['CONVEX_URL'] ?? 'https://your-deployment.convex.cloud';
|
|
106
|
-
const webUrl = baseUrl.replace('convex.cloud', 'specmarket.dev');
|
|
107
|
-
|
|
103
|
+
const webUrl = DEFAULT_WEB_URL;
|
|
108
104
|
const client = await getConvexClient();
|
|
109
105
|
|
|
110
106
|
let api: any;
|
|
@@ -58,9 +58,21 @@ const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
|
58
58
|
vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
59
59
|
|
|
60
60
|
import { handlePublish } from './publish.js';
|
|
61
|
+
import { SIDECAR_FILENAME } from '@specmarket/shared';
|
|
61
62
|
|
|
62
63
|
// --- Helpers ---
|
|
63
64
|
|
|
65
|
+
const VALID_SPECMARKET_YAML = `spec_format: specmarket
|
|
66
|
+
display_name: "Test Spec"
|
|
67
|
+
description: "A valid test spec with enough description length to pass."
|
|
68
|
+
output_type: web-app
|
|
69
|
+
primary_stack: nextjs-typescript
|
|
70
|
+
tags: []
|
|
71
|
+
estimated_tokens: 50000
|
|
72
|
+
estimated_cost_usd: 2.50
|
|
73
|
+
estimated_time_minutes: 30
|
|
74
|
+
`;
|
|
75
|
+
|
|
64
76
|
const VALID_SPEC_YAML = `name: test-spec
|
|
65
77
|
display_name: "Test Spec"
|
|
66
78
|
description: "A valid test spec with enough description length to pass."
|
|
@@ -109,8 +121,9 @@ describe('handlePublish', () => {
|
|
|
109
121
|
});
|
|
110
122
|
|
|
111
123
|
it('publishes a valid spec successfully', async () => {
|
|
112
|
-
// Write a valid spec
|
|
124
|
+
// Write a valid spec (specmarket.yaml required)
|
|
113
125
|
await Promise.all([
|
|
126
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
114
127
|
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
115
128
|
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild something.'),
|
|
116
129
|
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails here.'),
|
package/src/commands/publish.ts
CHANGED
|
@@ -102,6 +102,7 @@ export async function handlePublish(specPath: string, opts: { changelog?: string
|
|
|
102
102
|
specStorageId: storageId,
|
|
103
103
|
readme,
|
|
104
104
|
runner: specYaml.runner,
|
|
105
|
+
specFormat: validation.format,
|
|
105
106
|
minModel: specYaml.min_model,
|
|
106
107
|
estimatedTokens: specYaml.estimated_tokens,
|
|
107
108
|
estimatedCostUsd: specYaml.estimated_cost_usd,
|
package/src/commands/run.test.ts
CHANGED
|
@@ -60,9 +60,21 @@ const consoleSpy = vi.spyOn(console, 'log').mockImplementation(() => {});
|
|
|
60
60
|
vi.spyOn(console, 'error').mockImplementation(() => {});
|
|
61
61
|
|
|
62
62
|
import { handleRun } from './run.js';
|
|
63
|
+
import { SIDECAR_FILENAME } from '@specmarket/shared';
|
|
63
64
|
|
|
64
65
|
// --- Helpers ---
|
|
65
66
|
|
|
67
|
+
const VALID_SPECMARKET_YAML = `spec_format: specmarket
|
|
68
|
+
display_name: "Test Spec"
|
|
69
|
+
description: "A valid test spec with enough description length to pass."
|
|
70
|
+
output_type: web-app
|
|
71
|
+
primary_stack: nextjs-typescript
|
|
72
|
+
tags: []
|
|
73
|
+
estimated_tokens: 50000
|
|
74
|
+
estimated_cost_usd: 2.50
|
|
75
|
+
estimated_time_minutes: 30
|
|
76
|
+
`;
|
|
77
|
+
|
|
66
78
|
const VALID_SPEC_YAML = `name: test-spec
|
|
67
79
|
display_name: "Test Spec"
|
|
68
80
|
description: "A valid test spec with enough description length to pass."
|
|
@@ -114,6 +126,7 @@ describe('handleRun', () => {
|
|
|
114
126
|
|
|
115
127
|
it('runs a valid spec and prints summary', async () => {
|
|
116
128
|
await Promise.all([
|
|
129
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
117
130
|
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
118
131
|
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild it.'),
|
|
119
132
|
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails.'),
|
|
@@ -155,6 +168,7 @@ describe('handleRun', () => {
|
|
|
155
168
|
|
|
156
169
|
it('prints security warning before running', async () => {
|
|
157
170
|
await Promise.all([
|
|
171
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
158
172
|
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
159
173
|
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild it.'),
|
|
160
174
|
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails.'),
|
|
@@ -183,8 +197,200 @@ describe('handleRun', () => {
|
|
|
183
197
|
);
|
|
184
198
|
});
|
|
185
199
|
|
|
200
|
+
it('prints harness in run summary', async () => {
|
|
201
|
+
await Promise.all([
|
|
202
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
203
|
+
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
204
|
+
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild it.'),
|
|
205
|
+
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails.'),
|
|
206
|
+
writeFile(join(specDir, 'SUCCESS_CRITERIA.md'), VALID_SUCCESS_CRITERIA),
|
|
207
|
+
writeFile(join(specDir, 'stdlib', 'STACK.md'), '# Stack\nNext.js'),
|
|
208
|
+
]);
|
|
209
|
+
|
|
210
|
+
mockRunSpec.mockResolvedValue({
|
|
211
|
+
report: {
|
|
212
|
+
runId: 'run-123',
|
|
213
|
+
status: 'success',
|
|
214
|
+
loopCount: 1,
|
|
215
|
+
totalTokens: 1000,
|
|
216
|
+
totalCostUsd: 0.1,
|
|
217
|
+
totalTimeMinutes: 1,
|
|
218
|
+
successCriteriaResults: [],
|
|
219
|
+
},
|
|
220
|
+
outputDir: '/tmp/output',
|
|
221
|
+
});
|
|
222
|
+
mockSubmitTelemetry.mockResolvedValue(false);
|
|
223
|
+
|
|
224
|
+
await handleRun(specDir, { harness: 'codex' });
|
|
225
|
+
|
|
226
|
+
expect(consoleSpy).toHaveBeenCalledWith(
|
|
227
|
+
expect.stringContaining('codex')
|
|
228
|
+
);
|
|
229
|
+
expect(mockRunSpec).toHaveBeenCalledWith(
|
|
230
|
+
specDir,
|
|
231
|
+
expect.any(Object),
|
|
232
|
+
expect.objectContaining({ harness: 'codex' }),
|
|
233
|
+
expect.any(Function)
|
|
234
|
+
);
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
it('exits with validation error for unknown harness', async () => {
|
|
238
|
+
await Promise.all([
|
|
239
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
240
|
+
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
241
|
+
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild it.'),
|
|
242
|
+
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails.'),
|
|
243
|
+
writeFile(join(specDir, 'SUCCESS_CRITERIA.md'), VALID_SUCCESS_CRITERIA),
|
|
244
|
+
writeFile(join(specDir, 'stdlib', 'STACK.md'), '# Stack\nNext.js'),
|
|
245
|
+
]);
|
|
246
|
+
|
|
247
|
+
await expect(handleRun(specDir, { harness: 'unknown-harness' })).rejects.toThrow(
|
|
248
|
+
'process.exit called'
|
|
249
|
+
);
|
|
250
|
+
expect(consoleSpy).toHaveBeenCalledWith(
|
|
251
|
+
expect.stringContaining('Unknown harness')
|
|
252
|
+
);
|
|
253
|
+
});
|
|
254
|
+
|
|
255
|
+
it('passes --workdir to runSpec as workdir option', async () => {
|
|
256
|
+
await Promise.all([
|
|
257
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
258
|
+
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
259
|
+
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild it.'),
|
|
260
|
+
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails.'),
|
|
261
|
+
writeFile(join(specDir, 'SUCCESS_CRITERIA.md'), VALID_SUCCESS_CRITERIA),
|
|
262
|
+
writeFile(join(specDir, 'stdlib', 'STACK.md'), '# Stack\nNext.js'),
|
|
263
|
+
]);
|
|
264
|
+
|
|
265
|
+
mockRunSpec.mockResolvedValue({
|
|
266
|
+
report: {
|
|
267
|
+
runId: 'run-123',
|
|
268
|
+
status: 'success',
|
|
269
|
+
loopCount: 1,
|
|
270
|
+
totalTokens: 1000,
|
|
271
|
+
totalCostUsd: 0.1,
|
|
272
|
+
totalTimeMinutes: 1,
|
|
273
|
+
successCriteriaResults: [],
|
|
274
|
+
},
|
|
275
|
+
outputDir: '/tmp/myworkdir',
|
|
276
|
+
});
|
|
277
|
+
mockSubmitTelemetry.mockResolvedValue(false);
|
|
278
|
+
|
|
279
|
+
await handleRun(specDir, { workdir: '/tmp/myworkdir' });
|
|
280
|
+
|
|
281
|
+
expect(mockRunSpec).toHaveBeenCalledWith(
|
|
282
|
+
specDir,
|
|
283
|
+
expect.any(Object),
|
|
284
|
+
expect.objectContaining({ workdir: '/tmp/myworkdir' }),
|
|
285
|
+
expect.any(Function)
|
|
286
|
+
);
|
|
287
|
+
expect(consoleSpy).toHaveBeenCalledWith(
|
|
288
|
+
expect.stringContaining('/tmp/myworkdir')
|
|
289
|
+
);
|
|
290
|
+
});
|
|
291
|
+
|
|
292
|
+
it('passes steeringQueue to runSpec', async () => {
|
|
293
|
+
await Promise.all([
|
|
294
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
295
|
+
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
296
|
+
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild it.'),
|
|
297
|
+
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails.'),
|
|
298
|
+
writeFile(join(specDir, 'SUCCESS_CRITERIA.md'), VALID_SUCCESS_CRITERIA),
|
|
299
|
+
writeFile(join(specDir, 'stdlib', 'STACK.md'), '# Stack\nNext.js'),
|
|
300
|
+
]);
|
|
301
|
+
|
|
302
|
+
mockRunSpec.mockResolvedValue({
|
|
303
|
+
report: {
|
|
304
|
+
runId: 'run-123',
|
|
305
|
+
status: 'success',
|
|
306
|
+
loopCount: 2,
|
|
307
|
+
totalTokens: 5000,
|
|
308
|
+
totalCostUsd: 0.5,
|
|
309
|
+
totalTimeMinutes: 3,
|
|
310
|
+
steeringActionCount: 0,
|
|
311
|
+
successCriteriaResults: [],
|
|
312
|
+
},
|
|
313
|
+
outputDir: '/tmp/output',
|
|
314
|
+
});
|
|
315
|
+
mockSubmitTelemetry.mockResolvedValue(false);
|
|
316
|
+
|
|
317
|
+
await handleRun(specDir, {});
|
|
318
|
+
|
|
319
|
+
// runSpec must receive a steeringQueue array in opts
|
|
320
|
+
expect(mockRunSpec).toHaveBeenCalledWith(
|
|
321
|
+
specDir,
|
|
322
|
+
expect.any(Object),
|
|
323
|
+
expect.objectContaining({ steeringQueue: expect.any(Array) }),
|
|
324
|
+
expect.any(Function)
|
|
325
|
+
);
|
|
326
|
+
});
|
|
327
|
+
|
|
328
|
+
it('shows steering action count in summary when > 0', async () => {
|
|
329
|
+
await Promise.all([
|
|
330
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
331
|
+
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
332
|
+
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild it.'),
|
|
333
|
+
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails.'),
|
|
334
|
+
writeFile(join(specDir, 'SUCCESS_CRITERIA.md'), VALID_SUCCESS_CRITERIA),
|
|
335
|
+
writeFile(join(specDir, 'stdlib', 'STACK.md'), '# Stack\nNext.js'),
|
|
336
|
+
]);
|
|
337
|
+
|
|
338
|
+
mockRunSpec.mockResolvedValue({
|
|
339
|
+
report: {
|
|
340
|
+
runId: 'run-123',
|
|
341
|
+
status: 'success',
|
|
342
|
+
loopCount: 5,
|
|
343
|
+
totalTokens: 20000,
|
|
344
|
+
totalCostUsd: 2.0,
|
|
345
|
+
totalTimeMinutes: 10,
|
|
346
|
+
steeringActionCount: 3,
|
|
347
|
+
successCriteriaResults: [],
|
|
348
|
+
},
|
|
349
|
+
outputDir: '/tmp/output',
|
|
350
|
+
});
|
|
351
|
+
mockSubmitTelemetry.mockResolvedValue(false);
|
|
352
|
+
|
|
353
|
+
await handleRun(specDir, {});
|
|
354
|
+
|
|
355
|
+
expect(consoleSpy).toHaveBeenCalledWith(
|
|
356
|
+
expect.stringContaining('Steering Actions: 3')
|
|
357
|
+
);
|
|
358
|
+
});
|
|
359
|
+
|
|
360
|
+
it('omits steering count from summary when 0', async () => {
|
|
361
|
+
await Promise.all([
|
|
362
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
363
|
+
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
364
|
+
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild it.'),
|
|
365
|
+
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails.'),
|
|
366
|
+
writeFile(join(specDir, 'SUCCESS_CRITERIA.md'), VALID_SUCCESS_CRITERIA),
|
|
367
|
+
writeFile(join(specDir, 'stdlib', 'STACK.md'), '# Stack\nNext.js'),
|
|
368
|
+
]);
|
|
369
|
+
|
|
370
|
+
mockRunSpec.mockResolvedValue({
|
|
371
|
+
report: {
|
|
372
|
+
runId: 'run-123',
|
|
373
|
+
status: 'success',
|
|
374
|
+
loopCount: 2,
|
|
375
|
+
totalTokens: 5000,
|
|
376
|
+
totalCostUsd: 0.5,
|
|
377
|
+
totalTimeMinutes: 3,
|
|
378
|
+
steeringActionCount: 0,
|
|
379
|
+
successCriteriaResults: [],
|
|
380
|
+
},
|
|
381
|
+
outputDir: '/tmp/output',
|
|
382
|
+
});
|
|
383
|
+
mockSubmitTelemetry.mockResolvedValue(false);
|
|
384
|
+
|
|
385
|
+
await handleRun(specDir, {});
|
|
386
|
+
|
|
387
|
+
const calls = consoleSpy.mock.calls.map((c) => String(c[0]));
|
|
388
|
+
expect(calls.some((c) => c.includes('Steering Actions'))).toBe(false);
|
|
389
|
+
});
|
|
390
|
+
|
|
186
391
|
it('exits with budget_exceeded code on budget runs', async () => {
|
|
187
392
|
await Promise.all([
|
|
393
|
+
writeFile(join(specDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
188
394
|
writeFile(join(specDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
189
395
|
writeFile(join(specDir, 'PROMPT.md'), '# Prompt\nBuild it.'),
|
|
190
396
|
writeFile(join(specDir, 'SPEC.md'), '# Spec\nDetails.'),
|
package/src/commands/run.ts
CHANGED
|
@@ -4,7 +4,7 @@ import ora from 'ora';
|
|
|
4
4
|
import { readFile, mkdir, writeFile as writeFileFn } from 'fs/promises';
|
|
5
5
|
import { join, resolve, isAbsolute } from 'path';
|
|
6
6
|
import { parse as parseYaml } from 'yaml';
|
|
7
|
-
import { specYamlSchema, EXIT_CODES } from '@specmarket/shared';
|
|
7
|
+
import { specYamlSchema, EXIT_CODES, KNOWN_HARNESSES } from '@specmarket/shared';
|
|
8
8
|
import { validateSpec } from './validate.js';
|
|
9
9
|
import { loadCredentials, isAuthenticated } from '../lib/auth.js';
|
|
10
10
|
import { getConvexClient } from '../lib/convex-client.js';
|
|
@@ -46,6 +46,8 @@ export async function handleRun(
|
|
|
46
46
|
dryRun?: boolean;
|
|
47
47
|
resume?: string;
|
|
48
48
|
output?: string;
|
|
49
|
+
harness?: string;
|
|
50
|
+
workdir?: string;
|
|
49
51
|
}
|
|
50
52
|
): Promise<void> {
|
|
51
53
|
// Resolve spec directory (and registry spec ID if downloaded from registry)
|
|
@@ -94,9 +96,15 @@ export async function handleRun(
|
|
|
94
96
|
await promptTelemetryOptIn();
|
|
95
97
|
}
|
|
96
98
|
|
|
97
|
-
//
|
|
99
|
+
// Validate --harness value
|
|
100
|
+
if (opts.harness && !(KNOWN_HARNESSES as readonly string[]).includes(opts.harness)) {
|
|
101
|
+
console.log(chalk.red(`\n✗ Unknown harness "${opts.harness}". Supported: ${KNOWN_HARNESSES.join(', ')}`));
|
|
102
|
+
process.exit(EXIT_CODES.VALIDATION_ERROR);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Pre-flight check: Ensure the selected harness CLI is installed
|
|
98
106
|
try {
|
|
99
|
-
await checkClaudeCliInstalled();
|
|
107
|
+
await checkClaudeCliInstalled(opts.harness);
|
|
100
108
|
} catch (err) {
|
|
101
109
|
console.log(chalk.red(`\n✗ ${(err as Error).message}`));
|
|
102
110
|
process.exit(EXIT_CODES.VALIDATION_ERROR);
|
|
@@ -105,12 +113,46 @@ export async function handleRun(
|
|
|
105
113
|
const maxLoops = opts.maxLoops ? parseInt(opts.maxLoops, 10) : undefined;
|
|
106
114
|
const maxBudget = opts.maxBudget ? parseFloat(opts.maxBudget) : undefined;
|
|
107
115
|
|
|
116
|
+
const harness = opts.harness ?? 'claude-code';
|
|
108
117
|
console.log(chalk.cyan(`\nRunning spec: ${chalk.bold(specYaml.display_name)}`));
|
|
109
118
|
console.log(chalk.gray(` Version: ${specYaml.version}`));
|
|
110
119
|
console.log(chalk.gray(` Model: ${opts.model ?? specYaml.min_model}`));
|
|
120
|
+
console.log(chalk.gray(` Harness: ${harness}`));
|
|
121
|
+
if (opts.workdir) {
|
|
122
|
+
console.log(chalk.gray(` Working dir: ${opts.workdir}`));
|
|
123
|
+
}
|
|
111
124
|
console.log(chalk.gray(` Max loops: ${maxLoops ?? 50}`));
|
|
112
125
|
console.log(chalk.gray(` Estimated tokens: ${specYaml.estimated_tokens.toLocaleString()}`));
|
|
113
126
|
console.log(chalk.gray(` Estimated cost: $${specYaml.estimated_cost_usd.toFixed(2)}`));
|
|
127
|
+
|
|
128
|
+
// Set up steering input: collect lines from stdin and queue them for injection
|
|
129
|
+
// at the next iteration boundary. Works when stdin is a TTY (interactive) or
|
|
130
|
+
// a pipe (scripted input). Non-blocking — the run continues regardless.
|
|
131
|
+
const steeringQueue: string[] = [];
|
|
132
|
+
let steeringInputBuffer = '';
|
|
133
|
+
const steeringDataHandler = (chunk: Buffer | string): void => {
|
|
134
|
+
const data = typeof chunk === 'string' ? chunk : chunk.toString('utf-8');
|
|
135
|
+
steeringInputBuffer += data;
|
|
136
|
+
const lines = steeringInputBuffer.split('\n');
|
|
137
|
+
steeringInputBuffer = lines.pop() ?? '';
|
|
138
|
+
for (const line of lines) {
|
|
139
|
+
const trimmed = line.trim();
|
|
140
|
+
if (trimmed) {
|
|
141
|
+
steeringQueue.push(trimmed);
|
|
142
|
+
// Write to stderr so it doesn't overwrite the spinner on stdout
|
|
143
|
+
process.stderr.write(
|
|
144
|
+
`\n${chalk.cyan('[steering]')} Queued: "${trimmed.length > 60 ? trimmed.slice(0, 60) + '…' : trimmed}"\n`
|
|
145
|
+
);
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
};
|
|
149
|
+
|
|
150
|
+
if (!opts.dryRun) {
|
|
151
|
+
process.stdin.setEncoding('utf-8');
|
|
152
|
+
process.stdin.resume();
|
|
153
|
+
process.stdin.on('data', steeringDataHandler);
|
|
154
|
+
console.log(chalk.gray(' Tip: Type a message + Enter to steer the agent mid-run.'));
|
|
155
|
+
}
|
|
114
156
|
console.log('');
|
|
115
157
|
|
|
116
158
|
const spinner = ora({ text: 'Starting loop iteration 1...', spinner: 'dots' }).start();
|
|
@@ -134,13 +176,20 @@ export async function handleRun(
|
|
|
134
176
|
dryRun: opts.dryRun,
|
|
135
177
|
resumeRunId: opts.resume,
|
|
136
178
|
outputDir: opts.output,
|
|
179
|
+
harness: opts.harness,
|
|
180
|
+
workdir: opts.workdir,
|
|
137
181
|
cliVersion: CLI_VERSION,
|
|
182
|
+
steeringQueue,
|
|
138
183
|
},
|
|
139
184
|
(iteration: LoopIteration) => {
|
|
140
185
|
spinner.text = `Loop ${iteration.iteration}: ${iteration.tokens.toLocaleString()} tokens, ${(iteration.durationMs / 1000).toFixed(1)}s`;
|
|
141
186
|
}
|
|
142
187
|
);
|
|
143
188
|
|
|
189
|
+
// Stop collecting steering input now that the run is complete
|
|
190
|
+
process.stdin.removeListener('data', steeringDataHandler);
|
|
191
|
+
process.stdin.pause();
|
|
192
|
+
|
|
144
193
|
const { report } = result;
|
|
145
194
|
const statusColor =
|
|
146
195
|
report.status === 'success'
|
|
@@ -158,6 +207,9 @@ export async function handleRun(
|
|
|
158
207
|
console.log(` Tokens: ${report.totalTokens.toLocaleString()}`);
|
|
159
208
|
console.log(` Cost: $${report.totalCostUsd.toFixed(4)}`);
|
|
160
209
|
console.log(` Time: ${report.totalTimeMinutes.toFixed(1)} minutes`);
|
|
210
|
+
if (report.steeringActionCount && report.steeringActionCount > 0) {
|
|
211
|
+
console.log(` Steering Actions: ${report.steeringActionCount}`);
|
|
212
|
+
}
|
|
161
213
|
console.log(` Run ID: ${chalk.gray(report.runId)}`);
|
|
162
214
|
console.log(` Output: ${chalk.gray(result.outputDir)}`);
|
|
163
215
|
|
|
@@ -378,6 +430,14 @@ export function createRunCommand(): Command {
|
|
|
378
430
|
.option('--dry-run', 'Validate and show config without executing')
|
|
379
431
|
.option('--resume <run-id>', 'Resume a previous run from where it left off')
|
|
380
432
|
.option('--output <dir>', 'Custom output directory for run artifacts')
|
|
433
|
+
.option(
|
|
434
|
+
'--harness <harness>',
|
|
435
|
+
`Agentic harness to use (default: claude-code). One of: ${KNOWN_HARNESSES.join(', ')}`
|
|
436
|
+
)
|
|
437
|
+
.option(
|
|
438
|
+
'--workdir <dir>',
|
|
439
|
+
'Run in an existing directory instead of a fresh sandbox (spec files not copied)'
|
|
440
|
+
)
|
|
381
441
|
.action(async (pathOrId: string, opts) => {
|
|
382
442
|
try {
|
|
383
443
|
await handleRun(pathOrId, opts);
|
|
@@ -4,6 +4,18 @@ import { join } from 'path';
|
|
|
4
4
|
import { tmpdir } from 'os';
|
|
5
5
|
import { randomUUID } from 'crypto';
|
|
6
6
|
import { validateSpec, detectCircularReferences } from './validate.js';
|
|
7
|
+
import { SIDECAR_FILENAME } from '@specmarket/shared';
|
|
8
|
+
|
|
9
|
+
const VALID_SPECMARKET_YAML = `spec_format: specmarket
|
|
10
|
+
display_name: "Test Spec"
|
|
11
|
+
description: "A valid test spec with enough description length to pass."
|
|
12
|
+
output_type: web-app
|
|
13
|
+
primary_stack: nextjs-typescript
|
|
14
|
+
tags: []
|
|
15
|
+
estimated_tokens: 50000
|
|
16
|
+
estimated_cost_usd: 2.50
|
|
17
|
+
estimated_time_minutes: 30
|
|
18
|
+
`;
|
|
7
19
|
|
|
8
20
|
const VALID_SPEC_YAML = `name: test-spec
|
|
9
21
|
display_name: "Test Spec"
|
|
@@ -51,6 +63,7 @@ describe('validateSpec', () => {
|
|
|
51
63
|
|
|
52
64
|
async function writeValidSpec() {
|
|
53
65
|
await Promise.all([
|
|
66
|
+
writeFile(join(tmpDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML),
|
|
54
67
|
writeFile(join(tmpDir, 'spec.yaml'), VALID_SPEC_YAML),
|
|
55
68
|
writeFile(join(tmpDir, 'PROMPT.md'), VALID_PROMPT_MD),
|
|
56
69
|
writeFile(join(tmpDir, 'SPEC.md'), VALID_SPEC_MD),
|
|
@@ -84,6 +97,15 @@ describe('validateSpec', () => {
|
|
|
84
97
|
expect(result.errors.some((e) => e.includes('PROMPT.md'))).toBe(true);
|
|
85
98
|
});
|
|
86
99
|
|
|
100
|
+
it('reports error when specmarket.yaml is missing', async () => {
|
|
101
|
+
await writeValidSpec();
|
|
102
|
+
const { unlink } = await import('fs/promises');
|
|
103
|
+
await unlink(join(tmpDir, SIDECAR_FILENAME));
|
|
104
|
+
const result = await validateSpec(tmpDir);
|
|
105
|
+
expect(result.valid).toBe(false);
|
|
106
|
+
expect(result.errors.some((e) => e.includes(SIDECAR_FILENAME) && e.includes('required'))).toBe(true);
|
|
107
|
+
});
|
|
108
|
+
|
|
87
109
|
it('reports error for missing SUCCESS_CRITERIA.md', async () => {
|
|
88
110
|
await writeValidSpec();
|
|
89
111
|
const { unlink } = await import('fs/promises');
|
|
@@ -289,8 +311,9 @@ describe('validateSpec format-aware', () => {
|
|
|
289
311
|
await rm(tmpDir, { recursive: true, force: true });
|
|
290
312
|
});
|
|
291
313
|
|
|
292
|
-
it('reports format
|
|
314
|
+
it('reports format from sidecar for specmarket spec', async () => {
|
|
293
315
|
await mkdir(join(tmpDir, 'stdlib'), { recursive: true });
|
|
316
|
+
await writeFile(join(tmpDir, SIDECAR_FILENAME), VALID_SPECMARKET_YAML);
|
|
294
317
|
await writeFile(join(tmpDir, 'spec.yaml'), VALID_SPEC_YAML);
|
|
295
318
|
await writeFile(join(tmpDir, 'PROMPT.md'), VALID_PROMPT_MD);
|
|
296
319
|
await writeFile(join(tmpDir, 'SPEC.md'), VALID_SPEC_MD);
|
|
@@ -298,11 +321,19 @@ describe('validateSpec format-aware', () => {
|
|
|
298
321
|
await writeFile(join(tmpDir, 'stdlib', 'STACK.md'), VALID_STACK_MD);
|
|
299
322
|
const result = await validateSpec(tmpDir);
|
|
300
323
|
expect(result.valid).toBe(true);
|
|
301
|
-
expect(result.format).toBe('specmarket
|
|
302
|
-
expect(result.formatDetectedBy).toBe('
|
|
324
|
+
expect(result.format).toBe('specmarket');
|
|
325
|
+
expect(result.formatDetectedBy).toBe('sidecar');
|
|
303
326
|
});
|
|
304
327
|
|
|
328
|
+
const SIDECAR_SPECKIT = `spec_format: speckit
|
|
329
|
+
display_name: My Spec
|
|
330
|
+
description: A long enough description for the sidecar schema.
|
|
331
|
+
output_type: web-app
|
|
332
|
+
primary_stack: nextjs-typescript
|
|
333
|
+
`;
|
|
334
|
+
|
|
305
335
|
it('speckit dir validates successfully', async () => {
|
|
336
|
+
await writeFile(join(tmpDir, SIDECAR_FILENAME), SIDECAR_SPECKIT);
|
|
306
337
|
await writeFile(join(tmpDir, 'spec.md'), '# Spec\nContent here.');
|
|
307
338
|
await writeFile(join(tmpDir, 'tasks.md'), '# Tasks');
|
|
308
339
|
await mkdir(join(tmpDir, '.specify'), { recursive: true });
|
|
@@ -312,6 +343,7 @@ describe('validateSpec format-aware', () => {
|
|
|
312
343
|
});
|
|
313
344
|
|
|
314
345
|
it('speckit missing tasks.md and plan.md returns error', async () => {
|
|
346
|
+
await writeFile(join(tmpDir, SIDECAR_FILENAME), SIDECAR_SPECKIT);
|
|
315
347
|
await writeFile(join(tmpDir, 'spec.md'), '# Spec');
|
|
316
348
|
const result = await validateSpec(tmpDir);
|
|
317
349
|
expect(result.valid).toBe(false);
|
|
@@ -319,6 +351,15 @@ describe('validateSpec format-aware', () => {
|
|
|
319
351
|
});
|
|
320
352
|
|
|
321
353
|
it('bmad dir validates successfully', async () => {
|
|
354
|
+
await writeFile(
|
|
355
|
+
join(tmpDir, SIDECAR_FILENAME),
|
|
356
|
+
`spec_format: bmad
|
|
357
|
+
display_name: PRD Spec
|
|
358
|
+
description: A long enough description for the sidecar schema.
|
|
359
|
+
output_type: web-app
|
|
360
|
+
primary_stack: nextjs-typescript
|
|
361
|
+
`
|
|
362
|
+
);
|
|
322
363
|
await writeFile(join(tmpDir, 'prd.md'), '# PRD\nProduct requirements.');
|
|
323
364
|
await writeFile(join(tmpDir, 'story-1.md'), '# Story 1');
|
|
324
365
|
const result = await validateSpec(tmpDir);
|
|
@@ -327,6 +368,15 @@ describe('validateSpec format-aware', () => {
|
|
|
327
368
|
});
|
|
328
369
|
|
|
329
370
|
it('ralph dir validates successfully', async () => {
|
|
371
|
+
await writeFile(
|
|
372
|
+
join(tmpDir, SIDECAR_FILENAME),
|
|
373
|
+
`spec_format: ralph
|
|
374
|
+
display_name: Ralph Spec
|
|
375
|
+
description: A long enough description for the sidecar schema.
|
|
376
|
+
output_type: web-app
|
|
377
|
+
primary_stack: nextjs-typescript
|
|
378
|
+
`
|
|
379
|
+
);
|
|
330
380
|
await writeFile(
|
|
331
381
|
join(tmpDir, 'prd.json'),
|
|
332
382
|
JSON.stringify({ userStories: [{ title: 'As a user I want X' }] })
|
|
@@ -337,6 +387,15 @@ describe('validateSpec format-aware', () => {
|
|
|
337
387
|
});
|
|
338
388
|
|
|
339
389
|
it('ralph prd.json missing userStories returns error', async () => {
|
|
390
|
+
await writeFile(
|
|
391
|
+
join(tmpDir, SIDECAR_FILENAME),
|
|
392
|
+
`spec_format: ralph
|
|
393
|
+
display_name: Ralph Spec
|
|
394
|
+
description: A long enough description for the sidecar schema.
|
|
395
|
+
output_type: web-app
|
|
396
|
+
primary_stack: nextjs-typescript
|
|
397
|
+
`
|
|
398
|
+
);
|
|
340
399
|
await writeFile(join(tmpDir, 'prd.json'), JSON.stringify({ other: true }));
|
|
341
400
|
const result = await validateSpec(tmpDir);
|
|
342
401
|
expect(result.valid).toBe(false);
|
|
@@ -344,6 +403,15 @@ describe('validateSpec format-aware', () => {
|
|
|
344
403
|
});
|
|
345
404
|
|
|
346
405
|
it('custom dir with sufficient .md validates', async () => {
|
|
406
|
+
await writeFile(
|
|
407
|
+
join(tmpDir, SIDECAR_FILENAME),
|
|
408
|
+
`spec_format: custom
|
|
409
|
+
display_name: Custom Spec
|
|
410
|
+
description: A long enough description for the sidecar schema.
|
|
411
|
+
output_type: web-app
|
|
412
|
+
primary_stack: nextjs-typescript
|
|
413
|
+
`
|
|
414
|
+
);
|
|
347
415
|
const content =
|
|
348
416
|
'# Readme\n\nThis is a spec with enough content to pass the 100-byte minimum for custom format. Extra text here.';
|
|
349
417
|
expect(content.length).toBeGreaterThan(100);
|
|
@@ -354,6 +422,15 @@ describe('validateSpec format-aware', () => {
|
|
|
354
422
|
});
|
|
355
423
|
|
|
356
424
|
it('custom dir with only tiny .md files fails', async () => {
|
|
425
|
+
await writeFile(
|
|
426
|
+
join(tmpDir, SIDECAR_FILENAME),
|
|
427
|
+
`spec_format: custom
|
|
428
|
+
display_name: Custom Spec
|
|
429
|
+
description: A long enough description for the sidecar schema.
|
|
430
|
+
output_type: web-app
|
|
431
|
+
primary_stack: nextjs-typescript
|
|
432
|
+
`
|
|
433
|
+
);
|
|
357
434
|
await writeFile(join(tmpDir, 'tiny.md'), 'x');
|
|
358
435
|
const result = await validateSpec(tmpDir);
|
|
359
436
|
expect(result.valid).toBe(false);
|
|
@@ -364,19 +441,19 @@ describe('validateSpec format-aware', () => {
|
|
|
364
441
|
await writeFile(join(tmpDir, 'spec.md'), '# Spec');
|
|
365
442
|
await writeFile(join(tmpDir, 'tasks.md'), '# Tasks');
|
|
366
443
|
await writeFile(
|
|
367
|
-
join(tmpDir,
|
|
444
|
+
join(tmpDir, SIDECAR_FILENAME),
|
|
368
445
|
'spec_format: speckit\ndisplay_name: X\ndescription: short'
|
|
369
446
|
);
|
|
370
447
|
const result = await validateSpec(tmpDir);
|
|
371
448
|
expect(result.valid).toBe(false);
|
|
372
|
-
expect(result.errors.some((e) => e.includes(
|
|
449
|
+
expect(result.errors.some((e) => e.includes(SIDECAR_FILENAME))).toBe(true);
|
|
373
450
|
});
|
|
374
451
|
|
|
375
452
|
it('sidecar with valid schema passes and format is from sidecar', async () => {
|
|
376
453
|
await writeFile(join(tmpDir, 'spec.md'), '# Spec');
|
|
377
454
|
await writeFile(join(tmpDir, 'tasks.md'), '# Tasks');
|
|
378
455
|
await writeFile(
|
|
379
|
-
join(tmpDir,
|
|
456
|
+
join(tmpDir, SIDECAR_FILENAME),
|
|
380
457
|
`spec_format: speckit
|
|
381
458
|
display_name: My Spec
|
|
382
459
|
description: A long enough description for the sidecar schema.
|