erosolar-cli 1.7.38 → 1.7.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/cliTestHarness.d.ts +165 -0
- package/dist/core/cliTestHarness.d.ts.map +1 -0
- package/dist/core/cliTestHarness.js +517 -0
- package/dist/core/cliTestHarness.js.map +1 -0
- package/dist/core/isolatedVerifier.js +107 -6
- package/dist/core/isolatedVerifier.js.map +1 -1
- package/dist/shell/inputProcessor.d.ts +2 -1
- package/dist/shell/inputProcessor.d.ts.map +1 -1
- package/dist/shell/inputProcessor.js +6 -5
- package/dist/shell/inputProcessor.js.map +1 -1
- package/dist/shell/interactiveShell.d.ts.map +1 -1
- package/dist/shell/interactiveShell.js +5 -1
- package/dist/shell/interactiveShell.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLI Test Harness - PTY-based interactive CLI testing
|
|
3
|
+
*
|
|
4
|
+
* This module provides real runtime verification of CLI behavior by:
|
|
5
|
+
* 1. Spawning the CLI in a pseudo-terminal (PTY)
|
|
6
|
+
* 2. Sending simulated user input (including paste sequences)
|
|
7
|
+
* 3. Capturing and analyzing output
|
|
8
|
+
* 4. Verifying expected behaviors
|
|
9
|
+
*
|
|
10
|
+
* @license MIT
|
|
11
|
+
*/
|
|
12
|
+
import { EventEmitter } from 'node:events';
|
|
13
|
+
export interface CLITestConfig {
|
|
14
|
+
/** Working directory for the CLI */
|
|
15
|
+
cwd: string;
|
|
16
|
+
/** Environment variables */
|
|
17
|
+
env?: Record<string, string>;
|
|
18
|
+
/** Timeout for the entire test (ms) */
|
|
19
|
+
timeout?: number;
|
|
20
|
+
/** Whether to use PTY mode (requires node-pty) */
|
|
21
|
+
usePty?: boolean;
|
|
22
|
+
/** Path to CLI entry point */
|
|
23
|
+
cliPath?: string;
|
|
24
|
+
}
|
|
25
|
+
export interface TestInput {
|
|
26
|
+
/** Type of input to send */
|
|
27
|
+
type: 'text' | 'paste' | 'key' | 'wait';
|
|
28
|
+
/** The content to send */
|
|
29
|
+
content?: string;
|
|
30
|
+
/** For 'key' type: special key name */
|
|
31
|
+
key?: 'enter' | 'tab' | 'escape' | 'ctrl-c' | 'ctrl-d';
|
|
32
|
+
/** For 'wait' type: milliseconds to wait */
|
|
33
|
+
delay?: number;
|
|
34
|
+
}
|
|
35
|
+
export interface TestExpectation {
|
|
36
|
+
/** Type of expectation */
|
|
37
|
+
type: 'output_contains' | 'output_matches' | 'output_not_contains' | 'exit_code';
|
|
38
|
+
/** Pattern or value to check */
|
|
39
|
+
value: string | number | RegExp;
|
|
40
|
+
/** Description for error messages */
|
|
41
|
+
description?: string;
|
|
42
|
+
/** Timeout for this specific expectation (ms) */
|
|
43
|
+
timeout?: number;
|
|
44
|
+
}
|
|
45
|
+
export interface CLITestScenario {
|
|
46
|
+
/** Unique test identifier */
|
|
47
|
+
id: string;
|
|
48
|
+
/** Human-readable description */
|
|
49
|
+
description: string;
|
|
50
|
+
/** Category of test */
|
|
51
|
+
category: 'paste' | 'input' | 'command' | 'output' | 'behavior';
|
|
52
|
+
/** Sequence of inputs to send */
|
|
53
|
+
inputs: TestInput[];
|
|
54
|
+
/** Expected outcomes */
|
|
55
|
+
expectations: TestExpectation[];
|
|
56
|
+
/** Setup commands to run before test */
|
|
57
|
+
setup?: string[];
|
|
58
|
+
/** Cleanup commands to run after test */
|
|
59
|
+
cleanup?: string[];
|
|
60
|
+
}
|
|
61
|
+
export interface TestResult {
|
|
62
|
+
scenario: CLITestScenario;
|
|
63
|
+
passed: boolean;
|
|
64
|
+
duration: number;
|
|
65
|
+
output: string;
|
|
66
|
+
errors: string[];
|
|
67
|
+
expectations: Array<{
|
|
68
|
+
expectation: TestExpectation;
|
|
69
|
+
passed: boolean;
|
|
70
|
+
actual?: string;
|
|
71
|
+
reason?: string;
|
|
72
|
+
}>;
|
|
73
|
+
}
|
|
74
|
+
declare const SPECIAL_KEYS: Record<string, string>;
|
|
75
|
+
export declare class CLITestHarness extends EventEmitter {
|
|
76
|
+
private config;
|
|
77
|
+
private process;
|
|
78
|
+
private output;
|
|
79
|
+
private errors;
|
|
80
|
+
private exitCode;
|
|
81
|
+
private ptyModule;
|
|
82
|
+
constructor(config: CLITestConfig);
|
|
83
|
+
/**
|
|
84
|
+
* Try to load node-pty for true PTY support
|
|
85
|
+
*/
|
|
86
|
+
private loadPtyModule;
|
|
87
|
+
/**
|
|
88
|
+
* Start the CLI process
|
|
89
|
+
*/
|
|
90
|
+
start(): Promise<void>;
|
|
91
|
+
/**
|
|
92
|
+
* Start CLI with PTY (for interactive features like bracketed paste)
|
|
93
|
+
*/
|
|
94
|
+
private startWithPty;
|
|
95
|
+
/**
|
|
96
|
+
* Start CLI with standard stdio (fallback, limited interactive support)
|
|
97
|
+
*/
|
|
98
|
+
private startWithStdio;
|
|
99
|
+
/**
|
|
100
|
+
* Send input to the CLI
|
|
101
|
+
*/
|
|
102
|
+
write(input: string): void;
|
|
103
|
+
/**
|
|
104
|
+
* Send a bracketed paste sequence
|
|
105
|
+
*/
|
|
106
|
+
paste(content: string): void;
|
|
107
|
+
/**
|
|
108
|
+
* Send a special key
|
|
109
|
+
*/
|
|
110
|
+
sendKey(key: keyof typeof SPECIAL_KEYS): void;
|
|
111
|
+
/**
|
|
112
|
+
* Wait for output matching a pattern
|
|
113
|
+
*/
|
|
114
|
+
waitForOutput(pattern: string | RegExp, timeout?: number): Promise<string>;
|
|
115
|
+
/**
|
|
116
|
+
* Wait for a specified duration
|
|
117
|
+
*/
|
|
118
|
+
wait(ms: number): Promise<void>;
|
|
119
|
+
/**
|
|
120
|
+
* Get current output
|
|
121
|
+
*/
|
|
122
|
+
getOutput(): string;
|
|
123
|
+
/**
|
|
124
|
+
* Get current errors
|
|
125
|
+
*/
|
|
126
|
+
getErrors(): string;
|
|
127
|
+
/**
|
|
128
|
+
* Stop the CLI process
|
|
129
|
+
*/
|
|
130
|
+
stop(): Promise<number>;
|
|
131
|
+
/**
|
|
132
|
+
* Run a complete test scenario
|
|
133
|
+
*/
|
|
134
|
+
runScenario(scenario: CLITestScenario): Promise<TestResult>;
|
|
135
|
+
/**
|
|
136
|
+
* Execute a single input action
|
|
137
|
+
*/
|
|
138
|
+
private executeInput;
|
|
139
|
+
/**
|
|
140
|
+
* Check a single expectation
|
|
141
|
+
*/
|
|
142
|
+
private checkExpectation;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Create a paste handling test scenario
|
|
146
|
+
*/
|
|
147
|
+
export declare function createPasteTestScenario(content: string, expectedLineCount: number): CLITestScenario;
|
|
148
|
+
/**
|
|
149
|
+
* Create a multi-line input test scenario
|
|
150
|
+
*/
|
|
151
|
+
export declare function createMultiLineInputScenario(): CLITestScenario;
|
|
152
|
+
/**
|
|
153
|
+
* Create a slash command test scenario
|
|
154
|
+
*/
|
|
155
|
+
export declare function createSlashCommandScenario(command: string): CLITestScenario;
|
|
156
|
+
/**
|
|
157
|
+
* Run verification tests for a specific claim type
|
|
158
|
+
*/
|
|
159
|
+
export declare function runVerificationTests(claimType: string, workingDir: string): Promise<{
|
|
160
|
+
passed: boolean;
|
|
161
|
+
results: TestResult[];
|
|
162
|
+
summary: string;
|
|
163
|
+
}>;
|
|
164
|
+
export default CLITestHarness;
|
|
165
|
+
//# sourceMappingURL=cliTestHarness.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cliTestHarness.d.ts","sourceRoot":"","sources":["../../src/core/cliTestHarness.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAIH,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAM3C,MAAM,WAAW,aAAa;IAC5B,oCAAoC;IACpC,GAAG,EAAE,MAAM,CAAC;IACZ,4BAA4B;IAC5B,GAAG,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC7B,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,kDAAkD;IAClD,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,8BAA8B;IAC9B,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,SAAS;IACxB,4BAA4B;IAC5B,IAAI,EAAE,MAAM,GAAG,OAAO,GAAG,KAAK,GAAG,MAAM,CAAC;IACxC,0BAA0B;IAC1B,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,uCAAuC;IACvC,GAAG,CAAC,EAAE,OAAO,GAAG,KAAK,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;IACvD,4CAA4C;IAC5C,KAAK,CAAC,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,eAAe;IAC9B,0BAA0B;IAC1B,IAAI,EAAE,iBAAiB,GAAG,gBAAgB,GAAG,qBAAqB,GAAG,WAAW,CAAC;IACjF,gCAAgC;IAChC,KAAK,EAAE,MAAM,GAAG,MAAM,GAAG,MAAM,CAAC;IAChC,qCAAqC;IACrC,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,iDAAiD;IACjD,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,eAAe;IAC9B,6BAA6B;IAC7B,EAAE,EAAE,MAAM,CAAC;IACX,iCAAiC;IACjC,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB;IACvB,QAAQ,EAAE,OAAO,GAAG,OAAO,GAAG,SAAS,GAAG,QAAQ,GAAG,UAAU,CAAC;IAChE,iCAAiC;IACjC,MAAM,EAAE,SAAS,EAAE,CAAC;IACpB,wBAAwB;IACxB,YAAY,EAAE,eAAe,EAAE,CAAC;IAChC,wCAAwC;IACxC,KAAK,CAAC,EAAE,MAAM,EAAE,CAAC;IACjB,yCAAyC;IACzC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,EAAE,eAAe,CAAC;IAC1B,MAAM,EAAE,OAAO,CAAC;IAChB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,YAAY,EAAE,KAAK,CAAC;QAClB,WAAW,EAAE,eAAe,CAAC;QAC7B,MAAM,EAAE,OAAO,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,MAAM,CAAC,EAAE,MAAM,CAAC;KACjB,CAAC,CAAC;CACJ;AASD,QAAA,MAAM,YAAY,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAMxC,CAAC;AAMF,qBAAa,cAAe,SAAQ,YAAY;IAC9C,OAAO,CAAC,MAAM,CAA0B;IACxC,OAAO,CAAC,OAAO,CAA6B;IAC5C,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,QAAQ,CAAuB;IACvC,OAAO,CAAC,SAAS,CAAa;gBAElB,MAAM,EAAE,aAAa;IAWjC;;OAEG;YACW,aAAa;IAa3B;;OAEG;IACG,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsB5B;;OAEG;YACW,YAAY;IAsB1B;;OAEG;YACW,cAAc;IAyB5B;;OAEG;IACH,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IAY1B;;OAEG;IACH,KAAK,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAI5B;;OAEG;IACH,OAAO,CAAC,GAAG,EAAE,MAAM,OAAO,YAAY,GAAG,IAAI;IAO7C;;OAEG;IACG,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,EAAE,OAAO,GAAE,MAAa,GAAG,OAAO,CAAC,MAAM,CAAC;IAuBtF;;OAEG;IACG,IAAI,CAAC,EAAE,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAIrC;;OAEG;IACH,SAAS,IAAI,MAAM;IAInB;;OAEG;IACH,SAAS,IAAI,MAAM;IAInB;;OAEG;IACG,IAAI,IAAI,OAAO,CAAC,MAAM,CAAC;IAuB7B;;OAEG;IACG,WAAW,CAAC,QAAQ,EAAE,eAAe,GAAG,OAAO,CAAC,UAAU,CAAC;IAgDjE;;OAEG;YACW,YAAY;IA0B1B;;OAEG;YACW,gBAAgB;CAkF/B;AAMD;;GAEG;AACH,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,MAAM,EAAE,iBAAiB,EAAE,MAAM,GAAG,eAAe,CAoBnG;AAED;;GAEG;AACH,wBAAgB,4BAA4B,IAAI,eAAe,CA2B9D;AAED;;GAEG;AACH,wBAAgB,0BAA0B,CAAC,OAAO,EAAE,MAAM,GAAG,eAAe,CAmB3E;AAMD;;GAEG;AACH,wBAAsB,oBAAoB,CACxC,SAAS,EAAE,MAAM,EACjB,UAAU,EAAE,MAAM,GACjB,OAAO,CAAC;IACT,MAAM,EAAE,OAAO,CAAC;IAChB,OAAO,EAAE,UAAU,EAAE,CAAC;IACtB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC,CA8ED;AAED,eAAe,cAAc,CAAC"}
|
|
@@ -0,0 +1,517 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CLI Test Harness - PTY-based interactive CLI testing
|
|
3
|
+
*
|
|
4
|
+
* This module provides real runtime verification of CLI behavior by:
|
|
5
|
+
* 1. Spawning the CLI in a pseudo-terminal (PTY)
|
|
6
|
+
* 2. Sending simulated user input (including paste sequences)
|
|
7
|
+
* 3. Capturing and analyzing output
|
|
8
|
+
* 4. Verifying expected behaviors
|
|
9
|
+
*
|
|
10
|
+
* @license MIT
|
|
11
|
+
*/
|
|
12
|
+
import { spawn } from 'node:child_process';
|
|
13
|
+
import { join } from 'node:path';
|
|
14
|
+
import { EventEmitter } from 'node:events';
|
|
15
|
+
// ============================================================================
|
|
16
|
+
// CONSTANTS
|
|
17
|
+
// ============================================================================
|
|
18
|
+
const BRACKETED_PASTE_START = '\x1b[200~';
|
|
19
|
+
const BRACKETED_PASTE_END = '\x1b[201~';
|
|
20
|
+
const SPECIAL_KEYS = {
|
|
21
|
+
'enter': '\r',
|
|
22
|
+
'tab': '\t',
|
|
23
|
+
'escape': '\x1b',
|
|
24
|
+
'ctrl-c': '\x03',
|
|
25
|
+
'ctrl-d': '\x04',
|
|
26
|
+
};
|
|
27
|
+
// ============================================================================
|
|
28
|
+
// CLI TEST HARNESS
|
|
29
|
+
// ============================================================================
|
|
30
|
+
export class CLITestHarness extends EventEmitter {
|
|
31
|
+
config;
|
|
32
|
+
process = null;
|
|
33
|
+
output = '';
|
|
34
|
+
errors = '';
|
|
35
|
+
exitCode = null;
|
|
36
|
+
ptyModule = null;
|
|
37
|
+
constructor(config) {
|
|
38
|
+
super();
|
|
39
|
+
this.config = {
|
|
40
|
+
cwd: config.cwd,
|
|
41
|
+
env: config.env ?? {},
|
|
42
|
+
timeout: config.timeout ?? 30000,
|
|
43
|
+
usePty: config.usePty ?? false,
|
|
44
|
+
cliPath: config.cliPath ?? join(config.cwd, 'dist/bin/erosolar-optimized.js'),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Try to load node-pty for true PTY support
|
|
49
|
+
*/
|
|
50
|
+
async loadPtyModule() {
|
|
51
|
+
if (this.ptyModule)
|
|
52
|
+
return true;
|
|
53
|
+
try {
|
|
54
|
+
// Dynamic import to avoid hard dependency
|
|
55
|
+
// @ts-expect-error - node-pty is optional
|
|
56
|
+
this.ptyModule = await import('node-pty');
|
|
57
|
+
return true;
|
|
58
|
+
}
|
|
59
|
+
catch {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Start the CLI process
|
|
65
|
+
*/
|
|
66
|
+
async start() {
|
|
67
|
+
this.output = '';
|
|
68
|
+
this.errors = '';
|
|
69
|
+
this.exitCode = null;
|
|
70
|
+
const env = {
|
|
71
|
+
...process.env,
|
|
72
|
+
...this.config.env,
|
|
73
|
+
// Disable color output for easier parsing
|
|
74
|
+
NO_COLOR: '1',
|
|
75
|
+
FORCE_COLOR: '0',
|
|
76
|
+
// Set non-interactive mode hints
|
|
77
|
+
CI: '1',
|
|
78
|
+
};
|
|
79
|
+
if (this.config.usePty && await this.loadPtyModule()) {
|
|
80
|
+
await this.startWithPty(env);
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
await this.startWithStdio(env);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
/**
|
|
87
|
+
* Start CLI with PTY (for interactive features like bracketed paste)
|
|
88
|
+
*/
|
|
89
|
+
async startWithPty(env) {
|
|
90
|
+
const pty = this.ptyModule;
|
|
91
|
+
this.process = pty.spawn('node', [this.config.cliPath], {
|
|
92
|
+
name: 'xterm-256color',
|
|
93
|
+
cols: 120,
|
|
94
|
+
rows: 30,
|
|
95
|
+
cwd: this.config.cwd,
|
|
96
|
+
env,
|
|
97
|
+
});
|
|
98
|
+
this.process.onData((data) => {
|
|
99
|
+
this.output += data;
|
|
100
|
+
this.emit('output', data);
|
|
101
|
+
});
|
|
102
|
+
this.process.onExit(({ exitCode }) => {
|
|
103
|
+
this.exitCode = exitCode;
|
|
104
|
+
this.emit('exit', exitCode);
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Start CLI with standard stdio (fallback, limited interactive support)
|
|
109
|
+
*/
|
|
110
|
+
async startWithStdio(env) {
|
|
111
|
+
this.process = spawn('node', [this.config.cliPath], {
|
|
112
|
+
cwd: this.config.cwd,
|
|
113
|
+
env,
|
|
114
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
115
|
+
});
|
|
116
|
+
this.process.stdout?.on('data', (data) => {
|
|
117
|
+
const str = data.toString();
|
|
118
|
+
this.output += str;
|
|
119
|
+
this.emit('output', str);
|
|
120
|
+
});
|
|
121
|
+
this.process.stderr?.on('data', (data) => {
|
|
122
|
+
const str = data.toString();
|
|
123
|
+
this.errors += str;
|
|
124
|
+
this.emit('error', str);
|
|
125
|
+
});
|
|
126
|
+
this.process.on('exit', (code) => {
|
|
127
|
+
this.exitCode = code ?? 0;
|
|
128
|
+
this.emit('exit', code);
|
|
129
|
+
});
|
|
130
|
+
}
|
|
131
|
+
/**
|
|
132
|
+
* Send input to the CLI
|
|
133
|
+
*/
|
|
134
|
+
write(input) {
|
|
135
|
+
if (!this.process) {
|
|
136
|
+
throw new Error('CLI process not started');
|
|
137
|
+
}
|
|
138
|
+
if (this.config.usePty && this.ptyModule) {
|
|
139
|
+
this.process.write(input);
|
|
140
|
+
}
|
|
141
|
+
else {
|
|
142
|
+
this.process.stdin?.write(input);
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Send a bracketed paste sequence
|
|
147
|
+
*/
|
|
148
|
+
paste(content) {
|
|
149
|
+
this.write(BRACKETED_PASTE_START + content + BRACKETED_PASTE_END);
|
|
150
|
+
}
|
|
151
|
+
/**
|
|
152
|
+
* Send a special key
|
|
153
|
+
*/
|
|
154
|
+
sendKey(key) {
|
|
155
|
+
const sequence = SPECIAL_KEYS[key];
|
|
156
|
+
if (sequence) {
|
|
157
|
+
this.write(sequence);
|
|
158
|
+
}
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* Wait for output matching a pattern
|
|
162
|
+
*/
|
|
163
|
+
async waitForOutput(pattern, timeout = 5000) {
|
|
164
|
+
const startTime = Date.now();
|
|
165
|
+
const regex = typeof pattern === 'string' ? new RegExp(pattern, 'i') : pattern;
|
|
166
|
+
return new Promise((resolve, reject) => {
|
|
167
|
+
const check = () => {
|
|
168
|
+
if (regex.test(this.output)) {
|
|
169
|
+
resolve(this.output);
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
if (Date.now() - startTime > timeout) {
|
|
173
|
+
reject(new Error(`Timeout waiting for pattern: ${pattern}`));
|
|
174
|
+
return;
|
|
175
|
+
}
|
|
176
|
+
setTimeout(check, 100);
|
|
177
|
+
};
|
|
178
|
+
check();
|
|
179
|
+
});
|
|
180
|
+
}
|
|
181
|
+
/**
|
|
182
|
+
* Wait for a specified duration
|
|
183
|
+
*/
|
|
184
|
+
async wait(ms) {
|
|
185
|
+
return new Promise(resolve => setTimeout(resolve, ms));
|
|
186
|
+
}
|
|
187
|
+
/**
|
|
188
|
+
* Get current output
|
|
189
|
+
*/
|
|
190
|
+
getOutput() {
|
|
191
|
+
return this.output;
|
|
192
|
+
}
|
|
193
|
+
/**
|
|
194
|
+
* Get current errors
|
|
195
|
+
*/
|
|
196
|
+
getErrors() {
|
|
197
|
+
return this.errors;
|
|
198
|
+
}
|
|
199
|
+
/**
|
|
200
|
+
* Stop the CLI process
|
|
201
|
+
*/
|
|
202
|
+
async stop() {
|
|
203
|
+
if (!this.process) {
|
|
204
|
+
return this.exitCode ?? 0;
|
|
205
|
+
}
|
|
206
|
+
return new Promise((resolve) => {
|
|
207
|
+
const timeout = setTimeout(() => {
|
|
208
|
+
this.process?.kill('SIGKILL');
|
|
209
|
+
}, 5000);
|
|
210
|
+
this.process.on('exit', (code) => {
|
|
211
|
+
clearTimeout(timeout);
|
|
212
|
+
resolve(code ?? 0);
|
|
213
|
+
});
|
|
214
|
+
// Try graceful shutdown first
|
|
215
|
+
this.sendKey('ctrl-c');
|
|
216
|
+
setTimeout(() => {
|
|
217
|
+
this.sendKey('ctrl-d');
|
|
218
|
+
}, 500);
|
|
219
|
+
});
|
|
220
|
+
}
|
|
221
|
+
/**
|
|
222
|
+
* Run a complete test scenario
|
|
223
|
+
*/
|
|
224
|
+
async runScenario(scenario) {
|
|
225
|
+
const startTime = Date.now();
|
|
226
|
+
const result = {
|
|
227
|
+
scenario,
|
|
228
|
+
passed: true,
|
|
229
|
+
duration: 0,
|
|
230
|
+
output: '',
|
|
231
|
+
errors: [],
|
|
232
|
+
expectations: [],
|
|
233
|
+
};
|
|
234
|
+
try {
|
|
235
|
+
// Start the CLI
|
|
236
|
+
await this.start();
|
|
237
|
+
// Wait for initial startup
|
|
238
|
+
await this.wait(1000);
|
|
239
|
+
// Execute input sequence
|
|
240
|
+
for (const input of scenario.inputs) {
|
|
241
|
+
await this.executeInput(input);
|
|
242
|
+
}
|
|
243
|
+
// Wait for processing
|
|
244
|
+
await this.wait(500);
|
|
245
|
+
// Check expectations
|
|
246
|
+
for (const expectation of scenario.expectations) {
|
|
247
|
+
const expResult = await this.checkExpectation(expectation);
|
|
248
|
+
result.expectations.push(expResult);
|
|
249
|
+
if (!expResult.passed) {
|
|
250
|
+
result.passed = false;
|
|
251
|
+
result.errors.push(expResult.reason || 'Expectation failed');
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
}
|
|
255
|
+
catch (error) {
|
|
256
|
+
result.passed = false;
|
|
257
|
+
result.errors.push(error instanceof Error ? error.message : String(error));
|
|
258
|
+
}
|
|
259
|
+
finally {
|
|
260
|
+
await this.stop();
|
|
261
|
+
result.output = this.output;
|
|
262
|
+
result.duration = Date.now() - startTime;
|
|
263
|
+
}
|
|
264
|
+
return result;
|
|
265
|
+
}
|
|
266
|
+
/**
|
|
267
|
+
* Execute a single input action
|
|
268
|
+
*/
|
|
269
|
+
async executeInput(input) {
|
|
270
|
+
switch (input.type) {
|
|
271
|
+
case 'text':
|
|
272
|
+
if (input.content) {
|
|
273
|
+
this.write(input.content);
|
|
274
|
+
}
|
|
275
|
+
break;
|
|
276
|
+
case 'paste':
|
|
277
|
+
if (input.content) {
|
|
278
|
+
this.paste(input.content);
|
|
279
|
+
}
|
|
280
|
+
break;
|
|
281
|
+
case 'key':
|
|
282
|
+
if (input.key) {
|
|
283
|
+
this.sendKey(input.key);
|
|
284
|
+
}
|
|
285
|
+
break;
|
|
286
|
+
case 'wait':
|
|
287
|
+
await this.wait(input.delay ?? 100);
|
|
288
|
+
break;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
/**
|
|
292
|
+
* Check a single expectation
|
|
293
|
+
*/
|
|
294
|
+
async checkExpectation(expectation) {
|
|
295
|
+
const timeout = expectation.timeout ?? 5000;
|
|
296
|
+
try {
|
|
297
|
+
switch (expectation.type) {
|
|
298
|
+
case 'output_contains': {
|
|
299
|
+
const pattern = expectation.value;
|
|
300
|
+
try {
|
|
301
|
+
await this.waitForOutput(pattern, timeout);
|
|
302
|
+
return { expectation, passed: true, actual: this.output };
|
|
303
|
+
}
|
|
304
|
+
catch {
|
|
305
|
+
return {
|
|
306
|
+
expectation,
|
|
307
|
+
passed: false,
|
|
308
|
+
actual: this.output.slice(-500),
|
|
309
|
+
reason: `Output does not contain: "${pattern}"`,
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
case 'output_matches': {
|
|
314
|
+
const regex = expectation.value instanceof RegExp
|
|
315
|
+
? expectation.value
|
|
316
|
+
: new RegExp(expectation.value);
|
|
317
|
+
if (regex.test(this.output)) {
|
|
318
|
+
return { expectation, passed: true, actual: this.output };
|
|
319
|
+
}
|
|
320
|
+
return {
|
|
321
|
+
expectation,
|
|
322
|
+
passed: false,
|
|
323
|
+
actual: this.output.slice(-500),
|
|
324
|
+
reason: `Output does not match: ${regex}`,
|
|
325
|
+
};
|
|
326
|
+
}
|
|
327
|
+
case 'output_not_contains': {
|
|
328
|
+
const pattern = expectation.value;
|
|
329
|
+
if (!this.output.includes(pattern)) {
|
|
330
|
+
return { expectation, passed: true };
|
|
331
|
+
}
|
|
332
|
+
return {
|
|
333
|
+
expectation,
|
|
334
|
+
passed: false,
|
|
335
|
+
actual: this.output.slice(-500),
|
|
336
|
+
reason: `Output unexpectedly contains: "${pattern}"`,
|
|
337
|
+
};
|
|
338
|
+
}
|
|
339
|
+
case 'exit_code': {
|
|
340
|
+
const expected = expectation.value;
|
|
341
|
+
if (this.exitCode === expected) {
|
|
342
|
+
return { expectation, passed: true, actual: String(this.exitCode) };
|
|
343
|
+
}
|
|
344
|
+
return {
|
|
345
|
+
expectation,
|
|
346
|
+
passed: false,
|
|
347
|
+
actual: String(this.exitCode),
|
|
348
|
+
reason: `Exit code ${this.exitCode} !== expected ${expected}`,
|
|
349
|
+
};
|
|
350
|
+
}
|
|
351
|
+
default:
|
|
352
|
+
return {
|
|
353
|
+
expectation,
|
|
354
|
+
passed: false,
|
|
355
|
+
reason: `Unknown expectation type: ${expectation.type}`,
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
catch (error) {
|
|
360
|
+
return {
|
|
361
|
+
expectation,
|
|
362
|
+
passed: false,
|
|
363
|
+
reason: error instanceof Error ? error.message : String(error),
|
|
364
|
+
};
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
// ============================================================================
|
|
369
|
+
// PREDEFINED TEST SCENARIOS
|
|
370
|
+
// ============================================================================
|
|
371
|
+
/**
|
|
372
|
+
* Create a paste handling test scenario
|
|
373
|
+
*/
|
|
374
|
+
export function createPasteTestScenario(content, expectedLineCount) {
|
|
375
|
+
return {
|
|
376
|
+
id: `paste-${expectedLineCount}-lines`,
|
|
377
|
+
description: `Test pasting ${expectedLineCount} lines of content`,
|
|
378
|
+
category: 'paste',
|
|
379
|
+
inputs: [
|
|
380
|
+
{ type: 'wait', delay: 2000 }, // Wait for CLI to initialize
|
|
381
|
+
{ type: 'paste', content },
|
|
382
|
+
{ type: 'wait', delay: 500 },
|
|
383
|
+
{ type: 'key', key: 'enter' },
|
|
384
|
+
{ type: 'wait', delay: 1000 },
|
|
385
|
+
],
|
|
386
|
+
expectations: [
|
|
387
|
+
{
|
|
388
|
+
type: 'output_contains',
|
|
389
|
+
value: `${expectedLineCount} line`,
|
|
390
|
+
description: 'Should show line count in preview',
|
|
391
|
+
},
|
|
392
|
+
],
|
|
393
|
+
};
|
|
394
|
+
}
|
|
395
|
+
/**
|
|
396
|
+
* Create a multi-line input test scenario
|
|
397
|
+
*/
|
|
398
|
+
export function createMultiLineInputScenario() {
|
|
399
|
+
const multiLineContent = 'function test() {\n console.log("hello");\n return true;\n}';
|
|
400
|
+
return {
|
|
401
|
+
id: 'multi-line-input',
|
|
402
|
+
description: 'Test multi-line code input handling',
|
|
403
|
+
category: 'input',
|
|
404
|
+
inputs: [
|
|
405
|
+
{ type: 'wait', delay: 2000 },
|
|
406
|
+
{ type: 'paste', content: multiLineContent },
|
|
407
|
+
{ type: 'wait', delay: 500 },
|
|
408
|
+
{ type: 'key', key: 'enter' },
|
|
409
|
+
{ type: 'wait', delay: 2000 },
|
|
410
|
+
],
|
|
411
|
+
expectations: [
|
|
412
|
+
{
|
|
413
|
+
type: 'output_contains',
|
|
414
|
+
value: '4 line',
|
|
415
|
+
description: 'Should show 4 lines in preview',
|
|
416
|
+
},
|
|
417
|
+
{
|
|
418
|
+
type: 'output_not_contains',
|
|
419
|
+
value: 'error',
|
|
420
|
+
description: 'Should not show errors',
|
|
421
|
+
},
|
|
422
|
+
],
|
|
423
|
+
};
|
|
424
|
+
}
|
|
425
|
+
/**
|
|
426
|
+
* Create a slash command test scenario
|
|
427
|
+
*/
|
|
428
|
+
export function createSlashCommandScenario(command) {
|
|
429
|
+
return {
|
|
430
|
+
id: `slash-${command.replace('/', '')}`,
|
|
431
|
+
description: `Test /${command} slash command`,
|
|
432
|
+
category: 'command',
|
|
433
|
+
inputs: [
|
|
434
|
+
{ type: 'wait', delay: 2000 },
|
|
435
|
+
{ type: 'text', content: `/${command}` },
|
|
436
|
+
{ type: 'key', key: 'enter' },
|
|
437
|
+
{ type: 'wait', delay: 1000 },
|
|
438
|
+
],
|
|
439
|
+
expectations: [
|
|
440
|
+
{
|
|
441
|
+
type: 'output_not_contains',
|
|
442
|
+
value: 'Unknown command',
|
|
443
|
+
description: 'Command should be recognized',
|
|
444
|
+
},
|
|
445
|
+
],
|
|
446
|
+
};
|
|
447
|
+
}
|
|
448
|
+
// ============================================================================
|
|
449
|
+
// VERIFICATION INTEGRATION
|
|
450
|
+
// ============================================================================
|
|
451
|
+
/**
|
|
452
|
+
* Run verification tests for a specific claim type
|
|
453
|
+
*/
|
|
454
|
+
export async function runVerificationTests(claimType, workingDir) {
|
|
455
|
+
const harness = new CLITestHarness({
|
|
456
|
+
cwd: workingDir,
|
|
457
|
+
timeout: 60000,
|
|
458
|
+
usePty: true, // Try PTY first
|
|
459
|
+
});
|
|
460
|
+
const scenarios = [];
|
|
461
|
+
// Select scenarios based on claim type
|
|
462
|
+
switch (claimType) {
|
|
463
|
+
case 'paste_handling':
|
|
464
|
+
scenarios.push(createPasteTestScenario('line1\nline2\nline3', 3), createPasteTestScenario('a\nb\nc\nd\ne\nf\ng\nh\ni\nj', 10), createMultiLineInputScenario());
|
|
465
|
+
break;
|
|
466
|
+
case 'slash_commands':
|
|
467
|
+
scenarios.push(createSlashCommandScenario('help'), createSlashCommandScenario('model'), createSlashCommandScenario('clear'));
|
|
468
|
+
break;
|
|
469
|
+
case 'build_success':
|
|
470
|
+
// For build claims, we don't need PTY - just run npm build
|
|
471
|
+
scenarios.push({
|
|
472
|
+
id: 'build-check',
|
|
473
|
+
description: 'Verify project builds successfully',
|
|
474
|
+
category: 'command',
|
|
475
|
+
inputs: [],
|
|
476
|
+
expectations: [],
|
|
477
|
+
});
|
|
478
|
+
break;
|
|
479
|
+
default:
|
|
480
|
+
// Generic behavior test
|
|
481
|
+
scenarios.push({
|
|
482
|
+
id: 'startup-check',
|
|
483
|
+
description: 'Verify CLI starts without errors',
|
|
484
|
+
category: 'behavior',
|
|
485
|
+
inputs: [
|
|
486
|
+
{ type: 'wait', delay: 3000 },
|
|
487
|
+
{ type: 'key', key: 'ctrl-c' },
|
|
488
|
+
],
|
|
489
|
+
expectations: [
|
|
490
|
+
{
|
|
491
|
+
type: 'output_not_contains',
|
|
492
|
+
value: 'Error:',
|
|
493
|
+
description: 'Should not show errors on startup',
|
|
494
|
+
},
|
|
495
|
+
],
|
|
496
|
+
});
|
|
497
|
+
}
|
|
498
|
+
const results = [];
|
|
499
|
+
let allPassed = true;
|
|
500
|
+
for (const scenario of scenarios) {
|
|
501
|
+
const result = await harness.runScenario(scenario);
|
|
502
|
+
results.push(result);
|
|
503
|
+
if (!result.passed) {
|
|
504
|
+
allPassed = false;
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
const passed = results.filter(r => r.passed).length;
|
|
508
|
+
const failed = results.filter(r => !r.passed).length;
|
|
509
|
+
const summary = `${passed}/${results.length} tests passed${failed > 0 ? `, ${failed} failed` : ''}`;
|
|
510
|
+
return {
|
|
511
|
+
passed: allPassed,
|
|
512
|
+
results,
|
|
513
|
+
summary,
|
|
514
|
+
};
|
|
515
|
+
}
|
|
516
|
+
export default CLITestHarness;
|
|
517
|
+
//# sourceMappingURL=cliTestHarness.js.map
|