@tishlang/tish-lsp 3.6.0 → 3.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/Cargo.toml +1 -0
- package/bin/tish-lsp +0 -0
- package/crates/tish/Cargo.toml +4 -1
- package/crates/tish/src/cli_help.rs +84 -0
- package/crates/tish/src/main.rs +93 -1
- package/crates/tish/tests/test_runner.rs +673 -0
- package/crates/tish_bytecode/src/compiler.rs +25 -4
- package/crates/tish_compile/src/resolve.rs +45 -12
- package/crates/tish_core/src/lib.rs +103 -11
- package/crates/tish_eval/src/eval.rs +16 -0
- package/crates/tish_eval/src/value_convert.rs +10 -0
- package/crates/tish_lsp/Cargo.toml +1 -1
- package/crates/tish_test/Cargo.toml +47 -0
- package/crates/tish_test/src/assert.rs +435 -0
- package/crates/tish_test/src/config.rs +85 -0
- package/crates/tish_test/src/coverage.rs +237 -0
- package/crates/tish_test/src/deep_equal.rs +261 -0
- package/crates/tish_test/src/discovery.rs +70 -0
- package/crates/tish_test/src/expect.rs +977 -0
- package/crates/tish_test/src/instrument.rs +457 -0
- package/crates/tish_test/src/isolation.rs +14 -0
- package/crates/tish_test/src/lib.rs +22 -0
- package/crates/tish_test/src/load.rs +152 -0
- package/crates/tish_test/src/mocks.rs +270 -0
- package/crates/tish_test/src/module.rs +318 -0
- package/crates/tish_test/src/registry.rs +235 -0
- package/crates/tish_test/src/report.rs +312 -0
- package/crates/tish_test/src/runner.rs +797 -0
- package/crates/tish_test/src/snapshots.rs +197 -0
- package/crates/tish_vm/src/vm.rs +17 -14
- package/package.json +1 -1
- package/platform/darwin-arm64/tish-lsp +0 -0
- package/platform/darwin-x64/tish-lsp +0 -0
- package/platform/linux-arm64/tish-lsp +0 -0
- package/platform/linux-x64/tish-lsp +0 -0
- package/platform/win32-x64/tish-lsp.exe +0 -0
|
@@ -0,0 +1,797 @@
|
|
|
1
|
+
//! Execute a collected suite tree.
|
|
2
|
+
|
|
3
|
+
use std::path::PathBuf;
|
|
4
|
+
use std::time::{Duration, Instant};
|
|
5
|
+
|
|
6
|
+
use regex::Regex;
|
|
7
|
+
use tishlang_core::{has_pending_throw, take_pending_throw, value_call, Value};
|
|
8
|
+
|
|
9
|
+
use crate::isolation::reset_between_files;
|
|
10
|
+
use crate::registry::{only_mode, SuiteNode, TestCase, TestMode};
|
|
11
|
+
use crate::report::{
|
|
12
|
+
gh_annotation, write_junit_xml, ConsoleReporter, ReporterKind, RunSummary, TestResultRecord,
|
|
13
|
+
TestStatus,
|
|
14
|
+
};
|
|
15
|
+
use crate::snapshots::{set_ci_mode, set_current_file, set_current_test, set_update_snapshots};
|
|
16
|
+
|
|
17
|
+
#[derive(Clone, Debug)]
|
|
18
|
+
pub struct TestOptions {
|
|
19
|
+
pub roots: Vec<PathBuf>,
|
|
20
|
+
pub filters: Vec<String>,
|
|
21
|
+
pub name_pattern: Option<String>,
|
|
22
|
+
pub timeout_ms: u64,
|
|
23
|
+
pub bail: bool,
|
|
24
|
+
pub retry: u32,
|
|
25
|
+
pub update_snapshots: bool,
|
|
26
|
+
pub reporter: ReporterKind,
|
|
27
|
+
pub junit_path: Option<PathBuf>,
|
|
28
|
+
/// When true, only run tests marked `.only` (also implied by any `.only` registration).
|
|
29
|
+
pub only: bool,
|
|
30
|
+
pub backend: String,
|
|
31
|
+
pub no_optimize: bool,
|
|
32
|
+
pub features: Vec<String>,
|
|
33
|
+
/// `i/n` shard of discovered files (1-based index).
|
|
34
|
+
pub shard: Option<(usize, usize)>,
|
|
35
|
+
/// Shuffle file order.
|
|
36
|
+
pub randomize: bool,
|
|
37
|
+
/// Seed for `--randomize` (also implies randomize when set).
|
|
38
|
+
pub seed: Option<u64>,
|
|
39
|
+
/// Run the full suite this many times.
|
|
40
|
+
pub rerun_each: u32,
|
|
41
|
+
/// Only run tests that include at least one of these tags.
|
|
42
|
+
pub tags: Vec<String>,
|
|
43
|
+
/// Preload modules (absolute or cwd-relative) evaluated before each test file.
|
|
44
|
+
pub preload: Vec<PathBuf>,
|
|
45
|
+
/// Treat a missing snapshot as a failure instead of writing it (also on when `CI` is set).
|
|
46
|
+
pub ci: bool,
|
|
47
|
+
/// Collect line coverage via AST instrumentation (backend-agnostic).
|
|
48
|
+
pub coverage: bool,
|
|
49
|
+
/// Directory for `lcov.info` (implies coverage). Default: `coverage/`.
|
|
50
|
+
pub coverage_dir: Option<PathBuf>,
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
impl Default for TestOptions {
|
|
54
|
+
fn default() -> Self {
|
|
55
|
+
Self {
|
|
56
|
+
roots: vec![PathBuf::from(".")],
|
|
57
|
+
filters: Vec::new(),
|
|
58
|
+
name_pattern: None,
|
|
59
|
+
timeout_ms: 5_000,
|
|
60
|
+
bail: false,
|
|
61
|
+
retry: 0,
|
|
62
|
+
update_snapshots: false,
|
|
63
|
+
reporter: ReporterKind::Console,
|
|
64
|
+
junit_path: None,
|
|
65
|
+
only: false,
|
|
66
|
+
backend: "vm".into(),
|
|
67
|
+
no_optimize: false,
|
|
68
|
+
features: Vec::new(),
|
|
69
|
+
shard: None,
|
|
70
|
+
randomize: false,
|
|
71
|
+
seed: None,
|
|
72
|
+
rerun_each: 1,
|
|
73
|
+
tags: Vec::new(),
|
|
74
|
+
preload: Vec::new(),
|
|
75
|
+
ci: false,
|
|
76
|
+
coverage: false,
|
|
77
|
+
coverage_dir: None,
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
#[derive(Clone, Debug)]
|
|
83
|
+
pub struct TestRunResult {
|
|
84
|
+
pub summary: RunSummary,
|
|
85
|
+
pub exit_code: i32,
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/// Run an already-collected suite (typically one file's tree).
|
|
89
|
+
pub fn run_suite(
|
|
90
|
+
suite: &SuiteNode,
|
|
91
|
+
opts: &TestOptions,
|
|
92
|
+
summary: &mut RunSummary,
|
|
93
|
+
reporter: &mut ConsoleReporter,
|
|
94
|
+
) -> bool {
|
|
95
|
+
let pattern = opts.name_pattern.as_ref().and_then(|p| Regex::new(p).ok());
|
|
96
|
+
let enforce_only = opts.only || only_mode();
|
|
97
|
+
run_suite_inner(
|
|
98
|
+
suite,
|
|
99
|
+
&[],
|
|
100
|
+
&[],
|
|
101
|
+
opts,
|
|
102
|
+
pattern.as_ref(),
|
|
103
|
+
enforce_only,
|
|
104
|
+
false,
|
|
105
|
+
summary,
|
|
106
|
+
reporter,
|
|
107
|
+
)
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
/// `ancestor_only` is true once any enclosing suite was declared with `describe.only`, so a
|
|
111
|
+
/// `.only` on an outer suite selects everything nested beneath it (Jest/Bun semantics) rather
|
|
112
|
+
/// than only its direct children.
|
|
113
|
+
#[allow(clippy::too_many_arguments)]
|
|
114
|
+
fn run_suite_inner(
|
|
115
|
+
suite: &SuiteNode,
|
|
116
|
+
parent_before: &[Value],
|
|
117
|
+
parent_after: &[Value],
|
|
118
|
+
opts: &TestOptions,
|
|
119
|
+
pattern: Option<&Regex>,
|
|
120
|
+
enforce_only: bool,
|
|
121
|
+
ancestor_only: bool,
|
|
122
|
+
summary: &mut RunSummary,
|
|
123
|
+
reporter: &mut ConsoleReporter,
|
|
124
|
+
) -> bool {
|
|
125
|
+
if suite.mode == TestMode::Skip {
|
|
126
|
+
// Mark nested tests as skipped without running hooks.
|
|
127
|
+
skip_tree(suite, summary, reporter);
|
|
128
|
+
return true;
|
|
129
|
+
}
|
|
130
|
+
let ancestor_only = ancestor_only || suite.mode == TestMode::Only;
|
|
131
|
+
|
|
132
|
+
let mut before_each = parent_before.to_vec();
|
|
133
|
+
before_each.extend(suite.before_each.iter().cloned());
|
|
134
|
+
let mut after_each = suite.after_each.clone();
|
|
135
|
+
after_each.extend(parent_after.iter().cloned());
|
|
136
|
+
|
|
137
|
+
if let Err(msg) = run_hooks(&suite.before_all) {
|
|
138
|
+
let rec = TestResultRecord {
|
|
139
|
+
full_name: format!("{} (beforeAll)", suite.name),
|
|
140
|
+
file: first_file_in(suite),
|
|
141
|
+
status: TestStatus::Fail,
|
|
142
|
+
duration: Duration::ZERO,
|
|
143
|
+
message: Some(msg.clone()),
|
|
144
|
+
};
|
|
145
|
+
reporter.on_result(&rec);
|
|
146
|
+
summary.record(rec);
|
|
147
|
+
// Every test below this suite is now unrunnable. Report each one rather than
|
|
148
|
+
// dropping it, so the total count still reflects what the file declared.
|
|
149
|
+
fail_tree(
|
|
150
|
+
suite,
|
|
151
|
+
&format!("suite setup failed (beforeAll): {msg}"),
|
|
152
|
+
summary,
|
|
153
|
+
reporter,
|
|
154
|
+
);
|
|
155
|
+
// `afterAll` still runs: `beforeAll` may have acquired resources before it threw.
|
|
156
|
+
if let Err(after_msg) = run_hooks(&suite.after_all) {
|
|
157
|
+
let rec = TestResultRecord {
|
|
158
|
+
full_name: format!("{} (afterAll)", suite.name),
|
|
159
|
+
file: String::new(),
|
|
160
|
+
status: TestStatus::Fail,
|
|
161
|
+
duration: Duration::ZERO,
|
|
162
|
+
message: Some(after_msg),
|
|
163
|
+
};
|
|
164
|
+
reporter.on_result(&rec);
|
|
165
|
+
summary.record(rec);
|
|
166
|
+
}
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
let mut ok = true;
|
|
171
|
+
for test in &suite.tests {
|
|
172
|
+
match classify_test(
|
|
173
|
+
test,
|
|
174
|
+
suite,
|
|
175
|
+
pattern,
|
|
176
|
+
enforce_only,
|
|
177
|
+
ancestor_only,
|
|
178
|
+
&opts.tags,
|
|
179
|
+
) {
|
|
180
|
+
TestDisposition::Run => {}
|
|
181
|
+
disposition => {
|
|
182
|
+
let status = match disposition {
|
|
183
|
+
TestDisposition::Todo => TestStatus::Todo,
|
|
184
|
+
TestDisposition::Skip => TestStatus::Skip,
|
|
185
|
+
TestDisposition::Filtered => TestStatus::Filtered,
|
|
186
|
+
TestDisposition::Run => unreachable!(),
|
|
187
|
+
};
|
|
188
|
+
let rec = TestResultRecord {
|
|
189
|
+
full_name: test.full_name.clone(),
|
|
190
|
+
file: test.file.clone(),
|
|
191
|
+
status,
|
|
192
|
+
duration: Duration::ZERO,
|
|
193
|
+
message: None,
|
|
194
|
+
};
|
|
195
|
+
reporter.on_result(&rec);
|
|
196
|
+
summary.record(rec);
|
|
197
|
+
continue;
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
set_current_file(&test.file);
|
|
202
|
+
set_current_test(&test.full_name);
|
|
203
|
+
let timeout = test.timeout_ms.unwrap_or(opts.timeout_ms);
|
|
204
|
+
let mut attempts = 0;
|
|
205
|
+
let max_attempts = opts.retry + 1;
|
|
206
|
+
let mut last_fail: Option<(Duration, String)> = None;
|
|
207
|
+
|
|
208
|
+
while attempts < max_attempts {
|
|
209
|
+
attempts += 1;
|
|
210
|
+
let start = Instant::now();
|
|
211
|
+
let _ = take_pending_throw();
|
|
212
|
+
|
|
213
|
+
if let Err(msg) = run_hooks(&before_each) {
|
|
214
|
+
last_fail = Some((start.elapsed(), msg));
|
|
215
|
+
let _ = run_hooks(&after_each);
|
|
216
|
+
continue;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
crate::expect::begin_test_assertions();
|
|
220
|
+
let result = run_test_body(&test.callback, timeout);
|
|
221
|
+
// An `expect.assertions(n)` contract turns a body that silently returned early
|
|
222
|
+
// into a failure instead of a pass.
|
|
223
|
+
let result = result.and_then(|()| crate::expect::check_test_assertions());
|
|
224
|
+
let dur = start.elapsed();
|
|
225
|
+
let _ = run_hooks(&after_each);
|
|
226
|
+
|
|
227
|
+
match (result, test.mode) {
|
|
228
|
+
(Ok(()), TestMode::Failing) => {
|
|
229
|
+
last_fail = Some((
|
|
230
|
+
dur,
|
|
231
|
+
"Expected test to fail (test.failing), but it passed".into(),
|
|
232
|
+
));
|
|
233
|
+
}
|
|
234
|
+
(Ok(()), _) => {
|
|
235
|
+
let rec = TestResultRecord {
|
|
236
|
+
full_name: test.full_name.clone(),
|
|
237
|
+
file: test.file.clone(),
|
|
238
|
+
status: TestStatus::Pass,
|
|
239
|
+
duration: dur,
|
|
240
|
+
message: None,
|
|
241
|
+
};
|
|
242
|
+
reporter.on_result(&rec);
|
|
243
|
+
summary.record(rec);
|
|
244
|
+
last_fail = None;
|
|
245
|
+
break;
|
|
246
|
+
}
|
|
247
|
+
(Err(msg), TestMode::Failing) => {
|
|
248
|
+
let rec = TestResultRecord {
|
|
249
|
+
full_name: test.full_name.clone(),
|
|
250
|
+
file: test.file.clone(),
|
|
251
|
+
status: TestStatus::Pass,
|
|
252
|
+
duration: dur,
|
|
253
|
+
message: Some(format!("failed as expected: {msg}")),
|
|
254
|
+
};
|
|
255
|
+
reporter.on_result(&rec);
|
|
256
|
+
summary.record(rec);
|
|
257
|
+
last_fail = None;
|
|
258
|
+
break;
|
|
259
|
+
}
|
|
260
|
+
(Err(msg), _) => {
|
|
261
|
+
last_fail = Some((dur, msg));
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
}
|
|
265
|
+
|
|
266
|
+
if let Some((dur, msg)) = last_fail {
|
|
267
|
+
if std::env::var_os("GITHUB_ACTIONS").is_some() {
|
|
268
|
+
eprintln!(
|
|
269
|
+
"::error file={},title={}::{}",
|
|
270
|
+
test.file,
|
|
271
|
+
test.full_name,
|
|
272
|
+
gh_annotation(&msg)
|
|
273
|
+
);
|
|
274
|
+
}
|
|
275
|
+
let rec = TestResultRecord {
|
|
276
|
+
full_name: test.full_name.clone(),
|
|
277
|
+
file: test.file.clone(),
|
|
278
|
+
status: TestStatus::Fail,
|
|
279
|
+
duration: dur,
|
|
280
|
+
message: Some(msg),
|
|
281
|
+
};
|
|
282
|
+
reporter.on_result(&rec);
|
|
283
|
+
summary.record(rec);
|
|
284
|
+
ok = false;
|
|
285
|
+
if opts.bail {
|
|
286
|
+
let _ = run_hooks(&suite.after_all);
|
|
287
|
+
return false;
|
|
288
|
+
}
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
for child in &suite.children {
|
|
293
|
+
if !run_suite_inner(
|
|
294
|
+
child,
|
|
295
|
+
&before_each,
|
|
296
|
+
&after_each,
|
|
297
|
+
opts,
|
|
298
|
+
pattern,
|
|
299
|
+
enforce_only,
|
|
300
|
+
ancestor_only,
|
|
301
|
+
summary,
|
|
302
|
+
reporter,
|
|
303
|
+
) {
|
|
304
|
+
ok = false;
|
|
305
|
+
if opts.bail {
|
|
306
|
+
let _ = run_hooks(&suite.after_all);
|
|
307
|
+
return false;
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
if let Err(msg) = run_hooks(&suite.after_all) {
|
|
313
|
+
let rec = TestResultRecord {
|
|
314
|
+
full_name: format!("{} (afterAll)", suite.name),
|
|
315
|
+
file: String::new(),
|
|
316
|
+
status: TestStatus::Fail,
|
|
317
|
+
duration: Duration::ZERO,
|
|
318
|
+
message: Some(msg),
|
|
319
|
+
};
|
|
320
|
+
reporter.on_result(&rec);
|
|
321
|
+
summary.record(rec);
|
|
322
|
+
ok = false;
|
|
323
|
+
}
|
|
324
|
+
ok
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
enum TestDisposition {
|
|
328
|
+
Run,
|
|
329
|
+
Skip,
|
|
330
|
+
Todo,
|
|
331
|
+
Filtered,
|
|
332
|
+
}
|
|
333
|
+
|
|
334
|
+
#[allow(clippy::too_many_arguments)]
|
|
335
|
+
fn classify_test(
|
|
336
|
+
test: &TestCase,
|
|
337
|
+
suite: &SuiteNode,
|
|
338
|
+
pattern: Option<&Regex>,
|
|
339
|
+
enforce_only: bool,
|
|
340
|
+
ancestor_only: bool,
|
|
341
|
+
tag_filter: &[String],
|
|
342
|
+
) -> TestDisposition {
|
|
343
|
+
// Name / tag / only filters hide tests entirely (including skip/todo that don't match).
|
|
344
|
+
// `ancestor_only` already folds in `suite.mode`, and every enclosing suite besides.
|
|
345
|
+
let _ = suite;
|
|
346
|
+
if enforce_only && test.mode != TestMode::Only && !ancestor_only {
|
|
347
|
+
return TestDisposition::Filtered;
|
|
348
|
+
}
|
|
349
|
+
if let Some(re) = pattern {
|
|
350
|
+
if !re.is_match(&test.full_name) && !re.is_match(&test.name) {
|
|
351
|
+
return TestDisposition::Filtered;
|
|
352
|
+
}
|
|
353
|
+
}
|
|
354
|
+
if !tag_filter.is_empty() {
|
|
355
|
+
let hit = test.tags.iter().any(|t| tag_filter.iter().any(|f| f == t));
|
|
356
|
+
if !hit {
|
|
357
|
+
return TestDisposition::Filtered;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
if test.mode == TestMode::Todo {
|
|
361
|
+
return TestDisposition::Todo;
|
|
362
|
+
}
|
|
363
|
+
if test.mode == TestMode::Skip || suite.mode == TestMode::Skip {
|
|
364
|
+
return TestDisposition::Skip;
|
|
365
|
+
}
|
|
366
|
+
TestDisposition::Run
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
fn skip_tree(suite: &SuiteNode, summary: &mut RunSummary, reporter: &mut ConsoleReporter) {
|
|
370
|
+
mark_tree(suite, TestStatus::Skip, None, summary, reporter);
|
|
371
|
+
}
|
|
372
|
+
|
|
373
|
+
/// Report every test under `suite` as failed — used when a `beforeAll` makes them unrunnable.
|
|
374
|
+
fn fail_tree(
|
|
375
|
+
suite: &SuiteNode,
|
|
376
|
+
message: &str,
|
|
377
|
+
summary: &mut RunSummary,
|
|
378
|
+
reporter: &mut ConsoleReporter,
|
|
379
|
+
) {
|
|
380
|
+
mark_tree(suite, TestStatus::Fail, Some(message), summary, reporter);
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
fn mark_tree(
|
|
384
|
+
suite: &SuiteNode,
|
|
385
|
+
status: TestStatus,
|
|
386
|
+
message: Option<&str>,
|
|
387
|
+
summary: &mut RunSummary,
|
|
388
|
+
reporter: &mut ConsoleReporter,
|
|
389
|
+
) {
|
|
390
|
+
for test in &suite.tests {
|
|
391
|
+
let rec = TestResultRecord {
|
|
392
|
+
full_name: test.full_name.clone(),
|
|
393
|
+
file: test.file.clone(),
|
|
394
|
+
status,
|
|
395
|
+
duration: Duration::ZERO,
|
|
396
|
+
message: message.map(|m| m.to_string()),
|
|
397
|
+
};
|
|
398
|
+
reporter.on_result(&rec);
|
|
399
|
+
summary.record(rec);
|
|
400
|
+
}
|
|
401
|
+
for child in &suite.children {
|
|
402
|
+
mark_tree(child, status, message, summary, reporter);
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/// First test file path found in the subtree — gives hook failures a file to report against.
|
|
407
|
+
fn first_file_in(suite: &SuiteNode) -> String {
|
|
408
|
+
if let Some(t) = suite.tests.first() {
|
|
409
|
+
return t.file.clone();
|
|
410
|
+
}
|
|
411
|
+
for child in &suite.children {
|
|
412
|
+
let f = first_file_in(child);
|
|
413
|
+
if !f.is_empty() {
|
|
414
|
+
return f;
|
|
415
|
+
}
|
|
416
|
+
}
|
|
417
|
+
String::new()
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
fn run_hooks(hooks: &[Value]) -> Result<(), String> {
|
|
421
|
+
for h in hooks {
|
|
422
|
+
let _ = take_pending_throw();
|
|
423
|
+
let result = value_call(h, &[]);
|
|
424
|
+
if has_pending_throw() {
|
|
425
|
+
let err = take_pending_throw().unwrap_or(Value::Null);
|
|
426
|
+
return Err(err.to_display_string());
|
|
427
|
+
}
|
|
428
|
+
#[cfg(feature = "promise")]
|
|
429
|
+
{
|
|
430
|
+
if matches!(result, Value::Promise(_)) {
|
|
431
|
+
let _ = take_pending_throw();
|
|
432
|
+
let _ = tishlang_runtime::await_promise(result);
|
|
433
|
+
if has_pending_throw() {
|
|
434
|
+
let err = take_pending_throw().unwrap_or(Value::Null);
|
|
435
|
+
return Err(err.to_display_string());
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
#[cfg(not(feature = "promise"))]
|
|
440
|
+
{
|
|
441
|
+
let _ = result;
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
Ok(())
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
/// Run one test body under `timeout_ms`.
|
|
448
|
+
///
|
|
449
|
+
/// The backends poll [`tishlang_core::execution_deadline_exceeded`] on loop back-edges, so a
|
|
450
|
+
/// runaway loop is aborted rather than hanging the run. A body that blocks inside a single
|
|
451
|
+
/// native call (a synchronous socket read, say) still cannot be preempted; the elapsed-time
|
|
452
|
+
/// check after it returns catches that case.
|
|
453
|
+
fn run_test_body(body: &Value, timeout_ms: u64) -> Result<(), String> {
|
|
454
|
+
let _ = take_pending_throw();
|
|
455
|
+
let start = Instant::now();
|
|
456
|
+
let deadline = Duration::from_millis(timeout_ms);
|
|
457
|
+
let timed_out = |extra: &str| -> String {
|
|
458
|
+
if extra.is_empty() {
|
|
459
|
+
format!("Test timed out after {timeout_ms}ms")
|
|
460
|
+
} else {
|
|
461
|
+
format!("Test timed out after {timeout_ms}ms ({extra})")
|
|
462
|
+
}
|
|
463
|
+
};
|
|
464
|
+
|
|
465
|
+
tishlang_core::set_execution_deadline(Some(timeout_ms));
|
|
466
|
+
let result = value_call(body, &[]);
|
|
467
|
+
let tripped = tishlang_core::execution_deadline_tripped();
|
|
468
|
+
tishlang_core::set_execution_deadline(None);
|
|
469
|
+
|
|
470
|
+
if tripped {
|
|
471
|
+
let _ = take_pending_throw();
|
|
472
|
+
return Err(timed_out(""));
|
|
473
|
+
}
|
|
474
|
+
if has_pending_throw() {
|
|
475
|
+
let err = take_pending_throw().unwrap_or(Value::Null);
|
|
476
|
+
return Err(err.to_display_string());
|
|
477
|
+
}
|
|
478
|
+
if start.elapsed() > deadline {
|
|
479
|
+
return Err(timed_out("body returned late"));
|
|
480
|
+
}
|
|
481
|
+
// Await promise-returning tests when the promise feature is on.
|
|
482
|
+
#[cfg(feature = "promise")]
|
|
483
|
+
{
|
|
484
|
+
if matches!(result, Value::Promise(_)) {
|
|
485
|
+
let _ = take_pending_throw();
|
|
486
|
+
let remaining = deadline.saturating_sub(start.elapsed());
|
|
487
|
+
if remaining.is_zero() {
|
|
488
|
+
return Err(timed_out("before await"));
|
|
489
|
+
}
|
|
490
|
+
tishlang_core::set_execution_deadline(Some(remaining.as_millis() as u64));
|
|
491
|
+
let _ = tishlang_runtime::await_promise(result);
|
|
492
|
+
let tripped = tishlang_core::execution_deadline_tripped();
|
|
493
|
+
tishlang_core::set_execution_deadline(None);
|
|
494
|
+
if tripped {
|
|
495
|
+
let _ = take_pending_throw();
|
|
496
|
+
return Err(timed_out("awaiting"));
|
|
497
|
+
}
|
|
498
|
+
if has_pending_throw() {
|
|
499
|
+
let err = take_pending_throw().unwrap_or(Value::Null);
|
|
500
|
+
return Err(err.to_display_string());
|
|
501
|
+
}
|
|
502
|
+
if start.elapsed() > deadline {
|
|
503
|
+
return Err(timed_out("awaiting"));
|
|
504
|
+
}
|
|
505
|
+
}
|
|
506
|
+
}
|
|
507
|
+
#[cfg(not(feature = "promise"))]
|
|
508
|
+
{
|
|
509
|
+
let _ = result;
|
|
510
|
+
}
|
|
511
|
+
Ok(())
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
/// High-level entry: discover files, load each (when `runner` feature), execute, report.
|
|
515
|
+
pub fn run_tests(opts: TestOptions) -> TestRunResult {
|
|
516
|
+
let mut opts = opts;
|
|
517
|
+
apply_package_config(&mut opts);
|
|
518
|
+
|
|
519
|
+
let times = opts.rerun_each.max(1);
|
|
520
|
+
let mut last = TestRunResult {
|
|
521
|
+
summary: RunSummary::default(),
|
|
522
|
+
exit_code: 0,
|
|
523
|
+
};
|
|
524
|
+
// `--rerun-each` exists to surface flakes: ANY failing pass fails the command. Reporting only
|
|
525
|
+
// the final pass would make a run that failed once and passed twice exit 0.
|
|
526
|
+
let mut worst_exit = 0;
|
|
527
|
+
let mut failed_passes = Vec::new();
|
|
528
|
+
for i in 0..times {
|
|
529
|
+
if times > 1 {
|
|
530
|
+
eprintln!("── rerun-each {}/{} ──", i + 1, times);
|
|
531
|
+
}
|
|
532
|
+
last = run_tests_once(&opts);
|
|
533
|
+
if last.exit_code != 0 {
|
|
534
|
+
worst_exit = last.exit_code;
|
|
535
|
+
failed_passes.push(i + 1);
|
|
536
|
+
if opts.bail {
|
|
537
|
+
break;
|
|
538
|
+
}
|
|
539
|
+
}
|
|
540
|
+
}
|
|
541
|
+
if times > 1 && !failed_passes.is_empty() {
|
|
542
|
+
eprintln!(
|
|
543
|
+
"\n{} of {} passes failed (pass {}) — flaky or consistently failing.",
|
|
544
|
+
failed_passes.len(),
|
|
545
|
+
times,
|
|
546
|
+
failed_passes
|
|
547
|
+
.iter()
|
|
548
|
+
.map(|n| n.to_string())
|
|
549
|
+
.collect::<Vec<_>>()
|
|
550
|
+
.join(", ")
|
|
551
|
+
);
|
|
552
|
+
}
|
|
553
|
+
last.exit_code = worst_exit;
|
|
554
|
+
last
|
|
555
|
+
}
|
|
556
|
+
|
|
557
|
+
fn apply_package_config(opts: &mut TestOptions) {
|
|
558
|
+
let cfg =
|
|
559
|
+
crate::config::load_tish_test_config(opts.roots.first().unwrap_or(&PathBuf::from(".")));
|
|
560
|
+
if opts.roots == vec![PathBuf::from(".")] && !cfg.root.is_empty() {
|
|
561
|
+
opts.roots = cfg.root;
|
|
562
|
+
}
|
|
563
|
+
if opts.timeout_ms == 5_000 {
|
|
564
|
+
if let Some(t) = cfg.timeout_ms {
|
|
565
|
+
opts.timeout_ms = t;
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
for p in cfg.preload {
|
|
569
|
+
if !opts.preload.contains(&p) {
|
|
570
|
+
opts.preload.push(p);
|
|
571
|
+
}
|
|
572
|
+
}
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
fn run_tests_once(opts: &TestOptions) -> TestRunResult {
|
|
576
|
+
set_update_snapshots(opts.update_snapshots);
|
|
577
|
+
set_ci_mode(opts.ci);
|
|
578
|
+
set_ci_mode(opts.ci);
|
|
579
|
+
let coverage_on = opts.coverage || opts.coverage_dir.is_some();
|
|
580
|
+
if coverage_on {
|
|
581
|
+
crate::coverage::begin();
|
|
582
|
+
}
|
|
583
|
+
let mut summary = RunSummary::default();
|
|
584
|
+
let mut reporter = ConsoleReporter::new(opts.reporter);
|
|
585
|
+
|
|
586
|
+
let mut files = crate::discovery::discover_tests(&opts.roots, &opts.filters);
|
|
587
|
+
if opts.randomize || opts.seed.is_some() {
|
|
588
|
+
let seed = opts.seed.unwrap_or_else(|| {
|
|
589
|
+
std::time::SystemTime::now()
|
|
590
|
+
.duration_since(std::time::UNIX_EPOCH)
|
|
591
|
+
.map(|d| d.as_nanos() as u64)
|
|
592
|
+
.unwrap_or(1)
|
|
593
|
+
});
|
|
594
|
+
eprintln!("--seed={seed}");
|
|
595
|
+
shuffle_with_seed(&mut files, seed);
|
|
596
|
+
}
|
|
597
|
+
if let Some((index, total)) = opts.shard {
|
|
598
|
+
if total > 0 && index >= 1 && index <= total {
|
|
599
|
+
files = files
|
|
600
|
+
.into_iter()
|
|
601
|
+
.enumerate()
|
|
602
|
+
.filter(|(i, _)| i % total == index - 1)
|
|
603
|
+
.map(|(_, f)| f)
|
|
604
|
+
.collect();
|
|
605
|
+
}
|
|
606
|
+
}
|
|
607
|
+
if files.is_empty() {
|
|
608
|
+
eprintln!("No test files found.");
|
|
609
|
+
}
|
|
610
|
+
|
|
611
|
+
// GitHub Actions annotations on failure
|
|
612
|
+
let gh = std::env::var_os("GITHUB_ACTIONS").is_some();
|
|
613
|
+
|
|
614
|
+
for file in &files {
|
|
615
|
+
reset_between_files();
|
|
616
|
+
set_current_file(&file.to_string_lossy());
|
|
617
|
+
|
|
618
|
+
#[cfg(feature = "runner")]
|
|
619
|
+
{
|
|
620
|
+
match crate::load::load_and_collect_tests(
|
|
621
|
+
file,
|
|
622
|
+
&opts.preload,
|
|
623
|
+
&opts.backend,
|
|
624
|
+
opts.no_optimize,
|
|
625
|
+
&opts.features,
|
|
626
|
+
) {
|
|
627
|
+
Ok(loaded) => {
|
|
628
|
+
// A throw inside a `describe` body truncates registration silently. Report
|
|
629
|
+
// each one as a failure so a file that lost tests can never exit 0.
|
|
630
|
+
let had_collect_errors = !loaded.collect_errors.is_empty();
|
|
631
|
+
for err in &loaded.collect_errors {
|
|
632
|
+
if gh {
|
|
633
|
+
eprintln!("::error file={}::{}", file.display(), gh_annotation(err));
|
|
634
|
+
}
|
|
635
|
+
let rec = TestResultRecord {
|
|
636
|
+
full_name: format!("{} (collection error)", file.display()),
|
|
637
|
+
file: file.display().to_string(),
|
|
638
|
+
status: TestStatus::Fail,
|
|
639
|
+
duration: Duration::ZERO,
|
|
640
|
+
message: Some(err.clone()),
|
|
641
|
+
};
|
|
642
|
+
reporter.on_result(&rec);
|
|
643
|
+
summary.record(rec);
|
|
644
|
+
}
|
|
645
|
+
let suite_ok = run_suite(&loaded.suite, opts, &mut summary, &mut reporter);
|
|
646
|
+
if (!suite_ok || had_collect_errors) && opts.bail {
|
|
647
|
+
break;
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
Err(e) => {
|
|
651
|
+
if gh {
|
|
652
|
+
eprintln!("::error file={}::{}", file.display(), gh_annotation(&e));
|
|
653
|
+
}
|
|
654
|
+
let rec = TestResultRecord {
|
|
655
|
+
full_name: file.display().to_string(),
|
|
656
|
+
file: file.display().to_string(),
|
|
657
|
+
status: TestStatus::Fail,
|
|
658
|
+
duration: Duration::ZERO,
|
|
659
|
+
message: Some(e),
|
|
660
|
+
};
|
|
661
|
+
reporter.on_result(&rec);
|
|
662
|
+
summary.record(rec);
|
|
663
|
+
if opts.bail {
|
|
664
|
+
break;
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
}
|
|
668
|
+
}
|
|
669
|
+
|
|
670
|
+
#[cfg(not(feature = "runner"))]
|
|
671
|
+
{
|
|
672
|
+
let _ = file;
|
|
673
|
+
let rec = TestResultRecord {
|
|
674
|
+
full_name: "runner feature disabled".into(),
|
|
675
|
+
file: String::new(),
|
|
676
|
+
status: TestStatus::Fail,
|
|
677
|
+
duration: Duration::ZERO,
|
|
678
|
+
message: Some(
|
|
679
|
+
"tishlang_test built without `runner` feature; cannot load test files".into(),
|
|
680
|
+
),
|
|
681
|
+
};
|
|
682
|
+
reporter.on_result(&rec);
|
|
683
|
+
summary.record(rec);
|
|
684
|
+
break;
|
|
685
|
+
}
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
reporter.finish(&summary);
|
|
689
|
+
if let Some(path) = &opts.junit_path {
|
|
690
|
+
if let Err(e) = write_junit_xml(path, &summary) {
|
|
691
|
+
eprintln!("Failed to write junit report: {e}");
|
|
692
|
+
}
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
if coverage_on {
|
|
696
|
+
crate::coverage::retain_non_test_files();
|
|
697
|
+
let dir = opts
|
|
698
|
+
.coverage_dir
|
|
699
|
+
.clone()
|
|
700
|
+
.unwrap_or_else(crate::coverage::default_dir);
|
|
701
|
+
let lcov = dir.join("lcov.info");
|
|
702
|
+
if let Err(e) = crate::coverage::write_lcov(&lcov) {
|
|
703
|
+
eprintln!("Failed to write coverage report: {e}");
|
|
704
|
+
} else {
|
|
705
|
+
eprintln!("Wrote {}", lcov.display());
|
|
706
|
+
}
|
|
707
|
+
let _ = crate::coverage::print_summary(std::io::stderr());
|
|
708
|
+
}
|
|
709
|
+
|
|
710
|
+
TestRunResult {
|
|
711
|
+
exit_code: if summary.success() { 0 } else { 1 },
|
|
712
|
+
summary,
|
|
713
|
+
}
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
fn shuffle_with_seed(files: &mut [PathBuf], seed: u64) {
|
|
717
|
+
// Simple LCG shuffle (Fisher–Yates).
|
|
718
|
+
let mut state = seed;
|
|
719
|
+
for i in (1..files.len()).rev() {
|
|
720
|
+
state = state.wrapping_mul(6364136223846793005).wrapping_add(1);
|
|
721
|
+
let j = (state as usize) % (i + 1);
|
|
722
|
+
files.swap(i, j);
|
|
723
|
+
}
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
/// Paths a run writes into itself. Watching these would make `--watch --coverage` (or `-u`,
|
|
727
|
+
/// or `--reporter-outfile` inside the tree) re-trigger on its own output forever.
|
|
728
|
+
#[cfg(feature = "watch")]
|
|
729
|
+
fn is_self_written(path: &std::path::Path, opts: &TestOptions) -> bool {
|
|
730
|
+
if let Some(junit) = &opts.junit_path {
|
|
731
|
+
if path == junit.as_path() {
|
|
732
|
+
return true;
|
|
733
|
+
}
|
|
734
|
+
}
|
|
735
|
+
let coverage_dir = opts
|
|
736
|
+
.coverage_dir
|
|
737
|
+
.clone()
|
|
738
|
+
.unwrap_or_else(crate::coverage::default_dir);
|
|
739
|
+
path.components().any(|c| {
|
|
740
|
+
let s = c.as_os_str();
|
|
741
|
+
s == "__snapshots__" || s == coverage_dir.as_os_str() || s == ".git"
|
|
742
|
+
})
|
|
743
|
+
}
|
|
744
|
+
|
|
745
|
+
/// Run tests once, then re-run when files under `opts.roots` change (`--watch`).
|
|
746
|
+
///
|
|
747
|
+
/// With the `watch` feature (notify), blocks forever on filesystem events. Without it,
|
|
748
|
+
/// runs a single pass and returns (same as [`run_tests`]).
|
|
749
|
+
pub fn run_tests_watch(opts: TestOptions) -> i32 {
|
|
750
|
+
#[cfg(feature = "watch")]
|
|
751
|
+
{
|
|
752
|
+
use notify::{RecursiveMode, Watcher};
|
|
753
|
+
use std::sync::mpsc::channel;
|
|
754
|
+
use std::time::Duration;
|
|
755
|
+
|
|
756
|
+
let (tx, rx) = channel();
|
|
757
|
+
let mut watcher = match notify::recommended_watcher(move |res| {
|
|
758
|
+
let _ = tx.send(res);
|
|
759
|
+
}) {
|
|
760
|
+
Ok(w) => w,
|
|
761
|
+
Err(e) => {
|
|
762
|
+
eprintln!("tish test --watch: failed to start watcher ({e}); running once");
|
|
763
|
+
return run_tests(opts).exit_code;
|
|
764
|
+
}
|
|
765
|
+
};
|
|
766
|
+
for root in &opts.roots {
|
|
767
|
+
if let Err(e) = watcher.watch(root, RecursiveMode::Recursive) {
|
|
768
|
+
eprintln!("tish test --watch: cannot watch {}: {e}", root.display());
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
eprintln!("tish test --watch: watching for changes (Ctrl-C to stop)");
|
|
772
|
+
loop {
|
|
773
|
+
let result = run_tests(opts.clone());
|
|
774
|
+
// Block until a change we did not cause ourselves, then debounce briefly.
|
|
775
|
+
let relevant = loop {
|
|
776
|
+
match rx.recv() {
|
|
777
|
+
Ok(Ok(event)) if event.paths.iter().any(|p| !is_self_written(p, &opts)) => {
|
|
778
|
+
break true
|
|
779
|
+
}
|
|
780
|
+
// Our own snapshot / coverage / junit writes: keep waiting.
|
|
781
|
+
Ok(_) => continue,
|
|
782
|
+
Err(_) => break false,
|
|
783
|
+
}
|
|
784
|
+
};
|
|
785
|
+
if !relevant {
|
|
786
|
+
return result.exit_code;
|
|
787
|
+
}
|
|
788
|
+
while rx.recv_timeout(Duration::from_millis(150)).is_ok() {}
|
|
789
|
+
eprintln!("\n── file change; re-running ──");
|
|
790
|
+
}
|
|
791
|
+
}
|
|
792
|
+
#[cfg(not(feature = "watch"))]
|
|
793
|
+
{
|
|
794
|
+
eprintln!("tish test --watch: notify feature not enabled in this build; running once");
|
|
795
|
+
run_tests(opts).exit_code
|
|
796
|
+
}
|
|
797
|
+
}
|