@flemist/test-variants 3.0.3 → 5.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +287 -49
- package/build/browser/index.cjs +1 -0
- package/build/browser/index.d.ts +1 -0
- package/build/browser/index.mjs +4 -0
- package/build/common/-test/freezeProps.d.ts +2 -0
- package/build/common/garbage-collect/garbageCollect.d.ts +5 -0
- package/build/common/helpers/log.d.ts +5 -0
- package/build/common/index.cjs +1 -0
- package/build/common/index.d.ts +2 -0
- package/build/common/index.mjs +4 -0
- package/build/common/test-variants/-test/caches.d.ts +1 -0
- package/build/common/test-variants/-test/constants.d.ts +8 -0
- package/build/common/test-variants/-test/estimations/estimateCallCount.d.ts +4 -0
- package/build/common/test-variants/-test/estimations/estimateModeChanges.d.ts +4 -0
- package/build/common/test-variants/-test/generators/findBestError.d.ts +4 -0
- package/build/common/test-variants/-test/generators/primitives.d.ts +16 -0
- package/build/common/test-variants/-test/generators/run.d.ts +9 -0
- package/build/common/test-variants/-test/generators/template.d.ts +3 -0
- package/build/common/test-variants/-test/generators/testFunc.d.ts +3 -0
- package/build/common/test-variants/-test/helpers/CallController.d.ts +28 -0
- package/build/common/test-variants/-test/helpers/ErrorVariantController.d.ts +17 -0
- package/build/common/test-variants/-test/helpers/TestError.d.ts +2 -0
- package/build/common/test-variants/-test/helpers/deepEqualJsonLikeWithoutSeed.d.ts +1 -0
- package/build/common/test-variants/-test/helpers/deepFreezeJsonLike.d.ts +1 -0
- package/build/common/test-variants/-test/helpers/forEachVariant.d.ts +4 -0
- package/build/common/test-variants/-test/helpers/getMaxAttemptsPerVariant.d.ts +2 -0
- package/build/common/test-variants/-test/helpers/getParallelLimit.d.ts +7 -0
- package/build/common/test-variants/-test/helpers/getVariantArgs.d.ts +12 -0
- package/build/common/test-variants/-test/helpers/runWithTimeController.d.ts +7 -0
- package/build/common/test-variants/-test/invariants/CallCountInvariant.d.ts +18 -0
- package/build/common/test-variants/-test/invariants/CallOptionsInvariant.d.ts +32 -0
- package/build/common/test-variants/-test/invariants/ErrorBehaviorInvariant.d.ts +32 -0
- package/build/common/test-variants/-test/invariants/FindBestErrorInvariant.d.ts +37 -0
- package/build/common/test-variants/-test/invariants/IterationsInvariant.d.ts +21 -0
- package/build/common/test-variants/-test/invariants/LimitTimeInvariant.d.ts +26 -0
- package/build/common/test-variants/-test/invariants/LogInvariant.d.ts +126 -0
- package/build/common/test-variants/-test/invariants/OnErrorInvariant.d.ts +38 -0
- package/build/common/test-variants/-test/invariants/OnModeChangeInvariant.d.ts +32 -0
- package/build/common/test-variants/-test/invariants/ParallelInvariant.d.ts +25 -0
- package/build/common/test-variants/-test/log.d.ts +3 -0
- package/build/common/test-variants/-test/types.d.ts +34 -0
- package/build/common/test-variants/-test/variants.d.ts +3 -0
- package/build/common/test-variants/createTestRun.d.ts +3 -0
- package/build/common/test-variants/createTestVariants.d.ts +4 -0
- package/build/common/test-variants/iterator/createVariantsIterator.d.ts +4 -0
- package/build/common/test-variants/iterator/helpers/findValueIndex.d.ts +2 -0
- package/build/common/test-variants/iterator/helpers/mode.d.ts +3 -0
- package/build/common/test-variants/iterator/helpers/template.d.ts +7 -0
- package/build/common/test-variants/iterator/types.d.ts +106 -0
- package/build/common/test-variants/iterator/variant-navigation/-test/helpers/caches.d.ts +1 -0
- package/build/common/test-variants/iterator/variant-navigation/-test/helpers/check.d.ts +2 -0
- package/build/common/test-variants/iterator/variant-navigation/-test/helpers/create.d.ts +3 -0
- package/build/common/test-variants/iterator/variant-navigation/-test/helpers/format.d.ts +6 -0
- package/build/common/test-variants/iterator/variant-navigation/-test/helpers/parse.d.ts +8 -0
- package/build/common/test-variants/iterator/variant-navigation/-test/helpers/variants.d.ts +12 -0
- package/build/common/test-variants/iterator/variant-navigation/variantNavigation.d.ts +28 -0
- package/build/common/test-variants/log/format.d.ts +7 -0
- package/build/common/test-variants/log/getMemoryUsage.d.ts +2 -0
- package/build/common/test-variants/log/logOptions.d.ts +8 -0
- package/build/common/test-variants/run/AbortErrorSilent.d.ts +3 -0
- package/build/common/test-variants/run/RunContext.d.ts +21 -0
- package/build/common/test-variants/run/createRunResult.d.ts +6 -0
- package/build/common/test-variants/run/createRunState.d.ts +23 -0
- package/build/common/test-variants/run/errorHandlers.d.ts +20 -0
- package/build/common/test-variants/run/gcManager.d.ts +6 -0
- package/build/common/test-variants/run/resolveRunOptions.d.ts +23 -0
- package/build/common/test-variants/run/runIterationLoop.d.ts +9 -0
- package/build/common/test-variants/run/runLogger.d.ts +9 -0
- package/build/common/test-variants/run/types.d.ts +57 -0
- package/build/common/test-variants/testVariantsRun.d.ts +6 -0
- package/build/common/test-variants/types.d.ts +187 -0
- package/build/createTestVariants-BE_TQ9u5.mjs +1018 -0
- package/build/createTestVariants-Cmx68kHs.js +4 -0
- package/build/node/index.cjs +1 -0
- package/build/node/index.d.ts +8 -0
- package/build/node/index.mjs +107 -0
- package/build/node/test-variants/createSaveErrorVariantsStore.d.ts +5 -0
- package/{dist/lib → build/node}/test-variants/saveErrorVariants.d.ts +4 -3
- package/package.json +109 -68
- package/dist/bundle/browser.js +0 -669
- package/dist/lib/garbage-collect/garbageCollect.cjs +0 -30
- package/dist/lib/garbage-collect/garbageCollect.d.ts +0 -2
- package/dist/lib/garbage-collect/garbageCollect.mjs +0 -26
- package/dist/lib/index.cjs +0 -23
- package/dist/lib/index.d.ts +0 -7
- package/dist/lib/index.mjs +0 -13
- package/dist/lib/test-variants/argsToString.cjs +0 -17
- package/dist/lib/test-variants/argsToString.d.ts +0 -2
- package/dist/lib/test-variants/argsToString.mjs +0 -13
- package/dist/lib/test-variants/createTestVariants.cjs +0 -91
- package/dist/lib/test-variants/createTestVariants.d.ts +0 -8
- package/dist/lib/test-variants/createTestVariants.mjs +0 -87
- package/dist/lib/test-variants/createTestVariants.perf.cjs +0 -44
- package/dist/lib/test-variants/createTestVariants.perf.d.ts +0 -1
- package/dist/lib/test-variants/createTestVariants.perf.mjs +0 -42
- package/dist/lib/test-variants/prime.cjs +0 -65
- package/dist/lib/test-variants/prime.d.ts +0 -3
- package/dist/lib/test-variants/prime.mjs +0 -59
- package/dist/lib/test-variants/prime.perf.cjs +0 -30
- package/dist/lib/test-variants/prime.perf.d.ts +0 -1
- package/dist/lib/test-variants/prime.perf.mjs +0 -28
- package/dist/lib/test-variants/saveErrorVariants.cjs +0 -97
- package/dist/lib/test-variants/saveErrorVariants.mjs +0 -69
- package/dist/lib/test-variants/testVariantsCreateTestRun.cjs +0 -80
- package/dist/lib/test-variants/testVariantsCreateTestRun.d.ts +0 -22
- package/dist/lib/test-variants/testVariantsCreateTestRun.mjs +0 -76
- package/dist/lib/test-variants/testVariantsIterable.cjs +0 -70
- package/dist/lib/test-variants/testVariantsIterable.d.ts +0 -15
- package/dist/lib/test-variants/testVariantsIterable.mjs +0 -66
- package/dist/lib/test-variants/testVariantsIterator.cjs +0 -429
- package/dist/lib/test-variants/testVariantsIterator.d.ts +0 -67
- package/dist/lib/test-variants/testVariantsIterator.mjs +0 -425
- package/dist/lib/test-variants/testVariantsRun.cjs +0 -289
- package/dist/lib/test-variants/testVariantsRun.d.ts +0 -50
- package/dist/lib/test-variants/testVariantsRun.mjs +0 -265
- package/dist/lib/test-variants/types.cjs +0 -2
- package/dist/lib/test-variants/types.d.ts +0 -20
- package/dist/lib/test-variants/types.mjs +0 -1
package/README.md
CHANGED
|
@@ -1,66 +1,304 @@
|
|
|
1
1
|
[![NPM Version][npm-image]][npm-url]
|
|
2
2
|
[![NPM Downloads][downloads-image]][downloads-url]
|
|
3
3
|
[![Build Status][github-image]][github-url]
|
|
4
|
-
[![Test Coverage][coveralls-image]][coveralls-url]
|
|
4
|
+
<!-- [![Test Coverage][coveralls-image]][coveralls-url] -->
|
|
5
5
|
|
|
6
|
-
|
|
6
|
+
# @flemist/test-variants
|
|
7
7
|
|
|
8
|
-
|
|
8
|
+
TypeScript library for combinatorial randomized testing - runs a test function with all possible parameter combinations
|
|
9
|
+
|
|
10
|
+
## Terms
|
|
11
|
+
|
|
12
|
+
- `test` - function being tested with different parameter combinations
|
|
13
|
+
- `createTestVariants` - function that creates the testVariants function
|
|
14
|
+
- `testVariants` - function that runs the test iterating through parameter combinations
|
|
15
|
+
- `variant` - specific combination of parameters for the test
|
|
16
|
+
- `parameter template` - object where each test parameter corresponds to an array of possible values
|
|
17
|
+
- `seed` - value for initializing the pseudo-random generator inside the test, for reproducibility of randomized tests
|
|
18
|
+
- `iteration` - single test run with a specific parameter combination
|
|
19
|
+
- `iterating` - traversing parameter combinations (forward, backward, random)
|
|
20
|
+
- `async iteration` - iteration where the test function returns Promise
|
|
21
|
+
- `sync iteration` - iteration where the test function returns void or a value
|
|
22
|
+
- `iteration mode` - method of traversing variants (forward, backward, random)
|
|
23
|
+
- `full pass` - when all possible variants within constraints have been used at least once (possible only for forward and backward iteration modes)
|
|
24
|
+
- `variant attempts` - number of test runs with the same parameter variant before moving to the next variant
|
|
25
|
+
- `cycle` - full pass through all parameter variants
|
|
26
|
+
- `best error` - error that occurred on the lexicographically smallest parameter variant, i.e., the most convenient for debugging
|
|
27
|
+
- `IAbortSignalFast` - interface for aborting async operations (see @flemist/abort-controller-fast)
|
|
28
|
+
|
|
29
|
+
## Public API
|
|
9
30
|
|
|
10
|
-
## Sync only
|
|
11
31
|
```ts
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
32
|
+
// Creates a function for running tests with all parameter combinations
|
|
33
|
+
// The test function is passed as parameter, it can be: sync, async, or hybrid
|
|
34
|
+
const testVariants = createTestVariants(async (
|
|
35
|
+
// test parameters that will be iterated
|
|
36
|
+
{
|
|
37
|
+
arg1,
|
|
38
|
+
arg2,
|
|
39
|
+
arg3,
|
|
40
|
+
seed,
|
|
41
|
+
}: {
|
|
42
|
+
arg1: Type1,
|
|
43
|
+
arg2: Type2,
|
|
44
|
+
arg3: Type3,
|
|
45
|
+
// seed is generated automatically or by getSeed function,
|
|
46
|
+
// can be any type: number, string, object, function, etc.
|
|
47
|
+
// intended for pseudo-random generator and reproducibility of randomized tests
|
|
48
|
+
seed?: number | null,
|
|
49
|
+
},
|
|
50
|
+
{
|
|
51
|
+
// created by testVariants, combined with abortSignal from run options if provided
|
|
52
|
+
// use to abort async operations or check abortSignal.aborted
|
|
53
|
+
abortSignal,
|
|
54
|
+
// from run options timeController, or timeControllerDefault if not specified
|
|
55
|
+
// use for time-dependent operations: timeController.now(), delays, etc.
|
|
56
|
+
timeController,
|
|
57
|
+
}: TestVariantsTestOptions,
|
|
58
|
+
) => {
|
|
59
|
+
// test body
|
|
60
|
+
|
|
61
|
+
// Returns: void | number | { iterationsAsync: number, iterationsSync: number }
|
|
62
|
+
// Return iterationsAsync and iterationsSync if you need to count the total number
|
|
63
|
+
// of async and sync iterations inside the test
|
|
64
|
+
// number is equivalent to iterationsSync
|
|
15
65
|
})
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
66
|
+
|
|
67
|
+
const result = await testVariants({
|
|
68
|
+
// parameter templates
|
|
69
|
+
arg1: [value1, value2, value3],
|
|
70
|
+
arg2: (args) => [valueA, valueB], // args: { arg1 } - already assigned parameters
|
|
71
|
+
arg3: [valueX],
|
|
20
72
|
})({
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
})
|
|
73
|
+
// All parameters are optional
|
|
74
|
+
// Missing values or null mean default value is used
|
|
24
75
|
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
// [2, '3', false],
|
|
32
|
-
// [2, '4', true],
|
|
33
|
-
// [2, '4', false],
|
|
34
|
-
// ]
|
|
35
|
-
// count == 8
|
|
36
|
-
```
|
|
76
|
+
// Automatic garbage collection after N iterations or after time interval
|
|
77
|
+
// Useful for preventing timeout in karma for sync tests,
|
|
78
|
+
// or preventing hangs in Node.js due to Promise bugs
|
|
79
|
+
GC_Iterations: number, // default: 1000000
|
|
80
|
+
GC_IterationsAsync: number, // default: 10000
|
|
81
|
+
GC_Interval: number, // default: 1000 (milliseconds)
|
|
37
82
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
83
|
+
// Console output parameters, default: true
|
|
84
|
+
log: true, // all console output parameters default
|
|
85
|
+
log: boolean | {
|
|
86
|
+
// message about test start
|
|
87
|
+
start: boolean, // default: true
|
|
88
|
+
// every N milliseconds shows progress info and statistics; false/0 to disable
|
|
89
|
+
progress: number | false, // default: 5000 (milliseconds)
|
|
90
|
+
// message about test completion
|
|
91
|
+
completed: boolean, // default: true
|
|
92
|
+
// full error log with stack trace
|
|
93
|
+
error: boolean, // default: true
|
|
94
|
+
// message about iteration mode change, with info about current mode
|
|
95
|
+
modeChange: boolean, // default: true
|
|
96
|
+
// debug logging for internal behavior
|
|
97
|
+
debug: boolean, // default: false
|
|
98
|
+
// custom log function; receives log type and formatted message
|
|
99
|
+
func: (type: 'start' | 'progress' | 'completed' | 'error' | 'modeChange' | 'debug', message: string) => void,
|
|
100
|
+
},
|
|
101
|
+
|
|
102
|
+
// for aborting async operations inside the test
|
|
103
|
+
abortSignal: IAbortSignalFast,
|
|
104
|
+
|
|
105
|
+
// Parallel execution (for async tests)
|
|
106
|
+
parallel: boolean | number | ParallelOptions, // default: 1 - no parallel
|
|
107
|
+
parallel: true, // all parallel
|
|
108
|
+
parallel: 4, // maximum 4 parallel
|
|
109
|
+
parallel: false | 1, // sequential
|
|
110
|
+
parallel: {
|
|
111
|
+
count: 4, // maximum 4 parallel
|
|
112
|
+
sequentialOnError: true, // switch to sequential after first error (findBestError mode)
|
|
113
|
+
},
|
|
114
|
+
|
|
115
|
+
// Global limits
|
|
116
|
+
// Maximum total number of tests run (including attemptsPerVariant)
|
|
117
|
+
limitTests: number, // default: null - unlimited
|
|
118
|
+
// Maximum total runtime of testVariants
|
|
119
|
+
limitTime: number, // default: null - unlimited, (milliseconds)
|
|
120
|
+
// Test terminates when min(completedCount) across sequential modes (forward/backward) >= cycles
|
|
121
|
+
// Random mode is excluded from this calculation - it is limited only by global limits
|
|
122
|
+
// (limitTests, limitTime) or when no valid variants exist within current constraints
|
|
123
|
+
// Until termination conditions are met, iteration modes switch in a circle
|
|
124
|
+
// If all modes executed zero tests in their last round, the cycle terminates (stuck)
|
|
125
|
+
cycles: 3, // default: 1
|
|
126
|
+
|
|
127
|
+
// Iteration modes (variant traversal), default: forward
|
|
128
|
+
// All modes preserve their current positions between mode switches,
|
|
129
|
+
// so with multiple executions, they will eventually traverse all variants
|
|
130
|
+
// When traversal reaches the last variant and no termination conditions are met,
|
|
131
|
+
// traversal starts over
|
|
132
|
+
iterationModes: [
|
|
133
|
+
{
|
|
134
|
+
// Lexicographic traversal of variants (like numeric counting)
|
|
135
|
+
// from first (the very last argument in template)
|
|
136
|
+
// to last (the very first argument in template) or until limits reached
|
|
137
|
+
mode: 'forward',
|
|
138
|
+
// number of tests for each variant before moving to the next variant
|
|
139
|
+
attemptsPerVariant: number, // default: 1
|
|
140
|
+
// maximum number of attempted full passes of all variants, before mode switch
|
|
141
|
+
cycles: number, // default: 1
|
|
142
|
+
// maximum runtime before mode switch
|
|
143
|
+
limitTime: number, // default: null - unlimited, (milliseconds)
|
|
144
|
+
// maximum number of tests run before mode switch (including attemptsPerVariant)
|
|
145
|
+
limitTests: number, // default: null - unlimited
|
|
146
|
+
},
|
|
147
|
+
{
|
|
148
|
+
// Lexicographic traversal of variants in reverse order
|
|
149
|
+
// from the last possible or from current constraint to the first variant
|
|
150
|
+
// Same parameters as for 'forward'
|
|
151
|
+
mode: 'backward',
|
|
152
|
+
cycles: number,
|
|
153
|
+
attemptsPerVariant: number,
|
|
154
|
+
limitTime: number,
|
|
155
|
+
limitTests: number,
|
|
156
|
+
},
|
|
157
|
+
{
|
|
158
|
+
// Random traversal of variants within current constraints
|
|
159
|
+
mode: 'random',
|
|
160
|
+
limitTime: 10000,
|
|
161
|
+
limitTests: 1000,
|
|
162
|
+
},
|
|
163
|
+
],
|
|
164
|
+
|
|
165
|
+
// Iteration modes are best used in tandem with best error search
|
|
166
|
+
// Best error is the error that occurred on the lexicographically smallest variant
|
|
167
|
+
// Ideally the best error will be a variant with all argument values
|
|
168
|
+
// equal to the first value in the template
|
|
169
|
+
// Search is performed by repeated iteration and introducing new constraints
|
|
170
|
+
// when an error is found, thus the number of variants constantly decreases,
|
|
171
|
+
// and tests run faster
|
|
172
|
+
findBestError: {
|
|
173
|
+
equals: (a, b) => boolean,
|
|
174
|
+
// Extra per-arg constraint (does NOT replace lexicographic, both apply):
|
|
175
|
+
// each arg[i] <= errorArg[i]
|
|
176
|
+
limitArgOnError: boolean | Function, // default: false
|
|
177
|
+
limitArgOnError: true, // rule applies to all arguments
|
|
178
|
+
// Custom rule, whether to limit argument value
|
|
179
|
+
limitArgOnError: ({
|
|
180
|
+
name, // argument name
|
|
181
|
+
values, // all possible argument values in template
|
|
182
|
+
maxValueIndex, // current max value index limit for this arg; null if no limit
|
|
183
|
+
}) => boolean,
|
|
184
|
+
// the following is equivalent to limitArgOnError: true
|
|
185
|
+
limitArgOnError: () => true,
|
|
186
|
+
|
|
187
|
+
// Option intended only for system verification
|
|
188
|
+
// If true, iteration will include the last error variant
|
|
189
|
+
includeErrorVariant: boolean, // default: false
|
|
190
|
+
|
|
191
|
+
// If true, when testVariants completes, if an error was found,
|
|
192
|
+
// no exception will be thrown, instead
|
|
193
|
+
// all error info will be returned in the result
|
|
194
|
+
dontThrowIfError: boolean, // default: false
|
|
195
|
+
},
|
|
196
|
+
|
|
197
|
+
// Seed generation for pseudo-random generator
|
|
198
|
+
// Seed will be set in test parameters as seed field, even if it's null or undefined
|
|
199
|
+
// This seed will be used for exact reproduction of pseudo-random behavior inside the test
|
|
200
|
+
getSeed: ({ // default: null - seed disabled, not set in test arguments
|
|
201
|
+
// total number of tests run
|
|
202
|
+
tests,
|
|
203
|
+
}) => any,
|
|
204
|
+
getSeed: () => Math.random() * 0xFFFFFFFF, // example - random numeric seed
|
|
205
|
+
|
|
206
|
+
// Saving error variants to files for subsequent checks
|
|
207
|
+
// or continuing best error search
|
|
208
|
+
// Before iterating all variants, saved variants from files will be checked first
|
|
209
|
+
// in descending order of their save date (newest first)
|
|
210
|
+
saveErrorVariants: {
|
|
211
|
+
dir: './error-variants',
|
|
212
|
+
// Maximum number of checks for each saved variant
|
|
213
|
+
// Useful when error doesn't reproduce on first try
|
|
214
|
+
// due to factors independent of parameters or random generator
|
|
215
|
+
// If error is found, exception is thrown by default and testVariants terminates
|
|
216
|
+
attemptsPerVariant: 1, // default: 1
|
|
217
|
+
// Custom file path generation for saving variant
|
|
218
|
+
// Either relative to dir folder, or absolute path
|
|
219
|
+
// default: 2025-12-30_12-34-37_vw3h626wg7m.json
|
|
220
|
+
getFilePath: ({ sessionDate }) => string | null,
|
|
221
|
+
// Custom serialization, in case arguments are class instances
|
|
222
|
+
argsToJson: (args) => string | SavedArgs,
|
|
223
|
+
// Custom deserialization
|
|
224
|
+
jsonToArgs: (json) => Args,
|
|
225
|
+
// If true and findBestError is enabled, all files are checked,
|
|
226
|
+
// all errors from them are collected and used as initial constraints for findBestError
|
|
227
|
+
// Useful when you need to continue best error search after testVariants restart
|
|
228
|
+
useToFindBestError: false,
|
|
229
|
+
// Same as limitArgOnError
|
|
230
|
+
limitArg: boolean | Function, // default: false
|
|
231
|
+
// Extend template with extra args from limit if they are missing
|
|
232
|
+
extendTemplates: boolean, // default: false
|
|
233
|
+
},
|
|
234
|
+
|
|
235
|
+
// Called when an error occurs in the test
|
|
236
|
+
// before logging and throwing exception
|
|
237
|
+
onError: ({
|
|
238
|
+
error, // the error caught via try..catch
|
|
239
|
+
args, // test parameters that caused the error
|
|
240
|
+
tests, // number of tests run before the error (including attemptsPerVariant)
|
|
241
|
+
}) => void | Promise<void>,
|
|
242
|
+
|
|
243
|
+
// Called when iteration mode changes
|
|
244
|
+
// Invoked at test start and when switching to next mode
|
|
245
|
+
onModeChange: ({
|
|
246
|
+
mode, // current mode configuration (ModeConfig)
|
|
247
|
+
modeIndex, // current mode index in iterationModes array
|
|
248
|
+
tests, // number of tests run before this mode change
|
|
249
|
+
}) => void | Promise<void>,
|
|
250
|
+
|
|
251
|
+
// Time controller for all internal delays, timeouts and getting current time
|
|
252
|
+
// Used inside testVariants instead of direct setTimeout, Date.now calls, etc
|
|
253
|
+
// Intended only for testing and debugging the test-variants library itself
|
|
254
|
+
timeController: ITimeController, // default: null - use timeControllerDefault
|
|
44
255
|
})
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
256
|
+
|
|
257
|
+
// Result:
|
|
258
|
+
{
|
|
259
|
+
iterations: number,
|
|
260
|
+
// Best error found during testing; set when findBestError is enabled and an error occurred
|
|
261
|
+
// If dontThrowIfError is true, error is returned here instead of thrown
|
|
262
|
+
bestError: null | {
|
|
263
|
+
error: any, // the error caught via try..catch
|
|
264
|
+
args: { // test parameters that caused the error
|
|
265
|
+
arg1: Type1,
|
|
266
|
+
arg2: Type2,
|
|
267
|
+
arg3: Type3,
|
|
268
|
+
seed?: number | null,
|
|
269
|
+
},
|
|
270
|
+
tests: number, // number of tests run before the error (including attemptsPerVariant)
|
|
271
|
+
},
|
|
272
|
+
}
|
|
50
273
|
```
|
|
51
274
|
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
275
|
+
### Debug mode
|
|
276
|
+
|
|
277
|
+
When a test error occurs, the library automatically triggers JavaScript's `debugger` statement.
|
|
278
|
+
If you're running tests with a JS debugger attached:
|
|
279
|
+
|
|
280
|
+
1. Execution pauses at the `debugger` statement in the error handler
|
|
281
|
+
2. Set breakpoints in your test code where you want to investigate
|
|
282
|
+
3. Resume execution - if resuming took more than 50ms (meaning you were stepping through), the same failing variant will repeat
|
|
283
|
+
4. The variant repeats up to 5 times, allowing step-by-step debugging of the exact failing case
|
|
284
|
+
5. After 5 debug iterations or if you resume quickly (<50ms), the error is thrown normally
|
|
285
|
+
|
|
286
|
+
This enables debugging the exact parameter combination that caused the failure without manually recreating it.
|
|
287
|
+
|
|
288
|
+
### Sync mode optimization
|
|
289
|
+
|
|
290
|
+
The internal implementation operates in a faster synchronous mode (without await and Promise) when the test function is synchronous. The library detects whether each test invocation returns a Promise and switches to async handling only when necessary. This maximizes performance for sync tests while fully supporting async tests when needed.
|
|
291
|
+
|
|
292
|
+
### Logs format
|
|
293
|
+
```
|
|
294
|
+
[test-variants] start, memory: 139MB
|
|
295
|
+
[test-variants] mode[0]: random
|
|
296
|
+
[test-variants] cycle: 3, variant: 65 (1.0s), tests: 615 (5.0s), async: 12, memory: 148MB (+8.8MB)
|
|
297
|
+
[test-variants] mode[1]: backward, limitTests=10
|
|
298
|
+
[test-variants] cycle: 5, variant: 65/100 (2.0s), tests: 615 (6.0s), async: 123, memory: 139MB (-8.8MB)
|
|
299
|
+
[test-variants] mode[2]: forward, limitTests=100, limitTime=10.9m
|
|
300
|
+
[test-variants] end, tests: 815 (7.0s), async: 123, memory: 138MB (-1.0MB)
|
|
301
|
+
...
|
|
64
302
|
```
|
|
65
303
|
|
|
66
304
|
# License
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const e=require("../createTestVariants-Cmx68kHs.js");exports.createTestVariants=e.createTestVariants;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export * from '../common';
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"use strict";Object.defineProperty(exports,Symbol.toStringTag,{value:"Module"});const e=require("../createTestVariants-Cmx68kHs.js");exports.createTestVariants=e.createTestVariants;
|
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
export { type ErrorEvent, type GenerateErrorVariantFilePathOptions, type GetSeedParams, type LimitArgOnError, type LimitArgOnErrorOptions, type ModeChangeEvent, type ModeConfig, type OnErrorCallback, type OnModeChangeCallback, type ParallelOptions, type SaveErrorVariantsOptions, type TestVariantsBestError, type FindBestErrorOptions, type TestVariantsLogOptions, type TestVariantsRunOptions, type TestVariantsResult, type TestOptions, type TestVariantsTestResult, } from './test-variants/types';
|
|
2
|
+
export { createTestVariants } from './test-variants/createTestVariants';
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function getArgKey(index: number): string;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { ModeConfig } from '../..';
|
|
2
|
+
export declare const PARALLEL_MAX = 10;
|
|
3
|
+
export declare const TIME_MAX = 10;
|
|
4
|
+
export declare const LIMIT_MAX = 10000;
|
|
5
|
+
export declare const LIMIT_RANDOM_MODE_DEFAULT = 10;
|
|
6
|
+
export declare const ITERATIONS_SYNC = 10;
|
|
7
|
+
export declare const ITERATIONS_ASYNC = 1000000;
|
|
8
|
+
export declare const MODES_DEFAULT: readonly ModeConfig[];
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { TestVariantsRunOptions } from '../../..';
|
|
2
|
+
import { NumberRange } from '@flemist/simple-utils';
|
|
3
|
+
import { StressTestArgs, TestArgs } from '../types';
|
|
4
|
+
export declare function estimateCallCount(options: StressTestArgs, runOptions: TestVariantsRunOptions<TestArgs>, variantsCount: number, errorVariantIndex: number | null, withDelay: boolean): NumberRange;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { TestVariantsRunOptions } from '../../..';
|
|
2
|
+
import { NumberRange } from '@flemist/simple-utils';
|
|
3
|
+
import { StressTestArgs, TestArgs } from '../types';
|
|
4
|
+
export declare function estimateModeChanges(options: StressTestArgs, runOptions: TestVariantsRunOptions<TestArgs>, variantsCount: number, errorVariantIndex: number | null, withDelay: boolean, callCountRange: NumberRange): NumberRange;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { Random } from '@flemist/simple-utils';
|
|
2
|
+
import { StressTestArgs, Template } from '../types';
|
|
3
|
+
import { FindBestErrorOptions } from '../../..';
|
|
4
|
+
export declare function generateFindBestErrorOptions(rnd: Random, options: StressTestArgs, template: Template, argKeys: readonly string[]): FindBestErrorOptions | undefined;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { Random } from '@flemist/simple-utils';
|
|
2
|
+
import { StressTestArgs } from '../types';
|
|
3
|
+
export declare function generateBoolean(rnd: Random, booleanOption: boolean | null): boolean;
|
|
4
|
+
/**
|
|
5
|
+
* Generate integer with boundary values priority:
|
|
6
|
+
* 0, 1, 2, 3 - if max = 3
|
|
7
|
+
* 0, 1, 2, max, [3..max-1] - if max >= 4 and isMaxBoundary is true
|
|
8
|
+
* 0, 1, 2, [3..max] - if max >= 4 and isMaxBoundary is false
|
|
9
|
+
*/
|
|
10
|
+
export declare function generateBoundaryInt(rnd: Random, max: number, isMaxBoundary?: null | boolean): number;
|
|
11
|
+
/**
|
|
12
|
+
* Generate limit value with priority: 0, 1, max-1, max, max+1, [2..max-2]
|
|
13
|
+
*/
|
|
14
|
+
export declare function generateLimit(rnd: Random, max: number): number;
|
|
15
|
+
export declare function equals(a: unknown, b: unknown): boolean;
|
|
16
|
+
export declare function generateEquals(rnd: Random, options: StressTestArgs): ((a: unknown, b: unknown) => boolean) | undefined;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { Random } from '@flemist/simple-utils';
|
|
2
|
+
import { ModeChangeEvent, TestVariantsRunOptions } from '../../..';
|
|
3
|
+
import { StressTestArgs, Template, TestArgs } from '../types';
|
|
4
|
+
import { TestVariantsLogType } from '../../types';
|
|
5
|
+
export declare function generateRunOptions(rnd: Random, options: StressTestArgs, template: Template, argKeys: readonly string[], variantsCount: number, logFunc: (type: TestVariantsLogType, message: string) => void, onError: (event: {
|
|
6
|
+
error: unknown;
|
|
7
|
+
args: TestArgs;
|
|
8
|
+
tests: number;
|
|
9
|
+
}) => void, onModeChange: (event: ModeChangeEvent) => void): TestVariantsRunOptions<TestArgs>;
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
import { IAbortSignalFast } from '@flemist/abort-controller-fast';
|
|
2
|
+
import { TimeControllerMock } from '@flemist/time-controller';
|
|
3
|
+
import { PromiseOrValue } from '@flemist/async-utils';
|
|
4
|
+
import { TestFuncResult } from '../../run/types';
|
|
5
|
+
/**
|
|
6
|
+
* Emulates test function call with sync/async behavior
|
|
7
|
+
* Counts number of calls
|
|
8
|
+
*/
|
|
9
|
+
export declare class CallController {
|
|
10
|
+
private _callCount;
|
|
11
|
+
private _completedCount;
|
|
12
|
+
private _currentConcurrent;
|
|
13
|
+
private _maxConcurrent;
|
|
14
|
+
private readonly _isAsync;
|
|
15
|
+
private readonly _withDelay;
|
|
16
|
+
private readonly _abortController;
|
|
17
|
+
private readonly _timeController;
|
|
18
|
+
constructor(isAsync: boolean | null, withDelay: boolean);
|
|
19
|
+
/** Use inside test func */
|
|
20
|
+
call(start: () => void, end: () => void): PromiseOrValue<TestFuncResult>;
|
|
21
|
+
get callCount(): number;
|
|
22
|
+
get completedCount(): number;
|
|
23
|
+
get currentConcurrent(): number;
|
|
24
|
+
get maxConcurrent(): number;
|
|
25
|
+
get abortSignal(): IAbortSignalFast;
|
|
26
|
+
get timeController(): TimeControllerMock;
|
|
27
|
+
finalize(): void;
|
|
28
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { TestArgs } from '../types';
|
|
2
|
+
import { TestError } from './TestError';
|
|
3
|
+
/**
|
|
4
|
+
* Emulates error thrown for error variant after N retries
|
|
5
|
+
*/
|
|
6
|
+
export declare class ErrorVariantController {
|
|
7
|
+
private _retries;
|
|
8
|
+
private _lastThrownError;
|
|
9
|
+
private readonly _errorVariantArgs;
|
|
10
|
+
private readonly _retriesToError;
|
|
11
|
+
private _nextErrorId;
|
|
12
|
+
constructor(errorVariantArgs: TestArgs | null | undefined, retriesToError: number);
|
|
13
|
+
/** Use inside test func */
|
|
14
|
+
onCall(args: TestArgs): void;
|
|
15
|
+
get lastThrownError(): TestError | null;
|
|
16
|
+
get errorVariantArgs(): TestArgs | null;
|
|
17
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function deepEqualJsonLikeWithoutSeed(a: any, b: any): boolean;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function deepFreezeJsonLike(value: unknown): void;
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import { Template, TestArgs } from '../types';
|
|
2
|
+
/** @return true to break */
|
|
3
|
+
export type ForEachVariantCallback = (args: TestArgs, index: number) => boolean | null | undefined | void;
|
|
4
|
+
export declare function forEachVariant(template: Template, argKeys: string[], callback?: null | ForEachVariantCallback): number;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { ParallelOptions } from '../../types';
|
|
2
|
+
/**
|
|
3
|
+
* Extracts actual parallel limit from parallel option
|
|
4
|
+
*
|
|
5
|
+
* @returns 1 for sequential, Infinity for unlimited, or the specified count
|
|
6
|
+
*/
|
|
7
|
+
export declare function getParallelLimit(parallel: number | boolean | ParallelOptions | null | undefined): number;
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Template, TestArgs } from '../types';
|
|
2
|
+
export type Variant = {
|
|
3
|
+
index: number;
|
|
4
|
+
args: TestArgs;
|
|
5
|
+
};
|
|
6
|
+
export type Variants = {
|
|
7
|
+
first: null | Variant;
|
|
8
|
+
middle: null | Variant;
|
|
9
|
+
last: null | Variant;
|
|
10
|
+
error: null | Variant;
|
|
11
|
+
};
|
|
12
|
+
export declare function getVariantArgs(template: Template, argKeys: string[], variantsCount: number, errorIndex: number | null): Variants;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import { TimeControllerMock } from '@flemist/time-controller';
|
|
2
|
+
import { TestError } from './TestError';
|
|
3
|
+
export type AwaitResult<T> = {
|
|
4
|
+
result: T | null;
|
|
5
|
+
caughtError: TestError | null;
|
|
6
|
+
};
|
|
7
|
+
export declare function runWithTimeController<T>(timeController: TimeControllerMock, func: () => T | Promise<T>): Promise<AwaitResult<T>>;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { NumberRange } from '@flemist/simple-utils';
|
|
2
|
+
/**
|
|
3
|
+
* Validates call count bounds
|
|
4
|
+
*
|
|
5
|
+
* ## Applicability
|
|
6
|
+
* - Every test function call
|
|
7
|
+
*
|
|
8
|
+
* ## Invariants
|
|
9
|
+
* - callCount never exceeds callCountRange
|
|
10
|
+
*/
|
|
11
|
+
export declare class CallCountInvariant {
|
|
12
|
+
private readonly _callCountRange;
|
|
13
|
+
constructor(callCountRange: NumberRange);
|
|
14
|
+
/** Use inside test func */
|
|
15
|
+
onCall(callCount: number): void;
|
|
16
|
+
/** Run after test variants completion */
|
|
17
|
+
validateFinal(callCount: number): void;
|
|
18
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import { IAbortSignalFast } from '@flemist/abort-controller-fast';
|
|
2
|
+
import { ITimeController } from '@flemist/time-controller';
|
|
3
|
+
import { CallController } from '../helpers/CallController';
|
|
4
|
+
/**
|
|
5
|
+
* Validates TestOptions passed to test function
|
|
6
|
+
*
|
|
7
|
+
* ## Applicability
|
|
8
|
+
* Every test function call
|
|
9
|
+
*
|
|
10
|
+
* ## Invariants
|
|
11
|
+
* - abortSignal exists and is not null
|
|
12
|
+
* - timeController matches expected instance
|
|
13
|
+
* - abortSignal not aborted on first call
|
|
14
|
+
* - abortSignal aborted after CallController.finalize (combined signal depends on internal signal)
|
|
15
|
+
*
|
|
16
|
+
* ## Skipped Validations
|
|
17
|
+
* - abortSignal identity (library combines signals via combineAbortSignals)
|
|
18
|
+
* - abortSignal.aborted state during execution (depends on errors and sequentialOnError)
|
|
19
|
+
*/
|
|
20
|
+
export declare class CallOptionsInvariant {
|
|
21
|
+
private readonly _timeController;
|
|
22
|
+
private readonly _callController;
|
|
23
|
+
private _lastAbortSignal;
|
|
24
|
+
constructor(timeController: ITimeController, callController: CallController);
|
|
25
|
+
/** Use inside test func */
|
|
26
|
+
onCall(callOptions: {
|
|
27
|
+
abortSignal?: IAbortSignalFast;
|
|
28
|
+
timeController?: ITimeController;
|
|
29
|
+
}): void;
|
|
30
|
+
/** Run after test variants completion */
|
|
31
|
+
validateFinal(): void;
|
|
32
|
+
}
|