@flemist/test-variants 2.0.0-alpha → 2.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/bundle/browser.js +426 -237
- package/dist/lib/index.cjs +4 -0
- package/dist/lib/index.d.ts +3 -2
- package/dist/lib/index.mjs +3 -0
- package/dist/lib/test-variants/createTestVariants.cjs +3 -0
- package/dist/lib/test-variants/createTestVariants.d.ts +1 -1
- package/dist/lib/test-variants/createTestVariants.mjs +3 -0
- package/dist/lib/test-variants/createTestVariants.perf.cjs +3 -0
- package/dist/lib/test-variants/createTestVariants.perf.mjs +3 -0
- package/dist/lib/test-variants/saveErrorVariants.cjs +97 -0
- package/dist/lib/test-variants/saveErrorVariants.d.ts +9 -0
- package/dist/lib/test-variants/saveErrorVariants.mjs +69 -0
- package/dist/lib/test-variants/testVariantsRun.cjs +87 -14
- package/dist/lib/test-variants/testVariantsRun.d.ts +25 -4
- package/dist/lib/test-variants/testVariantsRun.mjs +68 -15
- package/dist/lib/test-variants/types.d.ts +17 -0
- package/package.json +22 -20
package/dist/lib/index.cjs
CHANGED
|
@@ -2,8 +2,11 @@
|
|
|
2
2
|
|
|
3
3
|
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
4
|
|
|
5
|
+
var testVariants_saveErrorVariants = require('./test-variants/saveErrorVariants.cjs');
|
|
5
6
|
var testVariants_createTestVariants = require('./test-variants/createTestVariants.cjs');
|
|
6
7
|
require('tslib');
|
|
8
|
+
require('fs');
|
|
9
|
+
require('path');
|
|
7
10
|
require('./test-variants/testVariantsIterable.cjs');
|
|
8
11
|
require('./test-variants/testVariantsCreateTestRun.cjs');
|
|
9
12
|
require('@flemist/async-utils');
|
|
@@ -15,4 +18,5 @@ require('./garbage-collect/garbageCollect.cjs');
|
|
|
15
18
|
|
|
16
19
|
|
|
17
20
|
|
|
21
|
+
exports.generateErrorVariantFilePath = testVariants_saveErrorVariants.generateErrorVariantFilePath;
|
|
18
22
|
exports.createTestVariants = testVariants_createTestVariants.createTestVariants;
|
package/dist/lib/index.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
|
-
export { type Obj, } from "./test-variants/types";
|
|
1
|
+
export { type Obj, type GenerateErrorVariantFilePathOptions, type SaveErrorVariantsOptions, } from "./test-variants/types";
|
|
2
|
+
export { generateErrorVariantFilePath, } from "./test-variants/saveErrorVariants";
|
|
2
3
|
export { type TestVariantsTemplate, type TestVariantsTemplates, type TestVariantsTemplatesExt, } from "./test-variants/testVariantsIterable";
|
|
3
4
|
export { type ErrorEvent, type OnErrorCallback, type TestVariantsTest, type TestVariantsTestResult, type TestVariantsCreateTestRunOptions, type TestVariantsTestRun, type TestVariantsTestRunResult, } from "./test-variants/testVariantsCreateTestRun";
|
|
4
|
-
export { type TestVariantsFindBestErrorOptions, type TestVariantsRunOptions, type TestVariantsBestError, type TestVariantsRunResult, } from "./test-variants/testVariantsRun";
|
|
5
|
+
export { type GetSeedParams, type TestVariantsFindBestErrorOptions, type TestVariantsRunOptions, type TestVariantsBestError, type TestVariantsRunResult, } from "./test-variants/testVariantsRun";
|
|
5
6
|
export { type TestVariantsSetArgs, type TestVariantsCall, createTestVariants, } from './test-variants/createTestVariants';
|
package/dist/lib/index.mjs
CHANGED
|
@@ -1,5 +1,8 @@
|
|
|
1
|
+
export { generateErrorVariantFilePath } from './test-variants/saveErrorVariants.mjs';
|
|
1
2
|
export { createTestVariants } from './test-variants/createTestVariants.mjs';
|
|
2
3
|
import 'tslib';
|
|
4
|
+
import 'fs';
|
|
5
|
+
import 'path';
|
|
3
6
|
import './test-variants/testVariantsIterable.mjs';
|
|
4
7
|
import './test-variants/testVariantsCreateTestRun.mjs';
|
|
5
8
|
import '@flemist/async-utils';
|
|
@@ -11,6 +11,9 @@ require('./argsToString.cjs');
|
|
|
11
11
|
require('@flemist/abort-controller-fast');
|
|
12
12
|
require('@flemist/time-limits');
|
|
13
13
|
require('../garbage-collect/garbageCollect.cjs');
|
|
14
|
+
require('./saveErrorVariants.cjs');
|
|
15
|
+
require('fs');
|
|
16
|
+
require('path');
|
|
14
17
|
|
|
15
18
|
function createTestVariants(test) {
|
|
16
19
|
return function testVariantsArgs(args) {
|
|
@@ -3,6 +3,6 @@ import { TestVariantsTemplatesExt } from "./testVariantsIterable";
|
|
|
3
3
|
import { TestVariantsCreateTestRunOptions, TestVariantsTest } from "./testVariantsCreateTestRun";
|
|
4
4
|
import { TestVariantsRunOptions, TestVariantsRunResult } from "./testVariantsRun";
|
|
5
5
|
import { Obj } from "./types";
|
|
6
|
-
export declare type TestVariantsCall<Args extends Obj> = (options?: null | TestVariantsRunOptions & TestVariantsCreateTestRunOptions<Args>) => PromiseOrValue<TestVariantsRunResult<Args>>;
|
|
6
|
+
export declare type TestVariantsCall<Args extends Obj> = <SavedArgs = Args>(options?: null | TestVariantsRunOptions<Args, SavedArgs> & TestVariantsCreateTestRunOptions<Args>) => PromiseOrValue<TestVariantsRunResult<Args>>;
|
|
7
7
|
export declare type TestVariantsSetArgs<Args extends Obj> = <ArgsExtra extends Obj>(args: TestVariantsTemplatesExt<Omit<Args, 'seed'>, Omit<ArgsExtra, 'seed'>>) => TestVariantsCall<Args>;
|
|
8
8
|
export declare function createTestVariants<Args extends Obj>(test: TestVariantsTest<Args>): TestVariantsSetArgs<Args>;
|
|
@@ -7,6 +7,9 @@ import './argsToString.mjs';
|
|
|
7
7
|
import '@flemist/abort-controller-fast';
|
|
8
8
|
import '@flemist/time-limits';
|
|
9
9
|
import '../garbage-collect/garbageCollect.mjs';
|
|
10
|
+
import './saveErrorVariants.mjs';
|
|
11
|
+
import 'fs';
|
|
12
|
+
import 'path';
|
|
10
13
|
|
|
11
14
|
function createTestVariants(test) {
|
|
12
15
|
return function testVariantsArgs(args) {
|
|
@@ -11,6 +11,9 @@ require('./testVariantsRun.cjs');
|
|
|
11
11
|
require('@flemist/abort-controller-fast');
|
|
12
12
|
require('@flemist/time-limits');
|
|
13
13
|
require('../garbage-collect/garbageCollect.cjs');
|
|
14
|
+
require('./saveErrorVariants.cjs');
|
|
15
|
+
require('fs');
|
|
16
|
+
require('path');
|
|
14
17
|
|
|
15
18
|
describe('test > testVariants perf', function () {
|
|
16
19
|
this.timeout(300000);
|
|
@@ -9,6 +9,9 @@ import './testVariantsRun.mjs';
|
|
|
9
9
|
import '@flemist/abort-controller-fast';
|
|
10
10
|
import '@flemist/time-limits';
|
|
11
11
|
import '../garbage-collect/garbageCollect.mjs';
|
|
12
|
+
import './saveErrorVariants.mjs';
|
|
13
|
+
import 'fs';
|
|
14
|
+
import 'path';
|
|
12
15
|
|
|
13
16
|
describe('test > testVariants perf', function () {
|
|
14
17
|
this.timeout(300000);
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
'use strict';
|
|
2
|
+
|
|
3
|
+
Object.defineProperty(exports, '__esModule', { value: true });
|
|
4
|
+
|
|
5
|
+
var tslib = require('tslib');
|
|
6
|
+
var fs = require('fs');
|
|
7
|
+
var path = require('path');
|
|
8
|
+
|
|
9
|
+
function _interopNamespace(e) {
|
|
10
|
+
if (e && e.__esModule) return e;
|
|
11
|
+
var n = Object.create(null);
|
|
12
|
+
if (e) {
|
|
13
|
+
Object.keys(e).forEach(function (k) {
|
|
14
|
+
if (k !== 'default') {
|
|
15
|
+
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
16
|
+
Object.defineProperty(n, k, d.get ? d : {
|
|
17
|
+
enumerable: true,
|
|
18
|
+
get: function () { return e[k]; }
|
|
19
|
+
});
|
|
20
|
+
}
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
n["default"] = e;
|
|
24
|
+
return Object.freeze(n);
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
var fs__namespace = /*#__PURE__*/_interopNamespace(fs);
|
|
28
|
+
var path__namespace = /*#__PURE__*/_interopNamespace(path);
|
|
29
|
+
|
|
30
|
+
/** Reads saved error variant files from directory, sorted by filename descending (newest first) */
|
|
31
|
+
function readErrorVariantFiles(dir) {
|
|
32
|
+
return tslib.__awaiter(this, void 0, void 0, function* () {
|
|
33
|
+
const stat = yield fs__namespace.promises.stat(dir).catch(() => null);
|
|
34
|
+
if (stat == null) {
|
|
35
|
+
return [];
|
|
36
|
+
}
|
|
37
|
+
if (!stat.isDirectory()) {
|
|
38
|
+
throw new Error(`[saveErrorVariants] path is not a directory: ${dir}`);
|
|
39
|
+
}
|
|
40
|
+
const files = yield fs__namespace.promises.readdir(dir);
|
|
41
|
+
const jsonFiles = files
|
|
42
|
+
.filter(file => file.endsWith('.json'))
|
|
43
|
+
.sort((a, b) => a > b ? -1 : a < b ? 1 : 0);
|
|
44
|
+
return jsonFiles.map(file => path__namespace.join(dir, file));
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
/** Parses saved error variant file and transforms JSON to args */
|
|
48
|
+
function parseErrorVariantFile(filePath, jsonToArgs) {
|
|
49
|
+
return tslib.__awaiter(this, void 0, void 0, function* () {
|
|
50
|
+
const content = yield fs__namespace.promises.readFile(filePath, 'utf-8');
|
|
51
|
+
let json;
|
|
52
|
+
try {
|
|
53
|
+
json = JSON.parse(content);
|
|
54
|
+
}
|
|
55
|
+
catch (err) {
|
|
56
|
+
throw new Error(`[saveErrorVariants] invalid JSON in file: ${filePath}`);
|
|
57
|
+
}
|
|
58
|
+
if (jsonToArgs) {
|
|
59
|
+
try {
|
|
60
|
+
return jsonToArgs(json);
|
|
61
|
+
}
|
|
62
|
+
catch (err) {
|
|
63
|
+
throw new Error(`[saveErrorVariants] jsonToArgs failed for file: ${filePath}`);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return json;
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
/** Generates default error variant file path: YYYY-MM-DD_HH-mm-ss.json (UTC) */
|
|
70
|
+
function generateErrorVariantFilePath(options) {
|
|
71
|
+
return options.sessionDate.toISOString().substring(0, 19).replace('T', '_').replaceAll(':', '-') + '.json';
|
|
72
|
+
}
|
|
73
|
+
/** Saves error-causing args to a JSON file, overwrites if file exists */
|
|
74
|
+
function saveErrorVariantFile(args, filePath, argsToJson) {
|
|
75
|
+
return tslib.__awaiter(this, void 0, void 0, function* () {
|
|
76
|
+
let content;
|
|
77
|
+
if (argsToJson) {
|
|
78
|
+
const result = argsToJson(args);
|
|
79
|
+
if (typeof result === 'string') {
|
|
80
|
+
content = result;
|
|
81
|
+
}
|
|
82
|
+
else {
|
|
83
|
+
content = JSON.stringify(result, null, 2);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
else {
|
|
87
|
+
content = JSON.stringify(args, null, 2);
|
|
88
|
+
}
|
|
89
|
+
yield fs__namespace.promises.mkdir(path__namespace.dirname(filePath), { recursive: true });
|
|
90
|
+
yield fs__namespace.promises.writeFile(filePath, content, 'utf-8');
|
|
91
|
+
});
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
exports.generateErrorVariantFilePath = generateErrorVariantFilePath;
|
|
95
|
+
exports.parseErrorVariantFile = parseErrorVariantFile;
|
|
96
|
+
exports.readErrorVariantFiles = readErrorVariantFiles;
|
|
97
|
+
exports.saveErrorVariantFile = saveErrorVariantFile;
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type GenerateErrorVariantFilePathOptions, type Obj } from "./types";
|
|
2
|
+
/** Reads saved error variant files from directory, sorted by filename descending (newest first) */
|
|
3
|
+
export declare function readErrorVariantFiles(dir: string): Promise<string[]>;
|
|
4
|
+
/** Parses saved error variant file and transforms JSON to args */
|
|
5
|
+
export declare function parseErrorVariantFile<Args extends Obj, SavedArgs>(filePath: string, jsonToArgs?: null | ((json: SavedArgs) => Args)): Promise<Args>;
|
|
6
|
+
/** Generates default error variant file path: YYYY-MM-DD_HH-mm-ss.json (UTC) */
|
|
7
|
+
export declare function generateErrorVariantFilePath(options: GenerateErrorVariantFilePathOptions): string;
|
|
8
|
+
/** Saves error-causing args to a JSON file, overwrites if file exists */
|
|
9
|
+
export declare function saveErrorVariantFile<Args extends Obj, SavedArgs>(args: Args, filePath: string, argsToJson?: null | ((args: Args) => string | SavedArgs)): Promise<void>;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { __awaiter } from 'tslib';
|
|
2
|
+
import * as fs from 'fs';
|
|
3
|
+
import * as path from 'path';
|
|
4
|
+
|
|
5
|
+
/** Reads saved error variant files from directory, sorted by filename descending (newest first) */
|
|
6
|
+
function readErrorVariantFiles(dir) {
|
|
7
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
8
|
+
const stat = yield fs.promises.stat(dir).catch(() => null);
|
|
9
|
+
if (stat == null) {
|
|
10
|
+
return [];
|
|
11
|
+
}
|
|
12
|
+
if (!stat.isDirectory()) {
|
|
13
|
+
throw new Error(`[saveErrorVariants] path is not a directory: ${dir}`);
|
|
14
|
+
}
|
|
15
|
+
const files = yield fs.promises.readdir(dir);
|
|
16
|
+
const jsonFiles = files
|
|
17
|
+
.filter(file => file.endsWith('.json'))
|
|
18
|
+
.sort((a, b) => a > b ? -1 : a < b ? 1 : 0);
|
|
19
|
+
return jsonFiles.map(file => path.join(dir, file));
|
|
20
|
+
});
|
|
21
|
+
}
|
|
22
|
+
/** Parses saved error variant file and transforms JSON to args */
|
|
23
|
+
function parseErrorVariantFile(filePath, jsonToArgs) {
|
|
24
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
25
|
+
const content = yield fs.promises.readFile(filePath, 'utf-8');
|
|
26
|
+
let json;
|
|
27
|
+
try {
|
|
28
|
+
json = JSON.parse(content);
|
|
29
|
+
}
|
|
30
|
+
catch (err) {
|
|
31
|
+
throw new Error(`[saveErrorVariants] invalid JSON in file: ${filePath}`);
|
|
32
|
+
}
|
|
33
|
+
if (jsonToArgs) {
|
|
34
|
+
try {
|
|
35
|
+
return jsonToArgs(json);
|
|
36
|
+
}
|
|
37
|
+
catch (err) {
|
|
38
|
+
throw new Error(`[saveErrorVariants] jsonToArgs failed for file: ${filePath}`);
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
return json;
|
|
42
|
+
});
|
|
43
|
+
}
|
|
44
|
+
/** Generates default error variant file path: YYYY-MM-DD_HH-mm-ss.json (UTC) */
|
|
45
|
+
function generateErrorVariantFilePath(options) {
|
|
46
|
+
return options.sessionDate.toISOString().substring(0, 19).replace('T', '_').replaceAll(':', '-') + '.json';
|
|
47
|
+
}
|
|
48
|
+
/** Saves error-causing args to a JSON file, overwrites if file exists */
|
|
49
|
+
function saveErrorVariantFile(args, filePath, argsToJson) {
|
|
50
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
51
|
+
let content;
|
|
52
|
+
if (argsToJson) {
|
|
53
|
+
const result = argsToJson(args);
|
|
54
|
+
if (typeof result === 'string') {
|
|
55
|
+
content = result;
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
content = JSON.stringify(result, null, 2);
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
else {
|
|
62
|
+
content = JSON.stringify(args, null, 2);
|
|
63
|
+
}
|
|
64
|
+
yield fs.promises.mkdir(path.dirname(filePath), { recursive: true });
|
|
65
|
+
yield fs.promises.writeFile(filePath, content, 'utf-8');
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export { generateErrorVariantFilePath, parseErrorVariantFile, readErrorVariantFiles, saveErrorVariantFile };
|
|
@@ -7,15 +7,57 @@ var abortControllerFast = require('@flemist/abort-controller-fast');
|
|
|
7
7
|
var asyncUtils = require('@flemist/async-utils');
|
|
8
8
|
var timeLimits = require('@flemist/time-limits');
|
|
9
9
|
var garbageCollect_garbageCollect = require('../garbage-collect/garbageCollect.cjs');
|
|
10
|
+
var testVariants_saveErrorVariants = require('./saveErrorVariants.cjs');
|
|
11
|
+
var path = require('path');
|
|
12
|
+
require('fs');
|
|
13
|
+
|
|
14
|
+
function _interopNamespace(e) {
|
|
15
|
+
if (e && e.__esModule) return e;
|
|
16
|
+
var n = Object.create(null);
|
|
17
|
+
if (e) {
|
|
18
|
+
Object.keys(e).forEach(function (k) {
|
|
19
|
+
if (k !== 'default') {
|
|
20
|
+
var d = Object.getOwnPropertyDescriptor(e, k);
|
|
21
|
+
Object.defineProperty(n, k, d.get ? d : {
|
|
22
|
+
enumerable: true,
|
|
23
|
+
get: function () { return e[k]; }
|
|
24
|
+
});
|
|
25
|
+
}
|
|
26
|
+
});
|
|
27
|
+
}
|
|
28
|
+
n["default"] = e;
|
|
29
|
+
return Object.freeze(n);
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
var path__namespace = /*#__PURE__*/_interopNamespace(path);
|
|
10
33
|
|
|
11
34
|
function testVariantsRun(testRun, variants, options = {}) {
|
|
12
|
-
var _a, _b, _c, _d, _e, _f;
|
|
35
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j;
|
|
13
36
|
return tslib.__awaiter(this, void 0, void 0, function* () {
|
|
14
|
-
const
|
|
15
|
-
const
|
|
16
|
-
const
|
|
17
|
-
const
|
|
18
|
-
|
|
37
|
+
const saveErrorVariants = options.saveErrorVariants;
|
|
38
|
+
const retriesPerVariant = (_a = saveErrorVariants === null || saveErrorVariants === void 0 ? void 0 : saveErrorVariants.retriesPerVariant) !== null && _a !== void 0 ? _a : 1;
|
|
39
|
+
const sessionDate = new Date();
|
|
40
|
+
const errorVariantFilePath = saveErrorVariants
|
|
41
|
+
? path__namespace.resolve(saveErrorVariants.dir, (_c = (_b = saveErrorVariants.getFilePath) === null || _b === void 0 ? void 0 : _b.call(saveErrorVariants, { sessionDate })) !== null && _c !== void 0 ? _c : testVariants_saveErrorVariants.generateErrorVariantFilePath({ sessionDate }))
|
|
42
|
+
: null;
|
|
43
|
+
// Replay phase: run previously saved error variants before normal iteration
|
|
44
|
+
if (saveErrorVariants) {
|
|
45
|
+
const files = yield testVariants_saveErrorVariants.readErrorVariantFiles(saveErrorVariants.dir);
|
|
46
|
+
for (const filePath of files) {
|
|
47
|
+
const args = yield testVariants_saveErrorVariants.parseErrorVariantFile(filePath, saveErrorVariants.jsonToArgs);
|
|
48
|
+
for (let retry = 0; retry < retriesPerVariant; retry++) {
|
|
49
|
+
const promiseOrResult = testRun(args, -1, null);
|
|
50
|
+
if (asyncUtils.isPromiseLike(promiseOrResult)) {
|
|
51
|
+
yield promiseOrResult;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
const GC_Iterations = (_d = options.GC_Iterations) !== null && _d !== void 0 ? _d : 1000000;
|
|
57
|
+
const GC_IterationsAsync = (_e = options.GC_IterationsAsync) !== null && _e !== void 0 ? _e : 10000;
|
|
58
|
+
const GC_Interval = (_f = options.GC_Interval) !== null && _f !== void 0 ? _f : 1000;
|
|
59
|
+
const logInterval = (_g = options.logInterval) !== null && _g !== void 0 ? _g : 5000;
|
|
60
|
+
const logCompleted = (_h = options.logCompleted) !== null && _h !== void 0 ? _h : true;
|
|
19
61
|
const abortSignalExternal = options.abortSignal;
|
|
20
62
|
const findBestError = options.findBestError;
|
|
21
63
|
const parallel = options.parallel === true
|
|
@@ -23,30 +65,55 @@ function testVariantsRun(testRun, variants, options = {}) {
|
|
|
23
65
|
: !options.parallel || options.parallel <= 0
|
|
24
66
|
? 1
|
|
25
67
|
: options.parallel;
|
|
26
|
-
const
|
|
27
|
-
let
|
|
68
|
+
const limitVariantsCount = (_j = options.limitVariantsCount) !== null && _j !== void 0 ? _j : null;
|
|
69
|
+
let cycleIndex = 0;
|
|
70
|
+
let repeatIndex = 0;
|
|
71
|
+
let seed = void 0;
|
|
28
72
|
let bestError = null;
|
|
29
73
|
let index = -1;
|
|
30
74
|
let args = {};
|
|
31
75
|
let variantsIterator = variants[Symbol.iterator]();
|
|
32
76
|
function nextVariant() {
|
|
33
77
|
while (true) {
|
|
78
|
+
// Try next repeat for current variant
|
|
79
|
+
if (findBestError && index >= 0 && (bestError == null || index < bestError.index)) {
|
|
80
|
+
repeatIndex++;
|
|
81
|
+
if (repeatIndex < findBestError.repeatsPerVariant) {
|
|
82
|
+
seed = findBestError.getSeed({
|
|
83
|
+
variantIndex: index,
|
|
84
|
+
cycleIndex,
|
|
85
|
+
repeatIndex,
|
|
86
|
+
totalIndex: cycleIndex * findBestError.repeatsPerVariant + repeatIndex,
|
|
87
|
+
});
|
|
88
|
+
return true;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
repeatIndex = 0;
|
|
34
92
|
index++;
|
|
35
|
-
if (
|
|
93
|
+
if (findBestError && cycleIndex >= findBestError.cycles) {
|
|
36
94
|
return false;
|
|
37
95
|
}
|
|
38
|
-
if (
|
|
96
|
+
if ((limitVariantsCount == null || index < limitVariantsCount)
|
|
97
|
+
&& (bestError == null || index < bestError.index)) {
|
|
39
98
|
const result = variantsIterator.next();
|
|
40
99
|
if (!result.done) {
|
|
41
100
|
args = result.value;
|
|
101
|
+
if (findBestError) {
|
|
102
|
+
seed = findBestError.getSeed({
|
|
103
|
+
variantIndex: index,
|
|
104
|
+
cycleIndex,
|
|
105
|
+
repeatIndex,
|
|
106
|
+
totalIndex: cycleIndex * findBestError.repeatsPerVariant + repeatIndex,
|
|
107
|
+
});
|
|
108
|
+
}
|
|
42
109
|
return true;
|
|
43
110
|
}
|
|
44
111
|
}
|
|
45
|
-
if (!
|
|
112
|
+
if (!findBestError) {
|
|
46
113
|
return false;
|
|
47
114
|
}
|
|
48
|
-
|
|
49
|
-
if (
|
|
115
|
+
cycleIndex++;
|
|
116
|
+
if (cycleIndex >= findBestError.cycles) {
|
|
50
117
|
return false;
|
|
51
118
|
}
|
|
52
119
|
index = -1;
|
|
@@ -75,7 +142,7 @@ function testVariantsRun(testRun, variants, options = {}) {
|
|
|
75
142
|
return tslib.__awaiter(this, void 0, void 0, function* () {
|
|
76
143
|
while (!(abortSignalExternal === null || abortSignalExternal === void 0 ? void 0 : abortSignalExternal.aborted) && (debug || nextVariant())) {
|
|
77
144
|
const _index = index;
|
|
78
|
-
const _args = Object.assign(Object.assign({}, args), { seed
|
|
145
|
+
const _args = Object.assign(Object.assign({}, args), { seed });
|
|
79
146
|
const now = (logInterval || GC_Interval) && Date.now();
|
|
80
147
|
if (logInterval && now - prevLogTime >= logInterval) {
|
|
81
148
|
// the log is required to prevent the karma browserNoActivityTimeout
|
|
@@ -109,6 +176,9 @@ function testVariantsRun(testRun, variants, options = {}) {
|
|
|
109
176
|
iterations += _iterationsSync + _iterationsAsync;
|
|
110
177
|
}
|
|
111
178
|
catch (err) {
|
|
179
|
+
if (errorVariantFilePath) {
|
|
180
|
+
yield testVariants_saveErrorVariants.saveErrorVariantFile(_args, errorVariantFilePath, saveErrorVariants.argsToJson);
|
|
181
|
+
}
|
|
112
182
|
if (findBestError) {
|
|
113
183
|
bestError = {
|
|
114
184
|
error: err,
|
|
@@ -146,6 +216,9 @@ function testVariantsRun(testRun, variants, options = {}) {
|
|
|
146
216
|
iterations += _iterationsSync + _iterationsAsync;
|
|
147
217
|
}
|
|
148
218
|
catch (err) {
|
|
219
|
+
if (errorVariantFilePath) {
|
|
220
|
+
yield testVariants_saveErrorVariants.saveErrorVariantFile(_args, errorVariantFilePath, saveErrorVariants.argsToJson);
|
|
221
|
+
}
|
|
149
222
|
if (findBestError) {
|
|
150
223
|
bestError = {
|
|
151
224
|
error: err,
|
|
@@ -1,10 +1,27 @@
|
|
|
1
1
|
import { TestVariantsTestRun } from './testVariantsCreateTestRun';
|
|
2
2
|
import { type IAbortSignalFast } from '@flemist/abort-controller-fast';
|
|
3
|
-
import { Obj } from "./types";
|
|
3
|
+
import { Obj, type SaveErrorVariantsOptions } from "./types";
|
|
4
|
+
/** Parameters passed to getSeed function for generating test seeds */
|
|
5
|
+
export declare type GetSeedParams = {
|
|
6
|
+
/** Index of current variant/parameter-combination being tested */
|
|
7
|
+
variantIndex: number;
|
|
8
|
+
/** Index of current cycle - full pass through all variants (0..cycles-1) */
|
|
9
|
+
cycleIndex: number;
|
|
10
|
+
/** Index of repeat for current variant within this cycle (0..repeatsPerVariant-1) */
|
|
11
|
+
repeatIndex: number;
|
|
12
|
+
/** Total index across all cycles: cycleIndex × repeatsPerVariant + repeatIndex */
|
|
13
|
+
totalIndex: number;
|
|
14
|
+
};
|
|
15
|
+
/** Options for finding the earliest failing variant across multiple test runs */
|
|
4
16
|
export declare type TestVariantsFindBestErrorOptions = {
|
|
5
|
-
|
|
17
|
+
/** Function to generate seed based on current iteration state */
|
|
18
|
+
getSeed: (params: GetSeedParams) => any;
|
|
19
|
+
/** Number of full passes through all variants */
|
|
20
|
+
cycles: number;
|
|
21
|
+
/** Number of repeat tests per variant within each cycle */
|
|
22
|
+
repeatsPerVariant: number;
|
|
6
23
|
};
|
|
7
|
-
export declare type TestVariantsRunOptions = {
|
|
24
|
+
export declare type TestVariantsRunOptions<Args extends Obj = Obj, SavedArgs = Args> = {
|
|
8
25
|
/** Wait for garbage collection after iterations */
|
|
9
26
|
GC_Iterations?: null | number;
|
|
10
27
|
/** Same as GC_Iterations but only for async test variants, required for 10000 and more of Promise rejections */
|
|
@@ -18,6 +35,10 @@ export declare type TestVariantsRunOptions = {
|
|
|
18
35
|
abortSignal?: null | IAbortSignalFast;
|
|
19
36
|
parallel?: null | number | boolean;
|
|
20
37
|
findBestError?: null | TestVariantsFindBestErrorOptions;
|
|
38
|
+
/** Save error-causing args to files and replay them before normal iteration */
|
|
39
|
+
saveErrorVariants?: null | SaveErrorVariantsOptions<Args, SavedArgs>;
|
|
40
|
+
/** Tests only first N variants, ignores the rest. If null or not specified, tests all variants */
|
|
41
|
+
limitVariantsCount?: null | number;
|
|
21
42
|
};
|
|
22
43
|
export declare type TestVariantsBestError<Args extends Obj> = {
|
|
23
44
|
error: any;
|
|
@@ -28,4 +49,4 @@ export declare type TestVariantsRunResult<Arg extends Obj> = {
|
|
|
28
49
|
iterations: number;
|
|
29
50
|
bestError: null | TestVariantsBestError<Arg>;
|
|
30
51
|
};
|
|
31
|
-
export declare function testVariantsRun<Args extends Obj>(testRun: TestVariantsTestRun<Args>, variants: Iterable<Args>, options?: TestVariantsRunOptions): Promise<TestVariantsRunResult<Args>>;
|
|
52
|
+
export declare function testVariantsRun<Args extends Obj, SavedArgs = Args>(testRun: TestVariantsTestRun<Args>, variants: Iterable<Args>, options?: TestVariantsRunOptions<Args, SavedArgs>): Promise<TestVariantsRunResult<Args>>;
|
|
@@ -1,17 +1,39 @@
|
|
|
1
1
|
import { __awaiter } from 'tslib';
|
|
2
2
|
import { AbortControllerFast } from '@flemist/abort-controller-fast';
|
|
3
|
-
import {
|
|
3
|
+
import { isPromiseLike, combineAbortSignals } from '@flemist/async-utils';
|
|
4
4
|
import { Pool } from '@flemist/time-limits';
|
|
5
5
|
import { garbageCollect } from '../garbage-collect/garbageCollect.mjs';
|
|
6
|
+
import { generateErrorVariantFilePath, readErrorVariantFiles, parseErrorVariantFile, saveErrorVariantFile } from './saveErrorVariants.mjs';
|
|
7
|
+
import * as path from 'path';
|
|
8
|
+
import 'fs';
|
|
6
9
|
|
|
7
10
|
function testVariantsRun(testRun, variants, options = {}) {
|
|
8
|
-
var _a, _b, _c, _d, _e, _f;
|
|
11
|
+
var _a, _b, _c, _d, _e, _f, _g, _h, _j;
|
|
9
12
|
return __awaiter(this, void 0, void 0, function* () {
|
|
10
|
-
const
|
|
11
|
-
const
|
|
12
|
-
const
|
|
13
|
-
const
|
|
14
|
-
|
|
13
|
+
const saveErrorVariants = options.saveErrorVariants;
|
|
14
|
+
const retriesPerVariant = (_a = saveErrorVariants === null || saveErrorVariants === void 0 ? void 0 : saveErrorVariants.retriesPerVariant) !== null && _a !== void 0 ? _a : 1;
|
|
15
|
+
const sessionDate = new Date();
|
|
16
|
+
const errorVariantFilePath = saveErrorVariants
|
|
17
|
+
? path.resolve(saveErrorVariants.dir, (_c = (_b = saveErrorVariants.getFilePath) === null || _b === void 0 ? void 0 : _b.call(saveErrorVariants, { sessionDate })) !== null && _c !== void 0 ? _c : generateErrorVariantFilePath({ sessionDate }))
|
|
18
|
+
: null;
|
|
19
|
+
// Replay phase: run previously saved error variants before normal iteration
|
|
20
|
+
if (saveErrorVariants) {
|
|
21
|
+
const files = yield readErrorVariantFiles(saveErrorVariants.dir);
|
|
22
|
+
for (const filePath of files) {
|
|
23
|
+
const args = yield parseErrorVariantFile(filePath, saveErrorVariants.jsonToArgs);
|
|
24
|
+
for (let retry = 0; retry < retriesPerVariant; retry++) {
|
|
25
|
+
const promiseOrResult = testRun(args, -1, null);
|
|
26
|
+
if (isPromiseLike(promiseOrResult)) {
|
|
27
|
+
yield promiseOrResult;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
const GC_Iterations = (_d = options.GC_Iterations) !== null && _d !== void 0 ? _d : 1000000;
|
|
33
|
+
const GC_IterationsAsync = (_e = options.GC_IterationsAsync) !== null && _e !== void 0 ? _e : 10000;
|
|
34
|
+
const GC_Interval = (_f = options.GC_Interval) !== null && _f !== void 0 ? _f : 1000;
|
|
35
|
+
const logInterval = (_g = options.logInterval) !== null && _g !== void 0 ? _g : 5000;
|
|
36
|
+
const logCompleted = (_h = options.logCompleted) !== null && _h !== void 0 ? _h : true;
|
|
15
37
|
const abortSignalExternal = options.abortSignal;
|
|
16
38
|
const findBestError = options.findBestError;
|
|
17
39
|
const parallel = options.parallel === true
|
|
@@ -19,30 +41,55 @@ function testVariantsRun(testRun, variants, options = {}) {
|
|
|
19
41
|
: !options.parallel || options.parallel <= 0
|
|
20
42
|
? 1
|
|
21
43
|
: options.parallel;
|
|
22
|
-
const
|
|
23
|
-
let
|
|
44
|
+
const limitVariantsCount = (_j = options.limitVariantsCount) !== null && _j !== void 0 ? _j : null;
|
|
45
|
+
let cycleIndex = 0;
|
|
46
|
+
let repeatIndex = 0;
|
|
47
|
+
let seed = void 0;
|
|
24
48
|
let bestError = null;
|
|
25
49
|
let index = -1;
|
|
26
50
|
let args = {};
|
|
27
51
|
let variantsIterator = variants[Symbol.iterator]();
|
|
28
52
|
function nextVariant() {
|
|
29
53
|
while (true) {
|
|
54
|
+
// Try next repeat for current variant
|
|
55
|
+
if (findBestError && index >= 0 && (bestError == null || index < bestError.index)) {
|
|
56
|
+
repeatIndex++;
|
|
57
|
+
if (repeatIndex < findBestError.repeatsPerVariant) {
|
|
58
|
+
seed = findBestError.getSeed({
|
|
59
|
+
variantIndex: index,
|
|
60
|
+
cycleIndex,
|
|
61
|
+
repeatIndex,
|
|
62
|
+
totalIndex: cycleIndex * findBestError.repeatsPerVariant + repeatIndex,
|
|
63
|
+
});
|
|
64
|
+
return true;
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
repeatIndex = 0;
|
|
30
68
|
index++;
|
|
31
|
-
if (
|
|
69
|
+
if (findBestError && cycleIndex >= findBestError.cycles) {
|
|
32
70
|
return false;
|
|
33
71
|
}
|
|
34
|
-
if (
|
|
72
|
+
if ((limitVariantsCount == null || index < limitVariantsCount)
|
|
73
|
+
&& (bestError == null || index < bestError.index)) {
|
|
35
74
|
const result = variantsIterator.next();
|
|
36
75
|
if (!result.done) {
|
|
37
76
|
args = result.value;
|
|
77
|
+
if (findBestError) {
|
|
78
|
+
seed = findBestError.getSeed({
|
|
79
|
+
variantIndex: index,
|
|
80
|
+
cycleIndex,
|
|
81
|
+
repeatIndex,
|
|
82
|
+
totalIndex: cycleIndex * findBestError.repeatsPerVariant + repeatIndex,
|
|
83
|
+
});
|
|
84
|
+
}
|
|
38
85
|
return true;
|
|
39
86
|
}
|
|
40
87
|
}
|
|
41
|
-
if (!
|
|
88
|
+
if (!findBestError) {
|
|
42
89
|
return false;
|
|
43
90
|
}
|
|
44
|
-
|
|
45
|
-
if (
|
|
91
|
+
cycleIndex++;
|
|
92
|
+
if (cycleIndex >= findBestError.cycles) {
|
|
46
93
|
return false;
|
|
47
94
|
}
|
|
48
95
|
index = -1;
|
|
@@ -71,7 +118,7 @@ function testVariantsRun(testRun, variants, options = {}) {
|
|
|
71
118
|
return __awaiter(this, void 0, void 0, function* () {
|
|
72
119
|
while (!(abortSignalExternal === null || abortSignalExternal === void 0 ? void 0 : abortSignalExternal.aborted) && (debug || nextVariant())) {
|
|
73
120
|
const _index = index;
|
|
74
|
-
const _args = Object.assign(Object.assign({}, args), { seed
|
|
121
|
+
const _args = Object.assign(Object.assign({}, args), { seed });
|
|
75
122
|
const now = (logInterval || GC_Interval) && Date.now();
|
|
76
123
|
if (logInterval && now - prevLogTime >= logInterval) {
|
|
77
124
|
// the log is required to prevent the karma browserNoActivityTimeout
|
|
@@ -105,6 +152,9 @@ function testVariantsRun(testRun, variants, options = {}) {
|
|
|
105
152
|
iterations += _iterationsSync + _iterationsAsync;
|
|
106
153
|
}
|
|
107
154
|
catch (err) {
|
|
155
|
+
if (errorVariantFilePath) {
|
|
156
|
+
yield saveErrorVariantFile(_args, errorVariantFilePath, saveErrorVariants.argsToJson);
|
|
157
|
+
}
|
|
108
158
|
if (findBestError) {
|
|
109
159
|
bestError = {
|
|
110
160
|
error: err,
|
|
@@ -142,6 +192,9 @@ function testVariantsRun(testRun, variants, options = {}) {
|
|
|
142
192
|
iterations += _iterationsSync + _iterationsAsync;
|
|
143
193
|
}
|
|
144
194
|
catch (err) {
|
|
195
|
+
if (errorVariantFilePath) {
|
|
196
|
+
yield saveErrorVariantFile(_args, errorVariantFilePath, saveErrorVariants.argsToJson);
|
|
197
|
+
}
|
|
145
198
|
if (findBestError) {
|
|
146
199
|
bestError = {
|
|
147
200
|
error: err,
|
|
@@ -1 +1,18 @@
|
|
|
1
1
|
export declare type Obj = Record<string, any>;
|
|
2
|
+
/** Options for generating error variant file path */
|
|
3
|
+
export declare type GenerateErrorVariantFilePathOptions = {
|
|
4
|
+
sessionDate: Date;
|
|
5
|
+
};
|
|
6
|
+
/** Options for saving and replaying error-causing parameter combinations */
|
|
7
|
+
export declare type SaveErrorVariantsOptions<Args, SavedArgs = Args> = {
|
|
8
|
+
/** Directory path for error variant JSON files */
|
|
9
|
+
dir: string;
|
|
10
|
+
/** Retry attempts per variant during replay phase (default: 1) */
|
|
11
|
+
retriesPerVariant?: null | number;
|
|
12
|
+
/** Custom file path generator; returns path relative to dir; null - use default path */
|
|
13
|
+
getFilePath?: null | ((options: GenerateErrorVariantFilePathOptions) => string | null);
|
|
14
|
+
/** Transform args before JSON serialization */
|
|
15
|
+
argsToJson?: null | ((args: Args) => string | SavedArgs);
|
|
16
|
+
/** Transform parsed JSON back to args */
|
|
17
|
+
jsonToArgs?: null | ((json: SavedArgs) => Args);
|
|
18
|
+
};
|