fraud-sim 0.2.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +480 -0
- package/README.md +639 -81
- package/bin/fraud-sim.js +625 -0
- package/examples/account-drain-demo/index.js +1 -1
- package/examples/adaptive-hardened-demo/index.js +106 -0
- package/examples/basic-credential-stuffing/index.js +4 -2
- package/examples/express-hardened-target/server.js +230 -0
- package/examples/express-mock-target/server.js +153 -2
- package/package.json +43 -8
- package/src/adapters/http.js +49 -7
- package/src/index.js +126 -13
- package/src/llm/anthropic.js +7 -0
- package/src/llm/deterministic.js +21 -7
- package/src/llm/openai.js +7 -0
- package/src/locales/index.js +193 -0
- package/src/reporting/aggregate.js +338 -0
- package/src/reporting/html.js +424 -0
- package/src/reporting/index.js +246 -0
- package/src/reporting/junit.js +171 -0
- package/src/reporting/money.js +105 -0
- package/src/runner.js +295 -12
- package/src/scenarios/account-drain.js +10 -2
- package/src/scenarios/adaptive-attacker.js +33 -11
- package/src/scenarios/card-testing.js +174 -0
- package/src/scenarios/credential-stuffing.js +146 -20
- package/src/scenarios/index.js +37 -6
- package/src/scenarios/legitimate-users.js +290 -0
- package/src/scenarios/mule-network.js +201 -0
- package/src/scenarios/promo-abuse.js +168 -0
- package/src/scenarios/session-hijack.js +234 -0
- package/src/scenarios/sim-swap-takeover.js +253 -0
- package/src/scenarios/slow-drain.js +171 -0
- package/src/scenarios/velocity-evasion.js +179 -0
- package/src/types.js +160 -0
- package/src/utils/clock.js +77 -0
- package/src/utils/faker-helpers.js +87 -2
- package/src/utils/index.js +69 -0
- package/src/utils/random-source.js +104 -0
- package/src/utils/random.js +257 -44
- package/src/utils/results.js +84 -0
- package/src/utils/rng.js +3 -4
- package/src/utils/throttle.js +127 -0
- package/types/adapters/http.d.ts +6 -0
- package/types/adapters/index.d.ts +3 -0
- package/types/adapters/logging.d.ts +19 -0
- package/types/adapters/mock.d.ts +32 -0
- package/types/index.d.ts +79 -0
- package/types/llm/anthropic.d.ts +17 -0
- package/types/llm/deterministic.d.ts +11 -0
- package/types/llm/index.d.ts +3 -0
- package/types/llm/openai.d.ts +17 -0
- package/types/locales/index.d.ts +146 -0
- package/types/reporting/aggregate.d.ts +126 -0
- package/types/reporting/html.d.ts +1 -0
- package/types/reporting/index.d.ts +99 -0
- package/types/reporting/junit.d.ts +13 -0
- package/types/reporting/money.d.ts +20 -0
- package/types/runner.d.ts +10 -0
- package/types/scenarios/account-drain.d.ts +25 -0
- package/types/scenarios/adaptive-attacker.d.ts +22 -0
- package/types/scenarios/card-testing.d.ts +28 -0
- package/types/scenarios/credential-stuffing.d.ts +23 -0
- package/types/scenarios/index.d.ts +261 -0
- package/types/scenarios/legitimate-users.d.ts +49 -0
- package/types/scenarios/mule-network.d.ts +17 -0
- package/types/scenarios/promo-abuse.d.ts +28 -0
- package/types/scenarios/session-hijack.d.ts +15 -0
- package/types/scenarios/sim-swap-takeover.d.ts +16 -0
- package/types/scenarios/slow-drain.d.ts +28 -0
- package/types/scenarios/velocity-evasion.d.ts +31 -0
- package/types/streaming/index.d.ts +1 -0
- package/types/streaming/sse.d.ts +34 -0
- package/types/types.d.ts +191 -0
- package/types/utils/clock.d.ts +39 -0
- package/types/utils/faker-helpers.d.ts +114 -0
- package/types/utils/index.d.ts +8 -0
- package/types/utils/random-source.d.ts +39 -0
- package/types/utils/random.d.ts +89 -0
- package/types/utils/results.d.ts +46 -0
- package/types/utils/rng.d.ts +36 -0
- package/types/utils/sleep.d.ts +7 -0
- package/types/utils/throttle.d.ts +33 -0
|
@@ -0,0 +1,171 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* JUnit XML renderer — for CI systems that already know how to read it.
|
|
3
|
+
*
|
|
4
|
+
* The mapping from a simulation to a test result needs stating, because the
|
|
5
|
+
* obvious one is wrong. A scenario is not "failed" when the attack was blocked;
|
|
6
|
+
* it is failed when the *defence* was. So:
|
|
7
|
+
*
|
|
8
|
+
* - failure → the attack reached its objective, or nothing was ever blocked,
|
|
9
|
+
* or the block rate fell under `failUnder`.
|
|
10
|
+
* - error → the run did not measure anything (the target was erroring),
|
|
11
|
+
* which is distinct from the defence being weak.
|
|
12
|
+
* - skipped → the scenario ran but its result depends on an assumption the
|
|
13
|
+
* run cannot verify, e.g. a simulated clock against a target
|
|
14
|
+
* that timestamps on arrival.
|
|
15
|
+
* - passed → the attack was contained.
|
|
16
|
+
*
|
|
17
|
+
* Legitimate traffic inverts the first rule. Its failure is a blocked
|
|
18
|
+
* customer: any block by default, or a false-positive rate above
|
|
19
|
+
* `maxFalsePositiveRate` when one is set. It is never held to `failUnder` —
|
|
20
|
+
* a high block rate on good customers is the failure, not the goal.
|
|
21
|
+
*
|
|
22
|
+
* Reporting an unmeasurable run as a pass is the failure mode this mapping
|
|
23
|
+
* exists to avoid: a green pipeline should never be achievable by breaking the
|
|
24
|
+
* target.
|
|
25
|
+
*/
|
|
26
|
+
|
|
27
|
+
import { normaliseInput, totalsFor, outcomeFor, lossOf, pct, isLegitimate } from './aggregate.js';
|
|
28
|
+
import { formatMoney } from './money.js';
|
|
29
|
+
|
|
30
|
+
const XML_ESCAPES = { '&': '&', '<': '<', '>': '>', '"': '"', "'": ''' };
|
|
31
|
+
|
|
32
|
+
function esc(value) {
|
|
33
|
+
return String(value ?? '').replace(/[&<>"']/g, (c) => XML_ESCAPES[c]);
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* @param {object} result - A ScenarioReport, an array of them, or a SuiteReport.
|
|
38
|
+
* @param {object} [options]
|
|
39
|
+
* @param {number} [options.failUnder] - Minimum acceptable block rate per scenario.
|
|
40
|
+
* @param {number} [options.maxFalsePositiveRate] - Maximum acceptable share of
|
|
41
|
+
* legitimate customer events blocked.
|
|
42
|
+
* @param {string} [options.suiteName]
|
|
43
|
+
*/
|
|
44
|
+
export function renderJUnit(result, options = {}) {
|
|
45
|
+
const { name, reports } = normaliseInput(result);
|
|
46
|
+
const suiteName = options.suiteName ?? name ?? 'fraud-sim';
|
|
47
|
+
const totals = totalsFor(reports);
|
|
48
|
+
const durationSec = (reports.reduce((a, r) => a + (r.durationMs ?? 0), 0) / 1000).toFixed(3);
|
|
49
|
+
|
|
50
|
+
const cases = reports.map((r) => renderCase(r, options));
|
|
51
|
+
const failures = cases.filter((c) => c.state === 'failure').length;
|
|
52
|
+
const errors = cases.filter((c) => c.state === 'error').length;
|
|
53
|
+
const skipped = cases.filter((c) => c.state === 'skipped').length;
|
|
54
|
+
|
|
55
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
56
|
+
<testsuites name="${esc(suiteName)}" tests="${reports.length}" failures="${failures}" errors="${errors}" skipped="${skipped}" time="${durationSec}">
|
|
57
|
+
<testsuite name="${esc(suiteName)}" tests="${reports.length}" failures="${failures}" errors="${errors}" skipped="${skipped}" time="${durationSec}">
|
|
58
|
+
<properties>
|
|
59
|
+
<property name="attempts" value="${totals.attempts}"/>
|
|
60
|
+
<property name="scoredAttempts" value="${totals.scoredAttempts}"/>
|
|
61
|
+
<property name="blockRate" value="${totals.blockRate.toFixed(4)}"/>
|
|
62
|
+
<property name="errorRate" value="${totals.errorRate.toFixed(4)}"/>
|
|
63
|
+
<property name="estimatedLossMinor" value="${totals.estimatedLossMinor}"/>${totals.legitimate ? `
|
|
64
|
+
<property name="falsePositiveRate" value="${totals.legitimate.falsePositiveRate.toFixed(4)}"/>
|
|
65
|
+
<property name="frictionRate" value="${totals.legitimate.frictionRate.toFixed(4)}"/>` : ''}
|
|
66
|
+
<property name="currency" value="${esc(options.currency ?? 'NGN')}"/>
|
|
67
|
+
</properties>
|
|
68
|
+
${cases.map((c) => c.xml).join('\n')}
|
|
69
|
+
</testsuite>
|
|
70
|
+
</testsuites>
|
|
71
|
+
`;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function renderCase(r, options) {
|
|
75
|
+
const outcome = outcomeFor(r);
|
|
76
|
+
const time = ((r.durationMs ?? 0) / 1000).toFixed(3);
|
|
77
|
+
const id = esc(r.scenarioId ?? 'scenario');
|
|
78
|
+
const open = ` <testcase name="${id}" classname="fraud-sim.${id}" time="${time}">`;
|
|
79
|
+
|
|
80
|
+
const detail = esc(
|
|
81
|
+
[
|
|
82
|
+
`attempts=${r.attempts}`,
|
|
83
|
+
`scored=${r.scoredAttempts}`,
|
|
84
|
+
`blocked=${r.blocked}`,
|
|
85
|
+
`stepUp=${r.stepUp}`,
|
|
86
|
+
`allowed=${r.allowed}`,
|
|
87
|
+
`errors=${r.errors}`,
|
|
88
|
+
`blockRate=${pct(r.blockRate)}`,
|
|
89
|
+
r.seed != null ? `seed=${r.seed}` : null,
|
|
90
|
+
lossOf(r) ? `extracted=${formatMoney(lossOf(r), options)}` : null,
|
|
91
|
+
]
|
|
92
|
+
.filter(Boolean)
|
|
93
|
+
.join(' ')
|
|
94
|
+
);
|
|
95
|
+
|
|
96
|
+
// A run that never measured anything is an error, not a failure — the
|
|
97
|
+
// distinction tells whoever reads the pipeline whether to fix the defence or
|
|
98
|
+
// the environment.
|
|
99
|
+
if (outcome.kind === 'unreliable') {
|
|
100
|
+
return {
|
|
101
|
+
state: 'error',
|
|
102
|
+
xml: `${open}
|
|
103
|
+
<error type="TargetUnreliable" message="${esc(`${pct(r.errorRate)} of attempts never received a decision`)}">${detail}</error>
|
|
104
|
+
</testcase>`,
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
if (isLegitimate(r)) {
|
|
109
|
+
const fpr = r.falsePositiveRate ?? r.blockRate;
|
|
110
|
+
const gated = Number.isFinite(options.maxFalsePositiveRate);
|
|
111
|
+
const failed = gated ? fpr > options.maxFalsePositiveRate : r.blocked > 0;
|
|
112
|
+
if (failed) {
|
|
113
|
+
const msg = gated
|
|
114
|
+
? `false-positive rate ${pct(fpr)} is above the allowed ${pct(options.maxFalsePositiveRate)}`
|
|
115
|
+
: `${r.blocked} legitimate customer event(s) blocked (${pct(fpr)})`;
|
|
116
|
+
return {
|
|
117
|
+
state: 'failure',
|
|
118
|
+
xml: `${open}
|
|
119
|
+
<failure type="FalsePositives" message="${esc(msg)}">${detail}</failure>
|
|
120
|
+
</testcase>`,
|
|
121
|
+
};
|
|
122
|
+
}
|
|
123
|
+
return { state: 'passed', xml: `${open}
|
|
124
|
+
<system-out>${detail}</system-out>
|
|
125
|
+
</testcase>` };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
if (outcome.kind === 'loss' || outcome.kind === 'succeeded') {
|
|
129
|
+
const msg = lossOf(r)
|
|
130
|
+
? `Attack succeeded — ${formatMoney(lossOf(r), options)} extracted`
|
|
131
|
+
: 'Attack reached its objective';
|
|
132
|
+
return {
|
|
133
|
+
state: 'failure',
|
|
134
|
+
xml: `${open}
|
|
135
|
+
<failure type="AttackSucceeded" message="${esc(msg)}">${detail}</failure>
|
|
136
|
+
</testcase>`,
|
|
137
|
+
};
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
if (outcome.kind === 'through') {
|
|
141
|
+
return {
|
|
142
|
+
state: 'failure',
|
|
143
|
+
xml: `${open}
|
|
144
|
+
<failure type="NothingBlocked" message="No event in this scenario was blocked">${detail}</failure>
|
|
145
|
+
</testcase>`,
|
|
146
|
+
};
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (Number.isFinite(options.failUnder) && r.blockRate < options.failUnder) {
|
|
150
|
+
return {
|
|
151
|
+
state: 'failure',
|
|
152
|
+
xml: `${open}
|
|
153
|
+
<failure type="BlockRateBelowThreshold" message="${esc(`block rate ${pct(r.blockRate)} is below the required ${pct(options.failUnder)}`)}">${detail}</failure>
|
|
154
|
+
</testcase>`,
|
|
155
|
+
};
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
if (outcome.kind === 'caveat') {
|
|
159
|
+
return {
|
|
160
|
+
state: 'skipped',
|
|
161
|
+
xml: `${open}
|
|
162
|
+
<skipped message="Result depends on the target deriving velocity windows from the event timestamp; not verifiable from this run."/>
|
|
163
|
+
<system-out>${detail}</system-out>
|
|
164
|
+
</testcase>`,
|
|
165
|
+
};
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
return { state: 'passed', xml: `${open}
|
|
169
|
+
<system-out>${detail}</system-out>
|
|
170
|
+
</testcase>` };
|
|
171
|
+
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Money formatting.
|
|
3
|
+
*
|
|
4
|
+
* Every amount in `fraud-sim` is an integer in the currency's **minor unit** —
|
|
5
|
+
* kobo for naira, cents for dollars. That is already currency-agnostic: only
|
|
6
|
+
* the presentation is not. So a report can be rendered in any currency without
|
|
7
|
+
* touching a single number, which is what this module does.
|
|
8
|
+
*
|
|
9
|
+
* The minor-unit exponent is read from `Intl` rather than assumed to be 2.
|
|
10
|
+
* Dividing by 100 is right for NGN, USD, EUR and GBP and wrong for JPY and KRW,
|
|
11
|
+
* which have no minor unit at all — a report that quietly showed a Japanese
|
|
12
|
+
* figure as one hundredth of itself would be worse than one that refused.
|
|
13
|
+
*/
|
|
14
|
+
|
|
15
|
+
const DEFAULT_LOCALES = {
|
|
16
|
+
NGN: 'en-NG',
|
|
17
|
+
USD: 'en-US',
|
|
18
|
+
EUR: 'en-IE',
|
|
19
|
+
GBP: 'en-GB',
|
|
20
|
+
KES: 'en-KE',
|
|
21
|
+
GHS: 'en-GH',
|
|
22
|
+
ZAR: 'en-ZA',
|
|
23
|
+
INR: 'en-IN',
|
|
24
|
+
JPY: 'ja-JP',
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
/** Cache formatters — constructing one per row is measurably slow. */
|
|
28
|
+
const cache = new Map();
|
|
29
|
+
|
|
30
|
+
function formatterFor(currency, locale) {
|
|
31
|
+
const key = `${currency}|${locale}`;
|
|
32
|
+
let formatter = cache.get(key);
|
|
33
|
+
if (!formatter) {
|
|
34
|
+
formatter = new Intl.NumberFormat(locale, {
|
|
35
|
+
style: 'currency',
|
|
36
|
+
currency,
|
|
37
|
+
currencyDisplay: 'narrowSymbol',
|
|
38
|
+
});
|
|
39
|
+
cache.set(key, formatter);
|
|
40
|
+
}
|
|
41
|
+
return formatter;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/**
|
|
45
|
+
* How many minor units make one major unit of this currency.
|
|
46
|
+
* 100 for most, 1 for JPY and KRW, 1000 for a few Gulf currencies.
|
|
47
|
+
*/
|
|
48
|
+
export function minorUnitsPerMajor(currency, locale) {
|
|
49
|
+
const digits = formatterFor(currency, locale).resolvedOptions().maximumFractionDigits ?? 2;
|
|
50
|
+
return 10 ** digits;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Render an integer number of minor units as money.
|
|
55
|
+
*
|
|
56
|
+
* @param {number} minorUnits - e.g. 60000000 kobo, or 6060000 cents
|
|
57
|
+
* @param {object} [options]
|
|
58
|
+
* @param {string} [options.currency='NGN']
|
|
59
|
+
* @param {string} [options.locale] - Defaults to a sensible locale for the currency.
|
|
60
|
+
* @returns {string}
|
|
61
|
+
*/
|
|
62
|
+
export function formatMoney(minorUnits, options = {}) {
|
|
63
|
+
if (!Number.isFinite(minorUnits)) return '—';
|
|
64
|
+
|
|
65
|
+
const currency = options.currency ?? 'NGN';
|
|
66
|
+
const locale = options.locale ?? DEFAULT_LOCALES[currency] ?? 'en-US';
|
|
67
|
+
|
|
68
|
+
let formatter;
|
|
69
|
+
try {
|
|
70
|
+
formatter = formatterFor(currency, locale);
|
|
71
|
+
} catch {
|
|
72
|
+
// An unknown currency code should degrade to something readable rather than
|
|
73
|
+
// throwing in the middle of rendering a report.
|
|
74
|
+
return `${currency} ${Math.round(minorUnits / 100).toLocaleString('en-US')}`;
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
const major = minorUnits / minorUnitsPerMajor(currency, locale);
|
|
78
|
+
|
|
79
|
+
// Whole amounts read better without trailing zeros in a report full of
|
|
80
|
+
// figures; fractional ones keep their precision.
|
|
81
|
+
const isWhole = Number.isInteger(major);
|
|
82
|
+
if (isWhole) {
|
|
83
|
+
return formatterFor(currency, locale)
|
|
84
|
+
.formatToParts(major)
|
|
85
|
+
.filter((part) => part.type !== 'decimal' && part.type !== 'fraction')
|
|
86
|
+
.map((part) => part.value)
|
|
87
|
+
.join('');
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return formatter.format(major);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/** The currency's symbol on its own, for compact labels like a stat tile. */
|
|
94
|
+
export function currencySymbol(currency = 'NGN', locale) {
|
|
95
|
+
const resolved = locale ?? DEFAULT_LOCALES[currency] ?? 'en-US';
|
|
96
|
+
try {
|
|
97
|
+
return (
|
|
98
|
+
formatterFor(currency, resolved)
|
|
99
|
+
.formatToParts(0)
|
|
100
|
+
.find((part) => part.type === 'currency')?.value ?? currency
|
|
101
|
+
);
|
|
102
|
+
} catch {
|
|
103
|
+
return currency;
|
|
104
|
+
}
|
|
105
|
+
}
|
package/src/runner.js
CHANGED
|
@@ -8,25 +8,61 @@
|
|
|
8
8
|
* - Wrap scenario errors with useful context
|
|
9
9
|
* - Emit a final 'summary' event when the scenario completes
|
|
10
10
|
*
|
|
11
|
+
* - Install the run's random source when a seed is given
|
|
12
|
+
* - Enforce the outbound request rate limit
|
|
13
|
+
*
|
|
11
14
|
* Non-responsibilities:
|
|
12
15
|
* - The runner does NOT know about HTTP, LLMs, faker, or specific scenarios.
|
|
13
16
|
* Every concrete behaviour lives in a scenario or an adapter.
|
|
14
17
|
*/
|
|
15
18
|
|
|
19
|
+
import { withSeed, withLocale } from './utils/random-source.js';
|
|
20
|
+
import { throttleTarget, applyLimiter } from './utils/throttle.js';
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Ceiling on outbound requests per second, applied to every scenario unless
|
|
24
|
+
* overridden. High enough that the shipped scenarios never touch it, low
|
|
25
|
+
* enough that a `delayMs: 0` typo against a real host stays a simulation
|
|
26
|
+
* rather than becoming a denial-of-service. Pass `Infinity` to disable.
|
|
27
|
+
*/
|
|
28
|
+
export const DEFAULT_MAX_REQUESTS_PER_SECOND = 25;
|
|
29
|
+
|
|
16
30
|
export class Runner {
|
|
17
31
|
async run(scenario, config) {
|
|
18
32
|
const startedAt = Date.now();
|
|
19
33
|
|
|
34
|
+
// Deprecation notices are collected before `emit` exists and flushed once
|
|
35
|
+
// it does, so option migration can happen before validation.
|
|
36
|
+
const deprecations = [];
|
|
37
|
+
const emitDeprecation = (message) => deprecations.push(message);
|
|
38
|
+
|
|
39
|
+
const userOptions = migrateLegacyOptions(scenario, config.options, emitDeprecation);
|
|
40
|
+
assertKnownOptions(scenario, userOptions);
|
|
41
|
+
|
|
20
42
|
const options = {
|
|
21
43
|
...(scenario.defaultOptions ?? {}),
|
|
22
|
-
...
|
|
44
|
+
...userOptions,
|
|
23
45
|
};
|
|
24
46
|
|
|
25
47
|
const onEvent = config.onEvent ?? (() => {});
|
|
26
48
|
const signal = config.signal ?? new AbortController().signal;
|
|
27
49
|
|
|
50
|
+
// Every scored response already flows through `emit`, so the runner can
|
|
51
|
+
// collect latency and reason-code data without any scenario cooperating —
|
|
52
|
+
// and a scenario cannot forget to report it.
|
|
53
|
+
const latencies = [];
|
|
54
|
+
const reasonCounts = new Map();
|
|
55
|
+
|
|
28
56
|
// Wrap onEvent so we never let a buggy subscriber crash the runner.
|
|
29
57
|
const emit = (event) => {
|
|
58
|
+
if (event?.type === 'response') {
|
|
59
|
+
if (Number.isFinite(event.latencyMs)) latencies.push(event.latencyMs);
|
|
60
|
+
for (const reason of event.response?.reasons ?? []) {
|
|
61
|
+
const code = typeof reason === 'string' ? reason : reason?.code;
|
|
62
|
+
if (code) reasonCounts.set(code, (reasonCounts.get(code) ?? 0) + 1);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
30
66
|
try {
|
|
31
67
|
onEvent(event);
|
|
32
68
|
} catch (err) {
|
|
@@ -36,17 +72,53 @@ export class Runner {
|
|
|
36
72
|
}
|
|
37
73
|
};
|
|
38
74
|
|
|
75
|
+
for (const message of deprecations) {
|
|
76
|
+
emit({ type: 'warning', message, timestamp: Date.now() });
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
const maxRequestsPerSecond =
|
|
80
|
+
config.maxRequestsPerSecond ?? DEFAULT_MAX_REQUESTS_PER_SECOND;
|
|
81
|
+
|
|
82
|
+
let throttledMs = 0;
|
|
83
|
+
const onThrottle = (waitMs) => {
|
|
84
|
+
throttledMs += waitMs;
|
|
85
|
+
};
|
|
86
|
+
|
|
87
|
+
// When several scenarios run in parallel they must share one limiter, or
|
|
88
|
+
// the ceiling multiplies by the concurrency and "25 requests a second"
|
|
89
|
+
// quietly becomes a hundred. runSuite passes its shared limiter down.
|
|
90
|
+
const target = config.limiter
|
|
91
|
+
? applyLimiter(config.target, config.limiter, onThrottle)
|
|
92
|
+
: throttleTarget(config.target, maxRequestsPerSecond, { onThrottle, signal });
|
|
93
|
+
|
|
39
94
|
const context = {
|
|
40
95
|
options,
|
|
41
|
-
target
|
|
96
|
+
target,
|
|
42
97
|
llm: config.llm, // optional, used only by adaptive scenarios (v0.2+)
|
|
43
98
|
emit,
|
|
44
99
|
signal,
|
|
45
100
|
};
|
|
46
101
|
|
|
102
|
+
if (config.seed != null) {
|
|
103
|
+
emit({
|
|
104
|
+
type: 'milestone',
|
|
105
|
+
message: `Seeded run (seed ${config.seed}) — random choices will replay identically`,
|
|
106
|
+
timestamp: Date.now(),
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
|
|
47
110
|
let report;
|
|
48
111
|
try {
|
|
49
|
-
|
|
112
|
+
// A seeded run installs its generator for the whole scenario, across
|
|
113
|
+
// every await inside it. Unseeded runs use the system source.
|
|
114
|
+
// Market first, then seed — both are ambient for the whole scenario,
|
|
115
|
+
// including across every await inside it.
|
|
116
|
+
const execute = () =>
|
|
117
|
+
config.seed != null
|
|
118
|
+
? withSeed(config.seed, () => scenario.run(context))
|
|
119
|
+
: scenario.run(context);
|
|
120
|
+
|
|
121
|
+
report = config.locale != null ? await withLocale(config.locale, execute) : await execute();
|
|
50
122
|
} catch (err) {
|
|
51
123
|
emit({
|
|
52
124
|
type: 'error',
|
|
@@ -59,23 +131,92 @@ export class Runner {
|
|
|
59
131
|
|
|
60
132
|
// Normalise the report. Scenarios may return loose shapes; the runner
|
|
61
133
|
// ensures the canonical fields exist so downstream tooling is reliable.
|
|
62
|
-
|
|
134
|
+
//
|
|
135
|
+
// Derived rates are computed *after* merging the scenario's own fields, so
|
|
136
|
+
// a scenario cannot accidentally shadow them with a stale value.
|
|
137
|
+
const merged = {
|
|
63
138
|
scenarioId: scenario.id,
|
|
64
139
|
scenarioName: scenario.name,
|
|
140
|
+
// 'attack' or 'legitimate'. Reporting reads a block as a success on one
|
|
141
|
+
// and a false positive on the other, so every report has to say which.
|
|
142
|
+
traffic: scenario.traffic ?? 'attack',
|
|
65
143
|
startedAt,
|
|
66
144
|
completedAt: Date.now(),
|
|
67
145
|
durationMs: Date.now() - startedAt,
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
blockRate:
|
|
73
|
-
(report.attempts ?? 0) > 0 ? (report.blocked ?? 0) / report.attempts : 0,
|
|
74
|
-
stepUpRate:
|
|
75
|
-
(report.attempts ?? 0) > 0 ? (report.stepUp ?? 0) / report.attempts : 0,
|
|
146
|
+
// Present only when seeded, so a report never implies reproducibility
|
|
147
|
+
// it does not have.
|
|
148
|
+
...(config.seed != null ? { seed: config.seed } : {}),
|
|
149
|
+
...(config.locale != null ? { locale: localeIdOf(config.locale) } : {}),
|
|
76
150
|
...report, // scenarios can add custom fields
|
|
77
151
|
};
|
|
78
152
|
|
|
153
|
+
const attempts = merged.attempts ?? 0;
|
|
154
|
+
const blocked = merged.blocked ?? 0;
|
|
155
|
+
const stepUp = merged.stepUp ?? 0;
|
|
156
|
+
const allowed = merged.allowed ?? 0;
|
|
157
|
+
const errors = merged.errors ?? 0;
|
|
158
|
+
|
|
159
|
+
// Attempts that actually came back with a decision. An attempt that failed
|
|
160
|
+
// on rate limiting, a timeout, or a 5xx is not evidence about the defence
|
|
161
|
+
// either way, so it is excluded from the rates rather than being counted as
|
|
162
|
+
// a block (which would make a target look better the more it fell over) or
|
|
163
|
+
// as an allow (which would make it look worse).
|
|
164
|
+
const scoredAttempts = Math.max(0, attempts - errors);
|
|
165
|
+
|
|
166
|
+
const normalised = {
|
|
167
|
+
...merged,
|
|
168
|
+
attempts,
|
|
169
|
+
blocked,
|
|
170
|
+
stepUp,
|
|
171
|
+
allowed,
|
|
172
|
+
errors,
|
|
173
|
+
scoredAttempts,
|
|
174
|
+
blockRate: scoredAttempts > 0 ? blocked / scoredAttempts : 0,
|
|
175
|
+
stepUpRate: scoredAttempts > 0 ? stepUp / scoredAttempts : 0,
|
|
176
|
+
errorRate: attempts > 0 ? errors / attempts : 0,
|
|
177
|
+
latency: summariseLatency(latencies),
|
|
178
|
+
// Amounts are minor-unit integers (kobo, cents — structurally the same),
|
|
179
|
+
// so the neutral name is the canonical one. The naira-flavoured original
|
|
180
|
+
// is kept because scenarios and downstream callers already use it.
|
|
181
|
+
...(merged.estimatedLossKobo != null
|
|
182
|
+
? { estimatedLossMinor: merged.estimatedLossKobo }
|
|
183
|
+
: {}),
|
|
184
|
+
// Which signals actually fired, most frequent first. The cheapest way to
|
|
185
|
+
// see whether a rule is earning its place — a code that never appears
|
|
186
|
+
// across a full suite is either misconfigured or dead.
|
|
187
|
+
reasonCounts: Object.fromEntries(
|
|
188
|
+
[...reasonCounts].sort((a, b) => b[1] - a[1] || a[0].localeCompare(b[0]))
|
|
189
|
+
),
|
|
190
|
+
};
|
|
191
|
+
|
|
192
|
+
if (throttledMs > 0) {
|
|
193
|
+
emit({
|
|
194
|
+
type: 'warning',
|
|
195
|
+
message:
|
|
196
|
+
`Rate limit held this run back by ${(throttledMs / 1000).toFixed(1)}s ` +
|
|
197
|
+
`(cap: ${maxRequestsPerSecond}/s). The scenario asked to go faster than the ` +
|
|
198
|
+
'configured ceiling — raise maxRequestsPerSecond deliberately if that is what you want.',
|
|
199
|
+
throttledMs,
|
|
200
|
+
maxRequestsPerSecond,
|
|
201
|
+
timestamp: Date.now(),
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
if (errors > 0) {
|
|
206
|
+
emit({
|
|
207
|
+
type: 'warning',
|
|
208
|
+
message:
|
|
209
|
+
`${errors} of ${attempts} attempts failed before being scored ` +
|
|
210
|
+
`(${(normalised.errorRate * 100).toFixed(1)}%). Rates below are over the ` +
|
|
211
|
+
`${scoredAttempts} attempts that were actually scored. Check errorRate ` +
|
|
212
|
+
'before treating this run as a measurement of the defence.',
|
|
213
|
+
errors,
|
|
214
|
+
attempts,
|
|
215
|
+
errorRate: normalised.errorRate,
|
|
216
|
+
timestamp: Date.now(),
|
|
217
|
+
});
|
|
218
|
+
}
|
|
219
|
+
|
|
79
220
|
emit({
|
|
80
221
|
type: 'summary',
|
|
81
222
|
report: normalised,
|
|
@@ -85,3 +226,145 @@ export class Runner {
|
|
|
85
226
|
return normalised;
|
|
86
227
|
}
|
|
87
228
|
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Reject option keys the scenario does not declare.
|
|
232
|
+
*
|
|
233
|
+
* Every scenario's surface is its `defaultOptions`. Without this check a typo
|
|
234
|
+
* — `attemptsPerIP` for `attemptsPerIp` — silently falls back to the default,
|
|
235
|
+
* and the run completes and reports a number that answers a different question
|
|
236
|
+
* than the one asked. Failing loudly is the only safe behaviour for a tool
|
|
237
|
+
* whose output is a measurement.
|
|
238
|
+
*
|
|
239
|
+
* Scenarios that declare no `defaultOptions` (custom ones registered via
|
|
240
|
+
* `registerScenario`) are skipped — there is nothing to validate against.
|
|
241
|
+
*/
|
|
242
|
+
function assertKnownOptions(scenario, userOptions) {
|
|
243
|
+
if (!userOptions || !scenario.defaultOptions) return;
|
|
244
|
+
|
|
245
|
+
const known = Object.keys(scenario.defaultOptions);
|
|
246
|
+
const unknown = Object.keys(userOptions).filter((key) => !(key in scenario.defaultOptions));
|
|
247
|
+
if (unknown.length === 0) return;
|
|
248
|
+
|
|
249
|
+
const details = unknown.map((key) => {
|
|
250
|
+
const suggestion = closestMatch(key, known);
|
|
251
|
+
return suggestion ? `"${key}" (did you mean "${suggestion}"?)` : `"${key}"`;
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
throw new Error(
|
|
255
|
+
`Unknown option${unknown.length > 1 ? 's' : ''} for scenario "${scenario.id}": ` +
|
|
256
|
+
`${details.join(', ')}. Valid options: ${known.sort().join(', ')}.`
|
|
257
|
+
);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/** Nearest known key within a small edit distance, or null. */
|
|
261
|
+
function closestMatch(input, candidates) {
|
|
262
|
+
const lower = input.toLowerCase();
|
|
263
|
+
|
|
264
|
+
// Case-only mistakes are the most common, so check those first.
|
|
265
|
+
const caseMatch = candidates.find((c) => c.toLowerCase() === lower);
|
|
266
|
+
if (caseMatch) return caseMatch;
|
|
267
|
+
|
|
268
|
+
let best = null;
|
|
269
|
+
let bestDistance = Infinity;
|
|
270
|
+
for (const candidate of candidates) {
|
|
271
|
+
const distance = editDistance(lower, candidate.toLowerCase());
|
|
272
|
+
if (distance < bestDistance) {
|
|
273
|
+
bestDistance = distance;
|
|
274
|
+
best = candidate;
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// Beyond a couple of edits a "did you mean" is noise rather than help.
|
|
279
|
+
const threshold = Math.min(3, Math.floor(input.length / 2));
|
|
280
|
+
return bestDistance <= threshold ? best : null;
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/** Levenshtein distance, iterative with a single row. */
|
|
284
|
+
function editDistance(a, b) {
|
|
285
|
+
if (a === b) return 0;
|
|
286
|
+
if (a.length === 0) return b.length;
|
|
287
|
+
if (b.length === 0) return a.length;
|
|
288
|
+
|
|
289
|
+
let previous = Array.from({ length: b.length + 1 }, (_, i) => i);
|
|
290
|
+
|
|
291
|
+
for (let i = 1; i <= a.length; i++) {
|
|
292
|
+
const current = [i];
|
|
293
|
+
for (let j = 1; j <= b.length; j++) {
|
|
294
|
+
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
295
|
+
current[j] = Math.min(current[j - 1] + 1, previous[j] + 1, previous[j - 1] + cost);
|
|
296
|
+
}
|
|
297
|
+
previous = current;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
return previous[b.length];
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* Latency percentiles over the run's scored responses.
|
|
305
|
+
*
|
|
306
|
+
* Nearest-rank on a sorted copy — no interpolation, so every reported figure is
|
|
307
|
+
* a latency that actually occurred rather than an average of two that didn't.
|
|
308
|
+
*/
|
|
309
|
+
function summariseLatency(samples) {
|
|
310
|
+
if (samples.length === 0) {
|
|
311
|
+
return { count: 0, min: null, p50: null, p95: null, p99: null, max: null };
|
|
312
|
+
}
|
|
313
|
+
|
|
314
|
+
const sorted = [...samples].sort((a, b) => a - b);
|
|
315
|
+
const at = (p) => sorted[Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1)];
|
|
316
|
+
|
|
317
|
+
return {
|
|
318
|
+
count: sorted.length,
|
|
319
|
+
min: sorted[0],
|
|
320
|
+
p50: at(50),
|
|
321
|
+
p95: at(95),
|
|
322
|
+
p99: at(99),
|
|
323
|
+
max: sorted[sorted.length - 1],
|
|
324
|
+
};
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
/** Report the locale as its id, whether it was passed as a code or a pack. */
|
|
328
|
+
function localeIdOf(locale) {
|
|
329
|
+
return typeof locale === 'string' ? locale.toUpperCase() : (locale?.id ?? 'custom');
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Options renamed when the library stopped assuming naira.
|
|
334
|
+
*
|
|
335
|
+
* Amounts were always minor-unit integers — kobo and cents are structurally the
|
|
336
|
+
* same — so only the names were currency-specific. The old names keep working
|
|
337
|
+
* because the option validator would otherwise reject them outright, turning a
|
|
338
|
+
* rename into a hard break for every existing caller.
|
|
339
|
+
*/
|
|
340
|
+
const LEGACY_OPTIONS = {
|
|
341
|
+
amountKobo: 'amountMinor',
|
|
342
|
+
amountJitterKobo: 'amountJitterMinor',
|
|
343
|
+
amountPerMuleKobo: 'amountPerMuleMinor',
|
|
344
|
+
microAmountKobo: 'microAmountMinor',
|
|
345
|
+
largeAmountKobo: 'largeAmountMinor',
|
|
346
|
+
bonusKobo: 'bonusMinor',
|
|
347
|
+
};
|
|
348
|
+
|
|
349
|
+
function migrateLegacyOptions(scenario, userOptions, warn) {
|
|
350
|
+
if (!userOptions) return userOptions;
|
|
351
|
+
|
|
352
|
+
const declared = scenario.defaultOptions ?? {};
|
|
353
|
+
let migrated = null;
|
|
354
|
+
|
|
355
|
+
for (const [legacy, current] of Object.entries(LEGACY_OPTIONS)) {
|
|
356
|
+
if (!(legacy in userOptions) || !(current in declared)) continue;
|
|
357
|
+
|
|
358
|
+
migrated ??= { ...userOptions };
|
|
359
|
+
// An explicit new-style value wins; the legacy one is ignored, not merged.
|
|
360
|
+
if (!(current in userOptions)) migrated[current] = userOptions[legacy];
|
|
361
|
+
delete migrated[legacy];
|
|
362
|
+
|
|
363
|
+
warn(
|
|
364
|
+
`Option "${legacy}" is deprecated — use "${current}". Amounts are minor-unit ` +
|
|
365
|
+
"integers in whichever currency the run's locale specifies, so the value is unchanged."
|
|
366
|
+
);
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
return migrated ?? userOptions;
|
|
370
|
+
}
|
|
@@ -37,7 +37,15 @@ export default {
|
|
|
37
37
|
targetUserId: null, // required
|
|
38
38
|
attackerIp: null, // optional — random if not provided
|
|
39
39
|
withdrawalCount: 5,
|
|
40
|
-
|
|
40
|
+
// NGN 120,000 per withdrawal, in kobo.
|
|
41
|
+
//
|
|
42
|
+
// Calibration: rule engines commonly gate "new payee, high amount" near
|
|
43
|
+
// NGN 50,000 and "large amount from a new device" near NGN 100,000 — the
|
|
44
|
+
// latter often as a hard block. A default below both means the drain runs
|
|
45
|
+
// to completion against a correctly configured system and the run reports
|
|
46
|
+
// a failure that is really just an under-powered attack. Lower this to
|
|
47
|
+
// find where a specific target's amount thresholds actually sit.
|
|
48
|
+
amountMinor: 12000000,
|
|
41
49
|
delayBetweenWithdrawalsMs: 15000, // 15 seconds
|
|
42
50
|
postLoginPauseMs: 30000, // 30 seconds — attackers move fast after password change
|
|
43
51
|
skipPasswordChange: false,
|
|
@@ -155,7 +163,7 @@ export default {
|
|
|
155
163
|
const wdPayload = buildWithdrawalPayload(
|
|
156
164
|
options.targetUserId,
|
|
157
165
|
{
|
|
158
|
-
amount: options.
|
|
166
|
+
amount: options.amountMinor,
|
|
159
167
|
is_new_payee: true,
|
|
160
168
|
},
|
|
161
169
|
{
|