@alex-dbs/coderuler 1.3.1 → 1.5.0

package/README.md

@@ -15,6 +15,7 @@ Install the CLI globally:
 ```sh
 npm install -g @alex-dbs/coderuler
 coderuler --help
+coderuler skill install
 ```
 
 The npm package is scoped. The command is `coderuler`. A global installation can load rules without a project-local dependency.
@@ -24,6 +25,7 @@ For a project-specific CLI version, install it locally:
 ```sh
 npm install --save-dev @alex-dbs/coderuler
 npx coderuler --help
+npx coderuler skill install
 ```
 
 The global installation is optional: with only the project-local dependency, run commands through `npx coderuler`.
@@ -114,14 +116,17 @@ export default class SafeResponseHeaders extends Rule {
   });
 
   check = ai({
-    rule: 'Report a violation only when this function writes request data into an HTTP response header without rejecting CR and LF first. Judge only this function. Do not assume unseen callers or helpers validate the data.',
+    instruction: `Reject CR and LF in untrusted values before inserting them into
+HTTP response headers or status lines.
+This policy covers response construction and writes visible in the selected function. Hidden helper behavior and unseen callers are outside its scope.
+Treat serialization arguments and statusMessage as untrusted. Fixed values and values checked for both CR and LF meet the requirement. Preserve fixed HTTP delimiters.`,
     input: subject => (subject as ScopeSubject).source,
     violationMessage: 'Reject CR and LF in request data before writing a response header.',
   });
 }
 ```
 
-This example shows an AI check for CWE-113. 
+This example shows an AI check for CWE-113.
 
 AI rules require a configured service and can incur charges.
 
@@ -155,7 +160,7 @@ A scope is a file or selected part of a file. CodeRuler validates selected rules
 The `settings` field sets AI options for one rule, such as its model, decision threshold, and input or output token prices.
 
 > [!TIP]
-> Browse the [`.coderuler` folder](.coderuler) for rule examples.
+> Browse the [`.coderuler.examples` folder](.coderuler.examples) for rule examples.
 
 ## Commands
 

package/dist/adapter/cli/commands/check.js

@@ -12,6 +12,7 @@ async function runCheck(target, options, context, services) {
         ? createProgress(context.writeError, () => context.terminal?.columns ?? 80)
         : undefined;
     let checkout;
+    let rulesCheckout;
     try {
         let reports;
         let durationMs;
@@ -24,21 +25,28 @@ async function runCheck(target, options, context, services) {
             }
             const area = await services.resolveTarget(checkout?.root ?? context.cwd, target);
             root = area.root;
-            let changedFiles = checkout?.changedFiles ?? (diffOnly
-                ? await services.changedFiles(area.root, options.base ?? 'HEAD')
-                : undefined);
-            const choice = options.rules === undefined
-                ? {}
-                : await services.resolveRuleSelector(options.rules, context.cwd, true);
+            let changes = checkout === undefined
+                ? (diffOnly ? await services.collectChanges(area.root, options.base ?? 'HEAD') : undefined)
+                : { changedFiles: checkout.changedFiles, changedLines: checkout.changedLines };
+            let choice = {};
+            if (options.rules !== undefined) {
+                rulesCheckout = await services.checkoutRuleRepository(options.rules);
+                choice = rulesCheckout === undefined
+                    ? await services.resolveRuleSelector(options.rules, context.cwd, true)
+                    : { explicit: rulesCheckout.root };
+            }
             const prepared = await planWalk(context.cwd, services, area.root, filepath => area.contains(filepath), choice, checkout?.rulesRoot ?? area.root);
             rules = options.tags.length === 0 && options.excludeTags.length === 0
                 ? prepared.rules
                 : prepared.rules.filter(({ rule }) => (options.tags.length === 0 ||
                     rule.tags?.some(tag => options.tags.includes(tag)) === true) &&
                     !rule.tags?.some(tag => options.excludeTags.includes(tag)));
-            if (changedFiles !== undefined) {
+            if (changes !== undefined) {
                 const ruleFiles = new Set(prepared.rules.map(entry => relative(area.root, resolve(entry.ruleFile)).split(sep).join('/')));
-                changedFiles = new Set([...changedFiles].filter(path => !ruleFiles.has(path)));
+                changes = {
+                    changedFiles: new Set([...changes.changedFiles].filter(path => !ruleFiles.has(path))),
+                    changedLines: new Map([...changes.changedLines].filter(([path]) => !ruleFiles.has(path))),
+                };
             }
             const concurrency = debug ? 1 : options.concurrency;
             walkthrough = verbose
@@ -59,7 +67,10 @@ async function runCheck(target, options, context, services) {
                 concurrency,
                 resolveDecision: services.decisionResolver(options.threshold),
                 ...(emit === undefined ? {} : { emit }),
-                ...(changedFiles === undefined ? {} : { changedFiles }),
+                ...(changes === undefined ? {} : {
+                    changedFiles: changes.changedFiles,
+                    changedLines: changes.changedLines,
+                }),
             };
             const startedAt = performance.now();
             reports = await services.runRules(rules, runContext);
@@ -87,7 +98,7 @@ async function runCheck(target, options, context, services) {
         }
     }
     finally {
-        await checkout?.cleanup();
+        await Promise.all([checkout?.cleanup(), rulesCheckout?.cleanup()]);
     }
 }
 export { runCheck };

package/dist/adapter/cli/commands/test.js

@@ -22,7 +22,7 @@ async function runTest(rulePath, options, context, services) {
         context.write(renderRuleTestHeading(ruleName, context.redact, color));
         previousRuleName = ruleName;
         for (let index = 0; index < failures.length; index += 1) {
-            context.write(renderRuleTestResult(failures[index], context.redact, color, index === failures.length - 1));
+            context.write(renderRuleTestResult(failures[index], context.redact, color, index === failures.length - 1, options.aiScores === true));
         }
         failures.length = 0;
     };
@@ -53,7 +53,7 @@ async function runTest(rulePath, options, context, services) {
                     context.write(renderRuleTestHeading(result.ruleName, context.redact, color));
                     previousRuleName = result.ruleName;
                 }
-                context.write(renderRuleTestResult(result, context.redact, color, lastInRule));
+                context.write(renderRuleTestResult(result, context.redact, color, lastInRule, options.aiScores === true));
             }
             progress?.message(`Running tests (${passed + failed} completed)`);
         }, options.timeout, options.concurrency, options.ai === true);

package/dist/adapter/cli/program.js

@@ -128,6 +128,7 @@ export function createProgram(services) {
         .description('Run the declared test cases for the selected rules')
         .argument('[rule]', 'rule name, module path, or directory')
         .option('--ai', 'include aiTests that use the configured decision backend')
+        .option('--ai-scores', 'show AI scores for aiTests')
         .option('--failures-only', 'show only failed test cases')
         .addOption(new Option('--timeout <duration>', 'per-test timeout (ms, s, or m)')
         .argParser(parseTimeout)
@@ -142,7 +143,7 @@ export function createProgram(services) {
         .command('check')
         .description('Evaluate every selected rule against the working tree')
         .argument('[path]', 'target file or folder that narrows selection', '.')
-        .option('--rules <rule>', 'rule name, module path, or directory')
+        .option('--rules <rule>', 'rule name, module path, directory, or Git repository URL[#revision]')
         .option('--tags <tags...>', 'run rules matching any listed tag', [])
         .option('--exclude-tags <tags...>', 'exclude rules matching any listed tag', [])
         .option('--json', 'print violations as {durationMs, reports} JSON on stdout')

package/dist/adapter/cli/reports.d.ts

@@ -4,7 +4,7 @@ import type { ViolationComment } from '../../app/violation-comments.js';
 import { type TextRedactor, type TextWriter } from './text.js';
 export type RuleHeadings = (name: string) => readonly string[];
 export declare function renderRuleTestHeading(ruleName: string, redact: TextRedactor, color: boolean): string;
-export declare function renderRuleTestResult(result: RuleTestResult, redact: TextRedactor, color: boolean, lastInRule: boolean): string;
+export declare function renderRuleTestResult(result: RuleTestResult, redact: TextRedactor, color: boolean, lastInRule: boolean, showAIScores?: boolean): string;
 export declare function renderRuleTestSummary(passed: number, failed: number, durationMs: number, redact: TextRedactor, color: boolean): string;
 export declare function createRuleHeadings(rules: readonly LoadedRule[]): RuleHeadings;
 export declare function createMetadataRenderer(write: TextWriter, redact: TextRedactor): (event: RunEvent) => void;

package/dist/adapter/cli/reports.js

@@ -36,7 +36,7 @@ function sourceText(line) {
 export function renderRuleTestHeading(ruleName, redact, color) {
     return `${paint(escapeText(redact(ruleName)), 'bold', color)}\n`;
 }
-export function renderRuleTestResult(result, redact, color, lastInRule) {
+export function renderRuleTestResult(result, redact, color, lastInRule, showAIScores = false) {
     const safe = (text) => escapeText(redact(text));
     const matches = result.expected === result.actual;
     const label = paint(safe(matches ? 'PASS' : 'FAIL'), matches ? 'green' : 'red', color);
@@ -44,8 +44,11 @@ export function renderRuleTestResult(result, redact, color, lastInRule) {
     const detail = matches
         ? ''
         : safe(` (expected ${result.expected}, actual ${result.actual})`);
+    const aiScores = showAIScores && result.aiScores !== undefined && result.aiScores.length > 0
+        ? ` (${result.aiScores.length === 1 ? 'AI score' : 'AI scores'}: ${result.aiScores.join(', ')})`
+        : '';
     const duration = paint(safe(` (${formatDuration(result.durationMs)})`), 'dim', color);
-    return `${branch}${label} ${safe(result.name)}${detail}${duration}\n`;
+    return `${branch}${label} ${safe(result.name)}${detail}${aiScores}${duration}\n`;
 }
 export function renderRuleTestSummary(passed, failed, durationMs, redact, color) {
     const safe = (text) => escapeText(redact(text));

package/dist/adapter/cli/services.d.ts

@@ -1,8 +1,9 @@
-import type { LoadedRule, Project, RuleReport, RuleSelection, RunContext } from '../../app/ports.js';
+import type { DiffChanges, LoadedRule, Project, RuleReport, RuleSelection, RunContext } from '../../app/ports.js';
 import type { RuleTestResult } from '../../app/rule-tests.js';
 import type { TokenPricing } from '../../app/rule.js';
 import type { ViolationComment } from '../../app/violation-comments.js';
 import type { PullRequest, PullRequestCheckout } from '../github/pull-request.js';
+import type { RuleRepositoryCheckout } from '../git/rules.js';
 import type { SkillOptions } from '../skills/cli.js';
 import type { TerminalCapabilities } from './progress.js';
 import type { TextRedactor, TextWriter } from './text.js';
@@ -44,6 +45,7 @@ export interface TestOptions {
     readonly concurrency: number;
     readonly ai?: boolean;
     readonly failuresOnly?: boolean;
+    readonly aiScores?: boolean;
 }
 export interface DependencyOptions {
     readonly global?: boolean;
@@ -70,8 +72,9 @@ export interface ExecutionServices extends RuleServices {
     readonly decisionResolver: (thresholdOverride?: number, pricingOverrides?: Partial<TokenPricing>) => RunContext['resolveDecision'];
 }
 export interface CheckServices extends ExecutionServices {
-    readonly changedFiles: (root: string, base: string) => Promise<ReadonlySet<string>>;
+    readonly collectChanges: (root: string, base: string) => Promise<DiffChanges>;
     readonly checkoutPullRequest: (selector: string, baseBranch?: string) => Promise<PullRequestCheckout>;
+    readonly checkoutRuleRepository: (selector: string) => Promise<RuleRepositoryCheckout | undefined>;
     readonly commentOnPullRequest: (root: string, comments: readonly ViolationComment[], pullRequest?: PullRequest) => Promise<void>;
 }
 export type InspectServices = ExecutionServices;

package/dist/adapter/decision/jev.js

@@ -1,6 +1,7 @@
 const ENDPOINT = 'v1/systemone';
 const QUESTION_ID = 'follows_instructions';
 const TIMEOUT_MS = 30_000;
+const RETRY_DELAYS_MS = [3_000, 10_000, 30_000];
 const REDACTED = '[REDACTED]';
 const URL_LABEL = 'Decision base URL (TYPESAFE_BASE_URL)';
 function isMapping(value) {
@@ -53,6 +54,14 @@ function requestFailure(error, axios) {
     }
     return new Error(`Decision request failed: ${error instanceof Error ? error.message : String(error)}`);
 }
+function shouldRetry(error, axios) {
+    if (!axios.isAxiosError(error))
+        return false;
+    const status = error.response?.status;
+    return status === undefined
+        ? error.request !== undefined
+        : status === 408 || status === 429 || (status >= 500 && status < 600);
+}
 function readDecision(data) {
     if (!isMapping(data)) {
         throw new Error('Decision response is not a JSON object');
@@ -114,24 +123,32 @@ export class JevAdapter {
             },
         };
         const wire = JSON.stringify(body);
-        observer?.({ type: 'request', body: redact(wire, this.#needles) });
         let data;
-        try {
-            const response = await client.post(ENDPOINT, wire, {
-                headers: { 'Content-Type': 'application/json' },
-            });
-            data = response.data;
-            observer?.({ type: 'response', status: response.status, body: redact(JSON.stringify(data) ?? '', this.#needles) });
-        }
-        catch (error) {
-            if (axios.isAxiosError(error) && error.response !== undefined) {
-                observer?.({
-                    type: 'response',
-                    status: error.response.status,
-                    body: redact(JSON.stringify(error.response.data) ?? '', this.#needles),
+        for (let retry = 0;; retry += 1) {
+            observer?.({ type: 'request', body: redact(wire, this.#needles) });
+            try {
+                const response = await client.post(ENDPOINT, wire, {
+                    headers: { 'Content-Type': 'application/json' },
                 });
+                data = response.data;
+                observer?.({ type: 'response', status: response.status, body: redact(JSON.stringify(data) ?? '', this.#needles) });
+                break;
+            }
+            catch (error) {
+                if (axios.isAxiosError(error) && error.response !== undefined) {
+                    observer?.({
+                        type: 'response',
+                        status: error.response.status,
+                        body: redact(JSON.stringify(error.response.data) ?? '', this.#needles),
+                    });
+                }
+                const delayMs = RETRY_DELAYS_MS[retry];
+                if (delayMs === undefined || !shouldRetry(error, axios))
+                    throw requestFailure(error, axios);
+                const delay = Promise.withResolvers();
+                setTimeout(delay.resolve, delayMs);
+                await delay.promise;
             }
-            throw requestFailure(error, axios);
         }
         return readDecision(data);
     }

package/dist/adapter/decision/openai.d.ts

@@ -0,0 +1,11 @@
+import type { DecisionBackend, DecisionObserver, DecisionOutcome, DecisionRequest } from '../../app/ports.js';
+export interface OpenAIOptions {
+    readonly model: string;
+    readonly baseUrl: string;
+    readonly apiKey: string;
+}
+export declare class OpenAIAdapter implements DecisionBackend {
+    #private;
+    constructor({ model, baseUrl, apiKey }: OpenAIOptions);
+    decide(request: DecisionRequest, observer?: DecisionObserver): Promise<DecisionOutcome>;
+}

package/dist/adapter/decision/openai.js

@@ -0,0 +1,160 @@
+import { isRecord } from '../../app/validate.js';
+const QUESTION_NAME = 'follows_instructions';
+const TIMEOUT_MS = 30_000;
+const REDACTED = '[REDACTED]';
+const URL_LABEL = 'Decision base URL (OPENAI_BASE_URL)';
+function baseUrlOf(baseUrl) {
+    let url;
+    try {
+        url = new URL(baseUrl);
+    }
+    catch {
+        throw new Error(`${URL_LABEL} is not an absolute URL`);
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+        throw new Error(`${URL_LABEL} must use http or https`);
+    }
+    if (baseUrl.includes('?') || baseUrl.includes('#')) {
+        throw new Error(`${URL_LABEL} must not carry a query or a fragment`);
+    }
+    if (url.username !== '' || url.password !== '') {
+        throw new Error(`${URL_LABEL} must not embed credentials`);
+    }
+    return baseUrl;
+}
+function redactionNeedles(apiKey) {
+    if (apiKey === '')
+        return [];
+    const escaped = JSON.stringify(apiKey).slice(1, -1);
+    return escaped === apiKey ? [apiKey] : [escaped, apiKey];
+}
+function redact(text, needles) {
+    let redacted = text;
+    for (const needle of needles)
+        redacted = redacted.replaceAll(needle, REDACTED);
+    return redacted;
+}
+function formattedInstructions(request) {
+    const { rule, question, examples } = request.instructions;
+    const sections = [`Rule:\n${rule}`];
+    if (examples?.good !== undefined && examples.good.length > 0) {
+        sections.push(`Good examples (the rule is respected):\n${JSON.stringify(examples.good)}`);
+    }
+    if (examples?.bad !== undefined && examples.bad.length > 0) {
+        sections.push(`Bad examples (the rule is not respected):\n${JSON.stringify(examples.bad)}`);
+    }
+    sections.push(question);
+    return sections.join('\n\n');
+}
+function readDecision(data) {
+    if (!isRecord(data))
+        throw new Error('Decision response is not a JSON object');
+    const model = data.model;
+    if (typeof model !== 'string' || model.trim() === '') {
+        throw new Error('Decision response carries no model');
+    }
+    const answers = data.answers;
+    if (!Array.isArray(answers))
+        throw new Error('Decision response carries no answers');
+    const answer = answers.find((candidate) => isRecord(candidate) && candidate.name === QUESTION_NAME);
+    if (!isRecord(answer)) {
+        throw new Error(`Decision response carries no '${QUESTION_NAME}' answer`);
+    }
+    if (answer.type === 'refusal') {
+        throw new Error(`Decision response '${QUESTION_NAME}' answer was refused`);
+    }
+    if (answer.type !== 'predicate') {
+        throw new Error(`Decision response '${QUESTION_NAME}' answer is not a predicate answer`);
+    }
+    const probability = answer.probability;
+    if (typeof probability !== 'number' || !Number.isFinite(probability) || probability < 0 || probability > 1) {
+        throw new Error(`Decision response '${QUESTION_NAME}' answer is not a probability between 0 and 1`);
+    }
+    const rawUsage = data.usage;
+    if (rawUsage === undefined)
+        return { probability, model };
+    if (!isRecord(rawUsage))
+        throw new Error('Decision response usage must be an object');
+    const inputTokens = rawUsage.input_tokens;
+    const outputTokens = rawUsage.output_tokens;
+    if (typeof inputTokens !== 'number' || !Number.isSafeInteger(inputTokens) || inputTokens < 0 ||
+        typeof outputTokens !== 'number' || !Number.isSafeInteger(outputTokens) || outputTokens < 0) {
+        throw new Error('Decision response usage must carry nonnegative safe integer input_tokens and output_tokens');
+    }
+    return { probability, model, usage: { inputTokens, outputTokens } };
+}
+export class OpenAIAdapter {
+    #client;
+    #apiKey;
+    #baseUrl;
+    #model;
+    #needles;
+    constructor({ model, baseUrl, apiKey }) {
+        this.#model = model;
+        this.#baseUrl = baseUrlOf(baseUrl);
+        this.#apiKey = apiKey;
+        this.#needles = redactionNeedles(apiKey);
+    }
+    async decide(request, observer) {
+        const payload = {
+            model: this.#model,
+            input: JSON.stringify(request.state),
+            questions: [{
+                    type: 'predicate',
+                    name: QUESTION_NAME,
+                    instructions: formattedInstructions(request),
+                }],
+        };
+        const observedPayload = {
+            ...payload,
+            input: redact(payload.input, this.#needles),
+            questions: payload.questions.map(question => ({
+                ...question,
+                instructions: redact(question.instructions, this.#needles),
+            })),
+        };
+        observer?.({
+            type: 'request',
+            body: redact(JSON.stringify(observedPayload), this.#needles),
+        });
+        // Load the SDK only when an AI decision runs, matching Jev's lazy provider dependency.
+        const sdk = await import('openai');
+        let data;
+        let status;
+        try {
+            const client = this.#client ??= new sdk.default({
+                apiKey: this.#apiKey,
+                baseURL: this.#baseUrl,
+                timeout: TIMEOUT_MS,
+                maxRetries: 2,
+                fetchOptions: { redirect: 'manual' },
+                logLevel: 'off',
+                defaultHeaders: { Authorization: `Bearer ${this.#apiKey}` },
+            });
+            const result = await client.decisions.create(payload).withResponse();
+            data = result.data;
+            status = result.response.status;
+        }
+        catch (error) {
+            if (error instanceof sdk.APIError && error.status !== undefined) {
+                const body = error.error === undefined ? error.message : JSON.stringify(error.error) ?? error.message;
+                observer?.({ type: 'response', status: error.status, body: redact(body, this.#needles) });
+                throw new Error(`Decision request failed with HTTP ${error.status}`);
+            }
+            if (error instanceof sdk.APIConnectionTimeoutError) {
+                throw new Error(`Decision request timed out after ${TIMEOUT_MS} ms`);
+            }
+            if (error instanceof sdk.APIConnectionError) {
+                throw new Error('Decision request failed: network error');
+            }
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`Decision request failed: ${redact(message, this.#needles)}`);
+        }
+        observer?.({
+            type: 'response',
+            status,
+            body: redact(JSON.stringify(data) ?? '', this.#needles),
+        });
+        return readDecision(data);
+    }
+}

package/dist/adapter/filesystem/rule-test-worker.d.ts

@@ -7,10 +7,15 @@ export interface RuleTestWorkerRequest {
     readonly files: RuleTestCase['files'];
     readonly ai: boolean;
     readonly dependencies?: PreparedRuleDependencies;
+    readonly includeDisabled?: boolean;
+}
+export interface RuleTestWorkerOutcome {
+    readonly actual: RuleTestCase['expected'];
+    readonly aiScores?: readonly number[];
 }
 export type RuleTestWorkerMessage = {
     readonly id: number;
-    readonly actual: RuleTestCase['expected'];
+    readonly outcome: RuleTestWorkerOutcome;
 } | {
     readonly id: number;
     readonly error: string;

package/dist/adapter/filesystem/rule-test-worker.js

@@ -12,7 +12,7 @@ function fixturePath(root, filepath) {
     }
     return target;
 }
-async function runCase({ root, ruleFile, files, ai, dependencies }) {
+async function runCase({ root, ruleFile, files, ai, dependencies, includeDisabled }) {
     for (const [filepath, source] of Object.entries(files)) {
         const target = fixturePath(root, filepath);
         await mkdir(dirname(target), { recursive: true });
@@ -21,6 +21,8 @@ async function runCase({ root, ruleFile, files, ai, dependencies }) {
     const rules = await createRuleLoader(dependencies === undefined ? {} : { preparedDependencies: dependencies }).load({ root, explicit: ruleFile, invocationCwd: root });
     if (rules.length !== 1)
         throw new Error('Expected exactly one freshly loaded rule');
+    if (includeDisabled === true)
+        rules[0].rule.enabled = true;
     const reports = await executeRules(rules, {
         project: createProject(root, { excludeFiles: [ruleFile] }),
         isTarget: () => true,
@@ -32,7 +34,15 @@ async function runCase({ root, ruleFile, files, ai, dependencies }) {
     const report = reports[0];
     if (reports.length !== 1 || report === undefined)
         throw new Error('Expected exactly one rule report');
-    return report.status;
+    let aiScores;
+    if (ai) {
+        for (const evaluation of report.evaluations) {
+            const score = evaluation.result.probability;
+            if (score !== undefined)
+                (aiScores ??= []).push(score);
+        }
+    }
+    return { actual: report.status, ...(aiScores === undefined ? {} : { aiScores }) };
 }
 const port = parentPort;
 if (port === null)
@@ -40,7 +50,7 @@ if (port === null)
 port.start();
 port.ref();
 port.on('message', (request) => {
-    void runCase(request).then(actual => port.postMessage({ id: request.id, actual }), error => port.postMessage({
+    void runCase(request).then(outcome => port.postMessage({ id: request.id, outcome }), error => port.postMessage({
         id: request.id,
         error: error instanceof Error ? error.message : String(error),
     }));

package/dist/adapter/filesystem/rule-tests.d.ts

@@ -1,3 +1,3 @@
 import type { LoadedRule } from '../../app/ports.js';
 import type { RuleTestResult } from '../../app/rule-tests.js';
-export declare function runRuleTests(rules: readonly LoadedRule[], onResult?: (result: RuleTestResult, lastInRule: boolean) => void, timeoutMs?: number, concurrency?: number, includeAI?: boolean): Promise<readonly RuleTestResult[]>;
+export declare function runRuleTests(rules: readonly LoadedRule[], onResult?: (result: RuleTestResult, lastInRule: boolean) => void, timeoutMs?: number, concurrency?: number, includeAI?: boolean, includeDisabled?: boolean): Promise<readonly RuleTestResult[]>;