@alex-dbs/coderuler 1.4.0 → 1.5.0

package/README.md

@@ -15,6 +15,7 @@ Install the CLI globally:
 ```sh
 npm install -g @alex-dbs/coderuler
 coderuler --help
+coderuler skill install
 ```
 
 The npm package is scoped. The command is `coderuler`. A global installation can load rules without a project-local dependency.
@@ -24,6 +25,7 @@ For a project-specific CLI version, install it locally:
 ```sh
 npm install --save-dev @alex-dbs/coderuler
 npx coderuler --help
+npx coderuler skill install
 ```
 
 The global installation is optional: with only the project-local dependency, run commands through `npx coderuler`.
@@ -158,7 +160,7 @@ A scope is a file or selected part of a file. CodeRuler validates selected rules
 The `settings` field sets AI options for one rule, such as its model, decision threshold, and input or output token prices.
 
 > [!TIP]
-> Browse the [`.coderuler` folder](.coderuler) for rule examples.
+> Browse the [`.coderuler.examples` folder](.coderuler.examples) for rule examples.
 
 ## Commands
 

package/dist/adapter/cli/commands/check.js

@@ -12,6 +12,7 @@ async function runCheck(target, options, context, services) {
         ? createProgress(context.writeError, () => context.terminal?.columns ?? 80)
         : undefined;
     let checkout;
+    let rulesCheckout;
     try {
         let reports;
         let durationMs;
@@ -24,21 +25,28 @@ async function runCheck(target, options, context, services) {
             }
             const area = await services.resolveTarget(checkout?.root ?? context.cwd, target);
             root = area.root;
-            let changedFiles = checkout?.changedFiles ?? (diffOnly
-                ? await services.changedFiles(area.root, options.base ?? 'HEAD')
-                : undefined);
-            const choice = options.rules === undefined
-                ? {}
-                : await services.resolveRuleSelector(options.rules, context.cwd, true);
+            let changes = checkout === undefined
+                ? (diffOnly ? await services.collectChanges(area.root, options.base ?? 'HEAD') : undefined)
+                : { changedFiles: checkout.changedFiles, changedLines: checkout.changedLines };
+            let choice = {};
+            if (options.rules !== undefined) {
+                rulesCheckout = await services.checkoutRuleRepository(options.rules);
+                choice = rulesCheckout === undefined
+                    ? await services.resolveRuleSelector(options.rules, context.cwd, true)
+                    : { explicit: rulesCheckout.root };
+            }
             const prepared = await planWalk(context.cwd, services, area.root, filepath => area.contains(filepath), choice, checkout?.rulesRoot ?? area.root);
             rules = options.tags.length === 0 && options.excludeTags.length === 0
                 ? prepared.rules
                 : prepared.rules.filter(({ rule }) => (options.tags.length === 0 ||
                     rule.tags?.some(tag => options.tags.includes(tag)) === true) &&
                     !rule.tags?.some(tag => options.excludeTags.includes(tag)));
-            if (changedFiles !== undefined) {
+            if (changes !== undefined) {
                 const ruleFiles = new Set(prepared.rules.map(entry => relative(area.root, resolve(entry.ruleFile)).split(sep).join('/')));
-                changedFiles = new Set([...changedFiles].filter(path => !ruleFiles.has(path)));
+                changes = {
+                    changedFiles: new Set([...changes.changedFiles].filter(path => !ruleFiles.has(path))),
+                    changedLines: new Map([...changes.changedLines].filter(([path]) => !ruleFiles.has(path))),
+                };
             }
             const concurrency = debug ? 1 : options.concurrency;
             walkthrough = verbose
@@ -59,7 +67,10 @@ async function runCheck(target, options, context, services) {
                 concurrency,
                 resolveDecision: services.decisionResolver(options.threshold),
                 ...(emit === undefined ? {} : { emit }),
-                ...(changedFiles === undefined ? {} : { changedFiles }),
+                ...(changes === undefined ? {} : {
+                    changedFiles: changes.changedFiles,
+                    changedLines: changes.changedLines,
+                }),
             };
             const startedAt = performance.now();
             reports = await services.runRules(rules, runContext);
@@ -87,7 +98,7 @@ async function runCheck(target, options, context, services) {
         }
     }
     finally {
-        await checkout?.cleanup();
+        await Promise.all([checkout?.cleanup(), rulesCheckout?.cleanup()]);
     }
 }
 export { runCheck };

package/dist/adapter/cli/program.js

@@ -128,7 +128,7 @@ export function createProgram(services) {
         .description('Run the declared test cases for the selected rules')
         .argument('[rule]', 'rule name, module path, or directory')
         .option('--ai', 'include aiTests that use the configured decision backend')
-        .option('--ai-scores', 'show Jev scores for aiTests')
+        .option('--ai-scores', 'show AI scores for aiTests')
         .option('--failures-only', 'show only failed test cases')
         .addOption(new Option('--timeout <duration>', 'per-test timeout (ms, s, or m)')
         .argParser(parseTimeout)
@@ -143,7 +143,7 @@ export function createProgram(services) {
         .command('check')
         .description('Evaluate every selected rule against the working tree')
         .argument('[path]', 'target file or folder that narrows selection', '.')
-        .option('--rules <rule>', 'rule name, module path, or directory')
+        .option('--rules <rule>', 'rule name, module path, directory, or Git repository URL[#revision]')
         .option('--tags <tags...>', 'run rules matching any listed tag', [])
         .option('--exclude-tags <tags...>', 'exclude rules matching any listed tag', [])
         .option('--json', 'print violations as {durationMs, reports} JSON on stdout')

package/dist/adapter/cli/reports.js

@@ -45,7 +45,7 @@ export function renderRuleTestResult(result, redact, color, lastInRule, showAISc
         ? ''
         : safe(` (expected ${result.expected}, actual ${result.actual})`);
     const aiScores = showAIScores && result.aiScores !== undefined && result.aiScores.length > 0
-        ? ` (${result.aiScores.length === 1 ? 'Jev score' : 'Jev scores'}: ${result.aiScores.join(', ')})`
+        ? ` (${result.aiScores.length === 1 ? 'AI score' : 'AI scores'}: ${result.aiScores.join(', ')})`
         : '';
     const duration = paint(safe(` (${formatDuration(result.durationMs)})`), 'dim', color);
     return `${branch}${label} ${safe(result.name)}${detail}${aiScores}${duration}\n`;

package/dist/adapter/cli/services.d.ts

@@ -1,8 +1,9 @@
-import type { LoadedRule, Project, RuleReport, RuleSelection, RunContext } from '../../app/ports.js';
+import type { DiffChanges, LoadedRule, Project, RuleReport, RuleSelection, RunContext } from '../../app/ports.js';
 import type { RuleTestResult } from '../../app/rule-tests.js';
 import type { TokenPricing } from '../../app/rule.js';
 import type { ViolationComment } from '../../app/violation-comments.js';
 import type { PullRequest, PullRequestCheckout } from '../github/pull-request.js';
+import type { RuleRepositoryCheckout } from '../git/rules.js';
 import type { SkillOptions } from '../skills/cli.js';
 import type { TerminalCapabilities } from './progress.js';
 import type { TextRedactor, TextWriter } from './text.js';
@@ -71,8 +72,9 @@ export interface ExecutionServices extends RuleServices {
     readonly decisionResolver: (thresholdOverride?: number, pricingOverrides?: Partial<TokenPricing>) => RunContext['resolveDecision'];
 }
 export interface CheckServices extends ExecutionServices {
-    readonly changedFiles: (root: string, base: string) => Promise<ReadonlySet<string>>;
+    readonly collectChanges: (root: string, base: string) => Promise<DiffChanges>;
     readonly checkoutPullRequest: (selector: string, baseBranch?: string) => Promise<PullRequestCheckout>;
+    readonly checkoutRuleRepository: (selector: string) => Promise<RuleRepositoryCheckout | undefined>;
     readonly commentOnPullRequest: (root: string, comments: readonly ViolationComment[], pullRequest?: PullRequest) => Promise<void>;
 }
 export type InspectServices = ExecutionServices;

package/dist/adapter/decision/jev.js

@@ -1,6 +1,7 @@
 const ENDPOINT = 'v1/systemone';
 const QUESTION_ID = 'follows_instructions';
 const TIMEOUT_MS = 30_000;
+const RETRY_DELAYS_MS = [3_000, 10_000, 30_000];
 const REDACTED = '[REDACTED]';
 const URL_LABEL = 'Decision base URL (TYPESAFE_BASE_URL)';
 function isMapping(value) {
@@ -53,6 +54,14 @@ function requestFailure(error, axios) {
     }
     return new Error(`Decision request failed: ${error instanceof Error ? error.message : String(error)}`);
 }
+function shouldRetry(error, axios) {
+    if (!axios.isAxiosError(error))
+        return false;
+    const status = error.response?.status;
+    return status === undefined
+        ? error.request !== undefined
+        : status === 408 || status === 429 || (status >= 500 && status < 600);
+}
 function readDecision(data) {
     if (!isMapping(data)) {
         throw new Error('Decision response is not a JSON object');
@@ -114,24 +123,32 @@ export class JevAdapter {
             },
         };
         const wire = JSON.stringify(body);
-        observer?.({ type: 'request', body: redact(wire, this.#needles) });
         let data;
-        try {
-            const response = await client.post(ENDPOINT, wire, {
-                headers: { 'Content-Type': 'application/json' },
-            });
-            data = response.data;
-            observer?.({ type: 'response', status: response.status, body: redact(JSON.stringify(data) ?? '', this.#needles) });
-        }
-        catch (error) {
-            if (axios.isAxiosError(error) && error.response !== undefined) {
-                observer?.({
-                    type: 'response',
-                    status: error.response.status,
-                    body: redact(JSON.stringify(error.response.data) ?? '', this.#needles),
+        for (let retry = 0;; retry += 1) {
+            observer?.({ type: 'request', body: redact(wire, this.#needles) });
+            try {
+                const response = await client.post(ENDPOINT, wire, {
+                    headers: { 'Content-Type': 'application/json' },
                 });
+                data = response.data;
+                observer?.({ type: 'response', status: response.status, body: redact(JSON.stringify(data) ?? '', this.#needles) });
+                break;
+            }
+            catch (error) {
+                if (axios.isAxiosError(error) && error.response !== undefined) {
+                    observer?.({
+                        type: 'response',
+                        status: error.response.status,
+                        body: redact(JSON.stringify(error.response.data) ?? '', this.#needles),
+                    });
+                }
+                const delayMs = RETRY_DELAYS_MS[retry];
+                if (delayMs === undefined || !shouldRetry(error, axios))
+                    throw requestFailure(error, axios);
+                const delay = Promise.withResolvers();
+                setTimeout(delay.resolve, delayMs);
+                await delay.promise;
             }
-            throw requestFailure(error, axios);
         }
         return readDecision(data);
     }

package/dist/adapter/decision/openai.d.ts

@@ -0,0 +1,11 @@
+import type { DecisionBackend, DecisionObserver, DecisionOutcome, DecisionRequest } from '../../app/ports.js';
+export interface OpenAIOptions {
+    readonly model: string;
+    readonly baseUrl: string;
+    readonly apiKey: string;
+}
+export declare class OpenAIAdapter implements DecisionBackend {
+    #private;
+    constructor({ model, baseUrl, apiKey }: OpenAIOptions);
+    decide(request: DecisionRequest, observer?: DecisionObserver): Promise<DecisionOutcome>;
+}

package/dist/adapter/decision/openai.js

@@ -0,0 +1,160 @@
+import { isRecord } from '../../app/validate.js';
+const QUESTION_NAME = 'follows_instructions';
+const TIMEOUT_MS = 30_000;
+const REDACTED = '[REDACTED]';
+const URL_LABEL = 'Decision base URL (OPENAI_BASE_URL)';
+function baseUrlOf(baseUrl) {
+    let url;
+    try {
+        url = new URL(baseUrl);
+    }
+    catch {
+        throw new Error(`${URL_LABEL} is not an absolute URL`);
+    }
+    if (url.protocol !== 'http:' && url.protocol !== 'https:') {
+        throw new Error(`${URL_LABEL} must use http or https`);
+    }
+    if (baseUrl.includes('?') || baseUrl.includes('#')) {
+        throw new Error(`${URL_LABEL} must not carry a query or a fragment`);
+    }
+    if (url.username !== '' || url.password !== '') {
+        throw new Error(`${URL_LABEL} must not embed credentials`);
+    }
+    return baseUrl;
+}
+function redactionNeedles(apiKey) {
+    if (apiKey === '')
+        return [];
+    const escaped = JSON.stringify(apiKey).slice(1, -1);
+    return escaped === apiKey ? [apiKey] : [escaped, apiKey];
+}
+function redact(text, needles) {
+    let redacted = text;
+    for (const needle of needles)
+        redacted = redacted.replaceAll(needle, REDACTED);
+    return redacted;
+}
+function formattedInstructions(request) {
+    const { rule, question, examples } = request.instructions;
+    const sections = [`Rule:\n${rule}`];
+    if (examples?.good !== undefined && examples.good.length > 0) {
+        sections.push(`Good examples (the rule is respected):\n${JSON.stringify(examples.good)}`);
+    }
+    if (examples?.bad !== undefined && examples.bad.length > 0) {
+        sections.push(`Bad examples (the rule is not respected):\n${JSON.stringify(examples.bad)}`);
+    }
+    sections.push(question);
+    return sections.join('\n\n');
+}
+function readDecision(data) {
+    if (!isRecord(data))
+        throw new Error('Decision response is not a JSON object');
+    const model = data.model;
+    if (typeof model !== 'string' || model.trim() === '') {
+        throw new Error('Decision response carries no model');
+    }
+    const answers = data.answers;
+    if (!Array.isArray(answers))
+        throw new Error('Decision response carries no answers');
+    const answer = answers.find((candidate) => isRecord(candidate) && candidate.name === QUESTION_NAME);
+    if (!isRecord(answer)) {
+        throw new Error(`Decision response carries no '${QUESTION_NAME}' answer`);
+    }
+    if (answer.type === 'refusal') {
+        throw new Error(`Decision response '${QUESTION_NAME}' answer was refused`);
+    }
+    if (answer.type !== 'predicate') {
+        throw new Error(`Decision response '${QUESTION_NAME}' answer is not a predicate answer`);
+    }
+    const probability = answer.probability;
+    if (typeof probability !== 'number' || !Number.isFinite(probability) || probability < 0 || probability > 1) {
+        throw new Error(`Decision response '${QUESTION_NAME}' answer is not a probability between 0 and 1`);
+    }
+    const rawUsage = data.usage;
+    if (rawUsage === undefined)
+        return { probability, model };
+    if (!isRecord(rawUsage))
+        throw new Error('Decision response usage must be an object');
+    const inputTokens = rawUsage.input_tokens;
+    const outputTokens = rawUsage.output_tokens;
+    if (typeof inputTokens !== 'number' || !Number.isSafeInteger(inputTokens) || inputTokens < 0 ||
+        typeof outputTokens !== 'number' || !Number.isSafeInteger(outputTokens) || outputTokens < 0) {
+        throw new Error('Decision response usage must carry nonnegative safe integer input_tokens and output_tokens');
+    }
+    return { probability, model, usage: { inputTokens, outputTokens } };
+}
+export class OpenAIAdapter {
+    #client;
+    #apiKey;
+    #baseUrl;
+    #model;
+    #needles;
+    constructor({ model, baseUrl, apiKey }) {
+        this.#model = model;
+        this.#baseUrl = baseUrlOf(baseUrl);
+        this.#apiKey = apiKey;
+        this.#needles = redactionNeedles(apiKey);
+    }
+    async decide(request, observer) {
+        const payload = {
+            model: this.#model,
+            input: JSON.stringify(request.state),
+            questions: [{
+                    type: 'predicate',
+                    name: QUESTION_NAME,
+                    instructions: formattedInstructions(request),
+                }],
+        };
+        const observedPayload = {
+            ...payload,
+            input: redact(payload.input, this.#needles),
+            questions: payload.questions.map(question => ({
+                ...question,
+                instructions: redact(question.instructions, this.#needles),
+            })),
+        };
+        observer?.({
+            type: 'request',
+            body: redact(JSON.stringify(observedPayload), this.#needles),
+        });
+        // Load the SDK only when an AI decision runs, matching Jev's lazy provider dependency.
+        const sdk = await import('openai');
+        let data;
+        let status;
+        try {
+            const client = this.#client ??= new sdk.default({
+                apiKey: this.#apiKey,
+                baseURL: this.#baseUrl,
+                timeout: TIMEOUT_MS,
+                maxRetries: 2,
+                fetchOptions: { redirect: 'manual' },
+                logLevel: 'off',
+                defaultHeaders: { Authorization: `Bearer ${this.#apiKey}` },
+            });
+            const result = await client.decisions.create(payload).withResponse();
+            data = result.data;
+            status = result.response.status;
+        }
+        catch (error) {
+            if (error instanceof sdk.APIError && error.status !== undefined) {
+                const body = error.error === undefined ? error.message : JSON.stringify(error.error) ?? error.message;
+                observer?.({ type: 'response', status: error.status, body: redact(body, this.#needles) });
+                throw new Error(`Decision request failed with HTTP ${error.status}`);
+            }
+            if (error instanceof sdk.APIConnectionTimeoutError) {
+                throw new Error(`Decision request timed out after ${TIMEOUT_MS} ms`);
+            }
+            if (error instanceof sdk.APIConnectionError) {
+                throw new Error('Decision request failed: network error');
+            }
+            const message = error instanceof Error ? error.message : String(error);
+            throw new Error(`Decision request failed: ${redact(message, this.#needles)}`);
+        }
+        observer?.({
+            type: 'response',
+            status,
+            body: redact(JSON.stringify(data) ?? '', this.#needles),
+        });
+        return readDecision(data);
+    }
+}

package/dist/adapter/filesystem/rule-test-worker.d.ts

@@ -7,6 +7,7 @@ export interface RuleTestWorkerRequest {
     readonly files: RuleTestCase['files'];
     readonly ai: boolean;
     readonly dependencies?: PreparedRuleDependencies;
+    readonly includeDisabled?: boolean;
 }
 export interface RuleTestWorkerOutcome {
     readonly actual: RuleTestCase['expected'];

package/dist/adapter/filesystem/rule-test-worker.js

@@ -12,7 +12,7 @@ function fixturePath(root, filepath) {
     }
     return target;
 }
-async function runCase({ root, ruleFile, files, ai, dependencies }) {
+async function runCase({ root, ruleFile, files, ai, dependencies, includeDisabled }) {
     for (const [filepath, source] of Object.entries(files)) {
         const target = fixturePath(root, filepath);
         await mkdir(dirname(target), { recursive: true });
@@ -21,6 +21,8 @@ async function runCase({ root, ruleFile, files, ai, dependencies }) {
     const rules = await createRuleLoader(dependencies === undefined ? {} : { preparedDependencies: dependencies }).load({ root, explicit: ruleFile, invocationCwd: root });
     if (rules.length !== 1)
         throw new Error('Expected exactly one freshly loaded rule');
+    if (includeDisabled === true)
+        rules[0].rule.enabled = true;
     const reports = await executeRules(rules, {
         project: createProject(root, { excludeFiles: [ruleFile] }),
         isTarget: () => true,

package/dist/adapter/filesystem/rule-tests.d.ts

@@ -1,3 +1,3 @@
 import type { LoadedRule } from '../../app/ports.js';
 import type { RuleTestResult } from '../../app/rule-tests.js';
-export declare function runRuleTests(rules: readonly LoadedRule[], onResult?: (result: RuleTestResult, lastInRule: boolean) => void, timeoutMs?: number, concurrency?: number, includeAI?: boolean): Promise<readonly RuleTestResult[]>;
+export declare function runRuleTests(rules: readonly LoadedRule[], onResult?: (result: RuleTestResult, lastInRule: boolean) => void, timeoutMs?: number, concurrency?: number, includeAI?: boolean, includeDisabled?: boolean): Promise<readonly RuleTestResult[]>;

package/dist/adapter/filesystem/rule-tests.js

@@ -1,10 +1,10 @@
-import { mkdtemp, realpath, rm } from 'node:fs/promises';
+import { lstat, mkdtemp, realpath, rm } from 'node:fs/promises';
 import { tmpdir } from 'node:os';
 import { join } from 'node:path';
 import { pathToFileURL } from 'node:url';
 import { Worker } from 'node:worker_threads';
 import { isRecord, validateJsonValue } from '../../app/validate.js';
-import { getRuleDependencies } from './rule-resolution.js';
+import { getRuleDependencies, registerRuleDependencies } from './rule-resolution.js';
 // A JavaScript entry lets inherited tsx hooks resolve the source worker's .js import.
 const WORKER_MODULE = new URL('./rule-test-worker.js', import.meta.url);
 const WORKER_ENTRY = new URL(`data:text/javascript,${encodeURIComponent(`import ${JSON.stringify(WORKER_MODULE.href)};`)}`);
@@ -24,6 +24,35 @@ function validateCase(value, label) {
     }
     return value;
 }
+async function loadExternalTests(ruleFile, testFile) {
+    try {
+        await lstat(testFile);
+    }
+    catch (error) {
+        if (error.code === 'ENOENT')
+            return undefined;
+        throw error;
+    }
+    try {
+        const dependencies = getRuleDependencies(ruleFile);
+        if (dependencies !== undefined)
+            registerRuleDependencies(testFile, dependencies);
+        // The companion path comes from the selected rule filename.
+        return await import(pathToFileURL(testFile).href);
+    }
+    catch (error) {
+        const detail = error instanceof Error ? error.message : String(error);
+        throw new Error(`Rule test module ${testFile} failed to import: ${detail}`, { cause: error });
+    }
+}
+function addTestGroup(groups, values, sourceLabel, exportName, ai) {
+    if (values === undefined)
+        return;
+    if (!Array.isArray(values)) {
+        throw new Error(`${sourceLabel} ${exportName}: expected an array of test cases`);
+    }
+    groups.push({ values, label: `${sourceLabel} ${exportName}`, ai });
+}
 class RuleTestWorkerSession {
     worker = new Worker(WORKER_ENTRY);
     nextId = 0;
@@ -114,7 +143,7 @@ async function stopSession(session) {
     if (session.current === current)
         session.current = undefined;
 }
-async function runCase(entry, example, ai, timeoutMs, session, onFailure) {
+async function runCase(entry, example, ai, timeoutMs, session, onFailure, includeDisabled) {
     const started = performance.now();
     const deadline = started + timeoutMs;
     let failure;
@@ -141,6 +170,7 @@ async function runCase(entry, example, ai, timeoutMs, session, onFailure) {
                 const dependencies = getRuleDependencies(entry.ruleFile);
                 const outcome = await active.run({
                     root, ruleFile: entry.ruleFile, files: example.files, ai,
+                    ...(includeDisabled ? { includeDisabled: true } : {}),
                     ...(dependencies === undefined ? {} : { dependencies }),
                 }, deadline, failCase);
                 if (typeof outcome === 'string')
@@ -180,7 +210,7 @@ async function runCase(entry, example, ai, timeoutMs, session, onFailure) {
         throw cause;
     }
 }
-export async function runRuleTests(rules, onResult, timeoutMs = 10_000, concurrency = 1, includeAI = false) {
+export async function runRuleTests(rules, onResult, timeoutMs = 10_000, concurrency = 1, includeAI = false, includeDisabled = false) {
     if (!Number.isSafeInteger(concurrency) || concurrency < 1) {
         throw new RangeError(`--concurrency must be a positive safe integer, received ${String(concurrency)}`);
     }
@@ -204,24 +234,28 @@ export async function runRuleTests(rules, onResult, timeoutMs = 10_000, concurre
         for (const rule of rules) {
             if (failureObserved)
                 throw failure;
-            if (!rule.rule.enabled)
+            if (!includeDisabled && !rule.rule.enabled)
                 continue;
             // Discovery selects the rule module path at runtime.
-            const { tests, aiTests } = await import(pathToFileURL(rule.ruleFile).href);
+            const ruleModule = await import(pathToFileURL(rule.ruleFile).href);
+            const testFile = rule.ruleFile.replace(/\.rule\.(ts|mts|js|mjs)$/, '.test.$1');
+            const externalModule = await loadExternalTests(rule.ruleFile, testFile);
+            const { tests, aiTests } = ruleModule;
             if (failureObserved)
                 throw failure;
-            if (!includeAI && aiTests !== undefined)
+            if (!includeAI && (aiTests !== undefined || externalModule?.aiTests !== undefined)) {
                 hasExcludedAITests = true;
-            const selectedAI = includeAI ? aiTests : undefined;
-            const label = `Rule "${rule.name}"`;
-            if (tests !== undefined && !Array.isArray(tests)) {
-                throw new Error(`${label} tests: expected an array of test cases`);
             }
-            if (selectedAI !== undefined && !Array.isArray(selectedAI)) {
-                throw new Error(`${label} aiTests: expected an array of test cases`);
+            const label = `Rule "${rule.name}"`;
+            const externalLabel = `${label} test module ${testFile}`;
+            const groups = [];
+            addTestGroup(groups, tests, label, 'tests', false);
+            addTestGroup(groups, externalModule?.tests, externalLabel, 'tests', false);
+            if (includeAI) {
+                addTestGroup(groups, aiTests, label, 'aiTests', true);
+                addTestGroup(groups, externalModule?.aiTests, externalLabel, 'aiTests', true);
             }
-            const regularCount = tests?.length ?? 0;
-            const count = regularCount + (selectedAI?.length ?? 0);
+            const count = groups.reduce((total, group) => total + group.values.length, 0);
             const completed = new Array(count);
             let nextIndex = 0;
             let nextResult = 0;
@@ -231,11 +265,15 @@ export async function runRuleTests(rules, onResult, timeoutMs = 10_000, concurre
                     if (index >= count)
                         return;
                     try {
-                        const ai = index >= regularCount;
-                        const caseIndex = ai ? index - regularCount : index;
-                        const value = ai ? selectedAI[caseIndex] : tests[caseIndex];
-                        const example = validateCase(value, `${label} ${ai ? 'aiTests' : 'tests'}[${caseIndex}]`);
-                        completed[index] = await runCase(rule, example, ai, timeoutMs, session, fail);
+                        let groupIndex = 0;
+                        let caseIndex = index;
+                        while (caseIndex >= groups[groupIndex].values.length) {
+                            caseIndex -= groups[groupIndex].values.length;
+                            groupIndex += 1;
+                        }
+                        const group = groups[groupIndex];
+                        const example = validateCase(group.values[caseIndex], `${group.label}[${caseIndex}]`);
+                        completed[index] = await runCase(rule, example, group.ai, timeoutMs, session, fail, includeDisabled);
                         if (failureObserved)
                             return;
                         while (completed[nextResult] !== undefined) {

package/dist/adapter/git/changes.d.ts

@@ -1 +1,2 @@
-export declare function changedFiles(root: string, base: string, env?: Readonly<Record<string, string | undefined>>): Promise<ReadonlySet<string>>;
+import type { DiffChanges } from '../../app/ports.js';
+export declare function collectChanges(root: string, base: string, env?: Readonly<Record<string, string | undefined>>): Promise<DiffChanges>;