dtp-caal 1.0.1 → 1.1.0

package/README.md

@@ -140,8 +140,31 @@ dtp-caal --url http://localhost:3000 --format json --output ./reports/a11y-resul
 | `--format <format>`| `-f` | `md` | Output format: `md` (Markdown) or `json` (JSON). |
 | `--auto-fix` | | `false` | Automatically attempt to locate and patch source files. |
 | `--src-dir <path>` | | `./` | Directory containing source files when `--auto-fix` is enabled. |
+| `--cache <path>` | | `./.caal-cache.json` | Verdict cache file. Unchanged elements reuse their previous verdict. |
+| `--no-cache` | | | Ignore the cache and analyze every element fresh (nothing is read or written). |
 | `--help` | `-h` | | Display help and argument descriptions. |
-| `--version` | `-v` | | Display CLI version. |
+| `--version` | `-V` | | Display CLI version. |
+
+---
+
+## 🔁 Consistent Results (Verdict Cache)
+
+LLMs are not fully deterministic: even with temperature `0` and a fixed seed, asking the model about the same element twice can occasionally produce a different verdict. To keep audits stable, `dtp-caal` caches every verdict:
+
+- Each element is fingerprinted from its HTML and its parent's HTML (plus the model and prompt version). Markup that changes on every page load without changing meaning (React `useId()` values, `nonce` attributes, HTML comments, whitespace) is ignored.
+- If an element is unchanged since a previous run, its cached verdict is reused and no API call is made. Only new or changed elements are sent to the model.
+- Identical elements within one run (e.g. the same nav link on every card) are analyzed once.
+- Entries not used for 30 days are dropped automatically.
+
+To share verdicts across machines, either commit `.caal-cache.json` to your repository or persist it in CI (see the GitHub Actions example below). Delete the file, or use `--no-cache`, to force a completely fresh audit.
+
+### Exit Codes
+
+| Code | Meaning |
+| :--- | :--- |
+| `0` | All analyzed elements passed. |
+| `1` | Accessibility issues were found (or the audit crashed). |
+| `2` | No issues found, but some elements could not be analyzed because of API errors (rate limits, invalid key). These are listed under **Not Analyzed** in the report and are never cached, so re-running retries only them. |
 
 ---
 
@@ -173,10 +196,10 @@ dtp-caal --url http://localhost:3000 --format json --output ./reports/a11y-resul
 ```
 
 1. **Extraction:** Playwright launches headless Chromium, navigates to the specified URL, waits for network idle, and queries target elements (`button`, `img`, `input`, `a`, `[role="button"]`, etc.). For each element, it extracts both the element's markup and sanitized parent container HTML.
-2. **Contextual Analysis:** Each element is analyzed using Groq's LLM endpoint. The prompt instructs the model to act as an accessibility engineer, identifying WCAG failures and synthesizing valid replacement markup.
+2. **Contextual Analysis:** Each element is first looked up in the verdict cache; only new or changed elements are analyzed using Groq's LLM endpoint (temperature `0`, fixed seed, strict JSON schema). The prompt instructs the model to act as an accessibility engineer, identifying WCAG failures and synthesizing valid replacement markup.
 3. **Reporting:** Results are structured into either a Markdown document or JSON file.
 4. **Remediation:** If `--auto-fix` is passed, the tool searches the specified `--src-dir` for files containing matching tokens, prompts the LLM to integrate the accessibility fix while preserving framework syntax (JSX, Vue, standard HTML), and updates the source files.
-5. **Exit Code:** If any element fails WCAG checks, the CLI terminates with exit code `1`, making it ideal for CI/CD gates.
+5. **Exit Code:** If any element fails WCAG checks, the CLI terminates with exit code `1`, making it ideal for CI/CD gates (see [Exit Codes](#exit-codes)).
 
 ---
 
@@ -216,6 +239,13 @@ jobs:
       - name: Install Playwright Browsers
         run: npx playwright install --with-deps chromium
 
+      - name: Restore Verdict Cache
+        uses: actions/cache/restore@v4
+        with:
+          path: .caal-cache.json
+          key: caal-verdicts-${{ github.run_id }}
+          restore-keys: caal-verdicts-
+
       - name: Run Accessibility Audit
         id: a11y_audit
         continue-on-error: true
@@ -224,6 +254,14 @@ jobs:
         run: |
           npx dtp-caal --url http://localhost:3000 --output ./caal-report.md --auto-fix --src-dir ./src
 
+      # Save even when the audit fails, so the next run reuses these verdicts
+      - name: Save Verdict Cache
+        if: always()
+        uses: actions/cache/save@v4
+        with:
+          path: .caal-cache.json
+          key: caal-verdicts-${{ github.run_id }}
+
       - name: Generate Fix Patch
         id: git_diff
         run: |

package/dist/analyzer.js

@@ -3,17 +3,41 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.analyzeElement = analyzeElement;
+exports.isUnanalysed = exports.isIssue = exports.DETERMINISTIC_PARAMS = exports.MODEL = void 0;
 exports.analyzeElements = analyzeElements;
 const groq_sdk_1 = __importDefault(require("groq-sdk"));
 const dotenv_1 = __importDefault(require("dotenv"));
+const cache_1 = require("./cache");
 dotenv_1.default.config();
 const apiKey = process.env.GROQ_API_KEY || '';
 const groq = new groq_sdk_1.default({ apiKey });
+exports.MODEL = 'openai/gpt-oss-120b';
+// Fixed sampling settings so the same input gets the same answer as often as the API allows
+exports.DETERMINISTIC_PARAMS = { temperature: 0, seed: 42, reasoning_effort: 'medium' };
+// Bump whenever the prompt or schema changes, so verdicts cached under the old prompt are not reused
+const PROMPT_VERSION = '2';
+const nullableString = { type: ['string', 'null'] };
+const VERDICT_SCHEMA = {
+    type: 'object',
+    properties: {
+        isAccessible: { type: 'boolean' },
+        issueTitle: nullableString,
+        explanation: nullableString,
+        suggestedFixCode: nullableString,
+        fixReasoning: nullableString
+    },
+    required: ['isAccessible', 'issueTitle', 'explanation', 'suggestedFixCode', 'fixReasoning'],
+    additionalProperties: false
+};
+// An element that could not be analysed (API error) is not an accessibility issue; it is reported separately
+const isIssue = (r) => !r.error && !r.isAccessible;
+exports.isIssue = isIssue;
+const isUnanalysed = (r) => Boolean(r.error);
+exports.isUnanalysed = isUnanalysed;
 const delay = (ms) => new Promise(res => setTimeout(res, ms));
-async function analyzeElement(element) {
+async function requestVerdict(element) {
     const prompt = `
-You are an expert accessibility engineer. Your task is to analyze an HTML/JSX element within its parent context to determine if it meets WCAG accessibility standards. 
+You are an expert accessibility engineer. Your task is to analyze an HTML/JSX element within its parent context to determine if it meets WCAG accessibility standards.
 You must output ONLY valid JSON without any markdown code blocks or conversational text.
 
 Context:
@@ -47,7 +71,7 @@ Return a JSON object with this exact structure:
 `;
     let retries = 3;
     let delayMs = 2000;
-    while (retries > 0) {
+    while (true) {
         try {
             const completion = await groq.chat.completions.create({
                 messages: [
@@ -60,60 +84,74 @@ Return a JSON object with this exact structure:
                         content: prompt
                     }
                 ],
-                model: 'openai/gpt-oss-120b',
-                response_format: { type: 'json_object' }
+                model: exports.MODEL,
+                ...exports.DETERMINISTIC_PARAMS,
+                response_format: {
+                    type: 'json_schema',
+                    json_schema: { name: 'accessibility_verdict', schema: VERDICT_SCHEMA, strict: true }
+                }
             });
             const text = completion.choices[0]?.message?.content;
-            if (text) {
-                const parsed = JSON.parse(text);
-                return {
-                    ...element,
-                    ...parsed
-                };
+            if (!text) {
+                throw new Error("No text in response");
+            }
+            const parsed = JSON.parse(text);
+            if (!(0, cache_1.isVerdict)(parsed)) {
+                throw new Error("Response did not match the verdict schema");
             }
-            throw new Error("No text in response");
+            return {
+                isAccessible: parsed.isAccessible,
+                issueTitle: parsed.issueTitle,
+                explanation: parsed.explanation,
+                suggestedFixCode: parsed.suggestedFixCode,
+                fixReasoning: parsed.fixReasoning
+            };
         }
         catch (error) {
-            if (error.status === 503 || error.status === 429 || error.message?.includes('503') || error.message?.includes('429')) {
-                console.warn(`[WARN] Rate limited. Retries left: ${retries - 1}. Retrying in ${delayMs}ms...`);
-                retries--;
-                if (retries === 0) {
-                    return {
-                        ...element,
-                        isAccessible: false,
-                        error: "Failed to analyze due to API rate limits."
-                    };
-                }
-                await delay(delayMs);
-                delayMs *= 2;
-            }
-            else {
-                return {
-                    ...element,
-                    isAccessible: false,
-                    error: error.message || "Unknown API error"
-                };
+            const retryable = error.status === 503 || error.status === 429 || error.message?.includes('503') || error.message?.includes('429');
+            retries--;
+            if (!retryable || retries === 0) {
+                throw retryable ? new Error("Failed to analyze due to API rate limits.") : error;
             }
+            console.warn(`[WARN] Rate limited. Retries left: ${retries}. Retrying in ${delayMs}ms...`);
+            await delay(delayMs);
+            delayMs *= 2;
         }
     }
-    return {
-        ...element,
-        isAccessible: false,
-        error: "Exhausted retries"
-    };
 }
-async function analyzeElements(elements) {
+async function analyzeElements(elements, cache) {
     const results = [];
+    let apiCalls = 0;
     console.log(`Analyzing ${elements.length} elements using Groq API...`);
     for (let i = 0; i < elements.length; i++) {
-        // Log progress
-        console.log(`Analyzing element ${i + 1}/${elements.length}: <${elements[i].tagName}>...`);
-        const result = await analyzeElement(elements[i]);
-        results.push(result);
+        const element = elements[i];
+        const key = (0, cache_1.cacheKey)(exports.MODEL, PROMPT_VERSION, element.elementHtml, element.parentHtml);
+        // Cache hits cover both earlier runs and identical elements earlier in this run
+        const cachedVerdict = cache.get(key);
+        if (cachedVerdict) {
+            console.log(`Analyzing element ${i + 1}/${elements.length}: <${element.tagName}> (cached)`);
+            results.push({ ...element, ...cachedVerdict, cached: true });
+            continue;
+        }
         // Wait briefly between requests to avoid rate limits
-        if (i < elements.length - 1) {
+        if (apiCalls > 0) {
             await delay(1000);
         }
+        apiCalls++;
+        console.log(`Analyzing element ${i + 1}/${elements.length}: <${element.tagName}>...`);
+        try {
+            const verdict = await requestVerdict(element);
+            cache.set(key, verdict);
+            results.push({ ...element, ...verdict, cached: false });
+        }
+        catch (error) {
+            // Errors are never cached, so the next run retries this element
+            results.push({ ...element, isAccessible: false, error: error.message || "Unknown API error" });
+        }
+        // Persist progress periodically so an interrupted run keeps what it already paid for
+        if (apiCalls % 20 === 0) {
+            cache.save();
+        }
     }
     return results;
 }

package/dist/cache.js

@@ -0,0 +1,99 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.VerdictCache = void 0;
+exports.normalizeHtml = normalizeHtml;
+exports.cacheKey = cacheKey;
+exports.isVerdict = isVerdict;
+const crypto_1 = __importDefault(require("crypto"));
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+// Entries not used for this long are dropped on save (their element most likely changed or was removed)
+const MAX_UNUSED_DAYS = 30;
+const today = () => new Date().toISOString().slice(0, 10);
+// Removes markup that differs between page loads without changing what the element is,
+// so the same component produces the same cache key on every run.
+function normalizeHtml(html) {
+    return html
+        .replace(/<!--[\s\S]*?-->/g, '')
+        .replace(/\s(?:data-caal-[\w-]+|nonce)="[^"]*"/g, '')
+        .replace(/:r[0-9a-z]+:|«r[0-9a-z]+»|_r_[0-9a-z]+_/g, ':id:') // React useId() values
+        .replace(/\s+/g, ' ')
+        .replace(/>\s+</g, '><')
+        .trim();
+}
+function cacheKey(model, promptVersion, elementHtml, parentHtml) {
+    return crypto_1.default
+        .createHash('sha256')
+        .update(JSON.stringify([promptVersion, model, normalizeHtml(elementHtml), normalizeHtml(parentHtml)]))
+        .digest('hex');
+}
+function isVerdict(value) {
+    const optionalString = (v) => v === null || typeof v === 'string';
+    return (typeof value === 'object' && value !== null &&
+        typeof value.isAccessible === 'boolean' &&
+        optionalString(value.issueTitle) &&
+        optionalString(value.explanation) &&
+        optionalString(value.suggestedFixCode) &&
+        optionalString(value.fixReasoning));
+}
+class VerdictCache {
+    // filePath === null disables persistence (the cache then only dedupes within a single run)
+    constructor(filePath) {
+        this.filePath = filePath;
+        this.entries = {};
+        this.dirty = false;
+        if (!filePath || !fs_1.default.existsSync(filePath))
+            return;
+        try {
+            const data = JSON.parse(fs_1.default.readFileSync(filePath, 'utf-8'));
+            if (data.version === 1 && data.entries) {
+                this.entries = data.entries;
+            }
+        }
+        catch {
+            console.warn(`[WARN] Ignoring unreadable cache file: ${filePath}`);
+        }
+    }
+    get size() {
+        return Object.keys(this.entries).length;
+    }
+    get(key) {
+        const entry = this.entries[key];
+        if (!entry || !isVerdict(entry.verdict))
+            return undefined;
+        if (entry.lastUsed !== today()) {
+            entry.lastUsed = today();
+            this.dirty = true;
+        }
+        return entry.verdict;
+    }
+    set(key, verdict) {
+        this.entries[key] = { verdict, lastUsed: today() };
+        this.dirty = true;
+    }
+    save() {
+        if (!this.filePath || !this.dirty)
+            return;
+        const cutoff = new Date(Date.now() - MAX_UNUSED_DAYS * 24 * 60 * 60 * 1000).toISOString().slice(0, 10);
+        const sorted = {};
+        for (const key of Object.keys(this.entries).sort()) {
+            if (this.entries[key].lastUsed >= cutoff) {
+                sorted[key] = this.entries[key];
+            }
+        }
+        this.entries = sorted;
+        const dir = path_1.default.dirname(this.filePath);
+        if (!fs_1.default.existsSync(dir)) {
+            fs_1.default.mkdirSync(dir, { recursive: true });
+        }
+        // Write to a temp file and rename so an interrupted run never leaves a half-written cache
+        const tmpPath = `${this.filePath}.tmp`;
+        fs_1.default.writeFileSync(tmpPath, JSON.stringify({ version: 1, entries: sorted }, null, 2) + '\n');
+        fs_1.default.renameSync(tmpPath, this.filePath);
+        this.dirty = false;
+    }
+}
+exports.VerdictCache = VerdictCache;

package/dist/index.js

@@ -8,19 +8,24 @@ const commander_1 = require("commander");
 const chalk_1 = __importDefault(require("chalk"));
 const scanner_1 = require("./scanner");
 const analyzer_1 = require("./analyzer");
+const cache_1 = require("./cache");
 const reporter_1 = require("./reporter");
 const remediator_1 = require("./remediator");
 const program = new commander_1.Command();
 program
-    .name('a11y-audit')
+    .name('dtp-caal')
     .description('Context-Aware Accessibility Linter CLI')
-    .version('1.0.1')
+    .version('1.1.0')
     .requiredOption('-u, --url <url>', 'URL to scan (e.g., http://localhost:3000)', 'http://localhost:3000')
     .option('-o, --output <path>', 'Output file path', './caal-report.md')
     .option('-f, --format <format>', 'Output format (json or md)', 'md')
     .option('--auto-fix', 'Automatically attempt to fix source files (beta)')
     .option('--src-dir <path>', 'Directory containing source files for auto-fix', './')
+    .option('--cache <path>', 'Verdict cache file; unchanged elements reuse their previous verdict', './.caal-cache.json')
+    .option('--no-cache', 'Ignore the cache file and analyze every element fresh')
     .action(async (options) => {
+    // options.cache is the file path, or false when --no-cache is passed
+    const cache = new cache_1.VerdictCache(options.cache || null);
     try {
         console.log(chalk_1.default.blue(`Starting accessibility audit for: ${options.url}`));
         // Step 1: Scan page and extract elements
@@ -29,12 +34,16 @@ program
             console.log(chalk_1.default.yellow('No relevant elements found to analyze.'));
             return;
         }
-        // Step 2: Analyze with LLM
+        // Step 2: Analyze with LLM (cached verdicts are reused)
         if (!process.env.GROQ_API_KEY) {
             console.error(chalk_1.default.red('Error: GROQ_API_KEY environment variable is not set.'));
             process.exit(1);
         }
-        const results = await (0, analyzer_1.analyzeElements)(scannedElements);
+        if (options.cache) {
+            console.log(chalk_1.default.gray(`Using verdict cache ${options.cache} (${cache.size} entries)`));
+        }
+        const results = await (0, analyzer_1.analyzeElements)(scannedElements, cache);
+        cache.save();
         // Step 3: Report
         const format = options.format.toLowerCase();
         if (format === 'json' || options.output.endsWith('.json')) {
@@ -48,17 +57,28 @@ program
             await (0, remediator_1.autoFix)(results, options.srcDir);
         }
         // Step 5: Exit with error code if issues found (useful for CI/CD)
-        const failedElements = results.filter(r => !r.isAccessible);
-        if (failedElements.length > 0) {
-            console.log(chalk_1.default.red(`\nFound ${failedElements.length} accessibility issues!`));
+        const cachedCount = results.filter(r => r.cached).length;
+        console.log(chalk_1.default.gray(`\n${cachedCount}/${results.length} verdicts came from the cache.`));
+        const issueCount = results.filter(analyzer_1.isIssue).length;
+        const unanalysedCount = results.filter(analyzer_1.isUnanalysed).length;
+        if (unanalysedCount > 0) {
+            console.log(chalk_1.default.yellow(`${unanalysedCount} elements could not be analyzed (API errors). Re-run to retry only those.`));
+        }
+        if (issueCount > 0) {
+            console.log(chalk_1.default.red(`\nFound ${issueCount} accessibility issues!`));
             process.exit(1);
         }
+        else if (unanalysedCount > 0) {
+            // Incomplete audit: not a pass, but distinguishable from real issues
+            process.exit(2);
+        }
         else {
             console.log(chalk_1.default.green('\nAll checks passed! 🎉'));
             process.exit(0);
         }
     }
     catch (error) {
+        cache.save();
         console.error(chalk_1.default.red('Audit failed:'), error);
         process.exit(1);
     }

package/dist/remediator.js

@@ -8,6 +8,7 @@ const fs_1 = __importDefault(require("fs"));
 const glob_1 = require("glob");
 const groq_sdk_1 = __importDefault(require("groq-sdk"));
 const chalk_1 = __importDefault(require("chalk"));
+const analyzer_1 = require("./analyzer");
 const dotenv_1 = __importDefault(require("dotenv"));
 dotenv_1.default.config();
 const apiKey = process.env.GROQ_API_KEY || '';
@@ -115,7 +116,8 @@ DO NOT wrap the response in markdown code blocks. OUTPUT ONLY THE RAW CODE.
                         content: prompt
                     }
                 ],
-                model: 'openai/gpt-oss-120b',
+                model: analyzer_1.MODEL,
+                ...analyzer_1.DETERMINISTIC_PARAMS,
             });
             let newContent = completion.choices[0]?.message?.content || '';
             // Just in case it wraps in markdown despite instructions

package/dist/reporter.js

@@ -7,6 +7,7 @@ exports.generateJsonReport = generateJsonReport;
 exports.generateMarkdownReport = generateMarkdownReport;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
+const analyzer_1 = require("./analyzer");
 function generateJsonReport(results, outputPath) {
     const outputDir = path_1.default.dirname(outputPath);
     if (!fs_1.default.existsSync(outputDir)) {
@@ -20,29 +21,39 @@ function generateMarkdownReport(results, outputPath) {
     if (!fs_1.default.existsSync(outputDir)) {
         fs_1.default.mkdirSync(outputDir, { recursive: true });
     }
-    const failedElements = results.filter(r => !r.isAccessible);
+    const failedElements = results.filter(analyzer_1.isIssue);
+    const unanalysedElements = results.filter(analyzer_1.isUnanalysed);
+    const cachedCount = results.filter(r => r.cached).length;
     let md = `# Accessibility Audit Report\n\n`;
     md += `**Total Elements Scanned:** ${results.length}\n`;
-    md += `**Accessibility Issues Found:** ${failedElements.length}\n\n`;
+    md += `**Accessibility Issues Found:** ${failedElements.length}\n`;
+    if (unanalysedElements.length > 0) {
+        md += `**Elements Not Analyzed (API errors):** ${unanalysedElements.length}\n`;
+    }
+    md += `**Verdicts Reused From Cache:** ${cachedCount}\n\n`;
     if (failedElements.length === 0) {
-        md += `🎉 Congratulations! No accessibility issues were found.\n`;
+        md += unanalysedElements.length === 0
+            ? `🎉 Congratulations! No accessibility issues were found.\n`
+            : `No accessibility issues were found in the analyzed elements, but the audit is incomplete.\n`;
     }
     else {
         md += `## Issues Detected\n\n`;
         failedElements.forEach((el, i) => {
             md += `### ${i + 1}. <${el.tagName}>: ${el.issueTitle || 'Issue'}\n\n`;
-            if (el.error) {
-                md += `**Error during analysis:** ${el.error}\n\n`;
-            }
-            else {
-                md += `**Explanation:** ${el.explanation}\n\n`;
-                md += `**Original Code:**\n\`\`\`html\n${el.elementHtml}\n\`\`\`\n\n`;
-                md += `**Suggested Fix:**\n\`\`\`html\n${el.suggestedFixCode}\n\`\`\`\n\n`;
-                md += `**Why this works:** ${el.fixReasoning}\n\n`;
-            }
+            md += `**Explanation:** ${el.explanation}\n\n`;
+            md += `**Original Code:**\n\`\`\`html\n${el.elementHtml}\n\`\`\`\n\n`;
+            md += `**Suggested Fix:**\n\`\`\`html\n${el.suggestedFixCode}\n\`\`\`\n\n`;
+            md += `**Why this works:** ${el.fixReasoning}\n\n`;
             md += `---\n\n`;
         });
     }
+    if (unanalysedElements.length > 0) {
+        md += `\n## Not Analyzed\n\n`;
+        md += `These elements could not be analyzed because of API errors. They are not cached, so re-running the audit retries only these.\n\n`;
+        unanalysedElements.forEach(el => {
+            md += `- \`<${el.tagName}>\`: ${el.error}\n`;
+        });
+    }
     fs_1.default.writeFileSync(outputPath, md);
     console.log(`Markdown report generated at ${outputPath}`);
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "dtp-caal",
-  "version": "1.0.1",
+  "version": "1.1.0",
   "description": "Context-Aware Accessibility Linter CLI powered by Playwright and LLMs",
   "main": "dist/index.js",
   "bin": {