@govtechsg/oobee 0.10.89 → 0.10.90

package/.github/workflows/bump-package-version.yml

@@ -28,19 +28,28 @@ jobs:
           current_version=$(node -p "require('./package.json').version")
           echo "version=$current_version" >> "$GITHUB_OUTPUT"
 
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Build
+        run: npm run build
+
       - name: Bump patch version
         id: bump
         run: |
           new_version=$(npm version patch --no-git-tag-version)
           echo "version=$new_version" >> "$GITHUB_OUTPUT"
 
+      - name: Generate oobee-client-scanner.js
+        run: node dist/generateOobeeClientScanner.js
+
       - name: Create branch and commit changes
         run: |
           BRANCH="bump/version-${{ steps.bump.outputs.version }}"
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
           git checkout -b "$BRANCH"
-          git add package.json package-lock.json
+          git add package.json package-lock.json oobee-client-scanner.js
           git commit -m "chore: bump version ${{ steps.current.outputs.version }} → ${{ steps.bump.outputs.version }}"
           git push origin "$BRANCH"
           echo "BRANCH=$BRANCH" >> $GITHUB_ENV

package/.github/workflows/docker-push-ghcr.yml

@@ -7,6 +7,8 @@ on:
         description: 'Release tag for the image (e.g. 0.10.87)'
         required: true
         type: string
+  release:
+    types: [published]
 
 permissions:
   contents: read
@@ -15,6 +17,8 @@ permissions:
 jobs:
   build-and-push:
     runs-on: ubuntu-latest
+    env:
+      RELEASE_TAG: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }}
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
@@ -43,7 +47,7 @@ jobs:
           platforms: linux/amd64,linux/arm64
           push: true
           tags: |
-            ghcr.io/${{ steps.repo.outputs.name }}:${{ inputs.release_tag }}
+            ghcr.io/${{ steps.repo.outputs.name }}:${{ env.RELEASE_TAG }}
             ghcr.io/${{ steps.repo.outputs.name }}:latest
           cache-from: type=gha
           cache-to: type=gha,mode=max

package/.github/workflows/publish.yml

@@ -22,13 +22,18 @@ jobs:
       - run: npm run build
         continue-on-error: false
 
-      - name: Create and push git tag
+      - name: Generate oobee-client-scanner.js
+        run: node dist/generateOobeeClientScanner.js
+
+      - name: Create and push git tags
         run: |
           VERSION=$(node -p "require('./package.json').version")
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
-          git tag -af "v${VERSION}" -m "Version ${VERSION}"
-          git push origin "v${VERSION}" --force
+          git tag -af "${VERSION}" -m "Version ${VERSION}"
+          git push origin "${VERSION}" --force
+          git tag -af "latest" -m "Latest release ${VERSION}"
+          git push origin "latest" --force
 
       - run: npm publish
         env:

package/README.md

@@ -6,6 +6,8 @@ This is the engine and command-line interface (CLI) for Oobee. The official appl
 
 For the **user-friendly desktop application**, check out [Oobee](https://go.gov.sg/oobee).
 
+[![DPG Badge](https://img.shields.io/badge/Verified-DPG-3333AB?logo=data:image/svg%2bxml;base64,PHN2ZyB3aWR0aD0iMzEiIGhlaWdodD0iMzMiIHZpZXdCb3g9IjAgMCAzMSAzMyIgZmlsbD0ibm9uZSIgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIj4KPHBhdGggZD0iTTE0LjIwMDggMjEuMzY3OEwxMC4xNzM2IDE4LjAxMjRMMTEuNTIxOSAxNi40MDAzTDEzLjk5MjggMTguNDU5TDE5LjYyNjkgMTIuMjExMUwyMS4xOTA5IDEzLjYxNkwxNC4yMDA4IDIxLjM2NzhaTTI0LjYyNDEgOS4zNTEyN0wyNC44MDcxIDMuMDcyOTdMMTguODgxIDUuMTg2NjJMMTUuMzMxNCAtMi4zMzA4MmUtMDVMMTEuNzgyMSA1LjE4NjYyTDUuODU2MDEgMy4wNzI5N0w2LjAzOTA2IDkuMzUxMjdMMCAxMS4xMTc3TDMuODQ1MjEgMTYuMDg5NUwwIDIxLjA2MTJMNi4wMzkwNiAyMi44Mjc3TDUuODU2MDEgMjkuMTA2TDExLjc4MjEgMjYuOTkyM0wxNS4zMzE0IDMyLjE3OUwxOC44ODEgMjYuOTkyM0wyNC44MDcxIDI5LjEwNkwyNC42MjQxIDIyLjgyNzdMMzAuNjYzMSAyMS4wNjEyTDI2LjgxNzYgMTYuMDg5NUwzMC42NjMxIDExLjExNzdMMjQuNjI0MSA5LjM1MTI3WiIgZmlsbD0id2hpdGUiLz4KPC9zdmc+Cg==)](https://digitalpublicgoods.net/r/oobee)
+
 ## Technology Stack
 
 1. [Crawlee](https://crawlee.dev/)

package/dist/mergeAxeResults/itemsStore.js

@@ -0,0 +1,51 @@
+import fs from 'fs-extra';
+import path from 'path';
+import readline from 'readline';
+export class ItemsStore {
+    constructor(storagePath) {
+        this.ensuredDirs = new Set();
+        this.basePath = path.join(storagePath, 'tmp-items');
+    }
+    sanitizeRuleId(ruleId) {
+        return ruleId.replace(/[^a-zA-Z0-9_-]/g, '_');
+    }
+    getRuleFilePath(category, ruleId) {
+        return path.join(this.basePath, category, `${this.sanitizeRuleId(ruleId)}.jsonl`);
+    }
+    async ensureDir(category) {
+        const dirPath = path.join(this.basePath, category);
+        if (!this.ensuredDirs.has(dirPath)) {
+            await fs.ensureDir(dirPath);
+            this.ensuredDirs.add(dirPath);
+        }
+    }
+    async appendPageItems(category, ruleId, entry) {
+        await this.ensureDir(category);
+        const filePath = this.getRuleFilePath(category, ruleId);
+        const line = JSON.stringify(entry) + '\n';
+        await fs.appendFile(filePath, line, 'utf8');
+    }
+    async *readRuleItems(category, ruleId) {
+        const filePath = this.getRuleFilePath(category, ruleId);
+        if (!fs.existsSync(filePath))
+            return;
+        const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' });
+        const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
+        for await (const line of rl) {
+            if (line.trim()) {
+                yield JSON.parse(line);
+            }
+        }
+    }
+    async readRuleItemsMap(category, ruleId) {
+        const map = new Map();
+        for await (const entry of this.readRuleItems(category, ruleId)) {
+            const key = entry.pageIndex != null ? String(entry.pageIndex) : entry.url;
+            map.set(key, entry);
+        }
+        return map;
+    }
+    async cleanup() {
+        await fs.rm(this.basePath, { recursive: true, force: true });
+    }
+}

package/dist/mergeAxeResults/jsonArtifacts.js

@@ -67,140 +67,130 @@ function writeLargeJsonToFile(obj, filePath) {
         write();
     });
 }
-const writeLargeScanItemsJsonToFile = async (obj, filePath) => {
-    return new Promise((resolve, reject) => {
-        const writeStream = fs.createWriteStream(filePath, { flags: 'a', encoding: 'utf8' });
-        const writeQueue = [];
-        let isWriting = false;
-        const processNextWrite = async () => {
-            if (isWriting || writeQueue.length === 0)
-                return;
-            isWriting = true;
-            const data = writeQueue.shift();
-            try {
-                if (!writeStream.write(data)) {
-                    await new Promise(resolve => {
-                        writeStream.once('drain', () => {
-                            resolve();
-                        });
-                    });
-                }
-            }
-            catch (error) {
-                writeStream.destroy(error);
-                return;
+const writeLargeScanItemsJsonToFile = async (obj, filePath, itemsStore) => {
+    const writeStream = fs.createWriteStream(filePath, { flags: 'a', encoding: 'utf8' });
+    const write = (data) => {
+        if (!writeStream.write(data)) {
+            return new Promise(resolve => writeStream.once('drain', resolve));
+        }
+        return Promise.resolve();
+    };
+    try {
+        await write('{\n');
+        const keys = Object.keys(obj);
+        for (let i = 0; i < keys.length; i++) {
+            const key = keys[i];
+            const value = obj[key];
+            if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+                await write(`  "${key}": ${JSON.stringify(value)}`);
             }
-            isWriting = false;
-            processNextWrite();
-        };
-        const queueWrite = (data) => {
-            writeQueue.push(data);
-            processNextWrite();
-        };
-        writeStream.on('error', error => {
-            consoleLogger.error(`Error writing object to JSON file: ${error}`);
-            reject(error);
-        });
-        writeStream.on('finish', () => {
-            consoleLogger.info(`JSON file written successfully: ${filePath}`);
-            resolve(true);
-        });
-        try {
-            queueWrite('{\n');
-            const keys = Object.keys(obj);
-            keys.forEach((key, i) => {
-                const value = obj[key];
-                if (value === null || typeof value !== 'object' || Array.isArray(value)) {
-                    queueWrite(`  "${key}": ${JSON.stringify(value)}`);
+            else {
+                await write(`  "${key}": {\n`);
+                const { rules, ...otherProperties } = value;
+                const otherKeys = Object.keys(otherProperties);
+                for (let j = 0; j < otherKeys.length; j++) {
+                    const propKey = otherKeys[j];
+                    const propValue = otherProperties[propKey];
+                    const propValueString = propValue === null ||
+                        typeof propValue === 'function' ||
+                        typeof propValue === 'undefined'
+                        ? 'null'
+                        : JSON.stringify(propValue);
+                    await write(`    "${propKey}": ${propValueString}`);
+                    if (j < otherKeys.length - 1 || (rules && rules.length >= 0)) {
+                        await write(',\n');
+                    }
+                    else {
+                        await write('\n');
+                    }
                 }
-                else {
-                    queueWrite(`  "${key}": {\n`);
-                    const { rules, ...otherProperties } = value;
-                    Object.entries(otherProperties).forEach(([propKey, propValue], j) => {
-                        const propValueString = propValue === null ||
-                            typeof propValue === 'function' ||
-                            typeof propValue === 'undefined'
-                            ? 'null'
-                            : JSON.stringify(propValue);
-                        queueWrite(`    "${propKey}": ${propValueString}`);
-                        if (j < Object.keys(otherProperties).length - 1 || (rules && rules.length >= 0)) {
-                            queueWrite(',\n');
-                        }
-                        else {
-                            queueWrite('\n');
+                if (rules && Array.isArray(rules)) {
+                    await write('    "rules": [\n');
+                    for (let j = 0; j < rules.length; j++) {
+                        const rule = rules[j];
+                        await write('      {\n');
+                        const { pagesAffected, ...otherRuleProperties } = rule;
+                        const ruleKeys = Object.keys(otherRuleProperties);
+                        for (let k = 0; k < ruleKeys.length; k++) {
+                            const ruleKey = ruleKeys[k];
+                            const ruleValue = otherRuleProperties[ruleKey];
+                            const ruleValueString = ruleValue === null ||
+                                typeof ruleValue === 'function' ||
+                                typeof ruleValue === 'undefined'
+                                ? 'null'
+                                : JSON.stringify(ruleValue);
+                            await write(`        "${ruleKey}": ${ruleValueString}`);
+                            if (k < ruleKeys.length - 1 || pagesAffected) {
+                                await write(',\n');
+                            }
+                            else {
+                                await write('\n');
+                            }
                         }
-                    });
-                    if (rules && Array.isArray(rules)) {
-                        queueWrite('    "rules": [\n');
-                        rules.forEach((rule, j) => {
-                            queueWrite('      {\n');
-                            const { pagesAffected, ...otherRuleProperties } = rule;
-                            Object.entries(otherRuleProperties).forEach(([ruleKey, ruleValue], k) => {
-                                const ruleValueString = ruleValue === null ||
-                                    typeof ruleValue === 'function' ||
-                                    typeof ruleValue === 'undefined'
-                                    ? 'null'
-                                    : JSON.stringify(ruleValue);
-                                queueWrite(`        "${ruleKey}": ${ruleValueString}`);
-                                if (k < Object.keys(otherRuleProperties).length - 1 || pagesAffected) {
-                                    queueWrite(',\n');
+                        if (pagesAffected && Array.isArray(pagesAffected)) {
+                            // Load items from disk for this rule if itemsStore is available
+                            let itemsMap = null;
+                            if (itemsStore && rule.rule) {
+                                itemsMap = await itemsStore.readRuleItemsMap(key, rule.rule);
+                            }
+                            await write('        "pagesAffected": [\n');
+                            for (let p = 0; p < pagesAffected.length; p++) {
+                                const page = pagesAffected[p];
+                                let fullPage = page;
+                                if (itemsMap) {
+                                    const lookupKey = page.pageIndex != null ? String(page.pageIndex) : page.url;
+                                    const entry = itemsMap.get(lookupKey);
+                                    if (entry) {
+                                        // Strip itemsCount to match original scanItems.json format
+                                        const { itemsCount: _ic, ...pageWithoutCount } = page;
+                                        fullPage = { ...pageWithoutCount, items: entry.items };
+                                    }
+                                }
+                                const pageJson = JSON.stringify(fullPage, null, 2)
+                                    .split('\n')
+                                    .map(line => `          ${line}`)
+                                    .join('\n');
+                                await write(pageJson);
+                                if (p < pagesAffected.length - 1) {
+                                    await write(',\n');
                                 }
                                 else {
-                                    queueWrite('\n');
+                                    await write('\n');
                                 }
-                            });
-                            if (pagesAffected && Array.isArray(pagesAffected)) {
-                                queueWrite('        "pagesAffected": [\n');
-                                pagesAffected.forEach((page, p) => {
-                                    const pageJson = JSON.stringify(page, null, 2)
-                                        .split('\n')
-                                        .map(line => `          ${line}`)
-                                        .join('\n');
-                                    queueWrite(pageJson);
-                                    if (p < pagesAffected.length - 1) {
-                                        queueWrite(',\n');
-                                    }
-                                    else {
-                                        queueWrite('\n');
-                                    }
-                                });
-                                queueWrite('        ]');
-                            }
-                            queueWrite('\n      }');
-                            if (j < rules.length - 1) {
-                                queueWrite(',\n');
-                            }
-                            else {
-                                queueWrite('\n');
                             }
-                        });
-                        queueWrite('    ]');
+                            await write('        ]');
+                        }
+                        await write('\n      }');
+                        if (j < rules.length - 1) {
+                            await write(',\n');
+                        }
+                        else {
+                            await write('\n');
+                        }
                     }
-                    queueWrite('\n  }');
-                }
-                if (i < keys.length - 1) {
-                    queueWrite(',\n');
-                }
-                else {
-                    queueWrite('\n');
+                    await write('    ]');
                 }
-            });
-            queueWrite('}\n');
-            const checkQueueAndEnd = () => {
-                if (writeQueue.length === 0 && !isWriting) {
-                    writeStream.end();
-                }
-                else {
-                    setTimeout(checkQueueAndEnd, 100);
-                }
-            };
-            checkQueueAndEnd();
-        }
-        catch (err) {
-            writeStream.destroy(err);
-            reject(err);
+                await write('\n  }');
+            }
+            if (i < keys.length - 1) {
+                await write(',\n');
+            }
+            else {
+                await write('\n');
+            }
         }
-    });
+        await write('}\n');
+    }
+    finally {
+        writeStream.end();
+        await new Promise((resolve, reject) => {
+            writeStream.on('finish', () => {
+                consoleLogger.info(`JSON file written successfully: ${filePath}`);
+                resolve();
+            });
+            writeStream.on('error', reject);
+        });
+    }
 };
 async function compressJsonFileStreaming(inputPath, outputPath) {
     const readStream = fs.createReadStream(inputPath);
@@ -210,12 +200,12 @@ async function compressJsonFileStreaming(inputPath, outputPath) {
     await pipeline(readStream, gzip, base64Encode, writeStream);
     consoleLogger.info(`File successfully compressed and saved to ${outputPath}`);
 }
-const writeJsonFileAndCompressedJsonFile = async (data, storagePath, filename) => {
+const writeJsonFileAndCompressedJsonFile = async (data, storagePath, filename, itemsStore) => {
     try {
         consoleLogger.info(`Writing JSON to ${filename}.json`);
         const jsonFilePath = path.join(storagePath, `${filename}.json`);
         if (filename === 'scanItems') {
-            await writeLargeScanItemsJsonToFile(data, jsonFilePath);
+            await writeLargeScanItemsJsonToFile(data, jsonFilePath, itemsStore);
         }
         else {
             await writeLargeJsonToFile(data, jsonFilePath);
@@ -234,10 +224,10 @@ const writeJsonFileAndCompressedJsonFile = async (data, storagePath, filename) =
         throw error;
     }
 };
-const writeJsonAndBase64Files = async (allIssues, storagePath) => {
+const writeJsonAndBase64Files = async (allIssues, storagePath, itemsStore) => {
     const { items, ...rest } = allIssues;
     const { jsonFilePath: scanDataJsonFilePath, base64FilePath: scanDataBase64FilePath } = await writeJsonFileAndCompressedJsonFile(rest, storagePath, 'scanData');
-    const { jsonFilePath: scanItemsJsonFilePath, base64FilePath: scanItemsBase64FilePath } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...items }, storagePath, 'scanItems');
+    const { jsonFilePath: scanItemsJsonFilePath, base64FilePath: scanItemsBase64FilePath } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...items }, storagePath, 'scanItems', itemsStore);
     ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
         if (items[category].rules && Array.isArray(items[category].rules)) {
             items[category].rules.forEach(rule => {
@@ -267,22 +257,22 @@ const writeJsonAndBase64Files = async (allIssues, storagePath) => {
     const { jsonFilePath: scanIssuesSummaryJsonFilePath, base64FilePath: scanIssuesSummaryBase64FilePath, } = await writeJsonFileAndCompressedJsonFile({ oobeeAppVersion: allIssues.oobeeAppVersion, ...scanIssuesSummary }, storagePath, 'scanIssuesSummary');
     items.mustFix.rules.forEach(rule => {
         rule.pagesAffected.forEach(page => {
-            page.itemsCount = page.items.length;
+            page.itemsCount = page.itemsCount ?? (Array.isArray(page.items) ? page.items.length : 0);
         });
     });
     items.goodToFix.rules.forEach(rule => {
         rule.pagesAffected.forEach(page => {
-            page.itemsCount = page.items.length;
+            page.itemsCount = page.itemsCount ?? (Array.isArray(page.items) ? page.items.length : 0);
         });
     });
     items.needsReview.rules.forEach(rule => {
         rule.pagesAffected.forEach(page => {
-            page.itemsCount = page.items.length;
+            page.itemsCount = page.itemsCount ?? (Array.isArray(page.items) ? page.items.length : 0);
         });
     });
     items.passed.rules.forEach(rule => {
         rule.pagesAffected.forEach(page => {
-            page.itemsCount = page.items.length;
+            page.itemsCount = page.itemsCount ?? (Array.isArray(page.items) ? page.items.length : 0);
         });
     });
     items.mustFix.totalRuleIssues = items.mustFix.rules.length;

package/dist/mergeAxeResults/scanPages.js

@@ -15,8 +15,8 @@ export default function populateScanPagesDetail(allIssues) {
         categoryData.rules.forEach(rule => {
             const { rule: ruleId, conformance = [] } = rule;
             rule.pagesAffected.forEach(p => {
-                const { url, pageTitle, items = [] } = p;
-                const itemsCount = items.length;
+                const { url, pageTitle, itemsCount: storedCount, items } = p;
+                const itemsCount = storedCount ?? (Array.isArray(items) ? items.length : 0);
                 if (!pagesMap[url]) {
                     pagesMap[url] = {
                         pageTitle,