@govtechsg/oobee 0.10.89 → 0.10.90

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@govtechsg/oobee",
   "main": "dist/npmIndex.js",
-  "version": "0.10.89",
+  "version": "0.10.90",
   "type": "module",
   "author": "Government Technology Agency <info@tech.gov.sg>",
   "bin": {

package/src/mergeAxeResults/itemsStore.ts

@@ -0,0 +1,73 @@
+import fs from 'fs-extra';
+import path from 'path';
+import readline from 'readline';
+import type { ItemsInfo } from './types.js';
+
+export interface ItemsStoreEntry {
+  url: string;
+  pageTitle: string;
+  items: ItemsInfo[];
+  filePath?: string;
+  pageIndex?: number;
+  pageImagePath?: string;
+  metadata?: string;
+}
+
+export class ItemsStore {
+  private basePath: string;
+  private ensuredDirs = new Set<string>();
+
+  constructor(storagePath: string) {
+    this.basePath = path.join(storagePath, 'tmp-items');
+  }
+
+  private sanitizeRuleId(ruleId: string): string {
+    return ruleId.replace(/[^a-zA-Z0-9_-]/g, '_');
+  }
+
+  private getRuleFilePath(category: string, ruleId: string): string {
+    return path.join(this.basePath, category, `${this.sanitizeRuleId(ruleId)}.jsonl`);
+  }
+
+  private async ensureDir(category: string): Promise<void> {
+    const dirPath = path.join(this.basePath, category);
+    if (!this.ensuredDirs.has(dirPath)) {
+      await fs.ensureDir(dirPath);
+      this.ensuredDirs.add(dirPath);
+    }
+  }
+
+  async appendPageItems(category: string, ruleId: string, entry: ItemsStoreEntry): Promise<void> {
+    await this.ensureDir(category);
+    const filePath = this.getRuleFilePath(category, ruleId);
+    const line = JSON.stringify(entry) + '\n';
+    await fs.appendFile(filePath, line, 'utf8');
+  }
+
+  async *readRuleItems(category: string, ruleId: string): AsyncGenerator<ItemsStoreEntry> {
+    const filePath = this.getRuleFilePath(category, ruleId);
+    if (!fs.existsSync(filePath)) return;
+
+    const fileStream = fs.createReadStream(filePath, { encoding: 'utf8' });
+    const rl = readline.createInterface({ input: fileStream, crlfDelay: Infinity });
+
+    for await (const line of rl) {
+      if (line.trim()) {
+        yield JSON.parse(line) as ItemsStoreEntry;
+      }
+    }
+  }
+
+  async readRuleItemsMap(category: string, ruleId: string): Promise<Map<string, ItemsStoreEntry>> {
+    const map = new Map<string, ItemsStoreEntry>();
+    for await (const entry of this.readRuleItems(category, ruleId)) {
+      const key = entry.pageIndex != null ? String(entry.pageIndex) : entry.url;
+      map.set(key, entry);
+    }
+    return map;
+  }
+
+  async cleanup(): Promise<void> {
+    await fs.rm(this.basePath, { recursive: true, force: true });
+  }
+}

package/src/mergeAxeResults/jsonArtifacts.ts

@@ -6,6 +6,7 @@ import { pipeline } from 'stream/promises';
 import { a11yRuleShortDescriptionMap } from '../constants/constants.js';
 import { consoleLogger } from '../logs.js';
 import type { AllIssues } from './types.js';
+import type { ItemsStore } from './itemsStore.js';
 
 function* serializeObject(obj: any, depth = 0, indent = '  ') {
   const currentIndent = indent.repeat(depth);
@@ -79,158 +80,151 @@ function writeLargeJsonToFile(obj: object, filePath: string) {
   });
 }
 
-const writeLargeScanItemsJsonToFile = async (obj: object, filePath: string) => {
-  return new Promise((resolve, reject) => {
-    const writeStream = fs.createWriteStream(filePath, { flags: 'a', encoding: 'utf8' });
-    const writeQueue: string[] = [];
-    let isWriting = false;
-
-    const processNextWrite = async () => {
-      if (isWriting || writeQueue.length === 0) return;
-
-      isWriting = true;
-      const data = writeQueue.shift()!;
-
-      try {
-        if (!writeStream.write(data)) {
-          await new Promise<void>(resolve => {
-            writeStream.once('drain', () => {
-              resolve();
-            });
-          });
-        }
-      } catch (error) {
-        writeStream.destroy(error as Error);
-        return;
-      }
+const writeLargeScanItemsJsonToFile = async (
+  obj: object,
+  filePath: string,
+  itemsStore?: ItemsStore,
+) => {
+  const writeStream = fs.createWriteStream(filePath, { flags: 'a', encoding: 'utf8' });
 
-      isWriting = false;
-      processNextWrite();
-    };
-
-    const queueWrite = (data: string) => {
-      writeQueue.push(data);
-      processNextWrite();
-    };
+  const write = (data: string): Promise<void> => {
+    if (!writeStream.write(data)) {
+      return new Promise<void>(resolve => writeStream.once('drain', resolve));
+    }
+    return Promise.resolve();
+  };
 
-    writeStream.on('error', error => {
-      consoleLogger.error(`Error writing object to JSON file: ${error}`);
-      reject(error);
-    });
+  try {
+    await write('{\n');
+    const keys = Object.keys(obj);
 
-    writeStream.on('finish', () => {
-      consoleLogger.info(`JSON file written successfully: ${filePath}`);
-      resolve(true);
-    });
+    for (let i = 0; i < keys.length; i++) {
+      const key = keys[i];
+      const value = obj[key];
+
+      if (value === null || typeof value !== 'object' || Array.isArray(value)) {
+        await write(`  "${key}": ${JSON.stringify(value)}`);
+      } else {
+        await write(`  "${key}": {\n`);
+
+        const { rules, ...otherProperties } = value;
+        const otherKeys = Object.keys(otherProperties);
+
+        for (let j = 0; j < otherKeys.length; j++) {
+          const propKey = otherKeys[j];
+          const propValue = otherProperties[propKey];
+          const propValueString =
+            propValue === null ||
+            typeof propValue === 'function' ||
+            typeof propValue === 'undefined'
+              ? 'null'
+              : JSON.stringify(propValue);
+          await write(`    "${propKey}": ${propValueString}`);
+          if (j < otherKeys.length - 1 || (rules && rules.length >= 0)) {
+            await write(',\n');
+          } else {
+            await write('\n');
+          }
+        }
 
-    try {
-      queueWrite('{\n');
-      const keys = Object.keys(obj);
-
-      keys.forEach((key, i) => {
-        const value = obj[key];
-
-        if (value === null || typeof value !== 'object' || Array.isArray(value)) {
-          queueWrite(`  "${key}": ${JSON.stringify(value)}`);
-        } else {
-          queueWrite(`  "${key}": {\n`);
-
-          const { rules, ...otherProperties } = value;
-
-          Object.entries(otherProperties).forEach(([propKey, propValue], j) => {
-            const propValueString =
-              propValue === null ||
-              typeof propValue === 'function' ||
-              typeof propValue === 'undefined'
-                ? 'null'
-                : JSON.stringify(propValue);
-            queueWrite(`    "${propKey}": ${propValueString}`);
-            if (j < Object.keys(otherProperties).length - 1 || (rules && rules.length >= 0)) {
-              queueWrite(',\n');
-            } else {
-              queueWrite('\n');
+        if (rules && Array.isArray(rules)) {
+          await write('    "rules": [\n');
+
+          for (let j = 0; j < rules.length; j++) {
+            const rule = rules[j];
+            await write('      {\n');
+            const { pagesAffected, ...otherRuleProperties } = rule;
+            const ruleKeys = Object.keys(otherRuleProperties);
+
+            for (let k = 0; k < ruleKeys.length; k++) {
+              const ruleKey = ruleKeys[k];
+              const ruleValue = otherRuleProperties[ruleKey];
+              const ruleValueString =
+                ruleValue === null ||
+                typeof ruleValue === 'function' ||
+                typeof ruleValue === 'undefined'
+                  ? 'null'
+                  : JSON.stringify(ruleValue);
+              await write(`        "${ruleKey}": ${ruleValueString}`);
+              if (k < ruleKeys.length - 1 || pagesAffected) {
+                await write(',\n');
+              } else {
+                await write('\n');
+              }
             }
-          });
-
-          if (rules && Array.isArray(rules)) {
-            queueWrite('    "rules": [\n');
-
-            rules.forEach((rule, j) => {
-              queueWrite('      {\n');
-              const { pagesAffected, ...otherRuleProperties } = rule;
-
-              Object.entries(otherRuleProperties).forEach(([ruleKey, ruleValue], k) => {
-                const ruleValueString =
-                  ruleValue === null ||
-                  typeof ruleValue === 'function' ||
-                  typeof ruleValue === 'undefined'
-                    ? 'null'
-                    : JSON.stringify(ruleValue);
-                queueWrite(`        "${ruleKey}": ${ruleValueString}`);
-                if (k < Object.keys(otherRuleProperties).length - 1 || pagesAffected) {
-                  queueWrite(',\n');
-                } else {
-                  queueWrite('\n');
-                }
-              });
 
-              if (pagesAffected && Array.isArray(pagesAffected)) {
-                queueWrite('        "pagesAffected": [\n');
+            if (pagesAffected && Array.isArray(pagesAffected)) {
+              // Load items from disk for this rule if itemsStore is available
+              let itemsMap: Map<string, any> | null = null;
+              if (itemsStore && rule.rule) {
+                itemsMap = await itemsStore.readRuleItemsMap(key, rule.rule);
+              }
 
-                pagesAffected.forEach((page, p) => {
-                  const pageJson = JSON.stringify(page, null, 2)
-                    .split('\n')
-                    .map(line => `          ${line}`)
-                    .join('\n');
+              await write('        "pagesAffected": [\n');
 
-                  queueWrite(pageJson);
+              for (let p = 0; p < pagesAffected.length; p++) {
+                const page = pagesAffected[p];
+                let fullPage = page;
 
-                  if (p < pagesAffected.length - 1) {
-                    queueWrite(',\n');
-                  } else {
-                    queueWrite('\n');
+                if (itemsMap) {
+                  const lookupKey =
+                    page.pageIndex != null ? String(page.pageIndex) : page.url;
+                  const entry = itemsMap.get(lookupKey);
+                  if (entry) {
+                    // Strip itemsCount to match original scanItems.json format
+                    const { itemsCount: _ic, ...pageWithoutCount } = page;
+                    fullPage = { ...pageWithoutCount, items: entry.items };
                   }
-                });
+                }
 
-                queueWrite('        ]');
-              }
+                const pageJson = JSON.stringify(fullPage, null, 2)
+                  .split('\n')
+                  .map(line => `          ${line}`)
+                  .join('\n');
 
-              queueWrite('\n      }');
-              if (j < rules.length - 1) {
-                queueWrite(',\n');
-              } else {
-                queueWrite('\n');
-              }
-            });
+                await write(pageJson);
 
-            queueWrite('    ]');
-          }
-          queueWrite('\n  }');
-        }
+                if (p < pagesAffected.length - 1) {
+                  await write(',\n');
+                } else {
+                  await write('\n');
+                }
+              }
 
-        if (i < keys.length - 1) {
-          queueWrite(',\n');
-        } else {
-          queueWrite('\n');
-        }
-      });
+              await write('        ]');
+            }
 
-      queueWrite('}\n');
+            await write('\n      }');
+            if (j < rules.length - 1) {
+              await write(',\n');
+            } else {
+              await write('\n');
+            }
+          }
 
-      const checkQueueAndEnd = () => {
-        if (writeQueue.length === 0 && !isWriting) {
-          writeStream.end();
-        } else {
-          setTimeout(checkQueueAndEnd, 100);
+          await write('    ]');
         }
-      };
+        await write('\n  }');
+      }
 
-      checkQueueAndEnd();
-    } catch (err) {
-      writeStream.destroy(err as Error);
-      reject(err);
+      if (i < keys.length - 1) {
+        await write(',\n');
+      } else {
+        await write('\n');
+      }
     }
-  });
+
+    await write('}\n');
+  } finally {
+    writeStream.end();
+    await new Promise<void>((resolve, reject) => {
+      writeStream.on('finish', () => {
+        consoleLogger.info(`JSON file written successfully: ${filePath}`);
+        resolve();
+      });
+      writeStream.on('error', reject);
+    });
+  }
 };
 
 async function compressJsonFileStreaming(inputPath: string, outputPath: string) {
@@ -247,12 +241,13 @@ const writeJsonFileAndCompressedJsonFile = async (
   data: object,
   storagePath: string,
   filename: string,
+  itemsStore?: ItemsStore,
 ): Promise<{ jsonFilePath: string; base64FilePath: string }> => {
   try {
     consoleLogger.info(`Writing JSON to ${filename}.json`);
     const jsonFilePath = path.join(storagePath, `${filename}.json`);
     if (filename === 'scanItems') {
-      await writeLargeScanItemsJsonToFile(data, jsonFilePath);
+      await writeLargeScanItemsJsonToFile(data, jsonFilePath, itemsStore);
     } else {
       await writeLargeJsonToFile(data, jsonFilePath);
     }
@@ -277,6 +272,7 @@ const writeJsonFileAndCompressedJsonFile = async (
 const writeJsonAndBase64Files = async (
   allIssues: AllIssues,
   storagePath: string,
+  itemsStore?: ItemsStore,
 ): Promise<{
   scanDataJsonFilePath: string;
   scanDataBase64FilePath: string;
@@ -301,6 +297,7 @@ const writeJsonAndBase64Files = async (
       { oobeeAppVersion: allIssues.oobeeAppVersion, ...items },
       storagePath,
       'scanItems',
+      itemsStore,
     );
 
   ['mustFix', 'goodToFix', 'needsReview', 'passed'].forEach(category => {
@@ -345,22 +342,22 @@ const writeJsonAndBase64Files = async (
 
   items.mustFix.rules.forEach(rule => {
     rule.pagesAffected.forEach(page => {
-      page.itemsCount = page.items.length;
+      page.itemsCount = page.itemsCount ?? (Array.isArray(page.items) ? page.items.length : 0);
     });
   });
   items.goodToFix.rules.forEach(rule => {
     rule.pagesAffected.forEach(page => {
-      page.itemsCount = page.items.length;
+      page.itemsCount = page.itemsCount ?? (Array.isArray(page.items) ? page.items.length : 0);
     });
   });
   items.needsReview.rules.forEach(rule => {
     rule.pagesAffected.forEach(page => {
-      page.itemsCount = page.items.length;
+      page.itemsCount = page.itemsCount ?? (Array.isArray(page.items) ? page.items.length : 0);
     });
   });
   items.passed.rules.forEach(rule => {
     rule.pagesAffected.forEach(page => {
-      page.itemsCount = page.items.length;
+      page.itemsCount = page.itemsCount ?? (Array.isArray(page.items) ? page.items.length : 0);
     });
   });
 

package/src/mergeAxeResults/scanPages.ts

@@ -40,8 +40,8 @@ export default function populateScanPagesDetail(allIssues: AllIssues): void {
       const { rule: ruleId, conformance = [] } = rule;
 
       rule.pagesAffected.forEach(p => {
-        const { url, pageTitle, items = [] } = p;
-        const itemsCount = items.length;
+        const { url, pageTitle, itemsCount: storedCount, items } = p as any;
+        const itemsCount: number = storedCount ?? (Array.isArray(items) ? items.length : 0);
 
         if (!pagesMap[url]) {
           pagesMap[url] = {