@detiq/crawler 1.0.7 → 1.0.9

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@detiq/crawler",
-  "version": "1.0.7",
+  "version": "1.0.9",
   "private": false,
   "publishConfig": {
     "access": "public",

package/src/crawler.ts

@@ -269,6 +269,7 @@ async function launchChromeCDP(
       '--remote-allow-origins=*',
       '--no-first-run',
       '--no-default-browser-check',
+      '--window-size=1280,900',
     ],
   });
 
@@ -2665,6 +2666,8 @@ export async function crawlProject(opts: {
         }
 
         const linkMappings = await extractLinkMappings(page, origin);
+        // Clear existing elements before re-saving — prevents duplicates on re-crawl
+        await AppBrain.clearScreenElements(opts.tenantId, opts.projectId, screen.id).catch(() => {});
         // Batch element saves: 4 concurrent DB writes instead of serial round-trips
         for (let _ei = 0; _ei < elements.length; _ei += 4) {
           const _batch = elements.slice(_ei, _ei + 4);
@@ -4040,8 +4043,15 @@ async function contentFallbackElements(page: any): Promise<Array<{ meaning: stri
     });
 
     // 8. Navigation links — explicitly captured (excluded from interactive element selector)
-    document.querySelectorAll('nav a, [role="navigation"] a, header nav a, footer nav a, [aria-label*="navigation" i] a, [aria-label*="menu" i] a').forEach((a: any) => {
-      if (!isVisible(a)) return;
+    // Use a lighter visibility check: skip offsetParent because nav is often inside a responsive
+    // container (e.g. hidden lg:grid) that may be display:none at the crawler's viewport,
+    // causing offsetParent to be null even though the link exists in the page's nav structure.
+    const isNavVisible = (el: Element) => {
+      const s = window.getComputedStyle(el);
+      return s.display !== 'none' && s.visibility !== 'hidden' && s.opacity !== '0';
+    };
+    document.querySelectorAll('nav a, [role="navigation"] a, header a, header nav a, footer nav a, [aria-label*="navigation" i] a, [aria-label*="menu" i] a').forEach((a: any) => {
+      if (!isNavVisible(a)) return;
       const label = (a.getAttribute('aria-label') || a.textContent?.replace(/\s+/g, ' ').trim() || '').slice(0, 60);
       const href = a.href;
       if (!label || !href || /^(javascript:|#$)/i.test(href)) return;

package/src/job-queue.ts

@@ -51,7 +51,7 @@ async function notifyAdmins(tenantId: string, data: {
   }
 }
 import { AppBrain, getProjectStorageConfig, buildArtifactKey, uploadArtifact, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from '@detiq/app-brain';
-import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile } from '@detiq/agents';
+import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from '@detiq/agents';
 import { crawlProject } from './crawler.js';
 import { httpCrawl } from './http-engine.js';
 import { hashContent, normalizeText } from './incremental-diff.js';
@@ -751,6 +751,43 @@ export async function processJob(jobId: string, config: any, opts?: ProcessJobOp
           }
           await generateTaxonomySkillFiles(tenantId, projectId, projectMap.screens as any, result.apiEndpoints, result.performanceMetrics);
 
+          // Auto-detect AppField catalog from crawled DOM snapshots.
+          // Runs before test generation so getFieldsForCase() returns typed fields (FORM mode).
+          // Capped at 20 screens and only where domSnapshotPath exists to bound LLM cost.
+          try {
+            const { readFile } = await import('node:fs/promises');
+            const { existsSync } = await import('node:fs');
+            const screensWithDom = (projectMap.screens as any[]).filter((s: any) => s.domSnapshotPath);
+            const detectLimit = Math.min(screensWithDom.length, 20);
+            let fieldDetectAdded = 0;
+            for (let si = 0; si < detectLimit; si++) {
+              const screen = screensWithDom[si];
+              try {
+                let htmlContent: string | null = null;
+                const domPath: string = screen.domSnapshotPath;
+                if (domPath.startsWith('https://') || domPath.startsWith('http://')) {
+                  const res = await fetch(domPath);
+                  if (res.ok) htmlContent = await res.text();
+                } else if (existsSync(domPath)) {
+                  htmlContent = await readFile(domPath, 'utf8');
+                }
+                if (!htmlContent) continue;
+                const detected = await detectFieldsFromHtml({ tenantId, projectId, html: htmlContent });
+                if (detected.length > 0) {
+                  const merged = await mergeIntoCatalog({ tenantId, projectId, detected });
+                  fieldDetectAdded += merged.added;
+                }
+              } catch (sErr) {
+                logger.warn({ screenId: screen.id, err: sErr }, '[field-detect] skipped screen');
+              }
+            }
+            if (fieldDetectAdded > 0 || detectLimit > 0) {
+              logger.info({ screens: detectLimit, added: fieldDetectAdded, projectId }, '[field-detect] catalog populated from crawl');
+            }
+          } catch (fdErr) {
+            logger.warn({ err: fdErr }, '[field-detect] post-crawl field detection failed');
+          }
+
           // Cloud storage: upload screenshots to tenant S3/R2 if configured
           const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
           if (tenantStorageCfg) {