@detiq/crawler 1.0.8 → 1.1.0

package/dist/job-queue.js

@@ -30,7 +30,7 @@ async function notifyAdmins(tenantId, data) {
   }
 }
 import { AppBrain, buildArtifactKey, getTenantStorageConfig, uploadScreenshotToTenantStorage, uploadFileToTenantStorage, notifyRunCompletion, recordLLMCall } from "@detiq/app-brain";
-import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile } from "@detiq/agents";
+import { generateTaxonomySkillFiles, groupScreensWithAI, generateExecutionPlan, refreshTestCasesSkillFile, refreshTestDataSkillFile, refreshTestScriptSkillFile, refreshRequirementsCoverageSkillFile, detectFieldsFromHtml, mergeIntoCatalog } from "@detiq/agents";
 import { crawlProject } from "./crawler.js";
 import { httpCrawl } from "./http-engine.js";
 import { hashContent, normalizeText } from "./incremental-diff.js";
@@ -519,6 +519,39 @@ async function processJob(jobId, config, opts) {
             );
           }
           await generateTaxonomySkillFiles(tenantId, projectId, projectMap.screens, result.apiEndpoints, result.performanceMetrics);
+          try {
+            const { readFile } = await import("node:fs/promises");
+            const { existsSync } = await import("node:fs");
+            const screensWithDom = projectMap.screens.filter((s) => s.domSnapshotPath);
+            const detectLimit = Math.min(screensWithDom.length, 20);
+            let fieldDetectAdded = 0;
+            for (let si = 0; si < detectLimit; si++) {
+              const screen = screensWithDom[si];
+              try {
+                let htmlContent = null;
+                const domPath = screen.domSnapshotPath;
+                if (domPath.startsWith("https://") || domPath.startsWith("http://")) {
+                  const res = await fetch(domPath);
+                  if (res.ok) htmlContent = await res.text();
+                } else if (existsSync(domPath)) {
+                  htmlContent = await readFile(domPath, "utf8");
+                }
+                if (!htmlContent) continue;
+                const detected = await detectFieldsFromHtml({ tenantId, projectId, html: htmlContent });
+                if (detected.length > 0) {
+                  const merged = await mergeIntoCatalog({ tenantId, projectId, detected });
+                  fieldDetectAdded += merged.added;
+                }
+              } catch (sErr) {
+                logger.warn({ screenId: screen.id, err: sErr }, "[field-detect] skipped screen");
+              }
+            }
+            if (fieldDetectAdded > 0 || detectLimit > 0) {
+              logger.info({ screens: detectLimit, added: fieldDetectAdded, projectId }, "[field-detect] catalog populated from crawl");
+            }
+          } catch (fdErr) {
+            logger.warn({ err: fdErr }, "[field-detect] post-crawl field detection failed");
+          }
           const tenantStorageCfg = await getTenantStorageConfig(tenantId).catch(() => null);
           if (tenantStorageCfg) {
             let uploaded = 0;