@detiq/crawler 1.1.14 → 1.1.15

package/dist/crawl-session-manager.js

@@ -0,0 +1,145 @@
+const LOGIN_PATH_PATTERNS = [
+  "/login",
+  "/sign-in",
+  "/signin",
+  "/auth/login",
+  "/auth/sign-in"
+];
+function createSessionManager(config) {
+  let healthy = true;
+  let refreshAttempts = 0;
+  let authFailure = false;
+  const events = [];
+  const maxAttempts = config.maxRefreshAttempts ?? 3;
+  function onResponse(url, status) {
+    if ((status === 401 || status === 403) && healthy) {
+      healthy = false;
+      events.push({ type: "EXPIRY_DETECTED", isoTimestamp: (/* @__PURE__ */ new Date()).toISOString(), url });
+    }
+  }
+  function onNavigation(url) {
+    const isLoginPage = LOGIN_PATH_PATTERNS.some((pattern) => url.includes(pattern));
+    if (isLoginPage && healthy) {
+      healthy = false;
+      events.push({ type: "EXPIRY_DETECTED", isoTimestamp: (/* @__PURE__ */ new Date()).toISOString(), url });
+    }
+  }
+  function isHealthy() {
+    return healthy;
+  }
+  function needsRefresh() {
+    return !healthy && !authFailure && refreshAttempts < maxAttempts;
+  }
+  function markHealthy(isoTimestamp) {
+    healthy = true;
+    events.push({ type: "HEALTHY", isoTimestamp });
+  }
+  async function tryRefreshEndpoint(page, isoTimestamp) {
+    if (!config.refreshEndpoint) return false;
+    events.push({
+      type: "REFRESH_ATTEMPT",
+      isoTimestamp,
+      strategy: "refreshEndpoint",
+      attemptCount: refreshAttempts + 1
+    });
+    try {
+      const result = await page.evaluate(async (endpoint) => {
+        try {
+          const res = await fetch(endpoint, { method: "POST", credentials: "include" });
+          if (res.ok) {
+            const data = await res.json().catch(() => ({}));
+            return { ok: true, token: data.access_token || data.token || null };
+          }
+          return { ok: false, token: null };
+        } catch {
+          return { ok: false, token: null };
+        }
+      }, config.refreshEndpoint);
+      if (result.ok) {
+        if (result.token && config.tokenLocalStorageKey) {
+          await page.evaluate(
+            ([key, val]) => localStorage.setItem(key, val),
+            [config.tokenLocalStorageKey, result.token]
+          );
+        }
+        refreshAttempts++;
+        events.push({ type: "REFRESH_SUCCESS", isoTimestamp, strategy: "refreshEndpoint" });
+        return true;
+      }
+    } catch {
+    }
+    return false;
+  }
+  async function tryReLogin(page, isoTimestamp) {
+    if (!config.loginUrl || !config.email || !config.password) return false;
+    events.push({
+      type: "REFRESH_ATTEMPT",
+      isoTimestamp,
+      strategy: "reLogin",
+      attemptCount: refreshAttempts + 1
+    });
+    try {
+      await page.goto(config.loginUrl, { waitUntil: "domcontentloaded" });
+      await page.locator('input[type="email"], input[name="email"], input[name="username"]').first().fill(config.email);
+      await page.locator('input[type="password"]').first().fill(config.password);
+      await page.locator(
+        'button[type="submit"], button:has-text("Sign In"), button:has-text("Log In"), button:has-text("Login")'
+      ).first().click();
+      await page.waitForFunction(
+        (loginUrl) => !location.href.includes(loginUrl),
+        config.loginUrl,
+        { timeout: 1e4 }
+      );
+      events.push({ type: "REFRESH_SUCCESS", isoTimestamp, strategy: "reLogin" });
+      return true;
+    } catch {
+      events.push({ type: "REFRESH_FAILED", isoTimestamp, strategy: "reLogin" });
+      return false;
+    }
+  }
+  async function refresh(page, isoTimestamp) {
+    if (config.refreshEndpoint) {
+      const ok = await tryRefreshEndpoint(page, isoTimestamp);
+      if (ok) return true;
+    }
+    if (config.loginUrl && config.email && config.password) {
+      const ok = await tryReLogin(page, isoTimestamp);
+      if (ok) return true;
+    }
+    refreshAttempts++;
+    events.push({ type: "REFRESH_FAILED", isoTimestamp, attemptCount: refreshAttempts });
+    if (refreshAttempts >= maxAttempts) {
+      authFailure = true;
+      events.push({ type: "AUTH_FAILURE", isoTimestamp, attemptCount: refreshAttempts });
+    }
+    return false;
+  }
+  function getEvents() {
+    return [...events];
+  }
+  function getStats() {
+    return { healthy, refreshAttempts, authFailure };
+  }
+  return {
+    onResponse,
+    onNavigation,
+    isHealthy,
+    needsRefresh,
+    markHealthy,
+    refresh,
+    getEvents,
+    getStats
+  };
+}
+function attachSessionWatcher(page, manager) {
+  const onResponse = (res) => {
+    manager.onResponse(res.url(), res.status());
+  };
+  page.on("response", onResponse);
+  return () => page.off("response", onResponse);
+}
+export {
+  attachSessionWatcher,
+  createSessionManager
+};
+//# sourceMappingURL=crawl-session-manager.js.map

package/dist/crawl-session-manager.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../src/crawl-session-manager.ts"],
+  "sourcesContent": ["/**\n * Session management for authenticated crawl sessions.\n * Detects auth expiry via HTTP status codes and login-page redirects,\n * then attempts recovery via a token refresh endpoint or re-login flow.\n * Designed to keep long-running crawls alive without manual intervention.\n */\n\nimport type { Page } from 'playwright';\n\nexport interface SessionConfig {\n  loginUrl?: string;\n  email?: string;\n  password?: string;\n  healthCheckUrl?: string;\n  refreshEndpoint?: string;\n  tokenLocalStorageKey?: string;\n  maxRefreshAttempts?: number;\n}\n\nexport type SessionEventType =\n  | 'HEALTHY'\n  | 'EXPIRY_DETECTED'\n  | 'REFRESH_ATTEMPT'\n  | 'REFRESH_SUCCESS'\n  | 'REFRESH_FAILED'\n  | 'AUTH_FAILURE';\n\nexport interface SessionEvent {\n  type: SessionEventType;\n  isoTimestamp: string;\n  url?: string;\n  strategy?: string;\n  attemptCount?: number;\n}\n\nexport interface SessionManager {\n  onResponse(url: string, status: number): void;\n  onNavigation(url: string): void;\n  isHealthy(): boolean;\n  needsRefresh(): boolean;\n  markHealthy(isoTimestamp: string): void;\n  refresh(page: Page, isoTimestamp: string): Promise<boolean>;\n  getEvents(): SessionEvent[];\n  getStats(): { healthy: boolean; refreshAttempts: number; authFailure: boolean };\n}\n\nconst LOGIN_PATH_PATTERNS = [\n  '/login',\n  '/sign-in',\n  '/signin',\n  '/auth/login',\n  '/auth/sign-in',\n];\n\nexport function createSessionManager(config: SessionConfig): SessionManager {\n  let healthy = true;\n  let refreshAttempts = 0;\n  let authFailure = false;\n  const events: SessionEvent[] = [];\n  const maxAttempts = config.maxRefreshAttempts ?? 3;\n\n  function onResponse(url: string, status: number): void {\n    if ((status === 401 || status === 403) && healthy) {\n      healthy = false;\n      events.push({ type: 'EXPIRY_DETECTED', isoTimestamp: new Date().toISOString(), url });\n    }\n  }\n\n  function onNavigation(url: string): void {\n    const isLoginPage = LOGIN_PATH_PATTERNS.some((pattern) => url.includes(pattern));\n    if (isLoginPage && healthy) {\n      healthy = false;\n      events.push({ type: 'EXPIRY_DETECTED', isoTimestamp: new Date().toISOString(), url });\n    }\n  }\n\n  function isHealthy(): boolean {\n    return healthy;\n  }\n\n  function needsRefresh(): boolean {\n    return !healthy && !authFailure && refreshAttempts < maxAttempts;\n  }\n\n  function markHealthy(isoTimestamp: string): void {\n    healthy = true;\n    events.push({ type: 'HEALTHY', isoTimestamp });\n  }\n\n  async function tryRefreshEndpoint(page: Page, isoTimestamp: string): Promise<boolean> {\n    if (!config.refreshEndpoint) return false;\n\n    events.push({\n      type: 'REFRESH_ATTEMPT',\n      isoTimestamp,\n      strategy: 'refreshEndpoint',\n      attemptCount: refreshAttempts + 1,\n    });\n\n    try {\n      const result = await page.evaluate(async (endpoint) => {\n        try {\n          const res = await fetch(endpoint, { method: 'POST', credentials: 'include' });\n          if (res.ok) {\n            const data = await res.json().catch(() => ({}));\n            return { ok: true, token: (data as any).access_token || (data as any).token || null };\n          }\n          return { ok: false, token: null };\n        } catch {\n          return { ok: false, token: null };\n        }\n      }, config.refreshEndpoint);\n\n      if (result.ok) {\n        if (result.token && config.tokenLocalStorageKey) {\n          await page.evaluate(\n            ([key, val]) => localStorage.setItem(key, val),\n            [config.tokenLocalStorageKey, result.token] as [string, string],\n          );\n        }\n        refreshAttempts++;\n        events.push({ type: 'REFRESH_SUCCESS', isoTimestamp, strategy: 'refreshEndpoint' });\n        return true;\n      }\n    } catch {\n      // page.evaluate or localStorage.setItem failed \u2014 fall through\n    }\n\n    return false;\n  }\n\n  async function tryReLogin(page: Page, isoTimestamp: string): Promise<boolean> {\n    if (!config.loginUrl || !config.email || !config.password) return false;\n\n    events.push({\n      type: 'REFRESH_ATTEMPT',\n      isoTimestamp,\n      strategy: 'reLogin',\n      attemptCount: refreshAttempts + 1,\n    });\n\n    try {\n      await page.goto(config.loginUrl, { waitUntil: 'domcontentloaded' });\n\n      await page\n        .locator('input[type=\"email\"], input[name=\"email\"], input[name=\"username\"]')\n        .first()\n        .fill(config.email);\n\n      await page\n        .locator('input[type=\"password\"]')\n        .first()\n        .fill(config.password);\n\n      await page\n        .locator(\n          'button[type=\"submit\"], button:has-text(\"Sign In\"), button:has-text(\"Log In\"), button:has-text(\"Login\")',\n        )\n        .first()\n        .click();\n\n      await page.waitForFunction(\n        (loginUrl) => !location.href.includes(loginUrl),\n        config.loginUrl,\n        { timeout: 10000 },\n      );\n\n      events.push({ type: 'REFRESH_SUCCESS', isoTimestamp, strategy: 'reLogin' });\n      return true;\n    } catch {\n      events.push({ type: 'REFRESH_FAILED', isoTimestamp, strategy: 'reLogin' });\n      return false;\n    }\n  }\n\n  async function refresh(page: Page, isoTimestamp: string): Promise<boolean> {\n    // Strategy 1: token refresh endpoint\n    if (config.refreshEndpoint) {\n      const ok = await tryRefreshEndpoint(page, isoTimestamp);\n      if (ok) return true;\n    }\n\n    // Strategy 2: re-login\n    if (config.loginUrl && config.email && config.password) {\n      const ok = await tryReLogin(page, isoTimestamp);\n      if (ok) return true;\n    }\n\n    // All strategies exhausted\n    refreshAttempts++;\n    events.push({ type: 'REFRESH_FAILED', isoTimestamp, attemptCount: refreshAttempts });\n\n    if (refreshAttempts >= maxAttempts) {\n      authFailure = true;\n      events.push({ type: 'AUTH_FAILURE', isoTimestamp, attemptCount: refreshAttempts });\n    }\n\n    return false;\n  }\n\n  function getEvents(): SessionEvent[] {\n    return [...events];\n  }\n\n  function getStats(): { healthy: boolean; refreshAttempts: number; authFailure: boolean } {\n    return { healthy, refreshAttempts, authFailure };\n  }\n\n  return {\n    onResponse,\n    onNavigation,\n    isHealthy,\n    needsRefresh,\n    markHealthy,\n    refresh,\n    getEvents,\n    getStats,\n  };\n}\n\n/**\n * Attaches a Playwright response listener to feed HTTP status codes into the\n * session manager. Returns a cleanup function to remove the listener.\n */\nexport function attachSessionWatcher(page: Page, manager: SessionManager): () => void {\n  const onResponse = (res: import('playwright').Response) => {\n    manager.onResponse(res.url(), res.status());\n  };\n  page.on('response', onResponse);\n  return () => page.off('response', onResponse);\n}\n"],
+  "mappings": "AA8CA,MAAM,sBAAsB;AAAA,EAC1B;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EACA;AACF;AAEO,SAAS,qBAAqB,QAAuC;AAC1E,MAAI,UAAU;AACd,MAAI,kBAAkB;AACtB,MAAI,cAAc;AAClB,QAAM,SAAyB,CAAC;AAChC,QAAM,cAAc,OAAO,sBAAsB;AAEjD,WAAS,WAAW,KAAa,QAAsB;AACrD,SAAK,WAAW,OAAO,WAAW,QAAQ,SAAS;AACjD,gBAAU;AACV,aAAO,KAAK,EAAE,MAAM,mBAAmB,eAAc,oBAAI,KAAK,GAAE,YAAY,GAAG,IAAI,CAAC;AAAA,IACtF;AAAA,EACF;AAEA,WAAS,aAAa,KAAmB;AACvC,UAAM,cAAc,oBAAoB,KAAK,CAAC,YAAY,IAAI,SAAS,OAAO,CAAC;AAC/E,QAAI,eAAe,SAAS;AAC1B,gBAAU;AACV,aAAO,KAAK,EAAE,MAAM,mBAAmB,eAAc,oBAAI,KAAK,GAAE,YAAY,GAAG,IAAI,CAAC;AAAA,IACtF;AAAA,EACF;AAEA,WAAS,YAAqB;AAC5B,WAAO;AAAA,EACT;AAEA,WAAS,eAAwB;AAC/B,WAAO,CAAC,WAAW,CAAC,eAAe,kBAAkB;AAAA,EACvD;AAEA,WAAS,YAAY,cAA4B;AAC/C,cAAU;AACV,WAAO,KAAK,EAAE,MAAM,WAAW,aAAa,CAAC;AAAA,EAC/C;AAEA,iBAAe,mBAAmB,MAAY,cAAwC;AACpF,QAAI,CAAC,OAAO,gBAAiB,QAAO;AAEpC,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN;AAAA,MACA,UAAU;AAAA,MACV,cAAc,kBAAkB;AAAA,IAClC,CAAC;AAED,QAAI;AACF,YAAM,SAAS,MAAM,KAAK,SAAS,OAAO,aAAa;AACrD,YAAI;AACF,gBAAM,MAAM,MAAM,MAAM,UAAU,EAAE,QAAQ,QAAQ,aAAa,UAAU,CAAC;AAC5E,cAAI,IAAI,IAAI;AACV,kBAAM,OAAO,MAAM,IAAI,KAAK,EAAE,MAAM,OAAO,CAAC,EAAE;AAC9C,mBAAO,EAAE,IAAI,MAAM,OAAQ,KAAa,gBAAiB,KAAa,SAAS,KAAK;AAAA,UACtF;AACA,iBAAO,EAAE,IAAI,OAAO,OAAO,KAAK;AAAA,QAClC,QAAQ;AACN,iBAAO,EAAE,IAAI,OAAO,OAAO,KAAK;AAAA,QAClC;AAAA,MACF,GAAG,OAAO,eAAe;AAEzB,UAAI,OAAO,IAAI;AACb,YAAI,OAAO,SAAS,OAAO,sBAAsB;AAC/C,gBAAM,KAAK;AAAA,YACT,CAAC,CAAC,KAAK,GAAG,MAAM,aAAa,QAAQ,KAAK,GAAG;AAAA,YAC7C,CAAC,OAAO,sBAAsB,OAAO,KAAK;AAAA,UAC5C;AAAA,QACF;AACA;AACA,eAAO,KAAK,EAAE,MAAM,mBAAmB,cAAc,UAAU,kBAAkB,CAAC;AAClF,eAAO;AAAA,MACT;AAAA,IACF,QAAQ;AAAA,IAER;AAEA,WAAO;AAAA,EACT;AAEA,iBAAe,WAAW,MAAY,cAAwC;AAC5E,QAAI,CAAC,OAAO,YAAY,CAAC,OAAO,SAAS,CAAC,OAAO,SAAU,QAAO;AAElE,WAAO,KAAK;AAAA,MACV,MAAM;AAAA,MACN;AAAA,MACA,UAAU;AAAA,MACV,cAAc,kBAAkB;AAAA,IAClC,CAAC;AAED,QAAI;AACF,YAAM,KAAK,KAAK,OAAO,UAAU,EAAE,WAAW,mBAAmB,CAAC;AAElE,YAAM,KACH,QAAQ,kEAAkE,EAC1E,MAAM,EACN,KAAK,OAAO,KAAK;AAEpB,YAAM,KACH,QAAQ,wBAAwB,EAChC,MAAM,EACN,KAAK,OAAO,QAAQ;AAEvB,YAAM,KACH;AAAA,QACC;AAAA,MACF,EACC,MAAM,EACN,MAAM;AAET,YAAM,KAAK;AAAA,QACT,CAAC,aAAa,CAAC,SAAS,KAAK,SAAS,QAAQ;AAAA,QAC9C,OAAO;AAAA,QACP,EAAE,SAAS,IAAM;AAAA,MACnB;AAEA,aAAO,KAAK,EAAE,MAAM,mBAAmB,cAAc,UAAU,UAAU,CAAC;AAC1E,aAAO;AAAA,IACT,QAAQ;AACN,aAAO,KAAK,EAAE,MAAM,kBAAkB,cAAc,UAAU,UAAU,CAAC;AACzE,aAAO;AAAA,IACT;AAAA,EACF;AAEA,iBAAe,QAAQ,MAAY,cAAwC;AAEzE,QAAI,OAAO,iBAAiB;AAC1B,YAAM,KAAK,MAAM,mBAAmB,MAAM,YAAY;AACtD,UAAI,GAAI,QAAO;AAAA,IACjB;AAGA,QAAI,OAAO,YAAY,OAAO,SAAS,OAAO,UAAU;AACtD,YAAM,KAAK,MAAM,WAAW,MAAM,YAAY;AAC9C,UAAI,GAAI,QAAO;AAAA,IACjB;AAGA;AACA,WAAO,KAAK,EAAE,MAAM,kBAAkB,cAAc,cAAc,gBAAgB,CAAC;AAEnF,QAAI,mBAAmB,aAAa;AAClC,oBAAc;AACd,aAAO,KAAK,EAAE,MAAM,gBAAgB,cAAc,cAAc,gBAAgB,CAAC;AAAA,IACnF;AAEA,WAAO;AAAA,EACT;AAEA,WAAS,YAA4B;AACnC,WAAO,CAAC,GAAG,MAAM;AAAA,EACnB;AAEA,WAAS,WAAgF;AACvF,WAAO,EAAE,SAAS,iBAAiB,YAAY;AAAA,EACjD;AAEA,SAAO;AAAA,IACL;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,IACA;AAAA,EACF;AACF;AAMO,SAAS,qBAAqB,MAAY,SAAqC;AACpF,QAAM,aAAa,CAAC,QAAuC;AACzD,YAAQ,WAAW,IAAI,IAAI,GAAG,IAAI,OAAO,CAAC;AAAA,EAC5C;AACA,OAAK,GAAG,YAAY,UAAU;AAC9B,SAAO,MAAM,KAAK,IAAI,YAAY,UAAU;AAC9C;",
+  "names": []
+}