agentlinkops 0.5.0 → 0.6.0

package/src/citations/browser/gen/lib/input/editor/clearInput.js

@@ -0,0 +1,52 @@
+import {
+  clickLocatorLikeUser,
+  getBrowserPrimaryModifier
+} from "../../browser/humanBehavior.js";
+async function clearEditorInput(page, input, options = {}) {
+  const {
+    clickTimeoutMs = 3e3,
+    dismissWithEscape = false,
+    waitAfterMs = 0
+  } = options;
+  const count = await input.count().catch(() => 0);
+  if (count === 0) return false;
+  try {
+    const existingValue = await input.readInputValue().catch(() => "");
+    if (existingValue.trim().length === 0) {
+      return true;
+    }
+    const clicked = await clickLocatorLikeUser(page, input, {
+      timeout: clickTimeoutMs
+    }).catch(() => false) || await clickLocatorLikeUser(page, input, {
+      force: true,
+      timeout: clickTimeoutMs
+    }).catch(() => false);
+    if (!clicked) {
+      return false;
+    }
+    const primaryModifier = await getBrowserPrimaryModifier(page);
+    const selectAllShortcuts = primaryModifier === "Control" ? ["Control+A"] : [`${primaryModifier}+A`, "Control+A"];
+    let clearedByKeyboard = false;
+    for (const shortcut of selectAllShortcuts) {
+      await input.press(shortcut).catch(() => null);
+      await input.press("Backspace").catch(() => null);
+      clearedByKeyboard = await input.readInputValue().then((value) => value.trim().length === 0).catch(() => false);
+      if (clearedByKeyboard) break;
+    }
+    if (!clearedByKeyboard) {
+      await input.setInputValue("");
+    }
+    if (dismissWithEscape) {
+      await input.press("Escape").catch(() => null);
+    }
+    if (waitAfterMs > 0) {
+      await page.waitForTimeout(waitAfterMs);
+    }
+    return true;
+  } catch {
+    return false;
+  }
+}
+export {
+  clearEditorInput
+};

package/src/citations/browser/gen/lib/input/editor/findEditor.js

@@ -0,0 +1,54 @@
+import { NotFoundError } from "../../../../shims/errors.js";
+import { PROVIDER_EDITOR_SELECTORS } from "../../../../shims/utils.js";
+async function findActiveEditorCandidateFromSelectors(page, selectors) {
+  for (const selector of selectors) {
+    const nodes = page.locator(selector);
+    const count = await nodes.count();
+    for (let i = 0; i < count; i++) {
+      const el = nodes.nth(i);
+      try {
+        const visible = await el.isVisible().catch(() => false);
+        if (!visible) {
+          continue;
+        }
+        const box = await el.boundingBox().catch(() => null);
+        if (!box || box.width < 8 || box.height < 8) {
+          continue;
+        }
+        await el.scrollIntoViewIfNeeded().catch(() => {
+        });
+        await el.focus().catch(() => {
+        });
+        const state = await el.getEditableState().catch(() => null);
+        if (!(state?.connected && state.visible && state.editable && state.enabled)) {
+          continue;
+        }
+        return { locator: el, selector };
+      } catch (_error) {
+      }
+    }
+  }
+  throw new NotFoundError("active prompt editor");
+}
+async function findActiveEditorFromSelectors(page, selectors) {
+  const candidate = await findActiveEditorCandidateFromSelectors(
+    page,
+    selectors
+  );
+  return candidate.locator;
+}
+async function findActiveEditor(page, provider) {
+  const candidate = await findActiveEditorCandidate(page, provider);
+  return candidate.locator;
+}
+async function findActiveEditorCandidate(page, provider) {
+  const fallbackSelectors = [
+    ...new Set(Object.values(PROVIDER_EDITOR_SELECTORS).flat())
+  ];
+  const selectors = provider ? PROVIDER_EDITOR_SELECTORS[provider] || fallbackSelectors : fallbackSelectors;
+  return findActiveEditorCandidateFromSelectors(page, selectors);
+}
+export {
+  findActiveEditorCandidate,
+  findActiveEditorCandidateFromSelectors
+};

package/src/citations/browser/gen/lib/input/editor/findSendButton.js

@@ -0,0 +1,21 @@
+import { PROVIDER_SUBMIT_BTN_SELECTORS } from "../../../../shims/utils.js";
+async function findEnabledSendButton(page, provider) {
+  const selectors = PROVIDER_SUBMIT_BTN_SELECTORS[provider] || [];
+  for (const selector of selectors) {
+    const buttons = page.locator(selector);
+    const count = await buttons.count();
+    for (let i = 0; i < count; i++) {
+      const btn = buttons.nth(i);
+      try {
+        if (await btn.isVisible() && await btn.isEnabled()) {
+          return btn;
+        }
+      } catch {
+      }
+    }
+  }
+  return null;
+}
+export {
+  findEnabledSendButton
+};

package/src/citations/browser/gen/lib/input/editor/promptInput.js

@@ -0,0 +1,117 @@
+import { ExternalServiceError } from "../../../../shims/errors.js";
+import { logger } from "../../../../shims/utils.js";
+import {
+  clickLocatorLikeUser,
+  pastePrompt,
+  randomBetween
+} from "../../browser/humanBehavior.js";
+import { clearEditorInput } from "./clearInput.js";
+function normalizePromptValue(text) {
+  return text.replace(/\r\n/g, "\n").replace(/\u00a0/g, " ").replace(/[\u200b-\u200d\ufeff]/g, "").trim();
+}
+async function focusEditorTarget(page, input) {
+  await input.scrollIntoViewIfNeeded().catch(() => null);
+  await clickLocatorLikeUser(page, input, {
+    timeout: 3e3,
+    delay: randomBetween(25, 80)
+  }).catch(() => null);
+  await page.waitForTimeout(randomBetween(40, 120));
+  await input.focus().catch(() => null);
+  await page.waitForTimeout(randomBetween(30, 90));
+}
+async function prepareEditorForPrompt(page, input, provider) {
+  const count = await input.count().catch(() => 0);
+  if (count === 0) {
+    throw new ExternalServiceError(
+      provider,
+      `Editor not ready for ${provider}: input locator is missing`
+    );
+  }
+  const state = await input.getEditableState().catch(() => null);
+  if (!(state?.connected && state.editable && state.enabled && state.acceptsTextInput)) {
+    throw new ExternalServiceError(
+      provider,
+      `Editor not ready for ${provider}: input is not editable`
+    );
+  }
+  await focusEditorTarget(page, input);
+  const existingValue = await input.readInputValue().catch(() => "");
+  if (normalizePromptValue(existingValue).length === 0) {
+    await focusEditorTarget(page, input);
+    return;
+  }
+  const cleared = await clearEditorInput(page, input, {
+    clickTimeoutMs: 3e3,
+    waitAfterMs: randomBetween(40, 120)
+  });
+  if (!cleared) {
+    throw new ExternalServiceError(
+      provider,
+      `Editor not ready for ${provider}: could not clear existing input`
+    );
+  }
+  const remainingValue = await input.readInputValue().catch(() => "");
+  if (normalizePromptValue(remainingValue).length > 0) {
+    throw new ExternalServiceError(
+      provider,
+      `Editor not ready for ${provider}: input retained content after clear`
+    );
+  }
+  await focusEditorTarget(page, input);
+}
+async function insertPromptOnce(page, input, prompt, strategy) {
+  if (strategy === "directSet") {
+    await input.setInputValue(prompt);
+    await page.waitForTimeout(randomBetween(40, 120));
+    return;
+  }
+  await pastePrompt(page, prompt);
+}
+async function waitForPromptValue(page, input, expectedValue, timeoutMs) {
+  const deadline = Date.now() + timeoutMs;
+  let lastValue = await input.readInputValue().catch(() => "");
+  while (Date.now() < deadline) {
+    if (normalizePromptValue(lastValue) === expectedValue) {
+      return lastValue;
+    }
+    await page.waitForTimeout(randomBetween(80, 140));
+    lastValue = await input.readInputValue().catch(() => "");
+  }
+  return lastValue;
+}
+async function insertPromptIntoEditor(page, input, prompt, provider) {
+  const expectedValue = normalizePromptValue(prompt);
+  const strategies = [
+    ...provider === "perplexity" ? [] : ["directSet"],
+    "pacedPaste"
+  ];
+  for (const strategy of strategies) {
+    for (let attempt = 1; attempt <= 2; attempt++) {
+      await prepareEditorForPrompt(page, input, provider);
+      await insertPromptOnce(page, input, prompt, strategy);
+      const rawValue = await waitForPromptValue(
+        page,
+        input,
+        expectedValue,
+        strategy === "directSet" ? attempt === 1 ? 800 : 1400 : attempt === 1 ? 1800 : 2500
+      );
+      if (normalizePromptValue(rawValue) === expectedValue) {
+        return { rawValue, strategy };
+      }
+      if (attempt === 1) {
+        logger.warn(
+          `[${provider}] prompt verification mismatch after ${strategy} \u2014 retrying once`
+        );
+      }
+    }
+  }
+  const finalValue = await input.readInputValue().catch(() => "");
+  throw new ExternalServiceError(
+    provider,
+    `Typing failed: normalized input mismatch after local retry (expected ${expectedValue.length} chars, got ${normalizePromptValue(finalValue).length})`
+  );
+}
+export {
+  insertPromptIntoEditor,
+  normalizePromptValue
+};

package/src/citations/browser/gen/lib/input/markdown/converter.js

@@ -0,0 +1,101 @@
+import TurndownService from "turndown";
+const turndown = new TurndownService({
+  headingStyle: "atx",
+  codeBlockStyle: "fenced",
+  bulletListMarker: "-",
+  br: "\n"
+});
+turndown.addRule("paragraph", {
+  filter: "p",
+  replacement(content) {
+    return `
+
+${content}
+
+`;
+  }
+});
+turndown.addRule("div", {
+  filter: "div",
+  replacement(content) {
+    return `
+
+${content.trim()}
+
+`;
+  }
+});
+turndown.addRule("heading", {
+  filter: ["h1", "h2", "h3", "h4", "h5", "h6"],
+  replacement(content, node) {
+    const level = Number(node.nodeName.charAt(1));
+    return `
+
+${"#".repeat(level)} ${content}
+
+`;
+  }
+});
+turndown.addRule("list", {
+  filter: ["ul", "ol"],
+  replacement(content, node) {
+    const parent = node.parentNode;
+    if (parent && (parent.nodeName === "LI" || parent.nodeName === "UL" || parent.nodeName === "OL")) {
+      return `
+${content}`;
+    }
+    return `
+
+${content}
+
+`;
+  }
+});
+turndown.addRule("table", {
+  filter: "table",
+  replacement(_content, node) {
+    const table = node;
+    const rows = Array.from(table.querySelectorAll("tr"));
+    if (rows.length === 0) return "";
+    const result = [];
+    for (let i = 0; i < rows.length; i++) {
+      const cells = Array.from(rows[i].querySelectorAll("th, td"));
+      const line = cells.map((c) => turndown.turndown(c.innerHTML ?? c.textContent ?? "").replace(/\n+/g, " ").trim()).join(" | ");
+      result.push(`| ${line} |`);
+      if (i === 0) {
+        result.push(`| ${cells.map(() => "---").join(" | ")} |`);
+      }
+    }
+    return `
+
+${result.join("\n")}
+
+`;
+  }
+});
+turndown.addRule("link", {
+  filter: "a",
+  replacement(content) {
+    const trimmed = content.trim();
+    if (!trimmed) return "";
+    if (/^\[?\d+\]?$/.test(trimmed)) return "";
+    return content;
+  }
+});
+turndown.addRule("image", { filter: "img", replacement: () => "" });
+turndown.addRule("figure", { filter: "figure", replacement: () => "" });
+turndown.addRule("picture", { filter: "picture", replacement: () => "" });
+turndown.addRule("video", { filter: "video", replacement: () => "" });
+turndown.addRule("iframe", { filter: "iframe", replacement: () => "" });
+turndown.addRule("sup", { filter: "sup", replacement: () => "" });
+turndown.addRule("carousel", {
+  filter(node) {
+    const el = node;
+    const cn = el.className || "";
+    return el.nodeName === "DIV" && (cn.includes("carousel") || cn.includes("gallery") || cn.includes("slider") || cn.includes("swiper"));
+  },
+  replacement: () => ""
+});
+export {
+  turndown
+};

package/src/citations/browser/gen/lib/input/markdown/toMarkdown.js

@@ -0,0 +1,14 @@
+import { PROVIDER_MODEL_RESPONSE_SELECTORS } from "../../../../shims/utils.js";
+import { turndown } from "./converter.js";
+async function extractAssistantMarkdown(page, provider) {
+  const html = await page.runDomOp("response-html", {
+    provider,
+    selectors: PROVIDER_MODEL_RESPONSE_SELECTORS[provider] || []
+  });
+  if (!html) return "";
+  const markdown = turndown.turndown(html);
+  return markdown.replace(/\n{3,}/g, "\n\n").trim();
+}
+export {
+  extractAssistantMarkdown
+};

package/src/citations/browser/gen/lib/input/response/detectBotPage.js

@@ -0,0 +1,13 @@
+import { ExternalServiceError } from "../../../../shims/errors.js";
+async function detectBotPage(page, provider) {
+  const state = await page.runDomOp("detect-bot-page").catch(() => ({ botDetected: false, reason: null }));
+  if (state.botDetected) {
+    throw new ExternalServiceError(
+      provider,
+      state.reason ?? "bot detection page detected"
+    );
+  }
+}
+export {
+  detectBotPage
+};

package/src/citations/browser/gen/lib/input/response/getText.js

@@ -0,0 +1,10 @@
+import { PROVIDER_MODEL_RESPONSE_SELECTORS } from "../../../../shims/utils.js";
+async function getText(page, provider) {
+  return await page.runDomOp("response-text", {
+    provider,
+    selectors: PROVIDER_MODEL_RESPONSE_SELECTORS[provider] || []
+  });
+}
+export {
+  getText
+};

package/src/citations/browser/gen/lib/input/response/isGenerating.js

@@ -0,0 +1,59 @@
+import {
+  PROVIDER_MODEL_RESPONSE_SELECTORS,
+  PROVIDER_RESPONSE_GENERATION_SELECTORS
+} from "../../../../shims/utils.js";
+async function getGenerationStateSignature(page, provider) {
+  return await page.evaluate(
+    (selectors) => (selectors || []).map((selector) => {
+      const parts = Array.from(document.querySelectorAll(selector)).map((node) => {
+        const element = node;
+        const style = window.getComputedStyle(element);
+        const visible = element.offsetParent !== null && style.visibility !== "hidden" && style.display !== "none";
+        const text = (element.textContent || "").trim();
+        const ariaLabel = element.getAttribute("aria-label") || "";
+        const disabled = element.getAttribute("disabled") ? "1" : "0";
+        return `${visible ? 1 : 0}:${text}:${ariaLabel}:${disabled}`;
+      });
+      return `${selector}=>${parts.join("|")}`;
+    }).join("||"),
+    PROVIDER_RESPONSE_GENERATION_SELECTORS[provider] || []
+  );
+}
+async function hasVisibleGenerationIndicator(page, provider) {
+  return await page.evaluate(
+    (selectors) => (selectors || []).some(
+      (selector) => Array.from(document.querySelectorAll(selector)).some((node) => {
+        const element = node;
+        const style = window.getComputedStyle(element);
+        return element.offsetParent !== null && style.visibility !== "hidden" && style.display !== "none";
+      })
+    ),
+    PROVIDER_RESPONSE_GENERATION_SELECTORS[provider] || []
+  );
+}
+async function getResponseStateSignature(page, provider) {
+  return await page.evaluate((selectors) => {
+    const visible = (element) => {
+      if (!(element instanceof HTMLElement)) return false;
+      const style = window.getComputedStyle(element);
+      return element.offsetParent !== null && style.visibility !== "hidden" && style.display !== "none";
+    };
+    const elements = (selectors || []).flatMap((selector) => Array.from(document.querySelectorAll(selector))).filter(
+      (el) => visible(el) && (el.innerText || "").trim().length > 0
+    );
+    const latest = elements.at(-1) ?? null;
+    if (!latest) {
+      return { signature: "", textLength: 0 };
+    }
+    const text = (latest.innerText || "").replace(/\s+/g, " ").trim();
+    return {
+      signature: `${text.length}:${latest.innerHTML.length}:${latest.childElementCount}:${text.slice(-120)}`,
+      textLength: text.length
+    };
+  }, PROVIDER_MODEL_RESPONSE_SELECTORS[provider] || []);
+}
+export {
+  getGenerationStateSignature,
+  getResponseStateSignature,
+  hasVisibleGenerationIndicator
+};

package/src/citations/browser/gen/lib/input/response/waitForFinish.js

@@ -0,0 +1,105 @@
+import { ExternalServiceError } from "../../../../shims/errors.js";
+import {
+  logger,
+  PROVIDER_FORCE_EXIT_STABLE_MS,
+  PROVIDER_NO_OUTPUT_TIMEOUT_MS
+} from "../../../../shims/utils.js";
+import {
+  getGenerationStateSignature,
+  getResponseStateSignature,
+  hasVisibleGenerationIndicator
+} from "./isGenerating.js";
+async function sleep(ms) {
+  let timer = null;
+  try {
+    await new Promise((resolve) => {
+      timer = setTimeout(resolve, ms);
+    });
+  } finally {
+    if (timer !== null) {
+      clearTimeout(timer);
+    }
+  }
+}
+async function pollUntilCondition(checkFn, pollInterval, maxWait, timeoutError) {
+  const start = Date.now();
+  while (Date.now() - start < maxWait) {
+    if (await checkFn()) return;
+    await sleep(pollInterval);
+  }
+  throw timeoutError;
+}
+async function waitForAssistantToFinish(page, provider) {
+  logger.debug(
+    provider === "ai-overview" ? "\u23F3 Waiting for AI Overview response container to stabilize\u2026" : "\u23F3 Waiting for assistant to finish\u2026"
+  );
+  const waitStart = Date.now();
+  let lastGenerationState = "";
+  let lastResponseState = "";
+  let lastChangeAt = Date.now();
+  let initialized = false;
+  let seenResponse = false;
+  await pollUntilCondition(
+    async () => {
+      const [currentGenerationState, currentResponseState, hasVisibleIndicator] = await Promise.all([
+        getGenerationStateSignature(page, provider),
+        getResponseStateSignature(page, provider),
+        hasVisibleGenerationIndicator(page, provider)
+      ]);
+      const waitedFor = Date.now() - waitStart;
+      const forceExitStableMs = PROVIDER_FORCE_EXIT_STABLE_MS[provider];
+      const responseStateChanged = currentResponseState.signature !== lastResponseState;
+      const generationStateChanged = currentGenerationState !== lastGenerationState;
+      const requiresContainerStabilityOnly = provider === "ai-overview";
+      if (!initialized) {
+        lastGenerationState = currentGenerationState;
+        lastResponseState = currentResponseState.signature;
+        seenResponse = currentResponseState.textLength > 0;
+        lastChangeAt = Date.now();
+        initialized = true;
+        return false;
+      }
+      if (currentResponseState.textLength > 0) {
+        seenResponse = true;
+      }
+      if (responseStateChanged || generationStateChanged) {
+        lastGenerationState = currentGenerationState;
+        lastResponseState = currentResponseState.signature;
+        lastChangeAt = Date.now();
+        return false;
+      }
+      const stableFor = Date.now() - lastChangeAt;
+      if (requiresContainerStabilityOnly) {
+        if (seenResponse && stableFor >= 2500) {
+          logger.debug("\u2705 AI Overview response container stabilized");
+          return true;
+        }
+      }
+      if (seenResponse && !hasVisibleIndicator && stableFor >= 1500) {
+        logger.debug("\u2705 Assistant finished");
+        return true;
+      }
+      const noOutputTimeoutMs = PROVIDER_NO_OUTPUT_TIMEOUT_MS[provider];
+      if (waitedFor >= noOutputTimeoutMs) {
+        logger.warn(
+          `Generation state did not stabilize within ${Math.round(noOutputTimeoutMs / 1e3)}s`
+        );
+      }
+      if (stableFor >= forceExitStableMs) {
+        logger.warn(
+          `${hasVisibleIndicator ? "Generation indicator still visible and " : ""}generation state stable for ${Math.round(forceExitStableMs / 1e3)}s \u2014 forcing exit`
+        );
+        return true;
+      }
+      return false;
+    },
+    280 + Math.floor(Math.random() * 60),
+    // Poll ~300ms with ±50ms jitter
+    5 * 60 * 1e3,
+    // 5 min max — if a response hasn't arrived by then, something is wrong
+    new ExternalServiceError(provider, "Assistant wait timed out")
+  );
+}
+export {
+  waitForAssistantToFinish
+};

package/src/citations/browser/gen/lib/input/sources/findButton.js

@@ -0,0 +1,41 @@
+async function findSourcesButton(page, _provider) {
+  const buttonCount = await page.locator("button, [role='button']").count();
+  if (buttonCount === 0) return null;
+  const bestIndex = await page.evaluate((_unused) => {
+    const normalize = (text) => text?.toLowerCase().replace(/\s+/g, " ").trim() || "";
+    const getVisibleText = (el) => normalize(
+      Array.from(el.querySelectorAll("*")).map((child) => child.textContent || "").join(" ")
+    );
+    return Array.from(
+      document.querySelectorAll("button, [role='button'], [role='tab']")
+    ).map((el, index) => {
+      if (!(el instanceof HTMLElement)) {
+        return null;
+      }
+      const role = el.getAttribute("role");
+      if (role === "tab" || el.hasAttribute("aria-controls") || el.id?.includes("trigger")) {
+        return null;
+      }
+      const isAction = el.tagName === "BUTTON" || role === "button";
+      if (!isAction) return null;
+      const style = window.getComputedStyle(el);
+      if (style.display === "none" || style.visibility === "hidden") {
+        return null;
+      }
+      const text = getVisibleText(el);
+      const aria = normalize(el.getAttribute("aria-label"));
+      let score = 0;
+      if (/\b\d+\s*sources?\b/.test(text)) score += 120;
+      if (aria.includes("sources")) score += 90;
+      if (text === "sources") score += 80;
+      if (text.includes("sources")) score += 50;
+      if (score <= 0) return null;
+      score += index * 1e-3;
+      return { index, score };
+    }).filter((match) => match !== null).sort((a, b) => b.score - a.score || b.index - a.index)[0]?.index ?? -1;
+  }, null);
+  return bestIndex >= 0 ? page.locator("button, [role='button']").nth(bestIndex) : null;
+}
+export {
+  findSourcesButton
+};

package/src/citations/browser/gen/lib/input/sources/openPanel.js

@@ -0,0 +1,11 @@
+import { clickButtonViaDispatch } from "../../extraction/sourceUtils.js";
+async function openSourcesPanel(page, btn) {
+  const clicked = await btn.scrollIntoViewIfNeeded().then(() => btn.click({ timeout: 3e3 })).then(() => true).catch(() => false);
+  if (!clicked) {
+    if (!await clickButtonViaDispatch(page, btn)) return;
+  }
+  await page.waitForTimeout(500);
+}
+export {
+  openSourcesPanel
+};