agentlinkops 0.5.0 → 0.5.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (63) hide show
  1. package/.claude-plugin/plugin.json +1 -1
  2. package/.codex-plugin/plugin.json +1 -1
  3. package/.mcp.json +8 -0
  4. package/CHANGELOG.md +15 -1
  5. package/README.md +2 -1
  6. package/cli/citation.js +19 -1
  7. package/cli/client.js +1 -1
  8. package/cli/doctor.js +19 -0
  9. package/package.json +9 -1
  10. package/src/citations/browser/engine.js +196 -0
  11. package/src/citations/browser/gen/core/providers/_shared/google.js +4 -0
  12. package/src/citations/browser/gen/core/providers/_shared/rawSourcesDom.js +15 -0
  13. package/src/citations/browser/gen/core/providers/_shared/resetProviderPage.js +22 -0
  14. package/src/citations/browser/gen/core/providers/_shared/sourceUtils.js +8 -0
  15. package/src/citations/browser/gen/core/providers/ai-overview/index.js +70 -0
  16. package/src/citations/browser/gen/core/providers/ai-overview/lib/expand.js +59 -0
  17. package/src/citations/browser/gen/core/providers/ai-overview/lib/extractSources.js +60 -0
  18. package/src/citations/browser/gen/core/providers/ai-overview/lib/pageLifecycle.js +18 -0
  19. package/src/citations/browser/gen/core/providers/ai-overview/lib/session.js +62 -0
  20. package/src/citations/browser/gen/core/providers/chatgpt/index.js +28 -0
  21. package/src/citations/browser/gen/core/providers/chatgpt/lib/dismissAuthModal.js +39 -0
  22. package/src/citations/browser/gen/core/providers/chatgpt/lib/extractSources.js +66 -0
  23. package/src/citations/browser/gen/core/providers/chatgpt/lib/pageLifecycle.js +11 -0
  24. package/src/citations/browser/gen/core/providers/claude/index.js +16 -0
  25. package/src/citations/browser/gen/core/providers/claude/lib/extractSources.js +54 -0
  26. package/src/citations/browser/gen/core/providers/claude/lib/pageLifecycle.js +9 -0
  27. package/src/citations/browser/gen/core/providers/gemini/index.js +40 -0
  28. package/src/citations/browser/gen/core/providers/gemini/lib/extractSources.js +65 -0
  29. package/src/citations/browser/gen/core/providers/gemini/lib/pageLifecycle.js +14 -0
  30. package/src/citations/browser/gen/core/providers/gemini/lib/session.js +45 -0
  31. package/src/citations/browser/gen/core/providers/index.js +15 -0
  32. package/src/citations/browser/gen/core/providers/perplexity/index.js +31 -0
  33. package/src/citations/browser/gen/core/providers/perplexity/lib/extractSources.js +84 -0
  34. package/src/citations/browser/gen/core/providers/perplexity/lib/navigation.js +38 -0
  35. package/src/citations/browser/gen/core/providers/types.js +0 -0
  36. package/src/citations/browser/gen/lib/browser/camoufox.js +587 -0
  37. package/src/citations/browser/gen/lib/browser/display.js +147 -0
  38. package/src/citations/browser/gen/lib/browser/domOps.js +339 -0
  39. package/src/citations/browser/gen/lib/browser/humanBehavior.js +121 -0
  40. package/src/citations/browser/gen/lib/browser/navigate.js +46 -0
  41. package/src/citations/browser/gen/lib/browser/playwrightCompat.js +321 -0
  42. package/src/citations/browser/gen/lib/browser/proxy/forwarder.js +251 -0
  43. package/src/citations/browser/gen/lib/browser/runtimeTypes.js +0 -0
  44. package/src/citations/browser/gen/lib/extraction/sourceUtils.js +71 -0
  45. package/src/citations/browser/gen/lib/input/editor/assertNotBlocked.js +54 -0
  46. package/src/citations/browser/gen/lib/input/editor/clearInput.js +52 -0
  47. package/src/citations/browser/gen/lib/input/editor/findEditor.js +54 -0
  48. package/src/citations/browser/gen/lib/input/editor/findSendButton.js +21 -0
  49. package/src/citations/browser/gen/lib/input/editor/promptInput.js +117 -0
  50. package/src/citations/browser/gen/lib/input/markdown/converter.js +101 -0
  51. package/src/citations/browser/gen/lib/input/markdown/toMarkdown.js +14 -0
  52. package/src/citations/browser/gen/lib/input/response/detectBotPage.js +13 -0
  53. package/src/citations/browser/gen/lib/input/response/getText.js +10 -0
  54. package/src/citations/browser/gen/lib/input/response/isGenerating.js +59 -0
  55. package/src/citations/browser/gen/lib/input/response/waitForFinish.js +105 -0
  56. package/src/citations/browser/gen/lib/input/sources/findButton.js +41 -0
  57. package/src/citations/browser/gen/lib/input/sources/openPanel.js +11 -0
  58. package/src/citations/browser/gen/utils/agent-constants.js +87 -0
  59. package/src/citations/browser/shims/errors.js +26 -0
  60. package/src/citations/browser/shims/types.js +6 -0
  61. package/src/citations/browser/shims/utils.js +32 -0
  62. package/src/citations/contract.js +9 -2
  63. package/src/citations/runner.js +21 -5
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agentlinkops",
3
- "version": "0.5.0",
3
+ "version": "0.5.1",
4
4
  "description": "Connect an agent to AgentLinkOps and run backlink work with it: the connect skill (views, the three discovery verbs, evidence rules), local ledger and campaign skills, a portable CRM, the source-cited content pipeline, user-managed disavow lists, HTML reports and monitoring-event sync.",
5
5
  "author": {
6
6
  "name": "AgentLinkOps"
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agentlinkops",
3
- "version": "0.5.0",
3
+ "version": "0.5.1",
4
4
  "description": "Connect an agent to AgentLinkOps and run backlink work with it: the connect skill (views, the three discovery verbs, evidence rules), local ledger and campaign skills, a portable CRM, the source-cited content pipeline, user-managed disavow lists, HTML reports and monitoring-event sync.",
5
5
  "author": {
6
6
  "name": "AgentLinkOps"
package/.mcp.json ADDED
@@ -0,0 +1,8 @@
1
+ {
2
+ "mcpServers": {
3
+ "agentlinkops": {
4
+ "type": "http",
5
+ "url": "https://app.agentlinkops.com/mcp"
6
+ }
7
+ }
8
+ }
package/CHANGELOG.md CHANGED
@@ -3,7 +3,21 @@
3
3
  All notable changes to the `agentlinkops` package are recorded here. The version follows the
4
4
  plugin manifests in `.claude-plugin/` and `.codex-plugin/`.
5
5
 
6
- ## 0.5.0 (unreleased; DP-0036 agent surface)
6
+ ## 0.5.1 (2026-09-17)
7
+
8
+ - The plugin's `.mcp.json` ships in the tarball. 0.5.0 left it out, so a marketplace install
9
+ got the skills but no MCP server entry; the package build now carries and requires it.
10
+ - Package identity: `author` is AgentLinkOps (https://agentlinkops.com); `repository` and
11
+ `bugs` point at `sunlover7/agentlinkops-cli`. No personal identity in the registry metadata.
12
+ - `agentlinkops citation run PANEL.json --engine chatgpt:web-own-browser`: measures the
13
+ consumer UI through the customer's own Camoufox browser instead of an API, with a screenshot
14
+ of every answer in the evidence. `--engine NAME[:PROVIDER]` overrides the panel's engine list.
15
+ `playwright-core` is an optional dependency; `doctor` reports the browser stack as a skip,
16
+ never a failure, when it is absent (API and mock engines need none of it).
17
+ - Repository: CI on every push and a tag-driven release workflow that publishes through npm
18
+ trusted publishing with provenance.
19
+
20
+ ## 0.5.0 (2026-09-16; DP-0036 agent surface)
7
21
 
8
22
  - `agentlinkops tools [TOOLSET]`, `agentlinkops describe NAME` and `agentlinkops call NAME`:
9
23
  the three discovery verbs, answered offline from the bundled catalog snapshot (`--refresh`
package/README.md CHANGED
@@ -37,7 +37,8 @@ Four steps, the same ones the docs and the landing page show. Node.js 22.13 or l
37
37
  The skill is a short door: the first time your agent uses AgentLinkOps in a session it reads
38
38
  the full agent reference, which `agentlinkops skill` prints and
39
39
  https://agentlinkops.com/SKILL.md serves. Hosted monitoring is an invitation-only pilot; the
40
- local commands below need no account.
40
+ local commands below need no account. The package is `agentlinkops` on npm, published by the
41
+ `agentlinkops` account; the source is https://github.com/sunlover7/agentlinkops-cli.
41
42
 
42
43
  ## Install the CLI on its own
43
44
 
package/cli/citation.js CHANGED
@@ -21,6 +21,10 @@ const USAGE = `agentlinkops citation — AI citation watches in this repository
21
21
  agentlinkops citation run PANEL.json [--dir DIR] [--samples N] [--max-usd USD] [--json]
22
22
  run one fixed-n epoch over a panel; evidence and
23
23
  tallies land under .agentlinkops/citations/
24
+ agentlinkops citation run PANEL.json --engine chatgpt:web-own-browser
25
+ measure the consumer UI through Camoufox
26
+ (dedicated account, personal cadence; evidence
27
+ includes a screenshot of every answer)
24
28
  agentlinkops citation help this help
25
29
 
26
30
  Panel (JSON): targets (domain or url, brand, aliases), prompts, engines
@@ -53,6 +57,15 @@ export async function citationMain(argv = [], { cwd = process.cwd(), out = conso
53
57
  const dir = resolve(cwd, args.dir ?? '.agentlinkops');
54
58
  if (args.samples) panel.samples = Number(args.samples);
55
59
  if (args['max-usd']) panel.maxUsd = Number(args['max-usd']);
60
+ // --engine NAME[:PROVIDER] overrides the panel's engine list. The provider
61
+ // suffix picks the surface: chatgpt:web-own-browser measures the consumer
62
+ // UI through Camoufox; chatgpt:api would measure the API. Never averaged.
63
+ if (args.engine) {
64
+ const [engineName, providerTag] = String(args.engine).split(':');
65
+ const via = providerTag === 'web-own-browser' ? 'browser' : 'api';
66
+ panel.engines = [{ engine: engineName, via }];
67
+ err(`engine override: ${engineIdentity(panel.engines[0])}`);
68
+ }
56
69
  // Overrides are stated, never silent: a number the panel did not ask for is a
57
70
  // configuration decision the operator should see in the receipt.
58
71
  if (args.samples || args['max-usd']) {
@@ -63,6 +76,11 @@ export async function citationMain(argv = [], { cwd = process.cwd(), out = conso
63
76
  for (const spec of panel.engines) {
64
77
  const identity = engineIdentity(spec);
65
78
  if (engines.has(identity)) continue;
79
+ if (spec.via === 'browser') {
80
+ const { createBrowserEngine } = await import('../src/citations/browser/engine.js');
81
+ engines.set(identity, createBrowserEngine({ engineName: spec.engine, env }));
82
+ continue;
83
+ }
66
84
  if (spec.engine === 'mock') {
67
85
  engines.set(identity, createMockEngine({ fixtures: panel.mockFixtures ?? {} }));
68
86
  } else if (spec.engine === 'perplexity') {
@@ -79,7 +97,7 @@ export async function citationMain(argv = [], { cwd = process.cwd(), out = conso
79
97
  }
80
98
 
81
99
  try {
82
- const result = await runEpoch(panel, { dir, engines, delayMs: 250 });
100
+ const result = await runEpoch(panel, { dir, engines, delayMs: 250, err });
83
101
  if (args.json) {
84
102
  out(JSON.stringify(result, null, 2));
85
103
  } else {
package/cli/client.js CHANGED
@@ -2,7 +2,7 @@ import {restResponseContract} from '../shared/response-contract.js';
2
2
  // A small HTTP client for the cloud. Scoped keys only; no human session ever reaches here.
3
3
  // DP-0036-T13: the CLI names itself so the usage counters can tell it from other REST callers.
4
4
  // The version follows the plugin manifests; bump it with them.
5
- export const CLI_VERSION = '0.5.0';
5
+ export const CLI_VERSION = '0.5.1';
6
6
  export const CLI_USER_AGENT = `agentlinkops-cli/${CLI_VERSION}`;
7
7
  export class CloudError extends Error {
8
8
  constructor(code, status, details = null) {
package/cli/doctor.js CHANGED
@@ -261,6 +261,25 @@ export async function doctorMain(argv = [], { cwd = process.cwd(), out = console
261
261
  checks.push({ status: 'skip', name: 'citations', detail: 'no engine credential set; mock citation runs work, live runs need PERPLEXITY_API_KEY' });
262
262
  }
263
263
 
264
+ // Browser engines: the Camoufox stack. Each missing piece names its fix; none
265
+ // of it fails doctor, because API and mock engines do not need it.
266
+ try {
267
+ await import('playwright-core');
268
+ const { execFile } = await import('node:child_process');
269
+ const camoufoxOk = await new Promise((resolve) => {
270
+ // `camoufox path` resolves the fetched browser and exits 0; the launcher
271
+ // has no --version flag and exits nonzero on it.
272
+ execFile('camoufox', ['path'], { timeout: 8000 }, (error) => resolve(!error));
273
+ });
274
+ if (camoufoxOk) {
275
+ checks.push({ status: 'ok', name: 'browser', detail: 'playwright-core + camoufox present; browser engines available' });
276
+ } else {
277
+ checks.push({ status: 'skip', name: 'browser', detail: 'playwright-core ok but camoufox missing', fix: 'python3 -m pip install cloverlabs-camoufox[geoip] && python3 -m camoufox fetch' });
278
+ }
279
+ } catch {
280
+ checks.push({ status: 'skip', name: 'browser', detail: 'browser engines unavailable: playwright-core not installed', fix: 'npm install playwright-core (API and mock engines are unaffected)' });
281
+ }
282
+
264
283
  out(`agentlinkops doctor — ${config.dir}`);
265
284
 
266
285
  // Record the probe outcomes beside the cursors they protect — BEFORE rendering, so the
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agentlinkops",
3
- "version": "0.5.0",
3
+ "version": "0.5.1",
4
4
  "description": "A backlink ledger that lives in your repository: local link checks with dated evidence, supplier-export imports and optional sync to the hosted AgentLinkOps service.",
5
5
  "author": "AgentLinkOps (https://agentlinkops.com)",
6
6
  "license": "Apache-2.0",
@@ -21,6 +21,7 @@
21
21
  "templates",
22
22
  "references",
23
23
  "scripts",
24
+ ".mcp.json",
24
25
  ".claude-plugin",
25
26
  ".codex-plugin",
26
27
  "README.md",
@@ -54,6 +55,13 @@
54
55
  "dependencies": {
55
56
  "@modelcontextprotocol/server": "2.0.0",
56
57
  "parse5": "8.0.1",
58
+ "turndown": "^7.2.4",
57
59
  "zod": "4.6.0"
60
+ },
61
+ "devDependencies": {
62
+ "esbuild": "^0.28.2"
63
+ },
64
+ "optionalDependencies": {
65
+ "playwright-core": "^1.63.0"
58
66
  }
59
67
  }
@@ -0,0 +1,196 @@
1
+ // The browser engine provider: one AI surface, measured through a real
2
+ // Camoufox-driven Firefox at consumer URLs, as our citation adapter contract.
3
+ //
4
+ // Everything here follows the DP-0031 contract's posture rules: a fresh
5
+ // browser context per sample (session history can never contaminate the
6
+ // panel), the vendored oneglanse per-engine configs for wait/extract/citations
7
+ // (MIT, attributed), our proxy from AGENTLINKOPS_PROXY_URL, and a screenshot
8
+ // captured into evidence on every run so an observation is never just a claim.
9
+ //
10
+ // This module imports playwright-core lazily: a CLI install without the
11
+ // browser tooling still runs every non-browser engine, and `citation doctor`
12
+ // reports the gap instead of the whole CLI failing to load.
13
+ import { mkdir } from 'node:fs/promises';
14
+ import { homedir } from 'node:os';
15
+ import { join } from 'node:path';
16
+
17
+ import { EngineError } from '../adapter.js';
18
+
19
+ const lazy = async (path) => import(path);
20
+
21
+ let playwrightFirefox = null;
22
+ async function firefox() {
23
+ if (!playwrightFirefox) {
24
+ try {
25
+ playwrightFirefox = (await lazy('playwright-core')).firefox;
26
+ } catch (cause) {
27
+ throw new EngineError('playwright-core is not installed; browser engines need it (npm install playwright-core)', { cause });
28
+ }
29
+ }
30
+ return playwrightFirefox;
31
+ }
32
+
33
+ export const BROWSER_PROVIDERS = Object.freeze(['chatgpt', 'perplexity', 'gemini', 'claude', 'ai-overview']);
34
+
35
+ // Nominal per-answer estimate: ~0.3 MB through a residential proxy at ~$7/GB,
36
+ // plus slack for the page itself. Checked BEFORE the run by the budget cap.
37
+ const NOMINAL_COST_USD = 0.004;
38
+
39
+ function parseProxyUrl(raw) {
40
+ if (!raw) return null;
41
+ try {
42
+ const url = new URL(raw);
43
+ return {
44
+ server: `${url.protocol}//${url.hostname}:${url.port}`,
45
+ username: decodeURIComponent(url.username),
46
+ password: decodeURIComponent(url.password),
47
+ };
48
+ } catch {
49
+ throw new EngineError(`AGENTLINKOPS_PROXY_URL is not a valid proxy URL (length ${raw.length}, value never printed)`);
50
+ }
51
+ }
52
+
53
+ export function createBrowserEngine({ engineName, env = process.env, onEvent = () => {} } = {}) {
54
+ if (!BROWSER_PROVIDERS.includes(engineName)) {
55
+ throw new EngineError(`unknown browser engine: ${engineName} (known: ${BROWSER_PROVIDERS.join(', ')})`);
56
+ }
57
+ const identity = { engine: engineName, provider: 'web-own-browser', model: engineName };
58
+ const sessionDir = join(homedir(), '.agentlinkops', 'citations', 'sessions');
59
+
60
+ let browserHandle = null; // { browser, cleanup } for the whole epoch
61
+ let displayHandle = null;
62
+ let providerConfig = null;
63
+
64
+ async function ensureProviderConfig() {
65
+ if (providerConfig) return providerConfig;
66
+ const { PROVIDER_CONFIGS } = await lazy('./gen/core/providers/index.js');
67
+ providerConfig = PROVIDER_CONFIGS[engineName];
68
+ if (!providerConfig) throw new EngineError(`no provider config for ${engineName}`);
69
+ return providerConfig;
70
+ }
71
+
72
+ async function ensureBrowser() {
73
+ if (browserHandle) return browserHandle;
74
+ const [{ resolveCamoufoxLaunchOptions }, display, { firefox: launchFirefox }] = [
75
+ await lazy('./gen/lib/browser/camoufox.js'),
76
+ await lazy('./gen/lib/browser/display.js'),
77
+ { firefox: await firefox() },
78
+ ];
79
+
80
+ // Linux without a display gets the vendored self-bootstrapped Xvfb — the
81
+ // engines that block headless render headfully inside a virtual display.
82
+ if (process.platform === 'linux' && !display.detectDisplay()) {
83
+ displayHandle = await display.ensureDisplay({ allowExistingDisplay: false });
84
+ onEvent(`display: ${displayHandle.display}`);
85
+ }
86
+
87
+ const proxy = parseProxyUrl(env.AGENTLINKOPS_PROXY_URL);
88
+ if (proxy) onEvent(`proxy: ${proxy.server}`);
89
+
90
+ const options = await resolveCamoufoxLaunchOptions({
91
+ provider: engineName,
92
+ proxy: proxy ?? undefined,
93
+ // "virtual" on Linux = headful Firefox inside Xvfb; true headless is the
94
+ // detection magnet the research warned about.
95
+ headlessMode: process.platform === 'linux' ? 'virtual' : 'headful',
96
+ });
97
+ const browser = await launchFirefox({ ...options, executablePath: options.executablePath, proxy });
98
+
99
+ browserHandle = {
100
+ browser,
101
+ cleanup: async () => {
102
+ await browser.close().catch(() => null);
103
+ await displayHandle?.cleanup?.().catch(() => null);
104
+ displayHandle = null;
105
+ },
106
+ };
107
+ return browserHandle;
108
+ }
109
+
110
+ async function loadStorageState() {
111
+ try {
112
+ const { readFile } = await lazy('node:fs/promises');
113
+ return JSON.parse(await readFile(join(sessionDir, `${engineName}.json`), 'utf8'));
114
+ } catch {
115
+ return undefined; // anonymous session: several surfaces work logged-out
116
+ }
117
+ }
118
+
119
+ return {
120
+ identity,
121
+ estimateCostUsd: () => NOMINAL_COST_USD,
122
+
123
+ async run({ prompt }) {
124
+ const config = await ensureProviderConfig();
125
+ const { browser, cleanup } = await ensureBrowser();
126
+ const context = await browser.newContext({
127
+ storageState: await loadStorageState(),
128
+ locale: 'en-US',
129
+ timezoneId: 'America/New_York',
130
+ });
131
+ try {
132
+ const page = await context.newPage();
133
+ page.setDefaultTimeout(45_000);
134
+
135
+ if (config.navigateToPrompt) {
136
+ await config.navigateToPrompt(page, prompt);
137
+ } else {
138
+ await page.goto(config.url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
139
+ await config.postNavigationHook?.(page);
140
+ await config.beforePromptHook?.(page);
141
+
142
+ const { findActiveEditorCandidate } = await lazy('./gen/lib/input/editor/findEditor.js');
143
+ const { PROVIDER_EDITOR_SELECTORS } = await lazy('./shims/utils.js');
144
+ const editor = await findActiveEditorCandidate(page, PROVIDER_EDITOR_SELECTORS[engineName]);
145
+ if (!editor) throw new EngineError(`${engineName}: no prompt editor found (surface may be gated or changed)`);
146
+
147
+ const { insertPromptIntoEditor } = await lazy('./gen/lib/input/editor/promptInput.js');
148
+ await insertPromptIntoEditor(page, editor.locator, prompt, engineName);
149
+ await config.afterTypingHook?.(page);
150
+ await config.beforeSubmitHook?.(page);
151
+
152
+ const { findEnabledSendButton } = await lazy('./gen/lib/input/editor/findSendButton.js');
153
+ const { PROVIDER_SUBMIT_BTN_SELECTORS } = await lazy('./shims/utils.js');
154
+ const send = await findEnabledSendButton(page, PROVIDER_SUBMIT_BTN_SELECTORS[engineName]);
155
+ if (send) { await send.click(); }
156
+ else { await editor.locator.press('Enter'); }
157
+ }
158
+ await config.afterSubmitHook?.(page);
159
+
160
+ await config.waitForResponse(page);
161
+ const answer = await config.extractResponse(page);
162
+ if (!answer || answer.trim().length === 0) {
163
+ throw new EngineError(`${engineName}: answer extracted empty`, { retriable: true });
164
+ }
165
+ const sources = await config.extractSources(page);
166
+ const screenshotPng = await page.screenshot({ fullPage: false }).catch(() => null);
167
+
168
+ return {
169
+ engine: engineName,
170
+ provider: 'web-own-browser',
171
+ model: engineName,
172
+ providerModelVersion: `${engineName}-web`,
173
+ answer,
174
+ citations: (sources ?? []).map((s) => ({ url: s.url, title: s.title ?? s.cited_text ?? null })).filter((c) => typeof c.url === 'string' && c.url.length > 0),
175
+ fanOut: [],
176
+ usage: { input_tokens: 0, output_tokens: 0 },
177
+ costEstimateUsd: NOMINAL_COST_USD,
178
+ screenshotPng,
179
+ };
180
+ } catch (cause) {
181
+ if (cause instanceof EngineError) throw cause;
182
+ const msg = String(cause?.message ?? cause);
183
+ const retriable = /timeout|timed out|ERR_|net::|closed|Target closed/i.test(msg);
184
+ throw new EngineError(`${engineName}: browser run failed: ${msg}`, { retriable, cause });
185
+ } finally {
186
+ await context.close().catch(() => null);
187
+ }
188
+ },
189
+
190
+ // Called by the runner at epoch end.
191
+ async close() {
192
+ await browserHandle?.cleanup?.();
193
+ browserHandle = null;
194
+ },
195
+ };
196
+ }
@@ -0,0 +1,4 @@
1
+ const GOOGLE_CONSENT_SELECTOR = "button#L2AGLb, button#W0wltc, form[action*='consent.google.com'] button";
2
+ export {
3
+ GOOGLE_CONSENT_SELECTOR
4
+ };
@@ -0,0 +1,15 @@
1
+ import { AI_OVERVIEW_RAW_SOURCES_DOM_EXTRACTOR } from "../ai-overview/lib/extractSources.js";
2
+ import { CHATGPT_RAW_SOURCES_DOM_EXTRACTOR } from "../chatgpt/lib/extractSources.js";
3
+ import { CLAUDE_RAW_SOURCES_DOM_EXTRACTOR } from "../claude/lib/extractSources.js";
4
+ import { GEMINI_RAW_SOURCES_DOM_EXTRACTOR } from "../gemini/lib/extractSources.js";
5
+ import { PERPLEXITY_RAW_SOURCES_DOM_EXTRACTOR } from "../perplexity/lib/extractSources.js";
6
+ const PROVIDER_RAW_SOURCES_DOM_EXTRACTORS = {
7
+ chatgpt: CHATGPT_RAW_SOURCES_DOM_EXTRACTOR,
8
+ perplexity: PERPLEXITY_RAW_SOURCES_DOM_EXTRACTOR,
9
+ gemini: GEMINI_RAW_SOURCES_DOM_EXTRACTOR,
10
+ claude: CLAUDE_RAW_SOURCES_DOM_EXTRACTOR,
11
+ "ai-overview": AI_OVERVIEW_RAW_SOURCES_DOM_EXTRACTOR
12
+ };
13
+ export {
14
+ PROVIDER_RAW_SOURCES_DOM_EXTRACTORS
15
+ };
@@ -0,0 +1,22 @@
1
+ import { logger } from "../../../../shims/utils.js";
2
+ import { preInteractionIdle, randomBetween } from "../../../lib/browser/humanBehavior.js";
3
+ import { navigateWithRetry } from "../../../lib/browser/navigate.js";
4
+ import { detectBotPage } from "../../../lib/input/response/detectBotPage.js";
5
+ async function resetProviderPage(page, provider, url, options = {}) {
6
+ logger.log(`[${provider}] resetting page for next prompt`);
7
+ await preInteractionIdle(page).catch(() => {
8
+ });
9
+ await page.waitForTimeout(randomBetween(600, 1400));
10
+ await navigateWithRetry(page, url, {
11
+ waitUntil: "domcontentloaded",
12
+ timeout: 3e4
13
+ });
14
+ logger.log(`[${provider}] redirected back to provider page: ${page.url()}`);
15
+ await detectBotPage(page, provider);
16
+ await options.postNavigationHook?.(page);
17
+ await page.waitForTimeout(randomBetween(1200, 2600));
18
+ logger.log(`[${provider}] page reset ready: ${page.url()}`);
19
+ }
20
+ export {
21
+ resetProviderPage
22
+ };
@@ -0,0 +1,8 @@
1
+ import {
2
+ buildSources,
3
+ clickButtonViaDispatch
4
+ } from "../../../lib/extraction/sourceUtils.js";
5
+ export {
6
+ buildSources,
7
+ clickButtonViaDispatch
8
+ };
@@ -0,0 +1,70 @@
1
+ import { PROVIDER_EDITOR_SELECTORS } from "../../../../shims/utils.js";
2
+ import { logger } from "../../../../shims/utils.js";
3
+ import { navigateWithRetry } from "../../../lib/browser/navigate.js";
4
+ import {
5
+ findActiveEditorCandidateFromSelectors
6
+ } from "../../../lib/input/editor/findEditor.js";
7
+ import { extractAssistantMarkdown } from "../../../lib/input/markdown/toMarkdown.js";
8
+ import { insertPromptIntoEditor } from "../../../lib/input/editor/promptInput.js";
9
+ import { waitForAssistantToFinish } from "../../../lib/input/response/waitForFinish.js";
10
+ import { extractAIOverviewSources } from "./lib/extractSources.js";
11
+ import { expandAIOverviewSources } from "./lib/expand.js";
12
+ import {
13
+ AI_OVERVIEW_URL,
14
+ resetAIOverviewPage
15
+ } from "./lib/pageLifecycle.js";
16
+ import {
17
+ assertAIOverviewPageNotBlocked,
18
+ dismissGoogleConsentDialog,
19
+ ensureAIOverviewGoogleSession,
20
+ waitForAIOverviewSearchResults
21
+ } from "./lib/session.js";
22
+ const aiOverviewConfig = {
23
+ url: AI_OVERVIEW_URL,
24
+ label: "AI Overview",
25
+ displayName: "AI Overview",
26
+ skipInitialNavigation: true,
27
+ navigateToPrompt: async (page, prompt) => {
28
+ await ensureAIOverviewGoogleSession(page);
29
+ await navigateWithRetry(page, AI_OVERVIEW_URL, {
30
+ waitUntil: "domcontentloaded",
31
+ timeout: 3e4
32
+ });
33
+ assertAIOverviewPageNotBlocked(page);
34
+ await dismissGoogleConsentDialog(page);
35
+ const searchInput = await findActiveEditorCandidateFromSelectors(page, [
36
+ ...PROVIDER_EDITOR_SELECTORS["ai-overview"]
37
+ ]);
38
+ logger.debug(`[ai-overview] using search selector: ${searchInput.selector}`);
39
+ logger.debug(`[ai-overview] pasting ${prompt.length} chars\u2026`);
40
+ await insertPromptIntoEditor(
41
+ page,
42
+ searchInput.locator,
43
+ prompt,
44
+ "ai-overview"
45
+ );
46
+ logger.debug(`[ai-overview] pasting ${prompt.length} chars complete`);
47
+ await page.waitForTimeout(400);
48
+ logger.debug("[ai-overview] attempting submission\u2026");
49
+ await page.keyboard.press("Enter");
50
+ await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(
51
+ () => {
52
+ }
53
+ );
54
+ await waitForAIOverviewSearchResults(page);
55
+ await dismissGoogleConsentDialog(page);
56
+ assertAIOverviewPageNotBlocked(page);
57
+ logger.log(`[ai-overview] search ready: ${page.url()}`);
58
+ },
59
+ waitForResponse: (page) => waitForAssistantToFinish(page, "ai-overview"),
60
+ extractResponse: (page) => extractAssistantMarkdown(page, "ai-overview"),
61
+ beforeRetryHook: resetAIOverviewPage,
62
+ betweenPromptsHook: resetAIOverviewPage,
63
+ extractSources: async (page) => {
64
+ await expandAIOverviewSources(page);
65
+ return extractAIOverviewSources(page);
66
+ }
67
+ };
68
+ export {
69
+ aiOverviewConfig
70
+ };
@@ -0,0 +1,59 @@
1
+ import { logger } from "../../../../../shims/utils.js";
2
+ const AI_OVERVIEW_SOURCE_ROOT_SELECTOR = '[data-container-id="rhs-col"]';
3
+ const AI_OVERVIEW_BUTTON_SELECTOR = '[role="button"], button';
4
+ function shouldClickSourceExpansionButton(text, aria) {
5
+ const t = `${text} ${aria}`.toLowerCase();
6
+ if (t.includes("videos")) return false;
7
+ if (t.includes("more")) return true;
8
+ if (t.includes("all")) return true;
9
+ if (t.includes("expand")) return true;
10
+ return false;
11
+ }
12
+ async function expandAIOverviewSources(page) {
13
+ const matches = await page.evaluate(
14
+ ({
15
+ rootSelector,
16
+ buttonSelector
17
+ }) => {
18
+ const root = document.querySelector(rootSelector);
19
+ if (!(root instanceof HTMLElement)) return [];
20
+ return Array.from(root.querySelectorAll(buttonSelector)).map((button, index) => {
21
+ if (!(button instanceof HTMLElement)) {
22
+ return null;
23
+ }
24
+ const style = window.getComputedStyle(button);
25
+ const rect = button.getBoundingClientRect();
26
+ const visible = button.isConnected && style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0" && rect.width > 0 && rect.height > 0;
27
+ if (!visible) return null;
28
+ return {
29
+ index,
30
+ text: (button.textContent || "").trim(),
31
+ aria: button.getAttribute("aria-label")?.trim() || ""
32
+ };
33
+ }).filter(
34
+ (match) => match !== null
35
+ );
36
+ },
37
+ {
38
+ rootSelector: AI_OVERVIEW_SOURCE_ROOT_SELECTOR,
39
+ buttonSelector: AI_OVERVIEW_BUTTON_SELECTOR
40
+ }
41
+ );
42
+ for (const match of matches) {
43
+ if (!shouldClickSourceExpansionButton(match.text, match.aria)) {
44
+ continue;
45
+ }
46
+ const button = page.locator(`${AI_OVERVIEW_SOURCE_ROOT_SELECTOR} ${AI_OVERVIEW_BUTTON_SELECTOR}`).nth(match.index);
47
+ const visible = await button.isVisible().catch(() => false);
48
+ if (!visible) continue;
49
+ logger.debug(
50
+ `[ai-overview] expanding sources text="${match.text}" aria="${match.aria}"`
51
+ );
52
+ await button.click({ timeout: 5e3 }).catch(() => {
53
+ });
54
+ await page.waitForTimeout(700);
55
+ }
56
+ }
57
+ export {
58
+ expandAIOverviewSources
59
+ };
@@ -0,0 +1,60 @@
1
+ import { toErrorMessage } from "../../../../../shims/errors.js";
2
+ import { logger } from "../../../../../shims/utils.js";
3
+ import { buildSources } from "../../_shared/sourceUtils.js";
4
+ const AI_OVERVIEW_RAW_SOURCES_DOM_EXTRACTOR = String.raw`(_helpers) => {
5
+ const results = [];
6
+ const rhsCol = document.querySelector('[data-container-id="rhs-col"]');
7
+ if (!rhsCol) {
8
+ return { rawSources: results, containerFound: false };
9
+ }
10
+
11
+ for (const card of Array.from(rhsCol.querySelectorAll("div[data-src-id]"))) {
12
+ if (!(card instanceof HTMLElement)) continue;
13
+
14
+ const link = card.querySelector('a[href^="http"]');
15
+ if (!(link instanceof HTMLAnchorElement)) continue;
16
+
17
+ const title =
18
+ link
19
+ .getAttribute("aria-label")
20
+ ?.replace(/\.\s*Opens in new tab\.?$/i, "")
21
+ .trim() || link.href;
22
+ const citedText =
23
+ card.querySelector("[data-crb-snippet-text]")?.textContent?.trim() || "";
24
+
25
+ results.push({
26
+ rawHref: link.href,
27
+ title,
28
+ citedText,
29
+ });
30
+ }
31
+
32
+ return { rawSources: results, containerFound: true };
33
+ }`;
34
+ function normalizeAIOverviewTitle(title) {
35
+ return title.replace(/\s*\.?\s*opens in new tab\.?\s*$/i, "").trim();
36
+ }
37
+ async function extractAIOverviewSources(page) {
38
+ try {
39
+ const { rawSources, containerFound } = await page.runDomOp("raw-sources", {
40
+ provider: "ai-overview"
41
+ });
42
+ if (!containerFound) {
43
+ logger.warn("AI Overview container not found \u2014 no sources extracted");
44
+ }
45
+ const normalizedSources = rawSources.map((source) => ({
46
+ ...source,
47
+ title: normalizeAIOverviewTitle(source.title ?? "") || source.rawHref
48
+ }));
49
+ return buildSources(normalizedSources, { provider: "ai-overview" });
50
+ } catch (err) {
51
+ logger.error(
52
+ `Failed to extract AI Overview sources: ${toErrorMessage(err)}`
53
+ );
54
+ return [];
55
+ }
56
+ }
57
+ export {
58
+ AI_OVERVIEW_RAW_SOURCES_DOM_EXTRACTOR,
59
+ extractAIOverviewSources
60
+ };
@@ -0,0 +1,18 @@
1
+ import { resetProviderPage } from "../../_shared/resetProviderPage.js";
2
+ import {
3
+ assertAIOverviewPageNotBlocked,
4
+ dismissGoogleConsentDialog
5
+ } from "./session.js";
6
+ const AI_OVERVIEW_URL = "https://www.google.com/";
7
+ async function resetAIOverviewPage(page) {
8
+ await resetProviderPage(page, "ai-overview", AI_OVERVIEW_URL, {
9
+ postNavigationHook: async (currentPage) => {
10
+ assertAIOverviewPageNotBlocked(currentPage);
11
+ await dismissGoogleConsentDialog(currentPage);
12
+ }
13
+ });
14
+ }
15
+ export {
16
+ AI_OVERVIEW_URL,
17
+ resetAIOverviewPage
18
+ };