agentlinkops 0.5.0 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/.mcp.json +8 -0
- package/CHANGELOG.md +15 -1
- package/README.md +2 -1
- package/cli/citation.js +19 -1
- package/cli/client.js +1 -1
- package/cli/doctor.js +19 -0
- package/package.json +9 -1
- package/src/citations/browser/engine.js +196 -0
- package/src/citations/browser/gen/core/providers/_shared/google.js +4 -0
- package/src/citations/browser/gen/core/providers/_shared/rawSourcesDom.js +15 -0
- package/src/citations/browser/gen/core/providers/_shared/resetProviderPage.js +22 -0
- package/src/citations/browser/gen/core/providers/_shared/sourceUtils.js +8 -0
- package/src/citations/browser/gen/core/providers/ai-overview/index.js +70 -0
- package/src/citations/browser/gen/core/providers/ai-overview/lib/expand.js +59 -0
- package/src/citations/browser/gen/core/providers/ai-overview/lib/extractSources.js +60 -0
- package/src/citations/browser/gen/core/providers/ai-overview/lib/pageLifecycle.js +18 -0
- package/src/citations/browser/gen/core/providers/ai-overview/lib/session.js +62 -0
- package/src/citations/browser/gen/core/providers/chatgpt/index.js +28 -0
- package/src/citations/browser/gen/core/providers/chatgpt/lib/dismissAuthModal.js +39 -0
- package/src/citations/browser/gen/core/providers/chatgpt/lib/extractSources.js +66 -0
- package/src/citations/browser/gen/core/providers/chatgpt/lib/pageLifecycle.js +11 -0
- package/src/citations/browser/gen/core/providers/claude/index.js +16 -0
- package/src/citations/browser/gen/core/providers/claude/lib/extractSources.js +54 -0
- package/src/citations/browser/gen/core/providers/claude/lib/pageLifecycle.js +9 -0
- package/src/citations/browser/gen/core/providers/gemini/index.js +40 -0
- package/src/citations/browser/gen/core/providers/gemini/lib/extractSources.js +65 -0
- package/src/citations/browser/gen/core/providers/gemini/lib/pageLifecycle.js +14 -0
- package/src/citations/browser/gen/core/providers/gemini/lib/session.js +45 -0
- package/src/citations/browser/gen/core/providers/index.js +15 -0
- package/src/citations/browser/gen/core/providers/perplexity/index.js +31 -0
- package/src/citations/browser/gen/core/providers/perplexity/lib/extractSources.js +84 -0
- package/src/citations/browser/gen/core/providers/perplexity/lib/navigation.js +38 -0
- package/src/citations/browser/gen/core/providers/types.js +0 -0
- package/src/citations/browser/gen/lib/browser/camoufox.js +587 -0
- package/src/citations/browser/gen/lib/browser/display.js +147 -0
- package/src/citations/browser/gen/lib/browser/domOps.js +339 -0
- package/src/citations/browser/gen/lib/browser/humanBehavior.js +121 -0
- package/src/citations/browser/gen/lib/browser/navigate.js +46 -0
- package/src/citations/browser/gen/lib/browser/playwrightCompat.js +321 -0
- package/src/citations/browser/gen/lib/browser/proxy/forwarder.js +251 -0
- package/src/citations/browser/gen/lib/browser/runtimeTypes.js +0 -0
- package/src/citations/browser/gen/lib/extraction/sourceUtils.js +71 -0
- package/src/citations/browser/gen/lib/input/editor/assertNotBlocked.js +54 -0
- package/src/citations/browser/gen/lib/input/editor/clearInput.js +52 -0
- package/src/citations/browser/gen/lib/input/editor/findEditor.js +54 -0
- package/src/citations/browser/gen/lib/input/editor/findSendButton.js +21 -0
- package/src/citations/browser/gen/lib/input/editor/promptInput.js +117 -0
- package/src/citations/browser/gen/lib/input/markdown/converter.js +101 -0
- package/src/citations/browser/gen/lib/input/markdown/toMarkdown.js +14 -0
- package/src/citations/browser/gen/lib/input/response/detectBotPage.js +13 -0
- package/src/citations/browser/gen/lib/input/response/getText.js +10 -0
- package/src/citations/browser/gen/lib/input/response/isGenerating.js +59 -0
- package/src/citations/browser/gen/lib/input/response/waitForFinish.js +105 -0
- package/src/citations/browser/gen/lib/input/sources/findButton.js +41 -0
- package/src/citations/browser/gen/lib/input/sources/openPanel.js +11 -0
- package/src/citations/browser/gen/utils/agent-constants.js +87 -0
- package/src/citations/browser/shims/errors.js +26 -0
- package/src/citations/browser/shims/types.js +6 -0
- package/src/citations/browser/shims/utils.js +32 -0
- package/src/citations/contract.js +9 -2
- package/src/citations/runner.js +21 -5
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agentlinkops",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"description": "Connect an agent to AgentLinkOps and run backlink work with it: the connect skill (views, the three discovery verbs, evidence rules), local ledger and campaign skills, a portable CRM, the source-cited content pipeline, user-managed disavow lists, HTML reports and monitoring-event sync.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "AgentLinkOps"
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agentlinkops",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"description": "Connect an agent to AgentLinkOps and run backlink work with it: the connect skill (views, the three discovery verbs, evidence rules), local ledger and campaign skills, a portable CRM, the source-cited content pipeline, user-managed disavow lists, HTML reports and monitoring-event sync.",
|
|
5
5
|
"author": {
|
|
6
6
|
"name": "AgentLinkOps"
|
package/.mcp.json
ADDED
package/CHANGELOG.md
CHANGED
|
@@ -3,7 +3,21 @@
|
|
|
3
3
|
All notable changes to the `agentlinkops` package are recorded here. The version follows the
|
|
4
4
|
plugin manifests in `.claude-plugin/` and `.codex-plugin/`.
|
|
5
5
|
|
|
6
|
-
## 0.5.
|
|
6
|
+
## 0.5.1 (2026-09-17)
|
|
7
|
+
|
|
8
|
+
- The plugin's `.mcp.json` ships in the tarball. 0.5.0 left it out, so a marketplace install
|
|
9
|
+
got the skills but no MCP server entry; the package build now carries and requires it.
|
|
10
|
+
- Package identity: `author` is AgentLinkOps (https://agentlinkops.com); `repository` and
|
|
11
|
+
`bugs` point at `sunlover7/agentlinkops-cli`. No personal identity in the registry metadata.
|
|
12
|
+
- `agentlinkops citation run PANEL.json --engine chatgpt:web-own-browser`: measures the
|
|
13
|
+
consumer UI through the customer's own Camoufox browser instead of an API, with a screenshot
|
|
14
|
+
of every answer in the evidence. `--engine NAME[:PROVIDER]` overrides the panel's engine list.
|
|
15
|
+
`playwright-core` is an optional dependency; `doctor` reports the browser stack as a skip,
|
|
16
|
+
never a failure, when it is absent (API and mock engines need none of it).
|
|
17
|
+
- Repository: CI on every push and a tag-driven release workflow that publishes through npm
|
|
18
|
+
trusted publishing with provenance.
|
|
19
|
+
|
|
20
|
+
## 0.5.0 (2026-09-16; DP-0036 agent surface)
|
|
7
21
|
|
|
8
22
|
- `agentlinkops tools [TOOLSET]`, `agentlinkops describe NAME` and `agentlinkops call NAME`:
|
|
9
23
|
the three discovery verbs, answered offline from the bundled catalog snapshot (`--refresh`
|
package/README.md
CHANGED
|
@@ -37,7 +37,8 @@ Four steps, the same ones the docs and the landing page show. Node.js 22.13 or l
|
|
|
37
37
|
The skill is a short door: the first time your agent uses AgentLinkOps in a session it reads
|
|
38
38
|
the full agent reference, which `agentlinkops skill` prints and
|
|
39
39
|
https://agentlinkops.com/SKILL.md serves. Hosted monitoring is an invitation-only pilot; the
|
|
40
|
-
local commands below need no account.
|
|
40
|
+
local commands below need no account. The package is `agentlinkops` on npm, published by the
|
|
41
|
+
`agentlinkops` account; the source is https://github.com/sunlover7/agentlinkops-cli.
|
|
41
42
|
|
|
42
43
|
## Install the CLI on its own
|
|
43
44
|
|
package/cli/citation.js
CHANGED
|
@@ -21,6 +21,10 @@ const USAGE = `agentlinkops citation — AI citation watches in this repository
|
|
|
21
21
|
agentlinkops citation run PANEL.json [--dir DIR] [--samples N] [--max-usd USD] [--json]
|
|
22
22
|
run one fixed-n epoch over a panel; evidence and
|
|
23
23
|
tallies land under .agentlinkops/citations/
|
|
24
|
+
agentlinkops citation run PANEL.json --engine chatgpt:web-own-browser
|
|
25
|
+
measure the consumer UI through Camoufox
|
|
26
|
+
(dedicated account, personal cadence; evidence
|
|
27
|
+
includes a screenshot of every answer)
|
|
24
28
|
agentlinkops citation help this help
|
|
25
29
|
|
|
26
30
|
Panel (JSON): targets (domain or url, brand, aliases), prompts, engines
|
|
@@ -53,6 +57,15 @@ export async function citationMain(argv = [], { cwd = process.cwd(), out = conso
|
|
|
53
57
|
const dir = resolve(cwd, args.dir ?? '.agentlinkops');
|
|
54
58
|
if (args.samples) panel.samples = Number(args.samples);
|
|
55
59
|
if (args['max-usd']) panel.maxUsd = Number(args['max-usd']);
|
|
60
|
+
// --engine NAME[:PROVIDER] overrides the panel's engine list. The provider
|
|
61
|
+
// suffix picks the surface: chatgpt:web-own-browser measures the consumer
|
|
62
|
+
// UI through Camoufox; chatgpt:api would measure the API. Never averaged.
|
|
63
|
+
if (args.engine) {
|
|
64
|
+
const [engineName, providerTag] = String(args.engine).split(':');
|
|
65
|
+
const via = providerTag === 'web-own-browser' ? 'browser' : 'api';
|
|
66
|
+
panel.engines = [{ engine: engineName, via }];
|
|
67
|
+
err(`engine override: ${engineIdentity(panel.engines[0])}`);
|
|
68
|
+
}
|
|
56
69
|
// Overrides are stated, never silent: a number the panel did not ask for is a
|
|
57
70
|
// configuration decision the operator should see in the receipt.
|
|
58
71
|
if (args.samples || args['max-usd']) {
|
|
@@ -63,6 +76,11 @@ export async function citationMain(argv = [], { cwd = process.cwd(), out = conso
|
|
|
63
76
|
for (const spec of panel.engines) {
|
|
64
77
|
const identity = engineIdentity(spec);
|
|
65
78
|
if (engines.has(identity)) continue;
|
|
79
|
+
if (spec.via === 'browser') {
|
|
80
|
+
const { createBrowserEngine } = await import('../src/citations/browser/engine.js');
|
|
81
|
+
engines.set(identity, createBrowserEngine({ engineName: spec.engine, env }));
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
66
84
|
if (spec.engine === 'mock') {
|
|
67
85
|
engines.set(identity, createMockEngine({ fixtures: panel.mockFixtures ?? {} }));
|
|
68
86
|
} else if (spec.engine === 'perplexity') {
|
|
@@ -79,7 +97,7 @@ export async function citationMain(argv = [], { cwd = process.cwd(), out = conso
|
|
|
79
97
|
}
|
|
80
98
|
|
|
81
99
|
try {
|
|
82
|
-
const result = await runEpoch(panel, { dir, engines, delayMs: 250 });
|
|
100
|
+
const result = await runEpoch(panel, { dir, engines, delayMs: 250, err });
|
|
83
101
|
if (args.json) {
|
|
84
102
|
out(JSON.stringify(result, null, 2));
|
|
85
103
|
} else {
|
package/cli/client.js
CHANGED
|
@@ -2,7 +2,7 @@ import {restResponseContract} from '../shared/response-contract.js';
|
|
|
2
2
|
// A small HTTP client for the cloud. Scoped keys only; no human session ever reaches here.
|
|
3
3
|
// DP-0036-T13: the CLI names itself so the usage counters can tell it from other REST callers.
|
|
4
4
|
// The version follows the plugin manifests; bump it with them.
|
|
5
|
-
export const CLI_VERSION = '0.5.
|
|
5
|
+
export const CLI_VERSION = '0.5.1';
|
|
6
6
|
export const CLI_USER_AGENT = `agentlinkops-cli/${CLI_VERSION}`;
|
|
7
7
|
export class CloudError extends Error {
|
|
8
8
|
constructor(code, status, details = null) {
|
package/cli/doctor.js
CHANGED
|
@@ -261,6 +261,25 @@ export async function doctorMain(argv = [], { cwd = process.cwd(), out = console
|
|
|
261
261
|
checks.push({ status: 'skip', name: 'citations', detail: 'no engine credential set; mock citation runs work, live runs need PERPLEXITY_API_KEY' });
|
|
262
262
|
}
|
|
263
263
|
|
|
264
|
+
// Browser engines: the Camoufox stack. Each missing piece names its fix; none
|
|
265
|
+
// of it fails doctor, because API and mock engines do not need it.
|
|
266
|
+
try {
|
|
267
|
+
await import('playwright-core');
|
|
268
|
+
const { execFile } = await import('node:child_process');
|
|
269
|
+
const camoufoxOk = await new Promise((resolve) => {
|
|
270
|
+
// `camoufox path` resolves the fetched browser and exits 0; the launcher
|
|
271
|
+
// has no --version flag and exits nonzero on it.
|
|
272
|
+
execFile('camoufox', ['path'], { timeout: 8000 }, (error) => resolve(!error));
|
|
273
|
+
});
|
|
274
|
+
if (camoufoxOk) {
|
|
275
|
+
checks.push({ status: 'ok', name: 'browser', detail: 'playwright-core + camoufox present; browser engines available' });
|
|
276
|
+
} else {
|
|
277
|
+
checks.push({ status: 'skip', name: 'browser', detail: 'playwright-core ok but camoufox missing', fix: 'python3 -m pip install cloverlabs-camoufox[geoip] && python3 -m camoufox fetch' });
|
|
278
|
+
}
|
|
279
|
+
} catch {
|
|
280
|
+
checks.push({ status: 'skip', name: 'browser', detail: 'browser engines unavailable: playwright-core not installed', fix: 'npm install playwright-core (API and mock engines are unaffected)' });
|
|
281
|
+
}
|
|
282
|
+
|
|
264
283
|
out(`agentlinkops doctor — ${config.dir}`);
|
|
265
284
|
|
|
266
285
|
// Record the probe outcomes beside the cursors they protect — BEFORE rendering, so the
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agentlinkops",
|
|
3
|
-
"version": "0.5.
|
|
3
|
+
"version": "0.5.1",
|
|
4
4
|
"description": "A backlink ledger that lives in your repository: local link checks with dated evidence, supplier-export imports and optional sync to the hosted AgentLinkOps service.",
|
|
5
5
|
"author": "AgentLinkOps (https://agentlinkops.com)",
|
|
6
6
|
"license": "Apache-2.0",
|
|
@@ -21,6 +21,7 @@
|
|
|
21
21
|
"templates",
|
|
22
22
|
"references",
|
|
23
23
|
"scripts",
|
|
24
|
+
".mcp.json",
|
|
24
25
|
".claude-plugin",
|
|
25
26
|
".codex-plugin",
|
|
26
27
|
"README.md",
|
|
@@ -54,6 +55,13 @@
|
|
|
54
55
|
"dependencies": {
|
|
55
56
|
"@modelcontextprotocol/server": "2.0.0",
|
|
56
57
|
"parse5": "8.0.1",
|
|
58
|
+
"turndown": "^7.2.4",
|
|
57
59
|
"zod": "4.6.0"
|
|
60
|
+
},
|
|
61
|
+
"devDependencies": {
|
|
62
|
+
"esbuild": "^0.28.2"
|
|
63
|
+
},
|
|
64
|
+
"optionalDependencies": {
|
|
65
|
+
"playwright-core": "^1.63.0"
|
|
58
66
|
}
|
|
59
67
|
}
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
// The browser engine provider: one AI surface, measured through a real
|
|
2
|
+
// Camoufox-driven Firefox at consumer URLs, as our citation adapter contract.
|
|
3
|
+
//
|
|
4
|
+
// Everything here follows the DP-0031 contract's posture rules: a fresh
|
|
5
|
+
// browser context per sample (session history can never contaminate the
|
|
6
|
+
// panel), the vendored oneglanse per-engine configs for wait/extract/citations
|
|
7
|
+
// (MIT, attributed), our proxy from AGENTLINKOPS_PROXY_URL, and a screenshot
|
|
8
|
+
// captured into evidence on every run so an observation is never just a claim.
|
|
9
|
+
//
|
|
10
|
+
// This module imports playwright-core lazily: a CLI install without the
|
|
11
|
+
// browser tooling still runs every non-browser engine, and `citation doctor`
|
|
12
|
+
// reports the gap instead of the whole CLI failing to load.
|
|
13
|
+
import { mkdir } from 'node:fs/promises';
|
|
14
|
+
import { homedir } from 'node:os';
|
|
15
|
+
import { join } from 'node:path';
|
|
16
|
+
|
|
17
|
+
import { EngineError } from '../adapter.js';
|
|
18
|
+
|
|
19
|
+
const lazy = async (path) => import(path);
|
|
20
|
+
|
|
21
|
+
let playwrightFirefox = null;
|
|
22
|
+
async function firefox() {
|
|
23
|
+
if (!playwrightFirefox) {
|
|
24
|
+
try {
|
|
25
|
+
playwrightFirefox = (await lazy('playwright-core')).firefox;
|
|
26
|
+
} catch (cause) {
|
|
27
|
+
throw new EngineError('playwright-core is not installed; browser engines need it (npm install playwright-core)', { cause });
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
return playwrightFirefox;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export const BROWSER_PROVIDERS = Object.freeze(['chatgpt', 'perplexity', 'gemini', 'claude', 'ai-overview']);
|
|
34
|
+
|
|
35
|
+
// Nominal per-answer estimate: ~0.3 MB through a residential proxy at ~$7/GB,
|
|
36
|
+
// plus slack for the page itself. Checked BEFORE the run by the budget cap.
|
|
37
|
+
const NOMINAL_COST_USD = 0.004;
|
|
38
|
+
|
|
39
|
+
function parseProxyUrl(raw) {
|
|
40
|
+
if (!raw) return null;
|
|
41
|
+
try {
|
|
42
|
+
const url = new URL(raw);
|
|
43
|
+
return {
|
|
44
|
+
server: `${url.protocol}//${url.hostname}:${url.port}`,
|
|
45
|
+
username: decodeURIComponent(url.username),
|
|
46
|
+
password: decodeURIComponent(url.password),
|
|
47
|
+
};
|
|
48
|
+
} catch {
|
|
49
|
+
throw new EngineError(`AGENTLINKOPS_PROXY_URL is not a valid proxy URL (length ${raw.length}, value never printed)`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
export function createBrowserEngine({ engineName, env = process.env, onEvent = () => {} } = {}) {
|
|
54
|
+
if (!BROWSER_PROVIDERS.includes(engineName)) {
|
|
55
|
+
throw new EngineError(`unknown browser engine: ${engineName} (known: ${BROWSER_PROVIDERS.join(', ')})`);
|
|
56
|
+
}
|
|
57
|
+
const identity = { engine: engineName, provider: 'web-own-browser', model: engineName };
|
|
58
|
+
const sessionDir = join(homedir(), '.agentlinkops', 'citations', 'sessions');
|
|
59
|
+
|
|
60
|
+
let browserHandle = null; // { browser, cleanup } for the whole epoch
|
|
61
|
+
let displayHandle = null;
|
|
62
|
+
let providerConfig = null;
|
|
63
|
+
|
|
64
|
+
async function ensureProviderConfig() {
|
|
65
|
+
if (providerConfig) return providerConfig;
|
|
66
|
+
const { PROVIDER_CONFIGS } = await lazy('./gen/core/providers/index.js');
|
|
67
|
+
providerConfig = PROVIDER_CONFIGS[engineName];
|
|
68
|
+
if (!providerConfig) throw new EngineError(`no provider config for ${engineName}`);
|
|
69
|
+
return providerConfig;
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
async function ensureBrowser() {
|
|
73
|
+
if (browserHandle) return browserHandle;
|
|
74
|
+
const [{ resolveCamoufoxLaunchOptions }, display, { firefox: launchFirefox }] = [
|
|
75
|
+
await lazy('./gen/lib/browser/camoufox.js'),
|
|
76
|
+
await lazy('./gen/lib/browser/display.js'),
|
|
77
|
+
{ firefox: await firefox() },
|
|
78
|
+
];
|
|
79
|
+
|
|
80
|
+
// Linux without a display gets the vendored self-bootstrapped Xvfb — the
|
|
81
|
+
// engines that block headless render headfully inside a virtual display.
|
|
82
|
+
if (process.platform === 'linux' && !display.detectDisplay()) {
|
|
83
|
+
displayHandle = await display.ensureDisplay({ allowExistingDisplay: false });
|
|
84
|
+
onEvent(`display: ${displayHandle.display}`);
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const proxy = parseProxyUrl(env.AGENTLINKOPS_PROXY_URL);
|
|
88
|
+
if (proxy) onEvent(`proxy: ${proxy.server}`);
|
|
89
|
+
|
|
90
|
+
const options = await resolveCamoufoxLaunchOptions({
|
|
91
|
+
provider: engineName,
|
|
92
|
+
proxy: proxy ?? undefined,
|
|
93
|
+
// "virtual" on Linux = headful Firefox inside Xvfb; true headless is the
|
|
94
|
+
// detection magnet the research warned about.
|
|
95
|
+
headlessMode: process.platform === 'linux' ? 'virtual' : 'headful',
|
|
96
|
+
});
|
|
97
|
+
const browser = await launchFirefox({ ...options, executablePath: options.executablePath, proxy });
|
|
98
|
+
|
|
99
|
+
browserHandle = {
|
|
100
|
+
browser,
|
|
101
|
+
cleanup: async () => {
|
|
102
|
+
await browser.close().catch(() => null);
|
|
103
|
+
await displayHandle?.cleanup?.().catch(() => null);
|
|
104
|
+
displayHandle = null;
|
|
105
|
+
},
|
|
106
|
+
};
|
|
107
|
+
return browserHandle;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
async function loadStorageState() {
|
|
111
|
+
try {
|
|
112
|
+
const { readFile } = await lazy('node:fs/promises');
|
|
113
|
+
return JSON.parse(await readFile(join(sessionDir, `${engineName}.json`), 'utf8'));
|
|
114
|
+
} catch {
|
|
115
|
+
return undefined; // anonymous session: several surfaces work logged-out
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
return {
|
|
120
|
+
identity,
|
|
121
|
+
estimateCostUsd: () => NOMINAL_COST_USD,
|
|
122
|
+
|
|
123
|
+
async run({ prompt }) {
|
|
124
|
+
const config = await ensureProviderConfig();
|
|
125
|
+
const { browser, cleanup } = await ensureBrowser();
|
|
126
|
+
const context = await browser.newContext({
|
|
127
|
+
storageState: await loadStorageState(),
|
|
128
|
+
locale: 'en-US',
|
|
129
|
+
timezoneId: 'America/New_York',
|
|
130
|
+
});
|
|
131
|
+
try {
|
|
132
|
+
const page = await context.newPage();
|
|
133
|
+
page.setDefaultTimeout(45_000);
|
|
134
|
+
|
|
135
|
+
if (config.navigateToPrompt) {
|
|
136
|
+
await config.navigateToPrompt(page, prompt);
|
|
137
|
+
} else {
|
|
138
|
+
await page.goto(config.url, { waitUntil: 'domcontentloaded', timeout: 60_000 });
|
|
139
|
+
await config.postNavigationHook?.(page);
|
|
140
|
+
await config.beforePromptHook?.(page);
|
|
141
|
+
|
|
142
|
+
const { findActiveEditorCandidate } = await lazy('./gen/lib/input/editor/findEditor.js');
|
|
143
|
+
const { PROVIDER_EDITOR_SELECTORS } = await lazy('./shims/utils.js');
|
|
144
|
+
const editor = await findActiveEditorCandidate(page, PROVIDER_EDITOR_SELECTORS[engineName]);
|
|
145
|
+
if (!editor) throw new EngineError(`${engineName}: no prompt editor found (surface may be gated or changed)`);
|
|
146
|
+
|
|
147
|
+
const { insertPromptIntoEditor } = await lazy('./gen/lib/input/editor/promptInput.js');
|
|
148
|
+
await insertPromptIntoEditor(page, editor.locator, prompt, engineName);
|
|
149
|
+
await config.afterTypingHook?.(page);
|
|
150
|
+
await config.beforeSubmitHook?.(page);
|
|
151
|
+
|
|
152
|
+
const { findEnabledSendButton } = await lazy('./gen/lib/input/editor/findSendButton.js');
|
|
153
|
+
const { PROVIDER_SUBMIT_BTN_SELECTORS } = await lazy('./shims/utils.js');
|
|
154
|
+
const send = await findEnabledSendButton(page, PROVIDER_SUBMIT_BTN_SELECTORS[engineName]);
|
|
155
|
+
if (send) { await send.click(); }
|
|
156
|
+
else { await editor.locator.press('Enter'); }
|
|
157
|
+
}
|
|
158
|
+
await config.afterSubmitHook?.(page);
|
|
159
|
+
|
|
160
|
+
await config.waitForResponse(page);
|
|
161
|
+
const answer = await config.extractResponse(page);
|
|
162
|
+
if (!answer || answer.trim().length === 0) {
|
|
163
|
+
throw new EngineError(`${engineName}: answer extracted empty`, { retriable: true });
|
|
164
|
+
}
|
|
165
|
+
const sources = await config.extractSources(page);
|
|
166
|
+
const screenshotPng = await page.screenshot({ fullPage: false }).catch(() => null);
|
|
167
|
+
|
|
168
|
+
return {
|
|
169
|
+
engine: engineName,
|
|
170
|
+
provider: 'web-own-browser',
|
|
171
|
+
model: engineName,
|
|
172
|
+
providerModelVersion: `${engineName}-web`,
|
|
173
|
+
answer,
|
|
174
|
+
citations: (sources ?? []).map((s) => ({ url: s.url, title: s.title ?? s.cited_text ?? null })).filter((c) => typeof c.url === 'string' && c.url.length > 0),
|
|
175
|
+
fanOut: [],
|
|
176
|
+
usage: { input_tokens: 0, output_tokens: 0 },
|
|
177
|
+
costEstimateUsd: NOMINAL_COST_USD,
|
|
178
|
+
screenshotPng,
|
|
179
|
+
};
|
|
180
|
+
} catch (cause) {
|
|
181
|
+
if (cause instanceof EngineError) throw cause;
|
|
182
|
+
const msg = String(cause?.message ?? cause);
|
|
183
|
+
const retriable = /timeout|timed out|ERR_|net::|closed|Target closed/i.test(msg);
|
|
184
|
+
throw new EngineError(`${engineName}: browser run failed: ${msg}`, { retriable, cause });
|
|
185
|
+
} finally {
|
|
186
|
+
await context.close().catch(() => null);
|
|
187
|
+
}
|
|
188
|
+
},
|
|
189
|
+
|
|
190
|
+
// Called by the runner at epoch end.
|
|
191
|
+
async close() {
|
|
192
|
+
await browserHandle?.cleanup?.();
|
|
193
|
+
browserHandle = null;
|
|
194
|
+
},
|
|
195
|
+
};
|
|
196
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { AI_OVERVIEW_RAW_SOURCES_DOM_EXTRACTOR } from "../ai-overview/lib/extractSources.js";
|
|
2
|
+
import { CHATGPT_RAW_SOURCES_DOM_EXTRACTOR } from "../chatgpt/lib/extractSources.js";
|
|
3
|
+
import { CLAUDE_RAW_SOURCES_DOM_EXTRACTOR } from "../claude/lib/extractSources.js";
|
|
4
|
+
import { GEMINI_RAW_SOURCES_DOM_EXTRACTOR } from "../gemini/lib/extractSources.js";
|
|
5
|
+
import { PERPLEXITY_RAW_SOURCES_DOM_EXTRACTOR } from "../perplexity/lib/extractSources.js";
|
|
6
|
+
const PROVIDER_RAW_SOURCES_DOM_EXTRACTORS = {
|
|
7
|
+
chatgpt: CHATGPT_RAW_SOURCES_DOM_EXTRACTOR,
|
|
8
|
+
perplexity: PERPLEXITY_RAW_SOURCES_DOM_EXTRACTOR,
|
|
9
|
+
gemini: GEMINI_RAW_SOURCES_DOM_EXTRACTOR,
|
|
10
|
+
claude: CLAUDE_RAW_SOURCES_DOM_EXTRACTOR,
|
|
11
|
+
"ai-overview": AI_OVERVIEW_RAW_SOURCES_DOM_EXTRACTOR
|
|
12
|
+
};
|
|
13
|
+
export {
|
|
14
|
+
PROVIDER_RAW_SOURCES_DOM_EXTRACTORS
|
|
15
|
+
};
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import { logger } from "../../../../shims/utils.js";
|
|
2
|
+
import { preInteractionIdle, randomBetween } from "../../../lib/browser/humanBehavior.js";
|
|
3
|
+
import { navigateWithRetry } from "../../../lib/browser/navigate.js";
|
|
4
|
+
import { detectBotPage } from "../../../lib/input/response/detectBotPage.js";
|
|
5
|
+
async function resetProviderPage(page, provider, url, options = {}) {
|
|
6
|
+
logger.log(`[${provider}] resetting page for next prompt`);
|
|
7
|
+
await preInteractionIdle(page).catch(() => {
|
|
8
|
+
});
|
|
9
|
+
await page.waitForTimeout(randomBetween(600, 1400));
|
|
10
|
+
await navigateWithRetry(page, url, {
|
|
11
|
+
waitUntil: "domcontentloaded",
|
|
12
|
+
timeout: 3e4
|
|
13
|
+
});
|
|
14
|
+
logger.log(`[${provider}] redirected back to provider page: ${page.url()}`);
|
|
15
|
+
await detectBotPage(page, provider);
|
|
16
|
+
await options.postNavigationHook?.(page);
|
|
17
|
+
await page.waitForTimeout(randomBetween(1200, 2600));
|
|
18
|
+
logger.log(`[${provider}] page reset ready: ${page.url()}`);
|
|
19
|
+
}
|
|
20
|
+
export {
|
|
21
|
+
resetProviderPage
|
|
22
|
+
};
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { PROVIDER_EDITOR_SELECTORS } from "../../../../shims/utils.js";
|
|
2
|
+
import { logger } from "../../../../shims/utils.js";
|
|
3
|
+
import { navigateWithRetry } from "../../../lib/browser/navigate.js";
|
|
4
|
+
import {
|
|
5
|
+
findActiveEditorCandidateFromSelectors
|
|
6
|
+
} from "../../../lib/input/editor/findEditor.js";
|
|
7
|
+
import { extractAssistantMarkdown } from "../../../lib/input/markdown/toMarkdown.js";
|
|
8
|
+
import { insertPromptIntoEditor } from "../../../lib/input/editor/promptInput.js";
|
|
9
|
+
import { waitForAssistantToFinish } from "../../../lib/input/response/waitForFinish.js";
|
|
10
|
+
import { extractAIOverviewSources } from "./lib/extractSources.js";
|
|
11
|
+
import { expandAIOverviewSources } from "./lib/expand.js";
|
|
12
|
+
import {
|
|
13
|
+
AI_OVERVIEW_URL,
|
|
14
|
+
resetAIOverviewPage
|
|
15
|
+
} from "./lib/pageLifecycle.js";
|
|
16
|
+
import {
|
|
17
|
+
assertAIOverviewPageNotBlocked,
|
|
18
|
+
dismissGoogleConsentDialog,
|
|
19
|
+
ensureAIOverviewGoogleSession,
|
|
20
|
+
waitForAIOverviewSearchResults
|
|
21
|
+
} from "./lib/session.js";
|
|
22
|
+
const aiOverviewConfig = {
|
|
23
|
+
url: AI_OVERVIEW_URL,
|
|
24
|
+
label: "AI Overview",
|
|
25
|
+
displayName: "AI Overview",
|
|
26
|
+
skipInitialNavigation: true,
|
|
27
|
+
navigateToPrompt: async (page, prompt) => {
|
|
28
|
+
await ensureAIOverviewGoogleSession(page);
|
|
29
|
+
await navigateWithRetry(page, AI_OVERVIEW_URL, {
|
|
30
|
+
waitUntil: "domcontentloaded",
|
|
31
|
+
timeout: 3e4
|
|
32
|
+
});
|
|
33
|
+
assertAIOverviewPageNotBlocked(page);
|
|
34
|
+
await dismissGoogleConsentDialog(page);
|
|
35
|
+
const searchInput = await findActiveEditorCandidateFromSelectors(page, [
|
|
36
|
+
...PROVIDER_EDITOR_SELECTORS["ai-overview"]
|
|
37
|
+
]);
|
|
38
|
+
logger.debug(`[ai-overview] using search selector: ${searchInput.selector}`);
|
|
39
|
+
logger.debug(`[ai-overview] pasting ${prompt.length} chars\u2026`);
|
|
40
|
+
await insertPromptIntoEditor(
|
|
41
|
+
page,
|
|
42
|
+
searchInput.locator,
|
|
43
|
+
prompt,
|
|
44
|
+
"ai-overview"
|
|
45
|
+
);
|
|
46
|
+
logger.debug(`[ai-overview] pasting ${prompt.length} chars complete`);
|
|
47
|
+
await page.waitForTimeout(400);
|
|
48
|
+
logger.debug("[ai-overview] attempting submission\u2026");
|
|
49
|
+
await page.keyboard.press("Enter");
|
|
50
|
+
await page.waitForLoadState("domcontentloaded", { timeout: 5e3 }).catch(
|
|
51
|
+
() => {
|
|
52
|
+
}
|
|
53
|
+
);
|
|
54
|
+
await waitForAIOverviewSearchResults(page);
|
|
55
|
+
await dismissGoogleConsentDialog(page);
|
|
56
|
+
assertAIOverviewPageNotBlocked(page);
|
|
57
|
+
logger.log(`[ai-overview] search ready: ${page.url()}`);
|
|
58
|
+
},
|
|
59
|
+
waitForResponse: (page) => waitForAssistantToFinish(page, "ai-overview"),
|
|
60
|
+
extractResponse: (page) => extractAssistantMarkdown(page, "ai-overview"),
|
|
61
|
+
beforeRetryHook: resetAIOverviewPage,
|
|
62
|
+
betweenPromptsHook: resetAIOverviewPage,
|
|
63
|
+
extractSources: async (page) => {
|
|
64
|
+
await expandAIOverviewSources(page);
|
|
65
|
+
return extractAIOverviewSources(page);
|
|
66
|
+
}
|
|
67
|
+
};
|
|
68
|
+
export {
|
|
69
|
+
aiOverviewConfig
|
|
70
|
+
};
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { logger } from "../../../../../shims/utils.js";
|
|
2
|
+
const AI_OVERVIEW_SOURCE_ROOT_SELECTOR = '[data-container-id="rhs-col"]';
|
|
3
|
+
const AI_OVERVIEW_BUTTON_SELECTOR = '[role="button"], button';
|
|
4
|
+
function shouldClickSourceExpansionButton(text, aria) {
|
|
5
|
+
const t = `${text} ${aria}`.toLowerCase();
|
|
6
|
+
if (t.includes("videos")) return false;
|
|
7
|
+
if (t.includes("more")) return true;
|
|
8
|
+
if (t.includes("all")) return true;
|
|
9
|
+
if (t.includes("expand")) return true;
|
|
10
|
+
return false;
|
|
11
|
+
}
|
|
12
|
+
async function expandAIOverviewSources(page) {
|
|
13
|
+
const matches = await page.evaluate(
|
|
14
|
+
({
|
|
15
|
+
rootSelector,
|
|
16
|
+
buttonSelector
|
|
17
|
+
}) => {
|
|
18
|
+
const root = document.querySelector(rootSelector);
|
|
19
|
+
if (!(root instanceof HTMLElement)) return [];
|
|
20
|
+
return Array.from(root.querySelectorAll(buttonSelector)).map((button, index) => {
|
|
21
|
+
if (!(button instanceof HTMLElement)) {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
const style = window.getComputedStyle(button);
|
|
25
|
+
const rect = button.getBoundingClientRect();
|
|
26
|
+
const visible = button.isConnected && style.display !== "none" && style.visibility !== "hidden" && style.opacity !== "0" && rect.width > 0 && rect.height > 0;
|
|
27
|
+
if (!visible) return null;
|
|
28
|
+
return {
|
|
29
|
+
index,
|
|
30
|
+
text: (button.textContent || "").trim(),
|
|
31
|
+
aria: button.getAttribute("aria-label")?.trim() || ""
|
|
32
|
+
};
|
|
33
|
+
}).filter(
|
|
34
|
+
(match) => match !== null
|
|
35
|
+
);
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
rootSelector: AI_OVERVIEW_SOURCE_ROOT_SELECTOR,
|
|
39
|
+
buttonSelector: AI_OVERVIEW_BUTTON_SELECTOR
|
|
40
|
+
}
|
|
41
|
+
);
|
|
42
|
+
for (const match of matches) {
|
|
43
|
+
if (!shouldClickSourceExpansionButton(match.text, match.aria)) {
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
const button = page.locator(`${AI_OVERVIEW_SOURCE_ROOT_SELECTOR} ${AI_OVERVIEW_BUTTON_SELECTOR}`).nth(match.index);
|
|
47
|
+
const visible = await button.isVisible().catch(() => false);
|
|
48
|
+
if (!visible) continue;
|
|
49
|
+
logger.debug(
|
|
50
|
+
`[ai-overview] expanding sources text="${match.text}" aria="${match.aria}"`
|
|
51
|
+
);
|
|
52
|
+
await button.click({ timeout: 5e3 }).catch(() => {
|
|
53
|
+
});
|
|
54
|
+
await page.waitForTimeout(700);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
export {
|
|
58
|
+
expandAIOverviewSources
|
|
59
|
+
};
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
import { toErrorMessage } from "../../../../../shims/errors.js";
|
|
2
|
+
import { logger } from "../../../../../shims/utils.js";
|
|
3
|
+
import { buildSources } from "../../_shared/sourceUtils.js";
|
|
4
|
+
const AI_OVERVIEW_RAW_SOURCES_DOM_EXTRACTOR = String.raw`(_helpers) => {
|
|
5
|
+
const results = [];
|
|
6
|
+
const rhsCol = document.querySelector('[data-container-id="rhs-col"]');
|
|
7
|
+
if (!rhsCol) {
|
|
8
|
+
return { rawSources: results, containerFound: false };
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
for (const card of Array.from(rhsCol.querySelectorAll("div[data-src-id]"))) {
|
|
12
|
+
if (!(card instanceof HTMLElement)) continue;
|
|
13
|
+
|
|
14
|
+
const link = card.querySelector('a[href^="http"]');
|
|
15
|
+
if (!(link instanceof HTMLAnchorElement)) continue;
|
|
16
|
+
|
|
17
|
+
const title =
|
|
18
|
+
link
|
|
19
|
+
.getAttribute("aria-label")
|
|
20
|
+
?.replace(/\.\s*Opens in new tab\.?$/i, "")
|
|
21
|
+
.trim() || link.href;
|
|
22
|
+
const citedText =
|
|
23
|
+
card.querySelector("[data-crb-snippet-text]")?.textContent?.trim() || "";
|
|
24
|
+
|
|
25
|
+
results.push({
|
|
26
|
+
rawHref: link.href,
|
|
27
|
+
title,
|
|
28
|
+
citedText,
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return { rawSources: results, containerFound: true };
|
|
33
|
+
}`;
|
|
34
|
+
function normalizeAIOverviewTitle(title) {
|
|
35
|
+
return title.replace(/\s*\.?\s*opens in new tab\.?\s*$/i, "").trim();
|
|
36
|
+
}
|
|
37
|
+
async function extractAIOverviewSources(page) {
|
|
38
|
+
try {
|
|
39
|
+
const { rawSources, containerFound } = await page.runDomOp("raw-sources", {
|
|
40
|
+
provider: "ai-overview"
|
|
41
|
+
});
|
|
42
|
+
if (!containerFound) {
|
|
43
|
+
logger.warn("AI Overview container not found \u2014 no sources extracted");
|
|
44
|
+
}
|
|
45
|
+
const normalizedSources = rawSources.map((source) => ({
|
|
46
|
+
...source,
|
|
47
|
+
title: normalizeAIOverviewTitle(source.title ?? "") || source.rawHref
|
|
48
|
+
}));
|
|
49
|
+
return buildSources(normalizedSources, { provider: "ai-overview" });
|
|
50
|
+
} catch (err) {
|
|
51
|
+
logger.error(
|
|
52
|
+
`Failed to extract AI Overview sources: ${toErrorMessage(err)}`
|
|
53
|
+
);
|
|
54
|
+
return [];
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
export {
|
|
58
|
+
AI_OVERVIEW_RAW_SOURCES_DOM_EXTRACTOR,
|
|
59
|
+
extractAIOverviewSources
|
|
60
|
+
};
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { resetProviderPage } from "../../_shared/resetProviderPage.js";
|
|
2
|
+
import {
|
|
3
|
+
assertAIOverviewPageNotBlocked,
|
|
4
|
+
dismissGoogleConsentDialog
|
|
5
|
+
} from "./session.js";
|
|
6
|
+
const AI_OVERVIEW_URL = "https://www.google.com/";
|
|
7
|
+
async function resetAIOverviewPage(page) {
|
|
8
|
+
await resetProviderPage(page, "ai-overview", AI_OVERVIEW_URL, {
|
|
9
|
+
postNavigationHook: async (currentPage) => {
|
|
10
|
+
assertAIOverviewPageNotBlocked(currentPage);
|
|
11
|
+
await dismissGoogleConsentDialog(currentPage);
|
|
12
|
+
}
|
|
13
|
+
});
|
|
14
|
+
}
|
|
15
|
+
export {
|
|
16
|
+
AI_OVERVIEW_URL,
|
|
17
|
+
resetAIOverviewPage
|
|
18
|
+
};
|