gologin-web-access 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +19 -0
- package/LICENSE +21 -0
- package/README.md +344 -0
- package/dist/cli.js +173 -0
- package/dist/commands/back.js +13 -0
- package/dist/commands/batch.js +81 -0
- package/dist/commands/batchChangeTrack.js +99 -0
- package/dist/commands/batchExtract.js +97 -0
- package/dist/commands/batchScrape.js +140 -0
- package/dist/commands/changeTrack.js +65 -0
- package/dist/commands/check.js +14 -0
- package/dist/commands/click.js +14 -0
- package/dist/commands/close.js +19 -0
- package/dist/commands/configInit.js +77 -0
- package/dist/commands/configShow.js +23 -0
- package/dist/commands/cookies.js +22 -0
- package/dist/commands/cookiesClear.js +13 -0
- package/dist/commands/cookiesImport.js +14 -0
- package/dist/commands/crawl.js +71 -0
- package/dist/commands/crawlErrors.js +20 -0
- package/dist/commands/crawlResult.js +27 -0
- package/dist/commands/crawlStart.js +56 -0
- package/dist/commands/crawlStatus.js +25 -0
- package/dist/commands/current.js +14 -0
- package/dist/commands/dblclick.js +14 -0
- package/dist/commands/eval.js +20 -0
- package/dist/commands/extract.js +44 -0
- package/dist/commands/fill.js +15 -0
- package/dist/commands/find.js +16 -0
- package/dist/commands/focus.js +14 -0
- package/dist/commands/forward.js +13 -0
- package/dist/commands/get.js +15 -0
- package/dist/commands/hover.js +14 -0
- package/dist/commands/jobs.js +47 -0
- package/dist/commands/map.js +61 -0
- package/dist/commands/open.js +22 -0
- package/dist/commands/parseDocument.js +34 -0
- package/dist/commands/pdf.js +14 -0
- package/dist/commands/press.js +15 -0
- package/dist/commands/read.js +51 -0
- package/dist/commands/reload.js +13 -0
- package/dist/commands/run.js +76 -0
- package/dist/commands/scrape.js +19 -0
- package/dist/commands/scrapeJson.js +24 -0
- package/dist/commands/scrapeMarkdown.js +37 -0
- package/dist/commands/scrapeScreenshot.js +65 -0
- package/dist/commands/scrapeText.js +37 -0
- package/dist/commands/screenshot.js +23 -0
- package/dist/commands/scroll.js +23 -0
- package/dist/commands/scrollIntoView.js +14 -0
- package/dist/commands/search.js +39 -0
- package/dist/commands/searchBrowser.js +28 -0
- package/dist/commands/select.js +15 -0
- package/dist/commands/sessions.js +14 -0
- package/dist/commands/shared.js +102 -0
- package/dist/commands/snapshot.js +18 -0
- package/dist/commands/storageClear.js +18 -0
- package/dist/commands/storageExport.js +26 -0
- package/dist/commands/storageImport.js +23 -0
- package/dist/commands/tabClose.js +18 -0
- package/dist/commands/tabFocus.js +15 -0
- package/dist/commands/tabOpen.js +19 -0
- package/dist/commands/tabs.js +13 -0
- package/dist/commands/type.js +15 -0
- package/dist/commands/uncheck.js +14 -0
- package/dist/commands/upload.js +15 -0
- package/dist/commands/wait.js +27 -0
- package/dist/config.js +260 -0
- package/dist/doctor.js +86 -0
- package/dist/internal-agent/cli.js +336 -0
- package/dist/internal-agent/commands/back.js +12 -0
- package/dist/internal-agent/commands/check.js +17 -0
- package/dist/internal-agent/commands/click.js +17 -0
- package/dist/internal-agent/commands/close.js +12 -0
- package/dist/internal-agent/commands/cookies.js +23 -0
- package/dist/internal-agent/commands/cookiesClear.js +12 -0
- package/dist/internal-agent/commands/cookiesImport.js +18 -0
- package/dist/internal-agent/commands/current.js +9 -0
- package/dist/internal-agent/commands/dblclick.js +17 -0
- package/dist/internal-agent/commands/doctor.js +53 -0
- package/dist/internal-agent/commands/eval.js +30 -0
- package/dist/internal-agent/commands/fill.js +18 -0
- package/dist/internal-agent/commands/find.js +86 -0
- package/dist/internal-agent/commands/focus.js +17 -0
- package/dist/internal-agent/commands/forward.js +12 -0
- package/dist/internal-agent/commands/get.js +19 -0
- package/dist/internal-agent/commands/hover.js +17 -0
- package/dist/internal-agent/commands/open.js +67 -0
- package/dist/internal-agent/commands/pdf.js +18 -0
- package/dist/internal-agent/commands/press.js +19 -0
- package/dist/internal-agent/commands/reload.js +12 -0
- package/dist/internal-agent/commands/screenshot.js +22 -0
- package/dist/internal-agent/commands/scroll.js +25 -0
- package/dist/internal-agent/commands/scrollIntoView.js +17 -0
- package/dist/internal-agent/commands/select.js +18 -0
- package/dist/internal-agent/commands/sessions.js +15 -0
- package/dist/internal-agent/commands/shared.js +51 -0
- package/dist/internal-agent/commands/snapshot.js +16 -0
- package/dist/internal-agent/commands/storageClear.js +13 -0
- package/dist/internal-agent/commands/storageExport.js +24 -0
- package/dist/internal-agent/commands/storageImport.js +20 -0
- package/dist/internal-agent/commands/tabClose.js +21 -0
- package/dist/internal-agent/commands/tabFocus.js +21 -0
- package/dist/internal-agent/commands/tabOpen.js +13 -0
- package/dist/internal-agent/commands/tabs.js +17 -0
- package/dist/internal-agent/commands/type.js +18 -0
- package/dist/internal-agent/commands/uncheck.js +17 -0
- package/dist/internal-agent/commands/upload.js +18 -0
- package/dist/internal-agent/commands/wait.js +41 -0
- package/dist/internal-agent/daemon/browser.js +818 -0
- package/dist/internal-agent/daemon/refStore.js +26 -0
- package/dist/internal-agent/daemon/server.js +330 -0
- package/dist/internal-agent/daemon/sessionManager.js +684 -0
- package/dist/internal-agent/daemon/snapshot.js +285 -0
- package/dist/internal-agent/lib/config.js +59 -0
- package/dist/internal-agent/lib/daemon.js +300 -0
- package/dist/internal-agent/lib/errors.js +63 -0
- package/dist/internal-agent/lib/types.js +2 -0
- package/dist/internal-agent/lib/utils.js +165 -0
- package/dist/jobRunner.js +56 -0
- package/dist/lib/agentCli.js +158 -0
- package/dist/lib/browserRead.js +125 -0
- package/dist/lib/browserStructured.js +77 -0
- package/dist/lib/changeTracking.js +117 -0
- package/dist/lib/cloudApi.js +41 -0
- package/dist/lib/concurrency.js +15 -0
- package/dist/lib/crawl.js +313 -0
- package/dist/lib/document.js +170 -0
- package/dist/lib/errors.js +55 -0
- package/dist/lib/extract.js +65 -0
- package/dist/lib/extractRunner.js +22 -0
- package/dist/lib/jobRegistry.js +164 -0
- package/dist/lib/output.js +122 -0
- package/dist/lib/readSource.js +297 -0
- package/dist/lib/runbooks.js +193 -0
- package/dist/lib/search.js +727 -0
- package/dist/lib/selfCli.js +136 -0
- package/dist/lib/structuredScrape.js +83 -0
- package/dist/lib/types.js +2 -0
- package/dist/lib/unlocker.js +383 -0
- package/package.json +67 -0
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildReloadCommand = buildReloadCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildReloadCommand() {
|
|
7
|
+
const command = new commander_1.Command("reload")
|
|
8
|
+
.description("Reload the active Cloud Browser tab.")
|
|
9
|
+
.action(async (options) => {
|
|
10
|
+
await (0, shared_1.runBrowserCommand)(["reload"], { session: options.session });
|
|
11
|
+
});
|
|
12
|
+
return (0, shared_1.addSessionOption)(command);
|
|
13
|
+
}
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.buildRunCommand = buildRunCommand;
|
|
7
|
+
const path_1 = __importDefault(require("path"));
|
|
8
|
+
const commander_1 = require("commander");
|
|
9
|
+
const config_1 = require("../config");
|
|
10
|
+
const jobRegistry_1 = require("../lib/jobRegistry");
|
|
11
|
+
const runbooks_1 = require("../lib/runbooks");
|
|
12
|
+
const output_1 = require("../lib/output");
|
|
13
|
+
function buildRunCommand() {
|
|
14
|
+
return new commander_1.Command("run")
|
|
15
|
+
.description("Execute a JSON runbook through gologin-web-access commands.")
|
|
16
|
+
.argument("<runbookPath>", "Path to the runbook JSON file")
|
|
17
|
+
.option("--session <id>", "Default session id for browser commands")
|
|
18
|
+
.option("--profile <id>", "Default profile id for browser commands that open sessions")
|
|
19
|
+
.option("--vars <path>", "Path to a JSON variables file")
|
|
20
|
+
.option("--name <name>", "Override the stored job name")
|
|
21
|
+
.option("--continue-on-error", "Continue after failed steps")
|
|
22
|
+
.option("--json", "Print the final job record as JSON")
|
|
23
|
+
.action(async (runbookPath, options) => {
|
|
24
|
+
const config = await (0, config_1.loadConfig)();
|
|
25
|
+
const runbook = (0, runbooks_1.loadRunbookDefinition)(process.cwd(), runbookPath);
|
|
26
|
+
const variables = options.vars ? (0, runbooks_1.loadVariablesFile)(process.cwd(), options.vars) : undefined;
|
|
27
|
+
const absoluteRunbookPath = path_1.default.resolve(runbookPath);
|
|
28
|
+
const job = await (0, jobRegistry_1.createJob)(config, {
|
|
29
|
+
kind: "run",
|
|
30
|
+
name: options.name ?? path_1.default.basename(absoluteRunbookPath, path_1.default.extname(absoluteRunbookPath)),
|
|
31
|
+
cwd: process.cwd(),
|
|
32
|
+
args: process.argv.slice(2),
|
|
33
|
+
metadata: {
|
|
34
|
+
runbookPath: absoluteRunbookPath,
|
|
35
|
+
sessionId: options.session,
|
|
36
|
+
profileId: options.profile
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
await (0, jobRegistry_1.markJobRunning)(config, job.jobId);
|
|
40
|
+
try {
|
|
41
|
+
const execution = await (0, runbooks_1.executeRunbook)(runbook, {
|
|
42
|
+
cwd: process.cwd(),
|
|
43
|
+
sessionId: options.session,
|
|
44
|
+
profileId: options.profile,
|
|
45
|
+
variables,
|
|
46
|
+
continueOnError: options.continueOnError === true
|
|
47
|
+
});
|
|
48
|
+
const failed = execution.steps.filter((step) => step.status === "failed").length;
|
|
49
|
+
const output = execution.steps
|
|
50
|
+
.map((step) => [`step=${step.command} status=${step.status} durationMs=${step.durationMs}`, step.stdout.trim(), step.stderr.trim()].filter(Boolean).join("\n"))
|
|
51
|
+
.join("\n\n");
|
|
52
|
+
const record = await (0, jobRegistry_1.finalizeJob)(config, job.jobId, {
|
|
53
|
+
status: failed > 0 ? "partial" : "ok",
|
|
54
|
+
output,
|
|
55
|
+
result: execution
|
|
56
|
+
});
|
|
57
|
+
if (options.json) {
|
|
58
|
+
(0, output_1.printJson)(record);
|
|
59
|
+
return;
|
|
60
|
+
}
|
|
61
|
+
(0, output_1.printText)(output);
|
|
62
|
+
}
|
|
63
|
+
catch (error) {
|
|
64
|
+
const record = await (0, jobRegistry_1.finalizeJob)(config, job.jobId, {
|
|
65
|
+
status: "failed",
|
|
66
|
+
error: error instanceof Error ? error.message : String(error)
|
|
67
|
+
});
|
|
68
|
+
if (options.json) {
|
|
69
|
+
(0, output_1.printJson)(record);
|
|
70
|
+
process.exitCode = 1;
|
|
71
|
+
return;
|
|
72
|
+
}
|
|
73
|
+
throw error;
|
|
74
|
+
}
|
|
75
|
+
});
|
|
76
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildScrapeCommand = buildScrapeCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const shared_1 = require("./shared");
|
|
7
|
+
const output_1 = require("../lib/output");
|
|
8
|
+
const unlocker_1 = require("../lib/unlocker");
|
|
9
|
+
function buildScrapeCommand() {
|
|
10
|
+
return (0, shared_1.addUnlockerRequestOptions)(new commander_1.Command("scrape")
|
|
11
|
+
.description("Fetch rendered HTML through Gologin Web Unlocker.")
|
|
12
|
+
.argument("<url>", "URL to scrape")
|
|
13
|
+
.action(async (url, options) => {
|
|
14
|
+
const config = await (0, config_1.loadConfig)();
|
|
15
|
+
const apiKey = (0, config_1.requireWebUnlockerKey)(config);
|
|
16
|
+
const result = await (0, unlocker_1.scrapeRenderedHtml)(url, apiKey, (0, shared_1.normalizeUnlockerRequestOptions)(options));
|
|
17
|
+
(0, output_1.printText)(result.content);
|
|
18
|
+
}));
|
|
19
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildScrapeJsonCommand = buildScrapeJsonCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const output_1 = require("../lib/output");
|
|
7
|
+
const structuredScrape_1 = require("../lib/structuredScrape");
|
|
8
|
+
const shared_1 = require("./shared");
|
|
9
|
+
function buildScrapeJsonCommand() {
|
|
10
|
+
return (0, shared_1.addProfileOption)((0, shared_1.addUnlockerRequestOptions)(new commander_1.Command("scrape-json")
|
|
11
|
+
.description("Fetch a page through Web Unlocker and print a structured JSON envelope.")
|
|
12
|
+
.argument("<url>", "URL to scrape")
|
|
13
|
+
.option("--fallback <mode>", "none or browser structured fallback for JS-heavy pages", "none")
|
|
14
|
+
.action(async (url, options) => {
|
|
15
|
+
const config = await (0, config_1.loadConfig)();
|
|
16
|
+
const apiKey = (0, config_1.requireWebUnlockerKey)(config);
|
|
17
|
+
const envelope = await (0, structuredScrape_1.scrapeStructuredJson)(url, config, apiKey, {
|
|
18
|
+
fallback: (0, structuredScrape_1.normalizeStructuredFallbackMode)(options.fallback),
|
|
19
|
+
profile: options.profile,
|
|
20
|
+
request: (0, shared_1.normalizeUnlockerRequestOptions)(options),
|
|
21
|
+
});
|
|
22
|
+
(0, output_1.printJson)(envelope);
|
|
23
|
+
})));
|
|
24
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildScrapeMarkdownCommand = buildScrapeMarkdownCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const readSource_1 = require("../lib/readSource");
|
|
7
|
+
const shared_1 = require("./shared");
|
|
8
|
+
const output_1 = require("../lib/output");
|
|
9
|
+
function buildScrapeMarkdownCommand() {
|
|
10
|
+
return (0, shared_1.addUnlockerRequestOptions)(new commander_1.Command("scrape-markdown")
|
|
11
|
+
.description("Fetch a page through Web Unlocker and print Markdown.")
|
|
12
|
+
.argument("<url>", "URL to scrape")
|
|
13
|
+
.option("--source <source>", "Read source: auto, unlocker, or browser", "auto")
|
|
14
|
+
.action(async (url, options) => {
|
|
15
|
+
const config = await (0, config_1.loadConfig)();
|
|
16
|
+
const source = (0, readSource_1.normalizeReadSourceMode)(options.source, "auto");
|
|
17
|
+
const apiKey = source === "browser" ? "" : (0, config_1.requireWebUnlockerKey)(config);
|
|
18
|
+
const result = await (0, readSource_1.readMarkdownContent)(url, config, apiKey, {
|
|
19
|
+
source,
|
|
20
|
+
request: (0, shared_1.normalizeUnlockerRequestOptions)(options),
|
|
21
|
+
});
|
|
22
|
+
emitReadNotice(result.fallbackAttempted, result.fallbackUsed, result.fallbackReason);
|
|
23
|
+
(0, output_1.printText)(result.content);
|
|
24
|
+
}));
|
|
25
|
+
}
|
|
26
|
+
function emitReadNotice(fallbackAttempted, fallbackUsed, fallbackReason) {
|
|
27
|
+
if (!fallbackAttempted) {
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (fallbackUsed) {
|
|
31
|
+
process.stderr.write(`JS-rendered page detected, retrying with browser. ${fallbackReason ?? ""}\n`);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (fallbackReason) {
|
|
35
|
+
process.stderr.write(`${fallbackReason}\n`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.buildScrapeScreenshotCommand = buildScrapeScreenshotCommand;
|
|
7
|
+
const path_1 = __importDefault(require("path"));
|
|
8
|
+
const crypto_1 = require("crypto");
|
|
9
|
+
const commander_1 = require("commander");
|
|
10
|
+
const config_1 = require("../config");
|
|
11
|
+
const agentCli_1 = require("../lib/agentCli");
|
|
12
|
+
function buildScrapeScreenshotCommand() {
|
|
13
|
+
return new commander_1.Command("scrape-screenshot")
|
|
14
|
+
.description("Capture a full-page screenshot of a URL through Gologin Cloud Browser in one command.")
|
|
15
|
+
.argument("<url>", "URL to capture")
|
|
16
|
+
.argument("[outputPath]", "Destination PNG path")
|
|
17
|
+
.option("--profile <id>", "Gologin profile ID to use")
|
|
18
|
+
.option("--annotate", "Annotate the screenshot with snapshot refs")
|
|
19
|
+
.option("--press-escape", "Press Escape before capturing")
|
|
20
|
+
.option("--keep-open", "Leave the browser session open instead of auto-closing it")
|
|
21
|
+
.action(async (url, outputPath, options) => {
|
|
22
|
+
const config = await (0, config_1.loadConfig)();
|
|
23
|
+
(0, config_1.requireCloudToken)(config);
|
|
24
|
+
const sessionId = `shot-${(0, crypto_1.randomUUID)().slice(0, 8)}`;
|
|
25
|
+
const resolvedProfile = (0, config_1.resolveProfileId)(config, options.profile);
|
|
26
|
+
const resolvedOutputPath = path_1.default.resolve(outputPath ?? defaultScreenshotPath(url));
|
|
27
|
+
const openArgs = ["open", url, "--session", sessionId];
|
|
28
|
+
if (resolvedProfile) {
|
|
29
|
+
openArgs.push("--profile", resolvedProfile);
|
|
30
|
+
}
|
|
31
|
+
const screenshotArgs = ["screenshot", resolvedOutputPath, "--session", sessionId];
|
|
32
|
+
if (options.annotate) {
|
|
33
|
+
screenshotArgs.push("--annotate");
|
|
34
|
+
}
|
|
35
|
+
if (options.pressEscape) {
|
|
36
|
+
screenshotArgs.push("--press-escape");
|
|
37
|
+
}
|
|
38
|
+
const openResult = await (0, agentCli_1.runAgentCommandCapture)(openArgs, config);
|
|
39
|
+
if (openResult.exitCode !== 0) {
|
|
40
|
+
throw new Error(openResult.stderr || openResult.stdout || "Failed to open Cloud Browser session");
|
|
41
|
+
}
|
|
42
|
+
try {
|
|
43
|
+
const shotResult = await (0, agentCli_1.runAgentCommandCapture)(screenshotArgs, config);
|
|
44
|
+
if (shotResult.exitCode !== 0) {
|
|
45
|
+
throw new Error(shotResult.stderr || shotResult.stdout || "Failed to capture screenshot");
|
|
46
|
+
}
|
|
47
|
+
process.stdout.write(`${resolvedOutputPath}\n`);
|
|
48
|
+
}
|
|
49
|
+
finally {
|
|
50
|
+
if (!options.keepOpen) {
|
|
51
|
+
await (0, agentCli_1.runAgentCommandCapture)(["close", "--session", sessionId], config).catch(() => undefined);
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
function defaultScreenshotPath(url) {
|
|
57
|
+
try {
|
|
58
|
+
const parsed = new URL(url);
|
|
59
|
+
const hostname = parsed.hostname.replace(/[^a-z0-9.-]+/gi, "-");
|
|
60
|
+
return `${hostname || "page"}.png`;
|
|
61
|
+
}
|
|
62
|
+
catch {
|
|
63
|
+
return "page.png";
|
|
64
|
+
}
|
|
65
|
+
}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildScrapeTextCommand = buildScrapeTextCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const readSource_1 = require("../lib/readSource");
|
|
7
|
+
const shared_1 = require("./shared");
|
|
8
|
+
const output_1 = require("../lib/output");
|
|
9
|
+
function buildScrapeTextCommand() {
|
|
10
|
+
return (0, shared_1.addUnlockerRequestOptions)(new commander_1.Command("scrape-text")
|
|
11
|
+
.description("Fetch a page through Web Unlocker and print plain text.")
|
|
12
|
+
.argument("<url>", "URL to scrape")
|
|
13
|
+
.option("--source <source>", "Read source: auto, unlocker, or browser", "auto")
|
|
14
|
+
.action(async (url, options) => {
|
|
15
|
+
const config = await (0, config_1.loadConfig)();
|
|
16
|
+
const source = (0, readSource_1.normalizeReadSourceMode)(options.source, "auto");
|
|
17
|
+
const apiKey = source === "browser" ? "" : (0, config_1.requireWebUnlockerKey)(config);
|
|
18
|
+
const result = await (0, readSource_1.readTextContent)(url, config, apiKey, {
|
|
19
|
+
source,
|
|
20
|
+
request: (0, shared_1.normalizeUnlockerRequestOptions)(options),
|
|
21
|
+
});
|
|
22
|
+
emitReadNotice(result.fallbackAttempted, result.fallbackUsed, result.fallbackReason);
|
|
23
|
+
(0, output_1.printText)(result.content);
|
|
24
|
+
}));
|
|
25
|
+
}
|
|
26
|
+
function emitReadNotice(fallbackAttempted, fallbackUsed, fallbackReason) {
|
|
27
|
+
if (!fallbackAttempted) {
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
if (fallbackUsed) {
|
|
31
|
+
process.stderr.write(`JS-rendered page detected, retrying with browser. ${fallbackReason ?? ""}\n`);
|
|
32
|
+
return;
|
|
33
|
+
}
|
|
34
|
+
if (fallbackReason) {
|
|
35
|
+
process.stderr.write(`${fallbackReason}\n`);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildScreenshotCommand = buildScreenshotCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildScreenshotCommand() {
|
|
7
|
+
const command = new commander_1.Command("screenshot")
|
|
8
|
+
.description("Save a full-page screenshot for the current session.")
|
|
9
|
+
.argument("<targetPath>", "Where to write the screenshot file")
|
|
10
|
+
.option("--annotate", "Annotate the screenshot with refs")
|
|
11
|
+
.option("--press-escape", "Press escape before taking the screenshot")
|
|
12
|
+
.action(async (targetPath, options) => {
|
|
13
|
+
const args = ["screenshot", (0, shared_1.resolveOutputPath)(targetPath)];
|
|
14
|
+
if (options.annotate) {
|
|
15
|
+
args.push("--annotate");
|
|
16
|
+
}
|
|
17
|
+
if (options.pressEscape) {
|
|
18
|
+
args.push("--press-escape");
|
|
19
|
+
}
|
|
20
|
+
await (0, shared_1.runBrowserCommand)(args, { session: options.session });
|
|
21
|
+
});
|
|
22
|
+
return (0, shared_1.addSessionOption)(command);
|
|
23
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildScrollCommand = buildScrollCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildScrollCommand() {
|
|
7
|
+
const command = new commander_1.Command("scroll")
|
|
8
|
+
.description("Scroll the page or a target element.")
|
|
9
|
+
.argument("<direction>", "up, down, left, or right")
|
|
10
|
+
.argument("[pixels]", "Optional number of pixels")
|
|
11
|
+
.option("--target <ref>", "Optional snapshot ref target")
|
|
12
|
+
.action(async (direction, pixels, options) => {
|
|
13
|
+
const args = ["scroll", direction];
|
|
14
|
+
if (pixels) {
|
|
15
|
+
args.push(pixels);
|
|
16
|
+
}
|
|
17
|
+
if (options.target) {
|
|
18
|
+
args.push("--target", options.target);
|
|
19
|
+
}
|
|
20
|
+
await (0, shared_1.runBrowserCommand)(args, { session: options.session });
|
|
21
|
+
});
|
|
22
|
+
return (0, shared_1.addSessionOption)(command);
|
|
23
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildScrollIntoViewCommand = buildScrollIntoViewCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildScrollIntoViewCommand() {
|
|
7
|
+
const command = new commander_1.Command("scrollintoview")
|
|
8
|
+
.description("Scroll a target element into view.")
|
|
9
|
+
.argument("<ref>", "Snapshot ref, for example e2")
|
|
10
|
+
.action(async (ref, options) => {
|
|
11
|
+
await (0, shared_1.runBrowserCommand)(["scrollintoview", ref], { session: options.session });
|
|
12
|
+
});
|
|
13
|
+
return (0, shared_1.addSessionOption)(command);
|
|
14
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildSearchCommand = buildSearchCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const output_1 = require("../lib/output");
|
|
7
|
+
const search_1 = require("../lib/search");
|
|
8
|
+
function buildSearchCommand() {
|
|
9
|
+
return new commander_1.Command("search")
|
|
10
|
+
.description("Search the web through Gologin and return structured results with automatic fallback.")
|
|
11
|
+
.argument("<query>", "Search query")
|
|
12
|
+
.option("--limit <count>", "Maximum number of results", "10")
|
|
13
|
+
.option("--country <country>", "Country code for Google search", "us")
|
|
14
|
+
.option("--language <language>", "Language for Google search", "en")
|
|
15
|
+
.option("--source <mode>", "Search path: auto, unlocker, or browser", "auto")
|
|
16
|
+
.action(async (query, options) => {
|
|
17
|
+
const config = await (0, config_1.loadConfig)();
|
|
18
|
+
const result = await (0, search_1.searchWeb)(query, config, {
|
|
19
|
+
limit: normalizeLimit(options.limit),
|
|
20
|
+
country: options.country,
|
|
21
|
+
language: options.language,
|
|
22
|
+
source: normalizeSource(options.source),
|
|
23
|
+
});
|
|
24
|
+
(0, output_1.printJson)(result);
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
function normalizeLimit(value) {
|
|
28
|
+
const parsed = Number(value);
|
|
29
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
30
|
+
return 10;
|
|
31
|
+
}
|
|
32
|
+
return Math.min(Math.floor(parsed), 100);
|
|
33
|
+
}
|
|
34
|
+
function normalizeSource(value) {
|
|
35
|
+
if (value === "auto" || value === "unlocker" || value === "browser") {
|
|
36
|
+
return value;
|
|
37
|
+
}
|
|
38
|
+
throw new Error(`Unsupported search source: ${value}`);
|
|
39
|
+
}
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildSearchBrowserCommand = buildSearchBrowserCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildSearchBrowserCommand() {
|
|
7
|
+
const command = new commander_1.Command("search-browser")
|
|
8
|
+
.description("Open a Google search results page inside Gologin Cloud Browser.")
|
|
9
|
+
.argument("<query>", "Search query")
|
|
10
|
+
.option("--country <country>", "Country code for Google search", "us")
|
|
11
|
+
.option("--language <language>", "Language for Google search", "en")
|
|
12
|
+
.option("--idle-timeout-ms <ms>", "Idle timeout for the browser session")
|
|
13
|
+
.option("--proxy-country <country>", "Request a country-based proxy when supported")
|
|
14
|
+
.option("--proxy-mode <mode>", "Custom proxy mode: http, socks4, or socks5")
|
|
15
|
+
.option("--proxy-host <host>", "Custom proxy host")
|
|
16
|
+
.option("--proxy-port <port>", "Custom proxy port")
|
|
17
|
+
.option("--proxy-user <username>", "Custom proxy username")
|
|
18
|
+
.option("--proxy-pass <password>", "Custom proxy password")
|
|
19
|
+
.action(async (query, options) => {
|
|
20
|
+
const url = new URL("https://www.google.com/search");
|
|
21
|
+
url.searchParams.set("q", query);
|
|
22
|
+
url.searchParams.set("hl", options.language);
|
|
23
|
+
url.searchParams.set("gl", options.country.toLowerCase());
|
|
24
|
+
await (0, shared_1.runOpenLikeCommand)(url.toString(), options);
|
|
25
|
+
});
|
|
26
|
+
(0, shared_1.addProfileOption)(command);
|
|
27
|
+
return (0, shared_1.addSessionOption)(command);
|
|
28
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildSelectCommand = buildSelectCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildSelectCommand() {
|
|
7
|
+
const command = new commander_1.Command("select")
|
|
8
|
+
.description("Select a value in an element by snapshot ref.")
|
|
9
|
+
.argument("<ref>", "Snapshot ref, for example e2")
|
|
10
|
+
.argument("<value>", "Value to select")
|
|
11
|
+
.action(async (ref, value, options) => {
|
|
12
|
+
await (0, shared_1.runBrowserCommand)(["select", ref, value], { session: options.session });
|
|
13
|
+
});
|
|
14
|
+
return (0, shared_1.addSessionOption)(command);
|
|
15
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildSessionsCommand = buildSessionsCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const config_1 = require("../config");
|
|
6
|
+
const agentCli_1 = require("../lib/agentCli");
|
|
7
|
+
function buildSessionsCommand() {
|
|
8
|
+
return new commander_1.Command("sessions")
|
|
9
|
+
.description("List active daemon-backed browser sessions.")
|
|
10
|
+
.action(async () => {
|
|
11
|
+
const config = await (0, config_1.loadConfig)();
|
|
12
|
+
await (0, agentCli_1.runAgentCommand)(["sessions"], config);
|
|
13
|
+
});
|
|
14
|
+
}
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.addSessionOption = addSessionOption;
|
|
7
|
+
exports.addProfileOption = addProfileOption;
|
|
8
|
+
exports.runBrowserCommand = runBrowserCommand;
|
|
9
|
+
exports.runOpenLikeCommand = runOpenLikeCommand;
|
|
10
|
+
exports.resolveOutputPath = resolveOutputPath;
|
|
11
|
+
exports.addUnlockerRequestOptions = addUnlockerRequestOptions;
|
|
12
|
+
exports.normalizeUnlockerRequestOptions = normalizeUnlockerRequestOptions;
|
|
13
|
+
const path_1 = __importDefault(require("path"));
|
|
14
|
+
const config_1 = require("../config");
|
|
15
|
+
const agentCli_1 = require("../lib/agentCli");
|
|
16
|
+
function addSessionOption(command) {
|
|
17
|
+
return command.option("--session <id>", "Session ID. Defaults to the current session.");
|
|
18
|
+
}
|
|
19
|
+
function addProfileOption(command) {
|
|
20
|
+
return command.option("--profile <id>", "Gologin profile ID to use");
|
|
21
|
+
}
|
|
22
|
+
async function runBrowserCommand(commandArgs, options) {
|
|
23
|
+
const config = await (0, config_1.loadConfig)();
|
|
24
|
+
if (options?.requiresToken !== false) {
|
|
25
|
+
(0, config_1.requireCloudToken)(config);
|
|
26
|
+
}
|
|
27
|
+
const args = [...commandArgs];
|
|
28
|
+
if (options?.session) {
|
|
29
|
+
args.push("--session", options.session);
|
|
30
|
+
}
|
|
31
|
+
await (0, agentCli_1.runAgentCommand)(args, config);
|
|
32
|
+
}
|
|
33
|
+
async function runOpenLikeCommand(url, options) {
|
|
34
|
+
const config = await (0, config_1.loadConfig)();
|
|
35
|
+
(0, config_1.requireCloudToken)(config);
|
|
36
|
+
const args = ["open", url];
|
|
37
|
+
const profileId = (0, config_1.resolveProfileId)(config, options.profile);
|
|
38
|
+
if (profileId) {
|
|
39
|
+
args.push("--profile", profileId);
|
|
40
|
+
}
|
|
41
|
+
if (options.session) {
|
|
42
|
+
args.push("--session", options.session);
|
|
43
|
+
}
|
|
44
|
+
if (options.idleTimeoutMs) {
|
|
45
|
+
args.push("--idle-timeout-ms", options.idleTimeoutMs);
|
|
46
|
+
}
|
|
47
|
+
if (options.proxyCountry) {
|
|
48
|
+
args.push("--proxy-country", options.proxyCountry);
|
|
49
|
+
}
|
|
50
|
+
if (options.proxyMode) {
|
|
51
|
+
args.push("--proxy-mode", options.proxyMode);
|
|
52
|
+
}
|
|
53
|
+
if (options.proxyHost) {
|
|
54
|
+
args.push("--proxy-host", options.proxyHost);
|
|
55
|
+
}
|
|
56
|
+
if (options.proxyPort) {
|
|
57
|
+
args.push("--proxy-port", options.proxyPort);
|
|
58
|
+
}
|
|
59
|
+
if (options.proxyUser) {
|
|
60
|
+
args.push("--proxy-user", options.proxyUser);
|
|
61
|
+
}
|
|
62
|
+
if (options.proxyPass) {
|
|
63
|
+
args.push("--proxy-pass", options.proxyPass);
|
|
64
|
+
}
|
|
65
|
+
await (0, agentCli_1.runAgentCommand)(args, config);
|
|
66
|
+
}
|
|
67
|
+
function resolveOutputPath(targetPath) {
|
|
68
|
+
return path_1.default.resolve(targetPath);
|
|
69
|
+
}
|
|
70
|
+
function addUnlockerRequestOptions(command) {
|
|
71
|
+
return command
|
|
72
|
+
.option("--retry <count>", "Retry attempts for timeout, 429, and 5xx responses")
|
|
73
|
+
.option("--backoff-ms <ms>", "Base exponential backoff in milliseconds for retried requests")
|
|
74
|
+
.option("--timeout-ms <ms>", "Per-request timeout in milliseconds");
|
|
75
|
+
}
|
|
76
|
+
function normalizeUnlockerRequestOptions(options) {
|
|
77
|
+
return {
|
|
78
|
+
maxRetries: normalizeOptionalNonNegativeInt(options.retry),
|
|
79
|
+
backoffMs: normalizeOptionalNonNegativeInt(options.backoffMs),
|
|
80
|
+
timeoutMs: normalizeOptionalPositiveInt(options.timeoutMs),
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
function normalizeOptionalNonNegativeInt(value) {
|
|
84
|
+
if (!value) {
|
|
85
|
+
return undefined;
|
|
86
|
+
}
|
|
87
|
+
const parsed = Number(value);
|
|
88
|
+
if (!Number.isFinite(parsed) || parsed < 0) {
|
|
89
|
+
throw new Error(`Expected a non-negative integer, got: ${value}`);
|
|
90
|
+
}
|
|
91
|
+
return Math.floor(parsed);
|
|
92
|
+
}
|
|
93
|
+
function normalizeOptionalPositiveInt(value) {
|
|
94
|
+
if (!value) {
|
|
95
|
+
return undefined;
|
|
96
|
+
}
|
|
97
|
+
const parsed = Number(value);
|
|
98
|
+
if (!Number.isFinite(parsed) || parsed <= 0) {
|
|
99
|
+
throw new Error(`Expected a positive integer, got: ${value}`);
|
|
100
|
+
}
|
|
101
|
+
return Math.floor(parsed);
|
|
102
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildSnapshotCommand = buildSnapshotCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildSnapshotCommand() {
|
|
7
|
+
const command = new commander_1.Command("snapshot")
|
|
8
|
+
.description("Capture the current page state and assign clickable refs.")
|
|
9
|
+
.option("-i, --interactive", "Use the agent interactive snapshot format")
|
|
10
|
+
.action(async (options) => {
|
|
11
|
+
const args = ["snapshot"];
|
|
12
|
+
if (options.interactive) {
|
|
13
|
+
args.push("--interactive");
|
|
14
|
+
}
|
|
15
|
+
await (0, shared_1.runBrowserCommand)(args, { session: options.session });
|
|
16
|
+
});
|
|
17
|
+
return (0, shared_1.addSessionOption)(command);
|
|
18
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildStorageClearCommand = buildStorageClearCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildStorageClearCommand() {
|
|
7
|
+
const command = new commander_1.Command("storage-clear")
|
|
8
|
+
.description("Clear localStorage/sessionStorage in the active Cloud Browser tab.")
|
|
9
|
+
.option("--scope <scope>", "Storage scope: local, session, or both")
|
|
10
|
+
.action(async (options) => {
|
|
11
|
+
const args = ["storage-clear"];
|
|
12
|
+
if (options.scope) {
|
|
13
|
+
args.push("--scope", options.scope);
|
|
14
|
+
}
|
|
15
|
+
await (0, shared_1.runBrowserCommand)(args, { session: options.session });
|
|
16
|
+
});
|
|
17
|
+
return (0, shared_1.addSessionOption)(command);
|
|
18
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildStorageExportCommand = buildStorageExportCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildStorageExportCommand() {
|
|
7
|
+
const command = new commander_1.Command("storage-export")
|
|
8
|
+
.description("Export localStorage/sessionStorage from the active Cloud Browser tab.")
|
|
9
|
+
.argument("[outputPath]", "Optional path for the exported storage JSON")
|
|
10
|
+
.option("--scope <scope>", "Storage scope: local, session, or both")
|
|
11
|
+
.option("--json", "Print storage state as JSON")
|
|
12
|
+
.action(async (outputPath, options) => {
|
|
13
|
+
const args = ["storage-export"];
|
|
14
|
+
if (outputPath) {
|
|
15
|
+
args.push(outputPath);
|
|
16
|
+
}
|
|
17
|
+
if (options.scope) {
|
|
18
|
+
args.push("--scope", options.scope);
|
|
19
|
+
}
|
|
20
|
+
if (options.json) {
|
|
21
|
+
args.push("--json");
|
|
22
|
+
}
|
|
23
|
+
await (0, shared_1.runBrowserCommand)(args, { session: options.session });
|
|
24
|
+
});
|
|
25
|
+
return (0, shared_1.addSessionOption)(command);
|
|
26
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildStorageImportCommand = buildStorageImportCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildStorageImportCommand() {
|
|
7
|
+
const command = new commander_1.Command("storage-import")
|
|
8
|
+
.description("Import localStorage/sessionStorage into the active Cloud Browser tab.")
|
|
9
|
+
.argument("<storagePath>", "Path to the storage JSON file")
|
|
10
|
+
.option("--scope <scope>", "Storage scope: local, session, or both")
|
|
11
|
+
.option("--clear", "Clear existing storage before importing")
|
|
12
|
+
.action(async (storagePath, options) => {
|
|
13
|
+
const args = ["storage-import", storagePath];
|
|
14
|
+
if (options.scope) {
|
|
15
|
+
args.push("--scope", options.scope);
|
|
16
|
+
}
|
|
17
|
+
if (options.clear) {
|
|
18
|
+
args.push("--clear");
|
|
19
|
+
}
|
|
20
|
+
await (0, shared_1.runBrowserCommand)(args, { session: options.session });
|
|
21
|
+
});
|
|
22
|
+
return (0, shared_1.addSessionOption)(command);
|
|
23
|
+
}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildTabCloseCommand = buildTabCloseCommand;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const shared_1 = require("./shared");
|
|
6
|
+
function buildTabCloseCommand() {
|
|
7
|
+
const command = new commander_1.Command("tabclose")
|
|
8
|
+
.description("Close the current tab or a specific tab in the active Cloud Browser session.")
|
|
9
|
+
.argument("[index]", "Optional 1-based tab index")
|
|
10
|
+
.action(async (index, options) => {
|
|
11
|
+
const args = ["tabclose"];
|
|
12
|
+
if (index) {
|
|
13
|
+
args.push(index);
|
|
14
|
+
}
|
|
15
|
+
await (0, shared_1.runBrowserCommand)(args, { session: options.session });
|
|
16
|
+
});
|
|
17
|
+
return (0, shared_1.addSessionOption)(command);
|
|
18
|
+
}
|