agent-vision-mcp 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +117 -0
  3. package/dist/browser/cdp/browser-cdp-discovery-service.d.ts +10 -0
  4. package/dist/browser/cdp/browser-cdp-discovery-service.js +28 -0
  5. package/dist/browser/cdp/browser-live-tab-service.d.ts +16 -0
  6. package/dist/browser/cdp/browser-live-tab-service.js +42 -0
  7. package/dist/browser/cdp/browser-see-service.d.ts +33 -0
  8. package/dist/browser/cdp/browser-see-service.js +76 -0
  9. package/dist/browser/cdp/browser-tab-context-service.d.ts +23 -0
  10. package/dist/browser/cdp/browser-tab-context-service.js +90 -0
  11. package/dist/browser/cdp/browser-tab-resolution-service.d.ts +9 -0
  12. package/dist/browser/cdp/browser-tab-resolution-service.js +65 -0
  13. package/dist/browser/cdp/browser-tab-screenshot-service.d.ts +20 -0
  14. package/dist/browser/cdp/browser-tab-screenshot-service.js +59 -0
  15. package/dist/browser/cdp/cdp-websocket-session.d.ts +9 -0
  16. package/dist/browser/cdp/cdp-websocket-session.js +99 -0
  17. package/dist/browser/cdp/chrome-cdp-client.d.ts +12 -0
  18. package/dist/browser/cdp/chrome-cdp-client.js +141 -0
  19. package/dist/browser/cdp/live-browser-tab-registry.d.ts +12 -0
  20. package/dist/browser/cdp/live-browser-tab-registry.js +96 -0
  21. package/dist/browser/cdp/png-metadata.d.ts +5 -0
  22. package/dist/browser/cdp/png-metadata.js +16 -0
  23. package/dist/browser/cdp/tab-model.d.ts +33 -0
  24. package/dist/browser/cdp/tab-model.js +15 -0
  25. package/dist/browser/cdp/tab-resolution.d.ts +27 -0
  26. package/dist/browser/cdp/tab-resolution.js +48 -0
  27. package/dist/browser/cdp/types.d.ts +71 -0
  28. package/dist/browser/cdp/types.js +1 -0
  29. package/dist/capture/capture-pipeline.d.ts +5 -0
  30. package/dist/capture/capture-pipeline.js +1 -0
  31. package/dist/capture/create-screen-capture-provider.d.ts +3 -0
  32. package/dist/capture/create-screen-capture-provider.js +8 -0
  33. package/dist/capture/in-memory-capture-pipeline.d.ts +13 -0
  34. package/dist/capture/in-memory-capture-pipeline.js +52 -0
  35. package/dist/capture/in-memory-image-compositor.d.ts +5 -0
  36. package/dist/capture/in-memory-image-compositor.js +34 -0
  37. package/dist/capture/linux-portal-screenshot-provider.d.ts +8 -0
  38. package/dist/capture/linux-portal-screenshot-provider.js +181 -0
  39. package/dist/capture/mock-screen-capture-provider.d.ts +5 -0
  40. package/dist/capture/mock-screen-capture-provider.js +22 -0
  41. package/dist/capture/png-metadata.d.ts +5 -0
  42. package/dist/capture/png-metadata.js +18 -0
  43. package/dist/capture/screen-capture-provider.d.ts +4 -0
  44. package/dist/capture/screen-capture-provider.js +1 -0
  45. package/dist/capture/types.d.ts +38 -0
  46. package/dist/capture/types.js +1 -0
  47. package/dist/cdp-demo.d.ts +1 -0
  48. package/dist/cdp-demo.js +41 -0
  49. package/dist/demo.d.ts +1 -0
  50. package/dist/demo.js +54 -0
  51. package/dist/desktop/capture-now.d.ts +1 -0
  52. package/dist/desktop/capture-now.js +48 -0
  53. package/dist/desktop/controller.d.ts +25 -0
  54. package/dist/desktop/controller.js +77 -0
  55. package/dist/desktop/main.d.ts +1 -0
  56. package/dist/desktop/main.js +80 -0
  57. package/dist/desktop/preload.d.ts +1 -0
  58. package/dist/desktop/preload.js +26 -0
  59. package/dist/desktop/types.d.ts +31 -0
  60. package/dist/desktop/types.js +1 -0
  61. package/dist/errors/app-error.d.ts +7 -0
  62. package/dist/errors/app-error.js +11 -0
  63. package/dist/flow/types.d.ts +48 -0
  64. package/dist/flow/types.js +1 -0
  65. package/dist/flow/visual-capture-flow.d.ts +13 -0
  66. package/dist/flow/visual-capture-flow.js +196 -0
  67. package/dist/index.d.ts +1 -0
  68. package/dist/index.js +3 -0
  69. package/dist/logging/logger.d.ts +15 -0
  70. package/dist/logging/logger.js +28 -0
  71. package/dist/mcp/stdio-server.d.ts +19 -0
  72. package/dist/mcp/stdio-server.js +272 -0
  73. package/dist/mcp/tool-registry.d.ts +21 -0
  74. package/dist/mcp/tool-registry.js +33 -0
  75. package/dist/mcp-stdio.d.ts +2 -0
  76. package/dist/mcp-stdio.js +8 -0
  77. package/dist/overlay/local-overlay-agent.d.ts +46 -0
  78. package/dist/overlay/local-overlay-agent.js +551 -0
  79. package/dist/overlay/overlay-bundle-factory.d.ts +4 -0
  80. package/dist/overlay/overlay-bundle-factory.js +24 -0
  81. package/dist/overlay/types.d.ts +83 -0
  82. package/dist/overlay/types.js +1 -0
  83. package/dist/server.d.ts +19 -0
  84. package/dist/server.js +158 -0
  85. package/dist/session/capture-session-service.d.ts +21 -0
  86. package/dist/session/capture-session-service.js +50 -0
  87. package/dist/session/session-manager.d.ts +29 -0
  88. package/dist/session/session-manager.js +217 -0
  89. package/dist/session/session-store.d.ts +8 -0
  90. package/dist/session/session-store.js +15 -0
  91. package/dist/session/session-waiter.d.ts +14 -0
  92. package/dist/session/session-waiter.js +102 -0
  93. package/dist/types/annotation.d.ts +32 -0
  94. package/dist/types/annotation.js +1 -0
  95. package/dist/types/capture.d.ts +33 -0
  96. package/dist/types/capture.js +1 -0
  97. package/dist/types/session.d.ts +36 -0
  98. package/dist/types/session.js +1 -0
  99. package/package.json +38 -0
@@ -0,0 +1,99 @@
1
+ import WebSocket from "ws";
2
+ const CDP_COMMAND_TIMEOUT_MS = 10_000;
3
+ const isCdpCommandFailure = (response) => "error" in response;
4
+ const waitForOpen = (socket) => new Promise((resolve, reject) => {
5
+ if (socket.readyState === WebSocket.OPEN) {
6
+ resolve();
7
+ return;
8
+ }
9
+ const onOpen = () => {
10
+ cleanup();
11
+ resolve();
12
+ };
13
+ const onError = (error) => {
14
+ cleanup();
15
+ reject(error);
16
+ };
17
+ const cleanup = () => {
18
+ socket.off("open", onOpen);
19
+ socket.off("error", onError);
20
+ };
21
+ socket.on("open", onOpen);
22
+ socket.on("error", onError);
23
+ });
24
+ const withTimeout = async (promise, timeoutMs, message) => {
25
+ let timeoutId;
26
+ try {
27
+ return await Promise.race([
28
+ promise,
29
+ new Promise((_, reject) => {
30
+ timeoutId = setTimeout(() => reject(new Error(message)), timeoutMs);
31
+ })
32
+ ]);
33
+ }
34
+ finally {
35
+ if (timeoutId) {
36
+ clearTimeout(timeoutId);
37
+ }
38
+ }
39
+ };
40
+ export class CdpWebSocketSession {
41
+ socket;
42
+ nextCommandId = 1;
43
+ constructor(socket) {
44
+ this.socket = socket;
45
+ }
46
+ async sendCommand(method, params, timeoutMs = CDP_COMMAND_TIMEOUT_MS) {
47
+ const id = this.nextCommandId++;
48
+ return await withTimeout(new Promise((resolve, reject) => {
49
+ const onMessage = (raw) => {
50
+ try {
51
+ const parsed = JSON.parse(raw.toString());
52
+ if (parsed.id !== id) {
53
+ return;
54
+ }
55
+ cleanup();
56
+ if (isCdpCommandFailure(parsed) && parsed.error) {
57
+ reject(new Error(parsed.error.message ?? `CDP command failed: ${method}`));
58
+ return;
59
+ }
60
+ resolve("result" in parsed ? parsed.result ?? {} : {});
61
+ }
62
+ catch (error) {
63
+ cleanup();
64
+ reject(error);
65
+ }
66
+ };
67
+ const onError = (error) => {
68
+ cleanup();
69
+ reject(error);
70
+ };
71
+ const onClose = () => {
72
+ cleanup();
73
+ reject(new Error(`CDP socket closed before ${method} completed`));
74
+ };
75
+ const cleanup = () => {
76
+ this.socket.off("message", onMessage);
77
+ this.socket.off("error", onError);
78
+ this.socket.off("close", onClose);
79
+ };
80
+ this.socket.on("message", onMessage);
81
+ this.socket.on("error", onError);
82
+ this.socket.on("close", onClose);
83
+ this.socket.send(JSON.stringify({ id, method, params }));
84
+ }), timeoutMs, `Timed out waiting for CDP response to ${method}`);
85
+ }
86
+ }
87
+ export const withCdpTabSession = async (tab, run) => {
88
+ if (!tab.webSocketDebuggerUrl) {
89
+ throw new Error(`Live browser tab ${tab.targetId} is missing webSocketDebuggerUrl`);
90
+ }
91
+ const socket = new WebSocket(tab.webSocketDebuggerUrl);
92
+ try {
93
+ await withTimeout(waitForOpen(socket), CDP_COMMAND_TIMEOUT_MS, "Timed out opening CDP tab websocket");
94
+ return await run(new CdpWebSocketSession(socket));
95
+ }
96
+ finally {
97
+ socket.close();
98
+ }
99
+ };
@@ -0,0 +1,12 @@
1
+ import type { Logger } from "../../logging/logger.js";
2
+ import type { CdpConnectionStatus, CdpTabDiscoveryResult } from "./types.js";
3
+ export declare class ChromeCdpClient {
4
+ private readonly logger;
5
+ private readonly defaultEndpoint;
6
+ private readonly timeoutMs;
7
+ constructor(logger: Logger, defaultEndpoint?: string, timeoutMs?: number);
8
+ resolveEndpoint(override?: string): string;
9
+ getConnectionStatus(endpointOverride?: string): Promise<CdpConnectionStatus>;
10
+ discoverTabs(endpointOverride?: string): Promise<CdpTabDiscoveryResult>;
11
+ private fetchJson;
12
+ }
@@ -0,0 +1,141 @@
1
+ const DEFAULT_CDP_ENDPOINT = "http://127.0.0.1:9222";
2
+ const DEFAULT_TIMEOUT_MS = 5_000;
3
+ const nowIso = () => new Date().toISOString();
4
+ const trimTrailingSlash = (value) => value.replace(/\/+$/, "");
5
+ const normalizeEndpoint = (endpoint) => {
6
+ const candidate = (endpoint ?? process.env.CHROME_CDP_ENDPOINT ?? DEFAULT_CDP_ENDPOINT).trim();
7
+ if (candidate === "") {
8
+ return DEFAULT_CDP_ENDPOINT;
9
+ }
10
+ return trimTrailingSlash(candidate);
11
+ };
12
+ const readBrowserName = (browser) => {
13
+ if (!browser) {
14
+ return undefined;
15
+ }
16
+ const [name] = browser.split("/");
17
+ return name || browser;
18
+ };
19
+ const toErrorMessage = (error, endpoint) => {
20
+ const message = error instanceof Error ? error.message : String(error);
21
+ const lowered = message.toLowerCase();
22
+ if (lowered.includes("timed out")) {
23
+ return {
24
+ errorMessage: message,
25
+ errorHint: `Chrome did not respond in time. Check that a debug-enabled browser is running at ${endpoint}.`
26
+ };
27
+ }
28
+ if (lowered.includes("econnrefused") ||
29
+ lowered.includes("fetch failed") ||
30
+ lowered.includes("failed to fetch") ||
31
+ lowered.includes("networkerror")) {
32
+ return {
33
+ errorMessage: message,
34
+ errorHint: `Could not reach the CDP endpoint at ${endpoint}. Start Chrome with --remote-debugging-port=9222 or set CHROME_CDP_ENDPOINT.`
35
+ };
36
+ }
37
+ if (lowered.includes("404") || lowered.includes("status 404")) {
38
+ return {
39
+ errorMessage: message,
40
+ errorHint: `The endpoint ${endpoint} responded, but it does not look like a Chrome DevTools Protocol server.`
41
+ };
42
+ }
43
+ return { errorMessage: message };
44
+ };
45
+ export class ChromeCdpClient {
46
+ logger;
47
+ defaultEndpoint;
48
+ timeoutMs;
49
+ constructor(logger, defaultEndpoint = normalizeEndpoint(), timeoutMs = DEFAULT_TIMEOUT_MS) {
50
+ this.logger = logger;
51
+ this.defaultEndpoint = defaultEndpoint;
52
+ this.timeoutMs = timeoutMs;
53
+ }
54
+ resolveEndpoint(override) {
55
+ return normalizeEndpoint(override ?? this.defaultEndpoint);
56
+ }
57
+ async getConnectionStatus(endpointOverride) {
58
+ const endpoint = this.resolveEndpoint(endpointOverride);
59
+ const checkedAt = nowIso();
60
+ try {
61
+ const version = await this.fetchJson(`${endpoint}/json/version`);
62
+ return {
63
+ endpoint,
64
+ connected: true,
65
+ checkedAt,
66
+ browser: version.Browser,
67
+ browserName: readBrowserName(version.Browser),
68
+ protocolVersion: version.ProtocolVersion,
69
+ userAgent: version.UserAgent,
70
+ webSocketDebuggerUrl: version.webSocketDebuggerUrl
71
+ };
72
+ }
73
+ catch (error) {
74
+ const diagnostic = toErrorMessage(error, endpoint);
75
+ this.logger.error("Failed CDP connection check", {
76
+ endpoint,
77
+ errorMessage: diagnostic.errorMessage,
78
+ errorHint: diagnostic.errorHint
79
+ });
80
+ return {
81
+ endpoint,
82
+ connected: false,
83
+ checkedAt,
84
+ errorMessage: diagnostic.errorMessage,
85
+ errorHint: diagnostic.errorHint
86
+ };
87
+ }
88
+ }
89
+ async discoverTabs(endpointOverride) {
90
+ const endpoint = this.resolveEndpoint(endpointOverride);
91
+ const discoveredAt = nowIso();
92
+ const version = await this.fetchJson(`${endpoint}/json/version`);
93
+ const rawTargets = await this.fetchJson(`${endpoint}/json/list`);
94
+ const browserName = readBrowserName(version.Browser);
95
+ const tabs = rawTargets
96
+ .filter((target) => target.type === "page" || target.type === "tab")
97
+ .map((target) => ({
98
+ id: target.id ?? "unknown-target",
99
+ type: target.type ?? "page",
100
+ title: target.title ?? target.url ?? "Untitled tab",
101
+ url: target.url,
102
+ attached: target.attached ?? false,
103
+ browserName,
104
+ discoveredAt,
105
+ webSocketDebuggerUrl: target.webSocketDebuggerUrl,
106
+ devtoolsFrontendUrl: target.devtoolsFrontendUrl
107
+ }));
108
+ this.logger.debug("Discovered browser tabs over CDP", {
109
+ endpoint,
110
+ tabCount: tabs.length,
111
+ browser: version.Browser
112
+ });
113
+ return {
114
+ endpoint,
115
+ discoveredAt,
116
+ browser: version.Browser,
117
+ browserName,
118
+ protocolVersion: version.ProtocolVersion,
119
+ tabs
120
+ };
121
+ }
122
+ async fetchJson(url) {
123
+ const controller = new AbortController();
124
+ const timeout = setTimeout(() => controller.abort(new Error(`Timed out fetching CDP endpoint ${url}`)), this.timeoutMs);
125
+ try {
126
+ const response = await fetch(url, {
127
+ headers: {
128
+ accept: "application/json"
129
+ },
130
+ signal: controller.signal
131
+ });
132
+ if (!response.ok) {
133
+ throw new Error(`CDP request failed for ${url} with status ${response.status}`);
134
+ }
135
+ return (await response.json());
136
+ }
137
+ finally {
138
+ clearTimeout(timeout);
139
+ }
140
+ }
141
+ }
@@ -0,0 +1,12 @@
1
+ import type { Logger } from "../../logging/logger.js";
2
+ import { type LiveBrowserTab, type UpsertLiveBrowserTabsInput } from "./tab-model.js";
3
+ export declare class LiveBrowserTabRegistry {
4
+ private readonly logger;
5
+ private readonly tabs;
6
+ constructor(logger: Logger);
7
+ upsertDiscovery(input: UpsertLiveBrowserTabsInput): LiveBrowserTab[];
8
+ pruneStale(maxAgeMs?: number, now?: Date): LiveBrowserTab[];
9
+ list(): LiveBrowserTab[];
10
+ get(targetId: string): LiveBrowserTab | undefined;
11
+ private toLiveTab;
12
+ }
@@ -0,0 +1,96 @@
1
+ import { toActiveHeuristic, toOrderHint } from "./tab-model.js";
2
+ const DEFAULT_STALE_TAB_MAX_AGE_MS = 5 * 60 * 1000;
3
+ const toMs = (value) => Date.parse(value);
4
+ export class LiveBrowserTabRegistry {
5
+ logger;
6
+ tabs = new Map();
7
+ constructor(logger) {
8
+ this.logger = logger;
9
+ }
10
+ upsertDiscovery(input) {
11
+ const seenIds = new Set();
12
+ for (const [index, target] of input.tabs.entries()) {
13
+ seenIds.add(target.id);
14
+ this.tabs.set(target.id, this.toLiveTab(target, index, input));
15
+ }
16
+ const removedTargetIds = [];
17
+ for (const existing of this.tabs.values()) {
18
+ if (existing.lastDiscoveryEndpoint !== input.endpoint) {
19
+ continue;
20
+ }
21
+ if (seenIds.has(existing.targetId)) {
22
+ continue;
23
+ }
24
+ this.tabs.delete(existing.targetId);
25
+ removedTargetIds.push(existing.targetId);
26
+ }
27
+ const ordered = this.list().map((tab, index) => ({
28
+ ...tab,
29
+ recencyRank: index + 1
30
+ }));
31
+ for (const tab of ordered) {
32
+ this.tabs.set(tab.targetId, tab);
33
+ }
34
+ this.logger.debug("Updated live browser tab registry", {
35
+ endpoint: input.endpoint,
36
+ discoveredAt: input.discoveredAt,
37
+ tabCount: ordered.length,
38
+ seenIds: [...seenIds],
39
+ removedTargetIds
40
+ });
41
+ return ordered;
42
+ }
43
+ pruneStale(maxAgeMs = DEFAULT_STALE_TAB_MAX_AGE_MS, now = new Date()) {
44
+ const cutoffMs = now.getTime() - maxAgeMs;
45
+ const removed = [];
46
+ for (const tab of this.tabs.values()) {
47
+ const lastSeenMs = toMs(tab.lastSeenAt);
48
+ if (!Number.isFinite(lastSeenMs) || lastSeenMs >= cutoffMs) {
49
+ continue;
50
+ }
51
+ this.tabs.delete(tab.targetId);
52
+ removed.push(tab);
53
+ }
54
+ if (removed.length > 0) {
55
+ this.logger.info("Pruned stale live browser tabs", {
56
+ maxAgeMs,
57
+ removedTargetIds: removed.map((tab) => tab.targetId),
58
+ removedCount: removed.length
59
+ });
60
+ }
61
+ return removed;
62
+ }
63
+ list() {
64
+ return [...this.tabs.values()].sort((left, right) => {
65
+ if (left.activeHeuristic !== right.activeHeuristic) {
66
+ return left.activeHeuristic ? -1 : 1;
67
+ }
68
+ if (left.orderHint !== right.orderHint) {
69
+ return left.orderHint - right.orderHint;
70
+ }
71
+ return right.lastSeenAt.localeCompare(left.lastSeenAt);
72
+ });
73
+ }
74
+ get(targetId) {
75
+ return this.tabs.get(targetId);
76
+ }
77
+ toLiveTab(target, index, input) {
78
+ const existing = this.tabs.get(target.id);
79
+ return {
80
+ targetId: target.id,
81
+ type: target.type,
82
+ title: target.title,
83
+ url: target.url,
84
+ browserName: target.browserName,
85
+ attached: target.attached,
86
+ webSocketDebuggerUrl: target.webSocketDebuggerUrl,
87
+ devtoolsFrontendUrl: target.devtoolsFrontendUrl,
88
+ firstSeenAt: existing?.firstSeenAt ?? input.discoveredAt,
89
+ lastSeenAt: input.discoveredAt,
90
+ lastDiscoveryEndpoint: input.endpoint,
91
+ activeHeuristic: toActiveHeuristic(target, index),
92
+ recencyRank: existing?.recencyRank ?? index + 1,
93
+ orderHint: toOrderHint(target, index)
94
+ };
95
+ }
96
+ }
@@ -0,0 +1,5 @@
1
+ export type PngDimensions = {
2
+ width: number;
3
+ height: number;
4
+ };
5
+ export declare const readPngDimensions: (buffer: Buffer) => PngDimensions;
@@ -0,0 +1,16 @@
1
+ const PNG_SIGNATURE = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a]);
2
+ export const readPngDimensions = (buffer) => {
3
+ if (buffer.byteLength < 24) {
4
+ throw new Error("PNG buffer is too small to contain dimensions");
5
+ }
6
+ if (!buffer.subarray(0, PNG_SIGNATURE.byteLength).equals(PNG_SIGNATURE)) {
7
+ throw new Error("Expected PNG signature in screenshot payload");
8
+ }
9
+ if (buffer.subarray(12, 16).toString("ascii") !== "IHDR") {
10
+ throw new Error("Expected IHDR chunk in screenshot payload");
11
+ }
12
+ return {
13
+ width: buffer.readUInt32BE(16),
14
+ height: buffer.readUInt32BE(20)
15
+ };
16
+ };
@@ -0,0 +1,33 @@
1
+ import type { CdpDiscoveredTarget, CdpTabDiscoveryResult } from "./types.js";
2
+ export type LiveBrowserTab = {
3
+ targetId: string;
4
+ type: string;
5
+ title: string;
6
+ url?: string;
7
+ browserName?: string;
8
+ attached: boolean;
9
+ webSocketDebuggerUrl?: string;
10
+ devtoolsFrontendUrl?: string;
11
+ firstSeenAt: string;
12
+ lastSeenAt: string;
13
+ lastDiscoveryEndpoint: string;
14
+ activeHeuristic: boolean;
15
+ recencyRank: number;
16
+ orderHint: number;
17
+ };
18
+ export type LiveBrowserTabDiscoveryResult = {
19
+ endpoint: string;
20
+ discoveredAt: string;
21
+ browser?: string;
22
+ browserName?: string;
23
+ protocolVersion?: string;
24
+ tabs: LiveBrowserTab[];
25
+ };
26
+ export type UpsertLiveBrowserTabsInput = {
27
+ endpoint: string;
28
+ discoveredAt: string;
29
+ tabs: CdpDiscoveredTarget[];
30
+ };
31
+ export declare const toActiveHeuristic: (target: CdpDiscoveredTarget, index: number) => boolean;
32
+ export declare const toOrderHint: (target: CdpDiscoveredTarget, index: number) => number;
33
+ export declare const toDiscoveryResult: (source: CdpTabDiscoveryResult, tabs: LiveBrowserTab[]) => LiveBrowserTabDiscoveryResult;
@@ -0,0 +1,15 @@
1
+ export const toActiveHeuristic = (target, index) => target.attached || index === 0;
2
+ export const toOrderHint = (target, index) => {
3
+ if (target.attached) {
4
+ return index;
5
+ }
6
+ return index + 100;
7
+ };
8
+ export const toDiscoveryResult = (source, tabs) => ({
9
+ endpoint: source.endpoint,
10
+ discoveredAt: source.discoveredAt,
11
+ browser: source.browser,
12
+ browserName: source.browserName,
13
+ protocolVersion: source.protocolVersion,
14
+ tabs
15
+ });
@@ -0,0 +1,27 @@
1
+ import type { LiveBrowserTab } from "./tab-model.js";
2
+ export type LiveBrowserTabCandidate = {
3
+ targetId: string;
4
+ title: string;
5
+ url?: string;
6
+ browserName?: string;
7
+ attached: boolean;
8
+ activeHeuristic: boolean;
9
+ recencyRank: number;
10
+ matchScore: number;
11
+ matchReason: string;
12
+ };
13
+ export type LiveBrowserTabResolution = {
14
+ status: "resolved";
15
+ query?: string;
16
+ tab: LiveBrowserTab;
17
+ matchedCandidate: LiveBrowserTabCandidate;
18
+ candidateCount: number;
19
+ } | {
20
+ status: "ambiguous" | "not_found";
21
+ query?: string;
22
+ candidates: LiveBrowserTabCandidate[];
23
+ message: string;
24
+ };
25
+ export declare const toCandidate: (tab: LiveBrowserTab, matchScore: number, matchReason: string) => LiveBrowserTabCandidate;
26
+ export declare const scoreLiveTab: (tab: LiveBrowserTab, query?: string) => LiveBrowserTabCandidate | undefined;
27
+ export declare const sortCandidates: (left: LiveBrowserTabCandidate, right: LiveBrowserTabCandidate) => number;
@@ -0,0 +1,48 @@
1
+ const normalize = (value) => (value ?? "").trim().toLowerCase();
2
+ export const toCandidate = (tab, matchScore, matchReason) => ({
3
+ targetId: tab.targetId,
4
+ title: tab.title,
5
+ url: tab.url,
6
+ browserName: tab.browserName,
7
+ attached: tab.attached,
8
+ activeHeuristic: tab.activeHeuristic,
9
+ recencyRank: tab.recencyRank,
10
+ matchScore,
11
+ matchReason
12
+ });
13
+ export const scoreLiveTab = (tab, query) => {
14
+ const normalizedQuery = normalize(query);
15
+ if (normalizedQuery === "") {
16
+ if (tab.activeHeuristic) {
17
+ return toCandidate(tab, tab.attached ? 130 : 120, tab.attached ? "attached-tab" : "active-tab");
18
+ }
19
+ return toCandidate(tab, Math.max(1, 20 - tab.recencyRank), "recent-tab");
20
+ }
21
+ const title = normalize(tab.title);
22
+ const url = normalize(tab.url);
23
+ if (title === normalizedQuery) {
24
+ return toCandidate(tab, tab.activeHeuristic ? 140 : 130, "exact-title");
25
+ }
26
+ if (url !== "" && url === normalizedQuery) {
27
+ return toCandidate(tab, tab.activeHeuristic ? 138 : 128, "exact-url");
28
+ }
29
+ if (title.startsWith(normalizedQuery)) {
30
+ return toCandidate(tab, tab.activeHeuristic ? 115 : 105, "title-prefix");
31
+ }
32
+ if (title.includes(normalizedQuery)) {
33
+ return toCandidate(tab, tab.activeHeuristic ? 95 : 85, "title-substring");
34
+ }
35
+ if (url !== "" && url.includes(normalizedQuery)) {
36
+ return toCandidate(tab, tab.activeHeuristic ? 90 : 80, "url-substring");
37
+ }
38
+ return undefined;
39
+ };
40
+ export const sortCandidates = (left, right) => {
41
+ if (left.matchScore !== right.matchScore) {
42
+ return right.matchScore - left.matchScore;
43
+ }
44
+ if (left.activeHeuristic !== right.activeHeuristic) {
45
+ return left.activeHeuristic ? -1 : 1;
46
+ }
47
+ return left.recencyRank - right.recencyRank;
48
+ };
@@ -0,0 +1,71 @@
1
+ export type CdpBrowserVersion = {
2
+ Browser?: string;
3
+ ProtocolVersion?: string;
4
+ UserAgent?: string;
5
+ V8Version?: string;
6
+ WebKitVersion?: string;
7
+ webSocketDebuggerUrl?: string;
8
+ };
9
+ export type CdpConnectionStatus = {
10
+ endpoint: string;
11
+ connected: boolean;
12
+ checkedAt: string;
13
+ browser?: string;
14
+ browserName?: string;
15
+ protocolVersion?: string;
16
+ userAgent?: string;
17
+ webSocketDebuggerUrl?: string;
18
+ errorMessage?: string;
19
+ errorHint?: string;
20
+ };
21
+ export type CdpDiscoveredTarget = {
22
+ id: string;
23
+ type: string;
24
+ title: string;
25
+ url?: string;
26
+ attached: boolean;
27
+ browserName?: string;
28
+ discoveredAt: string;
29
+ webSocketDebuggerUrl?: string;
30
+ devtoolsFrontendUrl?: string;
31
+ };
32
+ export type CdpTabDiscoveryResult = {
33
+ endpoint: string;
34
+ discoveredAt: string;
35
+ browser?: string;
36
+ browserName?: string;
37
+ protocolVersion?: string;
38
+ tabs: CdpDiscoveredTarget[];
39
+ };
40
+ export type BrowserTabScreenshot = {
41
+ targetId: string;
42
+ title: string;
43
+ url?: string;
44
+ browserName?: string;
45
+ mimeType: "image/png";
46
+ bytesBase64: string;
47
+ width: number;
48
+ height: number;
49
+ byteLength: number;
50
+ capturedAt: string;
51
+ backend: "cdp-page-capture";
52
+ };
53
+ export type BrowserTabStructuredContext = {
54
+ targetId: string;
55
+ title: string;
56
+ url?: string;
57
+ browserName?: string;
58
+ pageTitle?: string;
59
+ pageUrl?: string;
60
+ documentLanguage?: string;
61
+ contentType?: string;
62
+ visibleText: string;
63
+ visibleTextLength: number;
64
+ viewport?: {
65
+ width?: number;
66
+ height?: number;
67
+ devicePixelRatio?: number;
68
+ };
69
+ collectedAt: string;
70
+ backend: "cdp-runtime-evaluate";
71
+ };
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,5 @@
1
+ import type { CaptureBundle } from "../types/capture.js";
2
+ import type { CapturePipelineInput } from "./types.js";
3
+ export interface CapturePipeline {
4
+ createBundle(input: CapturePipelineInput): Promise<CaptureBundle>;
5
+ }
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,3 @@
1
+ import type { Logger } from "../logging/logger.js";
2
+ import type { ScreenCaptureProvider } from "./screen-capture-provider.js";
3
+ export declare const createScreenCaptureProvider: (logger: Logger) => ScreenCaptureProvider;
@@ -0,0 +1,8 @@
1
+ import { LinuxPortalScreenshotProvider } from "./linux-portal-screenshot-provider.js";
2
+ import { MockScreenCaptureProvider } from "./mock-screen-capture-provider.js";
3
+ export const createScreenCaptureProvider = (logger) => {
4
+ if (process.platform === "linux") {
5
+ return new LinuxPortalScreenshotProvider(logger);
6
+ }
7
+ return new MockScreenCaptureProvider();
8
+ };
@@ -0,0 +1,13 @@
1
+ import type { Logger } from "../logging/logger.js";
2
+ import type { CaptureBundle } from "../types/capture.js";
3
+ import type { CapturePipeline } from "./capture-pipeline.js";
4
+ import { InMemoryImageCompositor } from "./in-memory-image-compositor.js";
5
+ import type { ScreenCaptureProvider } from "./screen-capture-provider.js";
6
+ import type { CapturePipelineInput } from "./types.js";
7
+ export declare class InMemoryCapturePipeline implements CapturePipeline {
8
+ private readonly provider;
9
+ private readonly compositor;
10
+ private readonly logger;
11
+ constructor(provider: ScreenCaptureProvider, compositor: InMemoryImageCompositor, logger: Logger);
12
+ createBundle(input: CapturePipelineInput): Promise<CaptureBundle>;
13
+ }
@@ -0,0 +1,52 @@
1
+ export class InMemoryCapturePipeline {
2
+ provider;
3
+ compositor;
4
+ logger;
5
+ constructor(provider, compositor, logger) {
6
+ this.provider = provider;
7
+ this.compositor = compositor;
8
+ this.logger = logger;
9
+ }
10
+ async createBundle(input) {
11
+ const source = await this.provider.capture({
12
+ sessionId: input.sessionId,
13
+ command: input.command,
14
+ displayId: input.context?.displayId,
15
+ activeAppName: input.context?.activeAppName,
16
+ activeWindowTitle: input.context?.activeWindowTitle
17
+ });
18
+ const manifest = {
19
+ sessionId: input.sessionId,
20
+ command: input.command,
21
+ sourceImage: {
22
+ width: source.width,
23
+ height: source.height,
24
+ displayId: source.displayId,
25
+ backend: source.backend
26
+ },
27
+ crop: input.selection,
28
+ annotations: input.annotations,
29
+ context: input.context
30
+ };
31
+ const image = this.compositor.compose(source, manifest);
32
+ this.logger.debug("Created in-memory capture bundle", {
33
+ sessionId: input.sessionId,
34
+ backend: image.backend,
35
+ width: image.width,
36
+ height: image.height,
37
+ annotationCount: input.annotations.length
38
+ });
39
+ return {
40
+ sessionId: input.sessionId,
41
+ command: input.command,
42
+ image,
43
+ selection: input.selection,
44
+ annotations: input.annotations,
45
+ context: {
46
+ ...input.context,
47
+ capturedAt: source.capturedAt,
48
+ displayId: input.context?.displayId ?? source.displayId
49
+ }
50
+ };
51
+ }
52
+ }