@dyyz1993/agent-browser 0.9.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (187) hide show
  1. package/LICENSE +202 -0
  2. package/README.md +907 -0
  3. package/bin/agent-browser-darwin-arm64 +0 -0
  4. package/bin/agent-browser.js +120 -0
  5. package/dist/__tests__/e2e/utils/test-helpers.d.ts +5 -0
  6. package/dist/__tests__/e2e/utils/test-helpers.d.ts.map +1 -0
  7. package/dist/__tests__/e2e/utils/test-helpers.js +22 -0
  8. package/dist/__tests__/e2e/utils/test-helpers.js.map +1 -0
  9. package/dist/__tests__/test-iframe.d.ts +2 -0
  10. package/dist/__tests__/test-iframe.d.ts.map +1 -0
  11. package/dist/__tests__/test-iframe.js +52 -0
  12. package/dist/__tests__/test-iframe.js.map +1 -0
  13. package/dist/__tests__/utils/parseCli.d.ts +20 -0
  14. package/dist/__tests__/utils/parseCli.d.ts.map +1 -0
  15. package/dist/__tests__/utils/parseCli.js +1086 -0
  16. package/dist/__tests__/utils/parseCli.js.map +1 -0
  17. package/dist/actions.d.ts +50 -0
  18. package/dist/actions.d.ts.map +1 -0
  19. package/dist/actions.js +2164 -0
  20. package/dist/actions.js.map +1 -0
  21. package/dist/browser.d.ts +556 -0
  22. package/dist/browser.d.ts.map +1 -0
  23. package/dist/browser.js +2599 -0
  24. package/dist/browser.js.map +1 -0
  25. package/dist/cli/commands.d.ts +8 -0
  26. package/dist/cli/commands.d.ts.map +1 -0
  27. package/dist/cli/commands.js +1038 -0
  28. package/dist/cli/commands.js.map +1 -0
  29. package/dist/cli/connection.d.ts +50 -0
  30. package/dist/cli/connection.d.ts.map +1 -0
  31. package/dist/cli/connection.js +595 -0
  32. package/dist/cli/connection.js.map +1 -0
  33. package/dist/cli/flags.d.ts +36 -0
  34. package/dist/cli/flags.d.ts.map +1 -0
  35. package/dist/cli/flags.js +206 -0
  36. package/dist/cli/flags.js.map +1 -0
  37. package/dist/cli/help.d.ts +4 -0
  38. package/dist/cli/help.d.ts.map +1 -0
  39. package/dist/cli/help.js +1024 -0
  40. package/dist/cli/help.js.map +1 -0
  41. package/dist/cli/output.d.ts +14 -0
  42. package/dist/cli/output.d.ts.map +1 -0
  43. package/dist/cli/output.js +456 -0
  44. package/dist/cli/output.js.map +1 -0
  45. package/dist/cli-new.d.ts +3 -0
  46. package/dist/cli-new.d.ts.map +1 -0
  47. package/dist/cli-new.js +308 -0
  48. package/dist/cli-new.js.map +1 -0
  49. package/dist/cli-old.d.ts +3 -0
  50. package/dist/cli-old.d.ts.map +1 -0
  51. package/dist/cli-old.js +1101 -0
  52. package/dist/cli-old.js.map +1 -0
  53. package/dist/cli.d.ts +3 -0
  54. package/dist/cli.d.ts.map +1 -0
  55. package/dist/cli.js +403 -0
  56. package/dist/cli.js.map +1 -0
  57. package/dist/content-detection.d.ts +18 -0
  58. package/dist/content-detection.d.ts.map +1 -0
  59. package/dist/content-detection.js +68 -0
  60. package/dist/content-detection.js.map +1 -0
  61. package/dist/daemon.d.ts +55 -0
  62. package/dist/daemon.d.ts.map +1 -0
  63. package/dist/daemon.js +426 -0
  64. package/dist/daemon.js.map +1 -0
  65. package/dist/diff.d.ts +42 -0
  66. package/dist/diff.d.ts.map +1 -0
  67. package/dist/diff.js +166 -0
  68. package/dist/diff.js.map +1 -0
  69. package/dist/human-mouse.d.ts +31 -0
  70. package/dist/human-mouse.d.ts.map +1 -0
  71. package/dist/human-mouse.js +184 -0
  72. package/dist/human-mouse.js.map +1 -0
  73. package/dist/ios-actions.d.ts +11 -0
  74. package/dist/ios-actions.d.ts.map +1 -0
  75. package/dist/ios-actions.js +228 -0
  76. package/dist/ios-actions.js.map +1 -0
  77. package/dist/ios-manager.d.ts +266 -0
  78. package/dist/ios-manager.d.ts.map +1 -0
  79. package/dist/ios-manager.js +1076 -0
  80. package/dist/ios-manager.js.map +1 -0
  81. package/dist/message-bridge.d.ts +10 -0
  82. package/dist/message-bridge.d.ts.map +1 -0
  83. package/dist/message-bridge.js +60 -0
  84. package/dist/message-bridge.js.map +1 -0
  85. package/dist/protocol.d.ts +26 -0
  86. package/dist/protocol.d.ts.map +1 -0
  87. package/dist/protocol.js +912 -0
  88. package/dist/protocol.js.map +1 -0
  89. package/dist/recorder/binding.d.ts +24 -0
  90. package/dist/recorder/binding.d.ts.map +1 -0
  91. package/dist/recorder/binding.js +215 -0
  92. package/dist/recorder/binding.js.map +1 -0
  93. package/dist/recorder/index.d.ts +4 -0
  94. package/dist/recorder/index.d.ts.map +1 -0
  95. package/dist/recorder/index.js +4 -0
  96. package/dist/recorder/index.js.map +1 -0
  97. package/dist/recorder/inject.js +1913 -0
  98. package/dist/recorder/recorder.d.ts +19 -0
  99. package/dist/recorder/recorder.d.ts.map +1 -0
  100. package/dist/recorder/recorder.js +101 -0
  101. package/dist/recorder/recorder.js.map +1 -0
  102. package/dist/recorder/store.d.ts +22 -0
  103. package/dist/recorder/store.d.ts.map +1 -0
  104. package/dist/recorder/store.js +150 -0
  105. package/dist/recorder/store.js.map +1 -0
  106. package/dist/recorder/types.d.ts +73 -0
  107. package/dist/recorder/types.d.ts.map +1 -0
  108. package/dist/recorder/types.js +5 -0
  109. package/dist/recorder/types.js.map +1 -0
  110. package/dist/snapshot.d.ts +81 -0
  111. package/dist/snapshot.d.ts.map +1 -0
  112. package/dist/snapshot.js +1348 -0
  113. package/dist/snapshot.js.map +1 -0
  114. package/dist/stream-server-standalone.d.ts +38 -0
  115. package/dist/stream-server-standalone.d.ts.map +1 -0
  116. package/dist/stream-server-standalone.js +494 -0
  117. package/dist/stream-server-standalone.js.map +1 -0
  118. package/dist/stream-server.d.ts +214 -0
  119. package/dist/stream-server.d.ts.map +1 -0
  120. package/dist/stream-server.js +811 -0
  121. package/dist/stream-server.js.map +1 -0
  122. package/dist/types.d.ts +914 -0
  123. package/dist/types.d.ts.map +1 -0
  124. package/dist/types.js +4 -0
  125. package/dist/types.js.map +1 -0
  126. package/dist/viewer-html.d.ts +2 -0
  127. package/dist/viewer-html.d.ts.map +1 -0
  128. package/dist/viewer-html.js +185 -0
  129. package/dist/viewer-html.js.map +1 -0
  130. package/dist/viewer-script.d.ts +47 -0
  131. package/dist/viewer-script.d.ts.map +1 -0
  132. package/dist/viewer-script.js +586 -0
  133. package/dist/viewer-script.js.map +1 -0
  134. package/package.json +86 -0
  135. package/scripts/build-all-platforms.sh +68 -0
  136. package/scripts/check-version-sync.js +39 -0
  137. package/scripts/check_goods_container.js +35 -0
  138. package/scripts/check_page_content.js +36 -0
  139. package/scripts/click_applause_rate.js +30 -0
  140. package/scripts/copy-native.js +36 -0
  141. package/scripts/copy-recorder.js +21 -0
  142. package/scripts/e2e-test-recorder.ts +584 -0
  143. package/scripts/explore_jd_page.js +31 -0
  144. package/scripts/extract_all_jd_data.js +80 -0
  145. package/scripts/extract_jd_product_detail.js +62 -0
  146. package/scripts/extract_jd_products_correct_links.js +78 -0
  147. package/scripts/extract_jd_products_final.js +80 -0
  148. package/scripts/extract_jd_reviews.js +48 -0
  149. package/scripts/extract_jd_seafood_final.js +78 -0
  150. package/scripts/extract_multiple_products.js +77 -0
  151. package/scripts/extract_products_no_scroll.js +68 -0
  152. package/scripts/extract_products_simple.js +68 -0
  153. package/scripts/find_applause_rate.js +26 -0
  154. package/scripts/find_jd_links.js +28 -0
  155. package/scripts/find_main_content.js +20 -0
  156. package/scripts/find_product_cards.js +38 -0
  157. package/scripts/find_root_content.js +26 -0
  158. package/scripts/find_unique_products.js +55 -0
  159. package/scripts/get_jd_product_detail.js +16 -0
  160. package/scripts/get_jd_products.js +23 -0
  161. package/scripts/get_jd_seafood_products.js +44 -0
  162. package/scripts/get_product_details_from_images.js +54 -0
  163. package/scripts/postinstall.js +235 -0
  164. package/scripts/scroll_and_get_products.js +47 -0
  165. package/scripts/scroll_deep_and_find.js +45 -0
  166. package/scripts/sync-version.js +69 -0
  167. package/scripts/verify-baidu-enter.ts +116 -0
  168. package/skills/agent-browser/SKILL.md +310 -0
  169. package/skills/agent-browser/references/authentication.md +198 -0
  170. package/skills/agent-browser/references/commands.md +471 -0
  171. package/skills/agent-browser/references/data-extraction.md +377 -0
  172. package/skills/agent-browser/references/proxy-support.md +188 -0
  173. package/skills/agent-browser/references/session-management.md +197 -0
  174. package/skills/agent-browser/references/snapshot-refs.md +379 -0
  175. package/skills/agent-browser/references/video-recording.md +173 -0
  176. package/skills/agent-browser/templates/api-interception.sh +53 -0
  177. package/skills/agent-browser/templates/authenticated-session.sh +97 -0
  178. package/skills/agent-browser/templates/capture-workflow.sh +69 -0
  179. package/skills/agent-browser/templates/data-extraction.sh +210 -0
  180. package/skills/agent-browser/templates/form-automation.sh +62 -0
  181. package/skills/skill-creator/LICENSE.txt +202 -0
  182. package/skills/skill-creator/SKILL.md +356 -0
  183. package/skills/skill-creator/references/output-patterns.md +82 -0
  184. package/skills/skill-creator/references/workflows.md +28 -0
  185. package/skills/skill-creator/scripts/init_skill.py +303 -0
  186. package/skills/skill-creator/scripts/package_skill.py +113 -0
  187. package/skills/skill-creator/scripts/quick_validate.py +95 -0
@@ -0,0 +1,811 @@
1
+ import * as net from 'net';
2
+ import * as fs from 'fs';
3
+ import * as path from 'path';
4
+ import { WebSocketServer, WebSocket } from 'ws';
5
+ import sharp from 'sharp';
6
+ import { setScreencastFrameCallback, setEventCallbacks } from './actions.js';
7
+ import { executeCommand } from './actions.js';
8
+ import { errorResponse, serializeResponse } from './protocol.js';
9
+ import { getSocketDir, getSession, getInstanceId } from './daemon.js';
10
+ export const STATE_CONFIGS = {
11
+ user_interacting: { format: 'jpeg', quality: 80, maxFps: 60, scale: 0.4 },
12
+ screen_moving: { format: 'webp', quality: 50, maxFps: 1, scale: 0.6 },
13
+ static: { format: 'webp', quality: 80, maxFps: 0.5, scale: 1 },
14
+ };
15
+ export class StreamStateManager {
16
+ currentState = 'static';
17
+ isUserInteracting = false;
18
+ userInteractionTimer = null;
19
+ lastFrameTime = 0;
20
+ frameInterval = Infinity;
21
+ onStateChange = null;
22
+ staticTimer = null;
23
+ USER_INTERACTION_TIMEOUT_MS = 1000;
24
+ SCREEN_MOVING_THRESHOLD_MS = 1000;
25
+ STATIC_TIMEOUT_MS = 1500;
26
+ setStateChangeCallback(callback) {
27
+ this.onStateChange = callback;
28
+ }
29
+ setState(newState) {
30
+ if (newState !== this.currentState) {
31
+ const previousState = this.currentState;
32
+ this.currentState = newState;
33
+ this.onStateChange?.(newState, previousState);
34
+ }
35
+ }
36
+ resetStaticTimer() {
37
+ if (this.staticTimer) {
38
+ clearTimeout(this.staticTimer);
39
+ }
40
+ this.staticTimer = setTimeout(() => {
41
+ if (!this.isUserInteracting) {
42
+ this.setState('static');
43
+ }
44
+ }, this.STATIC_TIMEOUT_MS);
45
+ }
46
+ onUserInteraction() {
47
+ this.setState('user_interacting');
48
+ this.isUserInteracting = true;
49
+ this.resetUserInteractionTimeout();
50
+ this.resetStaticTimer();
51
+ }
52
+ resetUserInteractionTimeout() {
53
+ if (this.userInteractionTimer) {
54
+ clearTimeout(this.userInteractionTimer);
55
+ }
56
+ this.userInteractionTimer = setTimeout(() => {
57
+ this.isUserInteracting = false;
58
+ const newState = this.frameInterval < this.SCREEN_MOVING_THRESHOLD_MS ? 'screen_moving' : 'static';
59
+ this.setState(newState);
60
+ }, this.USER_INTERACTION_TIMEOUT_MS);
61
+ }
62
+ onFrameReceived() {
63
+ const now = Date.now();
64
+ this.frameInterval = now - this.lastFrameTime;
65
+ this.lastFrameTime = now;
66
+ if (!this.isUserInteracting) {
67
+ const newState = this.frameInterval < this.SCREEN_MOVING_THRESHOLD_MS ? 'screen_moving' : 'static';
68
+ this.setState(newState);
69
+ }
70
+ this.resetStaticTimer();
71
+ }
72
+ getConfig() {
73
+ return STATE_CONFIGS[this.currentState];
74
+ }
75
+ getState() {
76
+ return this.currentState;
77
+ }
78
+ getFrameInterval() {
79
+ return this.frameInterval;
80
+ }
81
+ getIsUserInteracting() {
82
+ return this.isUserInteracting;
83
+ }
84
+ }
85
+ export class FrameRateController {
86
+ lastSentTime = 0;
87
+ fpsFrameCount = 0;
88
+ fpsLastTime = Date.now();
89
+ currentFps = 0;
90
+ FPS_CALCULATION_INTERVAL_MS = 1000;
91
+ shouldSendFrame(maxFps) {
92
+ const now = Date.now();
93
+ const minInterval = 1000 / maxFps;
94
+ if (now - this.lastSentTime >= minInterval) {
95
+ this.lastSentTime = now;
96
+ this.fpsFrameCount++;
97
+ this.calculateFps();
98
+ return true;
99
+ }
100
+ return false;
101
+ }
102
+ calculateFps() {
103
+ const now = Date.now();
104
+ const elapsed = now - this.fpsLastTime;
105
+ if (elapsed >= this.FPS_CALCULATION_INTERVAL_MS) {
106
+ this.currentFps = Math.round((this.fpsFrameCount * 1000) / elapsed);
107
+ this.fpsFrameCount = 0;
108
+ this.fpsLastTime = now;
109
+ }
110
+ }
111
+ getCurrentFps() {
112
+ return this.currentFps;
113
+ }
114
+ reset() {
115
+ this.lastSentTime = 0;
116
+ this.fpsFrameCount = 0;
117
+ this.fpsLastTime = Date.now();
118
+ this.currentFps = 0;
119
+ }
120
+ }
121
+ export class FrameProcessor {
122
+ async process(data, config, viewportWidth, viewportHeight) {
123
+ const buffer = Buffer.from(data, 'base64');
124
+ let processed = sharp(buffer);
125
+ if (config.scale < 1 && viewportWidth && viewportHeight) {
126
+ const newWidth = Math.round(viewportWidth * config.scale);
127
+ const newHeight = Math.round(viewportHeight * config.scale);
128
+ processed = processed.resize(newWidth, newHeight);
129
+ }
130
+ if (config.format === 'jpeg') {
131
+ processed = processed.jpeg({ quality: config.quality });
132
+ }
133
+ else {
134
+ processed = processed.webp({ quality: config.quality });
135
+ }
136
+ return processed.toBuffer();
137
+ }
138
+ }
139
+ export function isAllowedOrigin(origin) {
140
+ if (!origin) {
141
+ return true;
142
+ }
143
+ if (origin.startsWith('file://')) {
144
+ return true;
145
+ }
146
+ try {
147
+ const url = new URL(origin);
148
+ const host = url.hostname;
149
+ if (host === 'localhost' || host === '127.0.0.1' || host === '::1' || host === '[::1]') {
150
+ return true;
151
+ }
152
+ }
153
+ catch {
154
+ // Invalid origin URL - reject
155
+ }
156
+ return false;
157
+ }
158
+ function isCommandMessage(msg) {
159
+ return 'id' in msg && 'action' in msg && !('type' in msg);
160
+ }
161
+ export class StreamServer {
162
+ wss = null;
163
+ clients = new Set();
164
+ browser;
165
+ port;
166
+ isScreencasting = false;
167
+ stateManager = new StreamStateManager();
168
+ frameRateController = new FrameRateController();
169
+ frameProcessor = new FrameProcessor();
170
+ lastFrameData = null;
171
+ lastFrameMetadata = null;
172
+ constructor(browser, port = parseInt(process.env.AGENT_BROWSER_STREAM_PORT || '5005', 10)) {
173
+ this.browser = browser;
174
+ this.port = port;
175
+ this.stateManager.setStateChangeCallback((newState, previousState) => {
176
+ this.onStateChange(newState, previousState);
177
+ });
178
+ }
179
+ async onStateChange(newState, previousState) {
180
+ if (this.lastFrameData && this.lastFrameMetadata) {
181
+ const config = STATE_CONFIGS[newState];
182
+ try {
183
+ const processedBuffer = await this.frameProcessor.process(this.lastFrameData, config, this.lastFrameMetadata.deviceWidth, this.lastFrameMetadata.deviceHeight);
184
+ const headerMessage = {
185
+ type: 'frame',
186
+ metadata: this.lastFrameMetadata,
187
+ format: config.format,
188
+ fps: this.frameRateController.getCurrentFps(),
189
+ state: newState,
190
+ };
191
+ for (const client of this.clients) {
192
+ if (client.readyState === WebSocket.OPEN) {
193
+ client.send(JSON.stringify(headerMessage));
194
+ client.send(processedBuffer);
195
+ }
196
+ }
197
+ }
198
+ catch {
199
+ // Ignore errors when reprocessing frame
200
+ }
201
+ }
202
+ }
203
+ start() {
204
+ return new Promise((resolve, reject) => {
205
+ try {
206
+ this.wss = new WebSocketServer({
207
+ port: this.port,
208
+ verifyClient: (info) => {
209
+ if (isAllowedOrigin(info.origin)) {
210
+ return true;
211
+ }
212
+ console.log(`[StreamServer] Rejected connection from origin: ${info.origin}`);
213
+ return false;
214
+ },
215
+ });
216
+ this.wss.on('connection', (ws) => {
217
+ this.handleConnection(ws);
218
+ });
219
+ this.wss.on('error', (error) => {
220
+ console.error('[StreamServer] WebSocket error:', error);
221
+ reject(error);
222
+ });
223
+ this.wss.on('listening', () => {
224
+ console.log(`[StreamServer] Listening on port ${this.port}`);
225
+ setScreencastFrameCallback((frame) => {
226
+ this.broadcastFrame(frame).catch((err) => {
227
+ console.error('[StreamServer] Failed to broadcast frame:', err);
228
+ });
229
+ });
230
+ setEventCallbacks({
231
+ onTabCreated: (event) => {
232
+ this.broadcastEvent({ type: 'tab_created', data: event });
233
+ },
234
+ onTabClosed: (event) => {
235
+ this.broadcastEvent({ type: 'tab_closed', data: event });
236
+ },
237
+ onTabSwitched: (event) => {
238
+ this.broadcastEvent({ type: 'tab_switched', data: event });
239
+ },
240
+ onNavigation: (event) => {
241
+ this.broadcastEvent({ type: 'navigation', data: event });
242
+ },
243
+ });
244
+ resolve();
245
+ });
246
+ }
247
+ catch (error) {
248
+ reject(error);
249
+ }
250
+ });
251
+ }
252
+ async stop() {
253
+ if (this.isScreencasting) {
254
+ await this.stopScreencast();
255
+ }
256
+ setScreencastFrameCallback(null);
257
+ setEventCallbacks({});
258
+ for (const client of this.clients) {
259
+ client.close();
260
+ }
261
+ this.clients.clear();
262
+ if (this.wss) {
263
+ return new Promise((resolve) => {
264
+ this.wss.close(() => {
265
+ this.wss = null;
266
+ resolve();
267
+ });
268
+ });
269
+ }
270
+ }
271
+ handleConnection(ws) {
272
+ console.log('[StreamServer] Client connected');
273
+ this.clients.add(ws);
274
+ this.sendStatus(ws);
275
+ if (this.clients.size === 1 && !this.isScreencasting) {
276
+ this.startScreencast().catch((error) => {
277
+ console.error('[StreamServer] Failed to start screencast:', error);
278
+ this.sendError(ws, error.message);
279
+ });
280
+ }
281
+ ws.on('message', (data) => {
282
+ try {
283
+ const message = JSON.parse(data.toString());
284
+ this.handleMessage(message, ws);
285
+ }
286
+ catch (error) {
287
+ console.error('[StreamServer] Failed to parse message:', error);
288
+ }
289
+ });
290
+ ws.on('close', () => {
291
+ console.log('[StreamServer] Client disconnected');
292
+ this.clients.delete(ws);
293
+ if (this.clients.size === 0 && this.isScreencasting) {
294
+ this.stopScreencast().catch((error) => {
295
+ console.error('[StreamServer] Failed to stop screencast:', error);
296
+ });
297
+ }
298
+ });
299
+ ws.on('error', (error) => {
300
+ console.error('[StreamServer] Client error:', error);
301
+ this.clients.delete(ws);
302
+ });
303
+ }
304
+ async handleMessage(message, ws) {
305
+ if (isCommandMessage(message)) {
306
+ try {
307
+ const response = await executeCommand(message, this.browser);
308
+ ws.send(serializeResponse(response));
309
+ }
310
+ catch (error) {
311
+ const errorMessage = error instanceof Error ? error.message : String(error);
312
+ ws.send(serializeResponse(errorResponse(message.id, errorMessage)));
313
+ }
314
+ return;
315
+ }
316
+ try {
317
+ switch (message.type) {
318
+ case 'input_mouse':
319
+ this.stateManager.onUserInteraction();
320
+ await this.browser.injectMouseEvent({
321
+ type: message.eventType,
322
+ x: message.x,
323
+ y: message.y,
324
+ button: message.button,
325
+ clickCount: message.clickCount,
326
+ deltaX: message.deltaX,
327
+ deltaY: message.deltaY,
328
+ modifiers: message.modifiers,
329
+ });
330
+ break;
331
+ case 'input_keyboard':
332
+ this.stateManager.onUserInteraction();
333
+ await this.browser.injectKeyboardEvent({
334
+ type: message.eventType,
335
+ key: message.key,
336
+ code: message.code,
337
+ text: message.text,
338
+ modifiers: message.modifiers,
339
+ });
340
+ break;
341
+ case 'input_touch':
342
+ this.stateManager.onUserInteraction();
343
+ await this.browser.injectTouchEvent({
344
+ type: message.eventType,
345
+ touchPoints: message.touchPoints,
346
+ modifiers: message.modifiers,
347
+ });
348
+ break;
349
+ case 'input_text':
350
+ this.stateManager.onUserInteraction();
351
+ await this.browser.insertText(message.text);
352
+ break;
353
+ case 'keyboard_down':
354
+ this.stateManager.onUserInteraction();
355
+ await this.browser.getPage().keyboard.down(message.key);
356
+ break;
357
+ case 'keyboard_up':
358
+ await this.browser.getPage().keyboard.up(message.key);
359
+ break;
360
+ case 'keyboard_insert_text':
361
+ this.stateManager.onUserInteraction();
362
+ await this.browser.getPage().keyboard.insertText(message.text);
363
+ break;
364
+ case 'user_activity':
365
+ this.stateManager.onUserInteraction();
366
+ break;
367
+ case 'status':
368
+ this.sendStatus(ws);
369
+ break;
370
+ }
371
+ }
372
+ catch (error) {
373
+ const errorMessage = error instanceof Error ? error.message : String(error);
374
+ this.sendError(ws, errorMessage);
375
+ }
376
+ }
377
+ async broadcastFrame(frame) {
378
+ this.lastFrameData = frame.data;
379
+ this.lastFrameMetadata = frame.metadata;
380
+ this.stateManager.onFrameReceived();
381
+ const config = this.stateManager.getConfig();
382
+ if (!this.frameRateController.shouldSendFrame(config.maxFps)) {
383
+ return;
384
+ }
385
+ let processedBuffer;
386
+ try {
387
+ processedBuffer = await this.frameProcessor.process(frame.data, config, frame.metadata.deviceWidth, frame.metadata.deviceHeight);
388
+ }
389
+ catch {
390
+ processedBuffer = Buffer.from(frame.data, 'base64');
391
+ }
392
+ const headerMessage = {
393
+ type: 'frame',
394
+ metadata: frame.metadata,
395
+ format: config.format,
396
+ fps: this.frameRateController.getCurrentFps(),
397
+ state: this.stateManager.getState(),
398
+ };
399
+ for (const client of this.clients) {
400
+ if (client.readyState === WebSocket.OPEN) {
401
+ client.send(JSON.stringify(headerMessage));
402
+ client.send(processedBuffer);
403
+ }
404
+ }
405
+ }
406
+ broadcastEvent(message) {
407
+ const payload = JSON.stringify(message);
408
+ for (const client of this.clients) {
409
+ if (client.readyState === WebSocket.OPEN) {
410
+ client.send(payload);
411
+ }
412
+ }
413
+ }
414
+ sendStatus(ws) {
415
+ let viewportWidth;
416
+ let viewportHeight;
417
+ try {
418
+ const page = this.browser.getPage();
419
+ const viewport = page.viewportSize();
420
+ viewportWidth = viewport?.width;
421
+ viewportHeight = viewport?.height;
422
+ }
423
+ catch {
424
+ // Browser not launched yet
425
+ }
426
+ const message = {
427
+ type: 'status',
428
+ connected: true,
429
+ screencasting: this.isScreencasting,
430
+ viewportWidth,
431
+ viewportHeight,
432
+ fps: this.frameRateController.getCurrentFps(),
433
+ state: this.stateManager.getState(),
434
+ };
435
+ if (ws.readyState === WebSocket.OPEN) {
436
+ ws.send(JSON.stringify(message));
437
+ }
438
+ }
439
+ sendError(ws, errorMessage) {
440
+ const message = {
441
+ type: 'error',
442
+ message: errorMessage,
443
+ };
444
+ if (ws.readyState === WebSocket.OPEN) {
445
+ ws.send(JSON.stringify(message));
446
+ }
447
+ }
448
+ async startScreencast() {
449
+ if (this.isScreencasting)
450
+ return;
451
+ this.isScreencasting = true;
452
+ try {
453
+ if (!this.browser.isLaunched()) {
454
+ throw new Error('Browser not launched');
455
+ }
456
+ await this.browser.startScreencast((frame) => this.broadcastFrame(frame), {
457
+ format: 'jpeg',
458
+ quality: 80,
459
+ maxWidth: 1280,
460
+ maxHeight: 720,
461
+ everyNthFrame: 1,
462
+ });
463
+ for (const client of this.clients) {
464
+ this.sendStatus(client);
465
+ }
466
+ }
467
+ catch (error) {
468
+ this.isScreencasting = false;
469
+ throw error;
470
+ }
471
+ }
472
+ async stopScreencast() {
473
+ if (!this.isScreencasting)
474
+ return;
475
+ await this.browser.stopScreencast();
476
+ this.isScreencasting = false;
477
+ for (const client of this.clients) {
478
+ this.sendStatus(client);
479
+ }
480
+ }
481
+ getPort() {
482
+ return this.port;
483
+ }
484
+ getClientCount() {
485
+ return this.clients.size;
486
+ }
487
+ getStateManager() {
488
+ return this.stateManager;
489
+ }
490
+ getFrameRateController() {
491
+ return this.frameRateController;
492
+ }
493
+ }
494
+ const STREAM_SERVER_IPC_FILE = 'stream-server.ipc';
495
+ export function getStreamServerIpcPath() {
496
+ return path.join(getSocketDir(), STREAM_SERVER_IPC_FILE);
497
+ }
498
+ export class StreamServerProxy {
499
+ browser;
500
+ ipcSocket = null;
501
+ ipcPath;
502
+ session;
503
+ isScreencasting = false;
504
+ stateManager = new StreamStateManager();
505
+ frameRateController = new FrameRateController();
506
+ frameProcessor = new FrameProcessor();
507
+ reconnectTimer = null;
508
+ lastFrameData = null;
509
+ lastFrameMetadata = null;
510
+ constructor(browser) {
511
+ this.browser = browser;
512
+ this.ipcPath = getStreamServerIpcPath();
513
+ this.session = getSession();
514
+ this.stateManager.setStateChangeCallback((newState, previousState) => {
515
+ this.onStateChange(newState, previousState);
516
+ });
517
+ }
518
+ async onStateChange(newState, previousState) {
519
+ if (this.lastFrameData && this.lastFrameMetadata) {
520
+ const config = STATE_CONFIGS[newState];
521
+ try {
522
+ const processedBuffer = await this.frameProcessor.process(this.lastFrameData, config, this.lastFrameMetadata.deviceWidth, this.lastFrameMetadata.deviceHeight);
523
+ this.send({
524
+ type: 'frame',
525
+ session: this.session,
526
+ metadata: this.lastFrameMetadata,
527
+ format: config.format,
528
+ fps: this.frameRateController.getCurrentFps(),
529
+ state: newState,
530
+ data: processedBuffer.toString('base64'),
531
+ });
532
+ }
533
+ catch {
534
+ // Ignore errors when reprocessing frame
535
+ }
536
+ }
537
+ }
538
+ async connect() {
539
+ return new Promise((resolve, reject) => {
540
+ if (!fs.existsSync(this.ipcPath)) {
541
+ reject(new Error(`Stream Server IPC not found at ${this.ipcPath}`));
542
+ return;
543
+ }
544
+ this.ipcSocket = net.createConnection({ path: this.ipcPath }, () => {
545
+ console.log(`[StreamServerProxy] Connected to Stream Server for session: ${this.session}`);
546
+ this.send({
547
+ type: 'register',
548
+ session: this.session,
549
+ instanceId: getInstanceId(),
550
+ socketPath: this.getDaemonSocketPath(),
551
+ });
552
+ this.setupFrameCallback();
553
+ resolve();
554
+ });
555
+ this.ipcSocket.on('error', (err) => {
556
+ console.error('[StreamServerProxy] IPC error:', err);
557
+ this.scheduleReconnect();
558
+ reject(err);
559
+ });
560
+ this.ipcSocket.on('close', () => {
561
+ console.log('[StreamServerProxy] IPC connection closed');
562
+ this.ipcSocket = null;
563
+ this.scheduleReconnect();
564
+ });
565
+ let buffer = '';
566
+ this.ipcSocket.on('data', (data) => {
567
+ buffer += data.toString();
568
+ while (buffer.includes('\n')) {
569
+ const newlineIdx = buffer.indexOf('\n');
570
+ const line = buffer.substring(0, newlineIdx);
571
+ buffer = buffer.substring(newlineIdx + 1);
572
+ if (line.trim()) {
573
+ this.handleMessage(line);
574
+ }
575
+ }
576
+ });
577
+ });
578
+ }
579
+ handleMessage(line) {
580
+ try {
581
+ const message = JSON.parse(line);
582
+ switch (message.type) {
583
+ case 'input_mouse':
584
+ case 'input_keyboard':
585
+ case 'input_touch':
586
+ case 'input_text':
587
+ case 'keyboard_down':
588
+ case 'keyboard_up':
589
+ case 'keyboard_insert_text':
590
+ case 'user_activity':
591
+ this.handleInputMessage(message);
592
+ break;
593
+ case 'client_connected':
594
+ this.handleClientConnected(message.session);
595
+ break;
596
+ case 'client_disconnected':
597
+ this.handleClientDisconnected(message.session);
598
+ break;
599
+ }
600
+ }
601
+ catch (error) {
602
+ console.error('[StreamServerProxy] Failed to parse message:', error);
603
+ }
604
+ }
605
+ async handleClientConnected(session) {
606
+ if (session !== this.session)
607
+ return;
608
+ console.log(`[StreamServerProxy] Client connected for session ${session}, starting screencast`);
609
+ try {
610
+ await this.startScreencast();
611
+ }
612
+ catch (error) {
613
+ console.error('[StreamServerProxy] Failed to start screencast on client connected:', error);
614
+ }
615
+ }
616
+ async handleClientDisconnected(session) {
617
+ if (session !== this.session)
618
+ return;
619
+ console.log(`[StreamServerProxy] Client disconnected for session ${session}, stopping screencast`);
620
+ try {
621
+ await this.stopScreencast();
622
+ }
623
+ catch (error) {
624
+ console.error('[StreamServerProxy] Failed to stop screencast on client disconnected:', error);
625
+ }
626
+ }
627
+ async handleInputMessage(message) {
628
+ try {
629
+ switch (message.type) {
630
+ case 'input_mouse':
631
+ this.stateManager.onUserInteraction();
632
+ await this.browser.injectMouseEvent({
633
+ type: message.eventType,
634
+ x: message.x,
635
+ y: message.y,
636
+ button: message.button,
637
+ clickCount: message.clickCount,
638
+ deltaX: message.deltaX,
639
+ deltaY: message.deltaY,
640
+ modifiers: message.modifiers,
641
+ });
642
+ break;
643
+ case 'input_keyboard':
644
+ this.stateManager.onUserInteraction();
645
+ await this.browser.injectKeyboardEvent({
646
+ type: message.eventType,
647
+ key: message.key,
648
+ code: message.code,
649
+ text: message.text,
650
+ modifiers: message.modifiers,
651
+ });
652
+ break;
653
+ case 'input_touch':
654
+ this.stateManager.onUserInteraction();
655
+ await this.browser.injectTouchEvent({
656
+ type: message.eventType,
657
+ touchPoints: message.touchPoints,
658
+ modifiers: message.modifiers,
659
+ });
660
+ break;
661
+ case 'input_text':
662
+ this.stateManager.onUserInteraction();
663
+ await this.browser.insertText(message.text);
664
+ break;
665
+ case 'keyboard_down':
666
+ this.stateManager.onUserInteraction();
667
+ await this.browser.getPage().keyboard.down(message.key);
668
+ break;
669
+ case 'keyboard_up':
670
+ await this.browser.getPage().keyboard.up(message.key);
671
+ break;
672
+ case 'keyboard_insert_text':
673
+ this.stateManager.onUserInteraction();
674
+ await this.browser.getPage().keyboard.insertText(message.text);
675
+ break;
676
+ case 'user_activity':
677
+ this.stateManager.onUserInteraction();
678
+ break;
679
+ }
680
+ }
681
+ catch (error) {
682
+ console.error('[StreamServerProxy] Failed to handle input:', error);
683
+ }
684
+ }
685
+ setupFrameCallback() {
686
+ setScreencastFrameCallback((frame) => {
687
+ this.sendFrame(frame).catch((err) => {
688
+ console.error('[StreamServerProxy] Failed to send frame:', err);
689
+ });
690
+ });
691
+ setEventCallbacks({
692
+ onTabCreated: (event) => {
693
+ this.send({ type: 'tab_created', session: this.session, data: event });
694
+ },
695
+ onTabClosed: (event) => {
696
+ this.send({ type: 'tab_closed', session: this.session, data: event });
697
+ },
698
+ onTabSwitched: (event) => {
699
+ this.send({ type: 'tab_switched', session: this.session, data: event });
700
+ },
701
+ onNavigation: (event) => {
702
+ this.send({ type: 'navigation', session: this.session, data: event });
703
+ },
704
+ });
705
+ }
706
+ async sendFrame(frame) {
707
+ this.lastFrameData = frame.data;
708
+ this.lastFrameMetadata = frame.metadata;
709
+ this.stateManager.onFrameReceived();
710
+ const config = this.stateManager.getConfig();
711
+ if (!this.frameRateController.shouldSendFrame(config.maxFps)) {
712
+ return;
713
+ }
714
+ let processedBuffer;
715
+ try {
716
+ processedBuffer = await this.frameProcessor.process(frame.data, config, frame.metadata.deviceWidth, frame.metadata.deviceHeight);
717
+ }
718
+ catch {
719
+ processedBuffer = Buffer.from(frame.data, 'base64');
720
+ }
721
+ this.send({
722
+ type: 'frame',
723
+ session: this.session,
724
+ metadata: frame.metadata,
725
+ format: config.format,
726
+ fps: this.frameRateController.getCurrentFps(),
727
+ state: this.stateManager.getState(),
728
+ data: processedBuffer.toString('base64'),
729
+ });
730
+ }
731
+ send(message) {
732
+ if (this.ipcSocket && !this.ipcSocket.destroyed) {
733
+ this.ipcSocket.write(JSON.stringify(message) + '\n');
734
+ }
735
+ }
736
+ getDaemonSocketPath() {
737
+ const isWindows = process.platform === 'win32';
738
+ if (isWindows) {
739
+ return `tcp://127.0.0.1:${this.getDaemonPort()}`;
740
+ }
741
+ return path.join(getSocketDir(), `${this.session}.sock`);
742
+ }
743
+ getDaemonPort() {
744
+ let hash = 0;
745
+ for (let i = 0; i < this.session.length; i++) {
746
+ hash = (hash << 5) - hash + this.session.charCodeAt(i);
747
+ hash |= 0;
748
+ }
749
+ return 49152 + (Math.abs(hash) % 16383);
750
+ }
751
+ scheduleReconnect() {
752
+ if (this.reconnectTimer) {
753
+ clearTimeout(this.reconnectTimer);
754
+ }
755
+ this.reconnectTimer = setTimeout(() => {
756
+ this.connect().catch((err) => {
757
+ console.error('[StreamServerProxy] Reconnect failed:', err);
758
+ });
759
+ }, 2000);
760
+ }
761
+ async startScreencast() {
762
+ if (this.isScreencasting)
763
+ return;
764
+ this.isScreencasting = true;
765
+ try {
766
+ if (!this.browser.isLaunched()) {
767
+ throw new Error('Browser not launched');
768
+ }
769
+ await this.browser.startScreencast((frame) => this.sendFrame(frame), {
770
+ format: 'jpeg',
771
+ quality: 80,
772
+ maxWidth: 1280,
773
+ maxHeight: 720,
774
+ everyNthFrame: 1,
775
+ });
776
+ }
777
+ catch (error) {
778
+ this.isScreencasting = false;
779
+ throw error;
780
+ }
781
+ }
782
+ async stopScreencast() {
783
+ if (!this.isScreencasting)
784
+ return;
785
+ await this.browser.stopScreencast();
786
+ this.isScreencasting = false;
787
+ }
788
+ async disconnect() {
789
+ if (this.reconnectTimer) {
790
+ clearTimeout(this.reconnectTimer);
791
+ this.reconnectTimer = null;
792
+ }
793
+ this.send({
794
+ type: 'unregister',
795
+ session: this.session,
796
+ });
797
+ setScreencastFrameCallback(null);
798
+ setEventCallbacks({});
799
+ if (this.isScreencasting) {
800
+ await this.stopScreencast();
801
+ }
802
+ if (this.ipcSocket) {
803
+ this.ipcSocket.destroy();
804
+ this.ipcSocket = null;
805
+ }
806
+ }
807
+ isConnected() {
808
+ return this.ipcSocket !== null && !this.ipcSocket.destroyed;
809
+ }
810
+ }
811
+ //# sourceMappingURL=stream-server.js.map