mecord-connect 1.0.0

package/runtime/app/src/capabilities/browser-cdp-connection.js

@@ -0,0 +1,199 @@
+import { OperatorError } from '../core/errors.js';
+
+const DEFAULT_TIMEOUT_MS = 8_000;
+const MAX_CDP_MESSAGE_BYTES = 4 * 1024 * 1024;
+
+                                              
+
+                         
+             
+                
+                 
+               
+                                
+  
+
+                
+                                    
+                                 
+                        
+  
+
+                                                                   
+
+export function assertLoopbackDebuggerUrl(raw        )       {
+  let url     ;
+  try { url = new URL(raw); } catch { throw new OperatorError('UNSAFE_CDP_WEBSOCKET', 'Invalid DevTools WebSocket URL.'); }
+  if (!['ws:', 'wss:'].includes(url.protocol) || !['127.0.0.1', 'localhost', '::1', '[::1]'].includes(url.hostname)) {
+    throw new OperatorError('UNSAFE_CDP_WEBSOCKET', 'DevTools WebSocket endpoint must be loopback-only.');
+  }
+  if (url.username || url.password || url.hash) {
+    throw new OperatorError('UNSAFE_CDP_WEBSOCKET', 'DevTools WebSocket endpoint must not contain credentials or a fragment.');
+  }
+}
+
+function sameWebSocketDestination(expectedRaw        , actualRaw        )          {
+  try { return new URL(expectedRaw).toString() === new URL(actualRaw).toString(); } catch { return false; }
+}
+
+export class CdpConnection {
+           targetId        ;
+           url        ;
+  #socket           ;
+  #nextId = 1;
+  #pending = new Map                 ();
+  #listeners = new Map                            ();
+  #ready               ;
+  #closed = false;
+
+  constructor(targetId        , url        ) {
+    assertLoopbackDebuggerUrl(url);
+    this.targetId = targetId;
+    this.url = url;
+    this.#socket = new WebSocket(url);
+    this.#ready = new Promise      ((resolve, reject) => {
+      let settled = false;
+      const finish = (callback            ) => {
+        if (settled) return;
+        settled = true;
+        clearTimeout(timeout);
+        callback();
+      };
+      const timeout = setTimeout(() => finish(() => {
+        this.#closed = true;
+        try { this.#socket.close(); } catch { /* timeout is already authoritative */ }
+        reject(new OperatorError('CDP_CONNECT_TIMEOUT', 'Timed out connecting to browser target.', { retryable: true }));
+      }), 4_000);
+      this.#socket.addEventListener('open', () => finish(() => {
+        if (!sameWebSocketDestination(url, this.#socket.url)) {
+          this.#closed = true;
+          try { this.#socket.close(); } catch { /* destination mismatch is already authoritative */ }
+          reject(new OperatorError('CDP_WEBSOCKET_DESTINATION_CHANGED', 'Opened DevTools WebSocket destination differs from the authorized endpoint.', { retryable: false }));
+          return;
+        }
+        resolve();
+      }), { once: true });
+      this.#socket.addEventListener('error', () => finish(() => {
+        this.#closed = true;
+        try { this.#socket.close(); } catch { /* connection error is already authoritative */ }
+        reject(new OperatorError('CDP_CONNECT_FAILED', 'Failed to connect to browser target.', { retryable: true }));
+      }), { once: true });
+    });
+
+    this.#socket.addEventListener('message', (event) => {
+      try {
+        const raw = typeof event.data === 'string' ? event.data : String(event.data);
+        if (Buffer.byteLength(raw, 'utf8') > MAX_CDP_MESSAGE_BYTES) {
+          this.close();
+          return;
+        }
+        const message = JSON.parse(raw)     
+                      
+                           
+                                                      
+                          
+                           
+                             
+         ;
+        if (typeof message.id === 'number') {
+          const pending = this.#pending.get(message.id);
+          if (!pending) return;
+          clearTimeout(pending.timer);
+          this.#pending.delete(message.id);
+          if (message.error) {
+            pending.reject(new OperatorError('CDP_COMMAND_FAILED', message.error.message ?? 'CDP command failed.', { retryable: false, details: { code: message.error.code } }));
+          } else {
+            pending.resolve(message.result ?? {});
+          }
+          return;
+        }
+        if (message.method) {
+          for (const listener of this.#listeners.get(message.method) ?? []) listener(message.params ?? {}, message.sessionId);
+        }
+      } catch {
+        // Ignore malformed browser event payloads. A later command/postcondition will expose the failure.
+      }
+    });
+
+    this.#socket.addEventListener('close', () => {
+      this.#closed = true;
+      const error = new OperatorError('CDP_CONNECTION_CLOSED', 'Browser target connection closed.', { retryable: true });
+      for (const pending of this.#pending.values()) {
+        clearTimeout(pending.timer);
+        pending.reject(error);
+      }
+      this.#pending.clear();
+    });
+  }
+
+  get closed()          { return this.#closed; }
+
+  async send(method        , params          = {}, timeoutMs = DEFAULT_TIMEOUT_MS)                   {
+    return this.#send(method, params, timeoutMs);
+  }
+
+  async sendInSession(sessionId        , method        , params          = {}, timeoutMs = DEFAULT_TIMEOUT_MS)                   {
+    if (!sessionId || sessionId.length > 256) {
+      throw new OperatorError('INVALID_CDP_SESSION', 'Flattened CDP session id must contain 1-256 characters.');
+    }
+    return this.#send(method, params, timeoutMs, sessionId);
+  }
+
+  async #send(method        , params         , timeoutMs        , sessionId         )                   {
+    await this.#ready;
+    if (this.#closed || this.#socket.readyState !== WebSocket.OPEN) {
+      throw new OperatorError('CDP_CONNECTION_CLOSED', 'Browser target connection is not open.', { retryable: true });
+    }
+    const id = this.#nextId++;
+    return await new Promise         ((resolve, reject) => {
+      const timer = setTimeout(() => {
+        this.#pending.delete(id);
+        reject(new OperatorError('CDP_COMMAND_TIMEOUT', `${method} timed out.`, { retryable: true }));
+      }, timeoutMs);
+      this.#pending.set(id, { resolve, reject, timer });
+      this.#socket.send(JSON.stringify({ id, method, params, ...(sessionId ? { sessionId } : {}) }));
+    });
+  }
+
+  on(method        , listener               )             {
+    const bucket = this.#listeners.get(method) ?? new Set               ();
+    bucket.add(listener);
+    this.#listeners.set(method, bucket);
+    return () => {
+      bucket.delete(listener);
+      if (bucket.size === 0) this.#listeners.delete(method);
+    };
+  }
+
+  close()       {
+    if (this.#closed) return;
+    this.#closed = true;
+    try { this.#socket.close(); } catch { /* noop */ }
+  }
+}
+
+export class CdpSessionManager {
+  #sessions = new Map                       ();
+
+  get(target           )                {
+    if (!target.webSocketDebuggerUrl) {
+      throw new OperatorError('CDP_TARGET_UNAVAILABLE', 'Target does not expose a DevTools WebSocket endpoint.', { retryable: true });
+    }
+    const existing = this.#sessions.get(target.id);
+    if (existing && !existing.closed && existing.url === target.webSocketDebuggerUrl) return existing;
+    existing?.close();
+    const created = new CdpConnection(target.id, target.webSocketDebuggerUrl);
+    this.#sessions.set(target.id, created);
+    return created;
+  }
+
+  forget(targetId        )       {
+    this.#sessions.get(targetId)?.close();
+    this.#sessions.delete(targetId);
+  }
+
+  closeAll()       {
+    for (const session of this.#sessions.values()) session.close();
+    this.#sessions.clear();
+  }
+}

package/runtime/app/src/capabilities/browser-cdp-frames.js

@@ -0,0 +1,288 @@
+import { OperatorError } from '../core/errors.js';
+                                                                          
+import { interactionFunction, semanticSnapshotFunction, unwrapRuntimeValue } from './browser-cdp-page.js';
+
+const MAX_OOPIF_SESSIONS = 16;
+const MAX_OOPIF_DEPTH = 4;
+const ATTACH_SETTLE_MS = 25;
+
+                      
+                    
+                   
+              
+                
+  
+
+                     
+                         
+                        
+  
+
+                             
+                          
+                        
+  
+
+const AUTO_ATTACH_ON = {
+  autoAttach: true,
+  waitForDebuggerOnStart: false,
+  flatten: true
+};
+
+const AUTO_ATTACH_OFF = {
+  autoAttach: false,
+  waitForDebuggerOnStart: false,
+  flatten: true
+};
+
+function delay(ms        )                {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}
+
+async function attachOopifSessions(session               )                                {
+  const frames = new Map                       ();
+  const depthBySession = new Map                ();
+  const armed = new Set        ();
+  const offAttached = session.on('Target.attachedToTarget', (params, parentSessionId) => {
+    const sessionId = typeof params.sessionId === 'string' ? params.sessionId : '';
+    const targetInfo = params.targetInfo && typeof params.targetInfo === 'object' ? params.targetInfo            : undefined;
+    const type = typeof targetInfo?.type === 'string' ? targetInfo.type : '';
+    if (!sessionId || type !== 'iframe' || frames.has(sessionId) || frames.size >= MAX_OOPIF_SESSIONS) return;
+    const parentDepth = parentSessionId ? depthBySession.get(parentSessionId) ?? 0 : 0;
+    const depth = parentDepth + 1;
+    if (depth > MAX_OOPIF_DEPTH) return;
+    const frame                = {
+      sessionId,
+      targetId: typeof targetInfo?.targetId === 'string' ? targetInfo.targetId : '',
+      url: typeof targetInfo?.url === 'string' ? targetInfo.url.slice(0, 2000) : '',
+      depth
+    };
+    frames.set(sessionId, frame);
+    depthBySession.set(sessionId, depth);
+  });
+
+  try {
+    await session.send('Target.setAutoAttach', AUTO_ATTACH_ON);
+  } catch {
+    offAttached();
+    return { frames: [], async stop() { /* target does not expose child-target auto-attach */ } };
+  }
+
+  for (let round = 0; round < MAX_OOPIF_DEPTH; round += 1) {
+    await delay(ATTACH_SETTLE_MS);
+    const toArm = [...frames.values()].filter((frame) => frame.depth < MAX_OOPIF_DEPTH && !armed.has(frame.sessionId));
+    if (toArm.length === 0 && round > 0) break;
+    await Promise.allSettled(toArm.map(async (frame) => {
+      armed.add(frame.sessionId);
+      await session.sendInSession(frame.sessionId, 'Target.setAutoAttach', AUTO_ATTACH_ON);
+    }));
+  }
+  await delay(ATTACH_SETTLE_MS);
+
+  return {
+    frames: [...frames.values()].sort((a, b) => a.depth - b.depth || a.targetId.localeCompare(b.targetId)),
+    async stop() {
+      offAttached();
+      const deepestFirst = [...frames.values()].sort((a, b) => b.depth - a.depth);
+      await Promise.allSettled(deepestFirst.map((frame) => session.sendInSession(frame.sessionId, 'Target.setAutoAttach', AUTO_ATTACH_OFF)));
+      await Promise.allSettled([session.send('Target.setAutoAttach', AUTO_ATTACH_OFF)]);
+    }
+  };
+}
+
+async function evaluate(
+  session               ,
+  context              ,
+  expression        ,
+  userGesture = false
+)                   {
+  const params = { expression, returnByValue: true, awaitPromise: true, userGesture };
+  return context.frame
+    ? session.sendInSession(context.frame.sessionId, 'Runtime.evaluate', params)
+    : session.send('Runtime.evaluate', params);
+}
+
+export function semanticLocatorFunction(target                                                               ) {
+  const trim = (value         ) => String(value ?? '').replace(/\s+/g, ' ').trim();
+  const deepQuery = (selector        , max = 1000) => {
+    const found                                                                                    = [];
+    const seenScopes = new Set         ();
+    let scanned = 0;
+    const visit = (scope                                 , frameDepth        , shadowDepth        ) => {
+      if (!scope || seenScopes.has(scope) || found.length >= max || scanned >= 5000) return;
+      seenScopes.add(scope);
+      let elements            = [];
+      try { elements = Array.from(scope.querySelectorAll('*')).slice(0, 2500); } catch { return; }
+      for (const element of elements) {
+        if (found.length >= max || scanned++ >= 5000) break;
+        try { if (element.matches(selector)) found.push({ element, context: { frameDepth, shadowDepth } }); } catch { /* invalid selector becomes no match */ }
+        const shadow = (element                                                ).shadowRoot;
+        if (shadow && shadowDepth < 8) visit(shadow, frameDepth, shadowDepth + 1);
+        if (element.tagName === 'IFRAME' && frameDepth < 4) {
+          try {
+            const frameDocument = (element                     ).contentDocument;
+            if (frameDocument?.documentElement) visit(frameDocument, frameDepth + 1, shadowDepth);
+          } catch { /* cross-origin iframe is handled through an attached CDP target */ }
+        }
+      }
+    };
+    visit(document, 0, 0);
+    return found;
+  };
+  const nameOf = (element         ) => {
+    const aria = element.getAttribute('aria-label');
+    if (aria) return trim(aria);
+    const labelledBy = element.getAttribute('aria-labelledby');
+    if (labelledBy) {
+      const root = element.getRootNode()                         ;
+      const labels = labelledBy.split(/\s+/).map((id) => (root            ).getElementById?.(id)?.textContent ?? '').filter(Boolean);
+      if (labels.length) return trim(labels.join(' '));
+    }
+    const control = element                                                    ;
+    if (control.labels?.length) return trim(Array.from(control.labels).map((label) => label.textContent).join(' '));
+    const nativeText = ['BUTTON', 'SUMMARY', 'A', 'OPTION'].includes(element.tagName) ? trim(element.textContent) : '';
+    return trim(
+      element.getAttribute('placeholder')
+      || nativeText
+      || element.getAttribute('title')
+      || element.getAttribute('name')
+      || element.id
+      || element.textContent
+    );
+  };
+  const roleOf = (element         ) => {
+    const explicit = trim(element.getAttribute('role')).toLowerCase();
+    if (explicit) return explicit;
+    const tag = element.tagName;
+    if (tag === 'A' && element.hasAttribute('href')) return 'link';
+    if (tag === 'BUTTON' || tag === 'SUMMARY') return 'button';
+    if (tag === 'TEXTAREA') return 'textbox';
+    if (tag === 'SELECT') return 'combobox';
+    if (tag === 'OPTION') return 'option';
+    if (tag === 'INPUT') {
+      const type = trim(element.getAttribute('type') || 'text').toLowerCase();
+      if (type === 'checkbox') return 'checkbox';
+      if (type === 'radio') return 'radio';
+      if (['button', 'submit', 'reset', 'image'].includes(type)) return 'button';
+      if (type === 'search') return 'searchbox';
+      return 'textbox';
+    }
+    return '';
+  };
+  const visible = (element         ) => {
+    const view = element.ownerDocument?.defaultView;
+    const style = view?.getComputedStyle?.(element);
+    const rect = (element                                                       ).getBoundingClientRect?.();
+    if (style && (style.visibility === 'hidden' || style.display === 'none')) return false;
+    return !rect || (rect.width > 0 && rect.height > 0);
+  };
+  const selector = target.css || 'button,a[href],input,textarea,select,option,summary,[role],[contenteditable="true"]';
+  const matches = deepQuery(selector, 1000).filter(({ element }) => {
+    if (!visible(element)) return false;
+    if (target.text && !trim(element.textContent).toLowerCase().includes(target.text.toLowerCase())) return false;
+    if (target.role && roleOf(element) !== target.role.toLowerCase()) return false;
+    if (target.name && nameOf(element).toLowerCase() !== target.name.toLowerCase()) return false;
+    return true;
+  });
+  return {
+    count: matches.length,
+    matches: matches.slice(0, 3).map(({ element, context }) => ({
+      tag: element.tagName.toLowerCase(),
+      role: roleOf(element),
+      name: nameOf(element),
+      context
+    }))
+  };
+}
+
+export async function inspectOopifFrames(session               )                 
+                   
+              
+                
+                
+                     
+                    
+    {
+  const scope = await attachOopifSessions(session);
+  try {
+    const output = [];
+    for (const frame of scope.frames) {
+      try {
+        await Promise.allSettled([
+          session.sendInSession(frame.sessionId, 'Runtime.enable'),
+          session.sendInSession(frame.sessionId, 'Accessibility.enable')
+        ]);
+        const [identityResult, semanticResult] = await Promise.all([
+          session.sendInSession(frame.sessionId, 'Runtime.evaluate', {
+            expression: '({url:location.href,title:document.title,readyState:document.readyState})',
+            returnByValue: true
+          }),
+          session.sendInSession(frame.sessionId, 'Runtime.evaluate', {
+            expression: `(${semanticSnapshotFunction.toString()})()`,
+            returnByValue: true
+          })
+        ]);
+        const identity = unwrapRuntimeValue(identityResult)                       ;
+        output.push({
+          targetId: frame.targetId,
+          url: typeof identity?.url === 'string' ? identity.url.slice(0, 2000) : frame.url,
+          depth: frame.depth,
+          title: typeof identity?.title === 'string' ? identity.title.slice(0, 500) : '',
+          readyState: typeof identity?.readyState === 'string' ? identity.readyState : '',
+          semantic: unwrapRuntimeValue(semanticResult) ?? null
+        });
+      } catch {
+        // A frame may disappear during navigation. Omit it rather than failing the bounded parent snapshot.
+      }
+    }
+    return output;
+  } finally {
+    await scope.stop();
+  }
+}
+
+export async function performSemanticInteraction(
+  session               ,
+  input                                                                                                              
+)                                                                                        {
+  const scope = await attachOopifSessions(session);
+  try {
+    const contexts                 = [{ kind: 'main' }, ...scope.frames.map((frame)               => ({ kind: 'oopif', frame }))];
+    const locateExpression = `(${semanticLocatorFunction.toString()})(${JSON.stringify(input.target)})`;
+    const matches                                                                    = [];
+
+    for (const context of contexts) {
+      try {
+        const located = unwrapRuntimeValue(await evaluate(session, context, locateExpression))                       ;
+        const count = typeof located?.count === 'number' ? located.count : 0;
+        if (count > 0) matches.push({ context, count, samples: located?.matches });
+      } catch {
+        // Cross-origin frame may disappear or deny execution while the page is changing; continue bounded search.
+      }
+    }
+
+    const totalMatches = matches.reduce((sum, match) => sum + match.count, 0);
+    if (totalMatches === 0) {
+      throw new OperatorError('BROWSER_ELEMENT_NOT_FOUND', 'No matching semantic element was found.', { retryable: false, details: { target: input.target } });
+    }
+    if (totalMatches > 1) {
+      throw new OperatorError('BROWSER_AMBIGUOUS_ELEMENT', 'Semantic browser target matched multiple elements across page/frame contexts; narrow the selector.', {
+        retryable: false,
+        details: { target: input.target, totalMatches, contexts: matches.map((match) => ({ count: match.count, frameTargetId: match.context.frame?.targetId, samples: match.samples })) }
+      });
+    }
+
+    const chosen = matches[0] .context;
+    const expression = `(${interactionFunction.toString()})(${JSON.stringify(input)})`;
+    const value = unwrapRuntimeValue(await evaluate(session, chosen, expression, true));
+    if (!value || typeof value !== 'object') {
+      throw new OperatorError('BROWSER_INTERACTION_FAILED', 'Browser interaction returned no semantic result.', { retryable: true });
+    }
+    return {
+      value: value           ,
+      ...(chosen.frame ? { frame: { targetId: chosen.frame.targetId, url: chosen.frame.url, depth: chosen.frame.depth } } : {})
+    };
+  } finally {
+    await scope.stop();
+  }
+}