@qontinui/ui-bridge-wrapper 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -39,6 +39,44 @@ await transport.dispatch('login', { email: 'a@b.c', password: '...' });
39
39
  await transport.close();
40
40
  ```
41
41
 
42
+ ## A live session the runner can capture
43
+
44
+ The runner's vision routes (`POST /ui-bridge/vision/capture` and the analyze,
45
+ OCR and audit routes beside it) frame a WebSocket-transport app by relaying
46
+ `getScreenshot` over its socket — and only to an app whose register frame
47
+ advertises the `screenshot` capability. A Node process that owns a Playwright
48
+ page opts in by handing that page to the live transport:
49
+
50
+ ```ts
51
+ import WebSocket from 'ws';
52
+ import { createTransport } from '@qontinui/ui-bridge-wrapper';
53
+
54
+ const transport = createTransport({
55
+ kind: 'live',
56
+ runnerUrl: 'ws://127.0.0.1:9876/ui-bridge/ws',
57
+ appId: 'my-app',
58
+ options: { webSocketCtor: WebSocket, screenshotPage: page },
59
+ });
60
+ await transport.ready();
61
+ ```
62
+
63
+ `screenshotPage` registers the `getScreenshot` handler and advertises
64
+ `screenshot` together, so the advertisement cannot drift from the behaviour.
65
+ The reply is `{ screenshot: <base64 png>, width, height }` (CSS pixels); a
66
+ capture is bounded at 10 s (`SCREENSHOT_TIMEOUT`, retryable) and a failed one
67
+ answers a typed `SCREENSHOT_FAILED`. Without `screenshotPage` the runner refuses
68
+ the app as a capture target by name (`FRAME_SOURCE_UNSUPPORTED`).
69
+ `registerScreenshotCapture(registrar, page)` registers the same handler on any
70
+ other transport that owns a page.
71
+
72
+ A browser bundle (an extension content script, a page) imports the
73
+ **browser-safe** entry `@qontinui/ui-bridge-wrapper/live` instead of the
74
+ package root: the root re-exports Node-only transports and cannot load where
75
+ there is no `require`. Both entries share one copy of `WrapperTransportError`,
76
+ `HandlerRegistry` and `LiveSessionTransport` within a module format, so they
77
+ may be mixed in one ESM (or one CJS) program — but not a `require` of one with
78
+ an `import` of the other, which load separate copies.
79
+
42
80
  ## Package layout
43
81
 
44
82
  - `src/index.ts` — public surface (transports, helpers, types).
@@ -759,6 +759,113 @@ function isRegistrationMetadata(value) {
759
759
  return typeof v.userId === "string" && typeof v.sessionId === "string";
760
760
  }
761
761
 
762
+ // src/frame-page.ts
763
+ var PNG_SIGNATURE = [137, 80, 78, 71, 13, 10, 26, 10];
764
+ function pngDimensions(buf) {
765
+ if (buf.length < 24 || PNG_SIGNATURE.some((b, i) => buf[i] !== b)) {
766
+ throw new Error("not a PNG (signature mismatch or truncated header)");
767
+ }
768
+ const chunkType = String.fromCharCode(buf[12], buf[13], buf[14], buf[15]);
769
+ if (chunkType !== "IHDR") {
770
+ throw new Error(`PNG does not start with an IHDR chunk (found ${JSON.stringify(chunkType)})`);
771
+ }
772
+ const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
773
+ return { width: view.getUint32(16), height: view.getUint32(20) };
774
+ }
775
+ async function readPageMetrics(page) {
776
+ return page.evaluate(() => ({
777
+ devicePixelRatio: window.devicePixelRatio,
778
+ innerWidth: window.innerWidth,
779
+ innerHeight: window.innerHeight,
780
+ colorScheme: window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light"
781
+ }));
782
+ }
783
+
784
+ // src/screenshot-capture.ts
785
+ var SCREENSHOT_CAPABILITY = "screenshot";
786
+ var SCREENSHOT_ACTION = "getScreenshot";
787
+ var SCREENSHOT_TIMEOUT_MS = 1e4;
788
+ var NO_COMPOSITOR_FRAME = /Unable to capture screenshot/;
789
+ var NO_COMPOSITOR_FRAME_RETRY_MS = 16;
790
+ function toBase64(bytes) {
791
+ const CHUNK = 32768;
792
+ let binary = "";
793
+ for (let i = 0; i < bytes.length; i += CHUNK) {
794
+ binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
795
+ }
796
+ return btoa(binary);
797
+ }
798
+ function errorText(err) {
799
+ return err instanceof Error ? err.message : String(err);
800
+ }
801
+ async function captureScreenshotReply(page, options = {}) {
802
+ const timeoutMs = options.timeoutMs ?? SCREENSHOT_TIMEOUT_MS;
803
+ const retryMs = options.retryMs ?? NO_COMPOSITOR_FRAME_RETRY_MS;
804
+ const deadline = Date.now() + timeoutMs;
805
+ let timedOut = false;
806
+ let timer;
807
+ const attempt = async () => {
808
+ let metrics;
809
+ try {
810
+ metrics = await readPageMetrics(page);
811
+ } catch (err) {
812
+ throw new WrapperTransportError(
813
+ "SCREENSHOT_FAILED",
814
+ `could not read the page's viewport: ${errorText(err)}`,
815
+ { cause: err }
816
+ );
817
+ }
818
+ for (; ; ) {
819
+ let png;
820
+ try {
821
+ png = await page.screenshot({ type: "png", scale: "device", fullPage: false });
822
+ } catch (err) {
823
+ const text = errorText(err);
824
+ const noFrameYet = NO_COMPOSITOR_FRAME.test(text);
825
+ if (noFrameYet && !timedOut && Date.now() + retryMs < deadline) {
826
+ await new Promise((r) => setTimeout(r, retryMs));
827
+ if (timedOut) throw err;
828
+ continue;
829
+ }
830
+ throw new WrapperTransportError("SCREENSHOT_FAILED", `screenshot failed: ${text}`, {
831
+ retryable: noFrameYet,
832
+ cause: err
833
+ });
834
+ }
835
+ try {
836
+ pngDimensions(png);
837
+ } catch (err) {
838
+ throw new WrapperTransportError(
839
+ "SCREENSHOT_FAILED",
840
+ `the screenshot is unreadable: ${errorText(err)}`,
841
+ { cause: err }
842
+ );
843
+ }
844
+ return { screenshot: toBase64(png), width: metrics.innerWidth, height: metrics.innerHeight };
845
+ }
846
+ };
847
+ const timeout = new Promise((_, reject) => {
848
+ timer = setTimeout(() => {
849
+ timedOut = true;
850
+ reject(
851
+ new WrapperTransportError(
852
+ "SCREENSHOT_TIMEOUT",
853
+ `screenshot did not complete within ${timeoutMs}ms`,
854
+ { retryable: true }
855
+ )
856
+ );
857
+ }, timeoutMs);
858
+ });
859
+ try {
860
+ return await Promise.race([attempt(), timeout]);
861
+ } finally {
862
+ clearTimeout(timer);
863
+ }
864
+ }
865
+ function registerScreenshotCapture(registrar, page, options = {}) {
866
+ registrar.register(SCREENSHOT_ACTION, () => captureScreenshotReply(page, options));
867
+ }
868
+
762
869
  // src/transports/live.ts
763
870
  var LiveSessionTransport = class extends BaseTransport {
764
871
  kind = "live";
@@ -778,6 +885,16 @@ var LiveSessionTransport = class extends BaseTransport {
778
885
  this.appId = config.appId;
779
886
  this.appName = config.appName;
780
887
  this.opts = config.options ?? {};
888
+ const page = this.opts.screenshotPage;
889
+ if (page !== void 0) {
890
+ if (page === null || typeof page.screenshot !== "function" || typeof page.evaluate !== "function") {
891
+ throw new WrapperTransportError(
892
+ "INVALID_CONFIG",
893
+ `options.screenshotPage must expose screenshot() and evaluate() (a Playwright Page does)`
894
+ );
895
+ }
896
+ registerScreenshotCapture(this.registry, page);
897
+ }
781
898
  const fromInjection = injected?.webSocketCtor;
782
899
  const fromGlobal = typeof globalThis.WebSocket === "function" ? globalThis.WebSocket : void 0;
783
900
  const resolved = fromInjection ?? fromGlobal;
@@ -807,7 +924,8 @@ var LiveSessionTransport = class extends BaseTransport {
807
924
  };
808
925
  if (this.opts.framework !== void 0) frame.framework = this.opts.framework;
809
926
  if (this.opts.version !== void 0) frame.version = this.opts.version;
810
- if (this.opts.capabilities !== void 0) frame.capabilities = this.opts.capabilities;
927
+ const capabilities = this.advertisedCapabilities();
928
+ if (capabilities !== void 0) frame.capabilities = capabilities;
811
929
  if (this.opts.origin !== void 0) frame.origin = this.opts.origin;
812
930
  if (this.opts.pageUrl !== void 0) frame.pageUrl = this.opts.pageUrl;
813
931
  ws.send(JSON.stringify(frame));
@@ -847,6 +965,18 @@ var LiveSessionTransport = class extends BaseTransport {
847
965
  }
848
966
  }
849
967
  }
968
+ /**
969
+ * The register frame's `capabilities`: the caller's list, plus
970
+ * `"screenshot"` exactly when this transport registered the handler for it.
971
+ * `undefined` (field omitted) when there is nothing to advertise.
972
+ */
973
+ advertisedCapabilities() {
974
+ const declared = this.opts.capabilities;
975
+ if (this.opts.screenshotPage === void 0) return declared;
976
+ const list = [...declared ?? []];
977
+ if (!list.includes(SCREENSHOT_CAPABILITY)) list.push(SCREENSHOT_CAPABILITY);
978
+ return list;
979
+ }
850
980
  async buildContext() {
851
981
  return { kind: "live", commandId: null };
852
982
  }
@@ -1148,10 +1278,10 @@ async function loginDrive(page, { email, password, timeoutMs, log = () => {
1148
1278
  { timeout: timeoutMs }
1149
1279
  );
1150
1280
  } catch {
1151
- const errorText2 = await scrapeErrorCard(page);
1281
+ const errorText3 = await scrapeErrorCard(page);
1152
1282
  const finalUrl2 = page.url();
1153
1283
  log(`stalled ${finalUrl2}`);
1154
- return { ok: false, finalUrl: finalUrl2, landingPath: pathOf(finalUrl2), errorText: errorText2 };
1284
+ return { ok: false, finalUrl: finalUrl2, landingPath: pathOf(finalUrl2), errorText: errorText3 };
1155
1285
  }
1156
1286
  }
1157
1287
  await page.waitForLoadState("networkidle").catch(() => {
@@ -1165,8 +1295,8 @@ async function loginDrive(page, { email, password, timeoutMs, log = () => {
1165
1295
  const landingPath = await page.evaluate(() => location.pathname).catch(() => pathOf(finalUrl));
1166
1296
  const ok = !isAppLoginPath(finalUrl) && !isCognito(finalUrl) && !/\/auth\/callback/.test(pathOf(finalUrl));
1167
1297
  log(ok ? `authed ${finalUrl}` : `stalled ${finalUrl}`);
1168
- const errorText = ok ? null : await scrapeErrorCard(page);
1169
- return { ok, finalUrl, landingPath, errorText };
1298
+ const errorText2 = ok ? null : await scrapeErrorCard(page);
1299
+ return { ok, finalUrl, landingPath, errorText: errorText2 };
1170
1300
  }
1171
1301
 
1172
1302
  // src/action-outcome.ts