@qontinui/ui-bridge-wrapper 0.14.0 → 0.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -39,6 +39,45 @@ await transport.dispatch('login', { email: 'a@b.c', password: '...' });
39
39
  await transport.close();
40
40
  ```
41
41
 
42
+ ## A live session the runner can capture
43
+
44
+ The runner's vision routes (`POST /ui-bridge/vision/capture` and the analyze,
45
+ OCR and audit routes beside it) frame a WebSocket-transport app by relaying
46
+ `getScreenshot` over its socket — and only to an app whose register frame
47
+ advertises the `screenshot` capability. A Node process that owns a Playwright
48
+ page opts in by handing that page to the live transport:
49
+
50
+ ```ts
51
+ import WebSocket from 'ws';
52
+ import { createTransport } from '@qontinui/ui-bridge-wrapper';
53
+
54
+ const transport = createTransport({
55
+ kind: 'live',
56
+ runnerUrl: 'ws://127.0.0.1:9876/ui-bridge/ws',
57
+ appId: 'my-app',
58
+ appName: 'My app', // optional; defaults to appId (the runner requires both)
59
+ options: { webSocketCtor: WebSocket, screenshotPage: page },
60
+ });
61
+ await transport.ready();
62
+ ```
63
+
64
+ `screenshotPage` registers the `getScreenshot` handler and advertises
65
+ `screenshot` together, so the advertisement cannot drift from the behaviour.
66
+ The reply is `{ screenshot: <base64 png>, width, height }` (CSS pixels); a
67
+ capture is bounded at 10 s (`SCREENSHOT_TIMEOUT`, retryable) and a failed one
68
+ answers a typed `SCREENSHOT_FAILED`. Without `screenshotPage` the runner refuses
69
+ the app as a capture target by name (`FRAME_SOURCE_UNSUPPORTED`).
70
+ `registerScreenshotCapture(registrar, page)` registers the same handler on any
71
+ other transport that owns a page.
72
+
73
+ A browser bundle (an extension content script, a page) imports the
74
+ **browser-safe** entry `@qontinui/ui-bridge-wrapper/live` instead of the
75
+ package root: the root re-exports Node-only transports and cannot load where
76
+ there is no `require`. Both entries share one copy of `WrapperTransportError`,
77
+ `HandlerRegistry` and `LiveSessionTransport` within a module format, so they
78
+ may be mixed in one ESM (or one CJS) program — but not a `require` of one with
79
+ an `import` of the other, which load separate copies.
80
+
42
81
  ## Package layout
43
82
 
44
83
  - `src/index.ts` — public surface (transports, helpers, types).
@@ -759,6 +759,113 @@ function isRegistrationMetadata(value) {
759
759
  return typeof v.userId === "string" && typeof v.sessionId === "string";
760
760
  }
761
761
 
762
+ // src/frame-page.ts
763
+ var PNG_SIGNATURE = [137, 80, 78, 71, 13, 10, 26, 10];
764
+ function pngDimensions(buf) {
765
+ if (buf.length < 24 || PNG_SIGNATURE.some((b, i) => buf[i] !== b)) {
766
+ throw new Error("not a PNG (signature mismatch or truncated header)");
767
+ }
768
+ const chunkType = String.fromCharCode(buf[12], buf[13], buf[14], buf[15]);
769
+ if (chunkType !== "IHDR") {
770
+ throw new Error(`PNG does not start with an IHDR chunk (found ${JSON.stringify(chunkType)})`);
771
+ }
772
+ const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
773
+ return { width: view.getUint32(16), height: view.getUint32(20) };
774
+ }
775
+ async function readPageMetrics(page) {
776
+ return page.evaluate(() => ({
777
+ devicePixelRatio: window.devicePixelRatio,
778
+ innerWidth: window.innerWidth,
779
+ innerHeight: window.innerHeight,
780
+ colorScheme: window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light"
781
+ }));
782
+ }
783
+
784
+ // src/screenshot-capture.ts
785
+ var SCREENSHOT_CAPABILITY = "screenshot";
786
+ var SCREENSHOT_ACTION = "getScreenshot";
787
+ var SCREENSHOT_TIMEOUT_MS = 1e4;
788
+ var NO_COMPOSITOR_FRAME = /Unable to capture screenshot/;
789
+ var NO_COMPOSITOR_FRAME_RETRY_MS = 16;
790
+ function toBase64(bytes) {
791
+ const CHUNK = 32768;
792
+ let binary = "";
793
+ for (let i = 0; i < bytes.length; i += CHUNK) {
794
+ binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
795
+ }
796
+ return btoa(binary);
797
+ }
798
+ function errorText(err) {
799
+ return err instanceof Error ? err.message : String(err);
800
+ }
801
+ async function captureScreenshotReply(page, options = {}) {
802
+ const timeoutMs = options.timeoutMs ?? SCREENSHOT_TIMEOUT_MS;
803
+ const retryMs = options.retryMs ?? NO_COMPOSITOR_FRAME_RETRY_MS;
804
+ const deadline = Date.now() + timeoutMs;
805
+ let timedOut = false;
806
+ let timer;
807
+ const attempt = async () => {
808
+ let metrics;
809
+ try {
810
+ metrics = await readPageMetrics(page);
811
+ } catch (err) {
812
+ throw new WrapperTransportError(
813
+ "SCREENSHOT_FAILED",
814
+ `could not read the page's viewport: ${errorText(err)}`,
815
+ { cause: err }
816
+ );
817
+ }
818
+ for (; ; ) {
819
+ let png;
820
+ try {
821
+ png = await page.screenshot({ type: "png", scale: "device", fullPage: false });
822
+ } catch (err) {
823
+ const text = errorText(err);
824
+ const noFrameYet = NO_COMPOSITOR_FRAME.test(text);
825
+ if (noFrameYet && !timedOut && Date.now() + retryMs < deadline) {
826
+ await new Promise((r) => setTimeout(r, retryMs));
827
+ if (timedOut) throw err;
828
+ continue;
829
+ }
830
+ throw new WrapperTransportError("SCREENSHOT_FAILED", `screenshot failed: ${text}`, {
831
+ retryable: noFrameYet,
832
+ cause: err
833
+ });
834
+ }
835
+ try {
836
+ pngDimensions(png);
837
+ } catch (err) {
838
+ throw new WrapperTransportError(
839
+ "SCREENSHOT_FAILED",
840
+ `the screenshot is unreadable: ${errorText(err)}`,
841
+ { cause: err }
842
+ );
843
+ }
844
+ return { screenshot: toBase64(png), width: metrics.innerWidth, height: metrics.innerHeight };
845
+ }
846
+ };
847
+ const timeout = new Promise((_, reject) => {
848
+ timer = setTimeout(() => {
849
+ timedOut = true;
850
+ reject(
851
+ new WrapperTransportError(
852
+ "SCREENSHOT_TIMEOUT",
853
+ `screenshot did not complete within ${timeoutMs}ms`,
854
+ { retryable: true }
855
+ )
856
+ );
857
+ }, timeoutMs);
858
+ });
859
+ try {
860
+ return await Promise.race([attempt(), timeout]);
861
+ } finally {
862
+ clearTimeout(timer);
863
+ }
864
+ }
865
+ function registerScreenshotCapture(registrar, page, options = {}) {
866
+ registrar.register(SCREENSHOT_ACTION, () => captureScreenshotReply(page, options));
867
+ }
868
+
762
869
  // src/transports/live.ts
763
870
  var LiveSessionTransport = class extends BaseTransport {
764
871
  kind = "live";
@@ -774,10 +881,26 @@ var LiveSessionTransport = class extends BaseTransport {
774
881
  if (!config.runnerUrl) {
775
882
  throw new WrapperTransportError("INVALID_CONFIG", `Transport kind 'live' requires runnerUrl`);
776
883
  }
884
+ if (!config.appId?.trim()) {
885
+ throw new WrapperTransportError(
886
+ "INVALID_CONFIG",
887
+ `Transport kind 'live' requires a non-blank appId (the runner refuses a register frame without one)`
888
+ );
889
+ }
777
890
  this.url = config.runnerUrl;
778
891
  this.appId = config.appId;
779
892
  this.appName = config.appName;
780
893
  this.opts = config.options ?? {};
894
+ const page = this.opts.screenshotPage;
895
+ if (page !== void 0) {
896
+ if (page === null || typeof page.screenshot !== "function" || typeof page.evaluate !== "function") {
897
+ throw new WrapperTransportError(
898
+ "INVALID_CONFIG",
899
+ `options.screenshotPage must expose screenshot() and evaluate() (a Playwright Page does)`
900
+ );
901
+ }
902
+ registerScreenshotCapture(this.registry, page);
903
+ }
781
904
  const fromInjection = injected?.webSocketCtor;
782
905
  const fromGlobal = typeof globalThis.WebSocket === "function" ? globalThis.WebSocket : void 0;
783
906
  const resolved = fromInjection ?? fromGlobal;
@@ -802,12 +925,14 @@ var LiveSessionTransport = class extends BaseTransport {
802
925
  type: "register",
803
926
  transport: "websocket",
804
927
  appId: this.appId,
805
- appName: this.appName,
928
+ // appName defaults to appId: the runner requires a non-blank one.
929
+ appName: this.appName?.trim() || this.appId,
806
930
  appType: this.opts.appType ?? "wrapper"
807
931
  };
808
932
  if (this.opts.framework !== void 0) frame.framework = this.opts.framework;
809
933
  if (this.opts.version !== void 0) frame.version = this.opts.version;
810
- if (this.opts.capabilities !== void 0) frame.capabilities = this.opts.capabilities;
934
+ const capabilities = this.advertisedCapabilities();
935
+ if (capabilities !== void 0) frame.capabilities = capabilities;
811
936
  if (this.opts.origin !== void 0) frame.origin = this.opts.origin;
812
937
  if (this.opts.pageUrl !== void 0) frame.pageUrl = this.opts.pageUrl;
813
938
  ws.send(JSON.stringify(frame));
@@ -847,6 +972,18 @@ var LiveSessionTransport = class extends BaseTransport {
847
972
  }
848
973
  }
849
974
  }
975
+ /**
976
+ * The register frame's `capabilities`: the caller's list, plus
977
+ * `"screenshot"` exactly when this transport registered the handler for it.
978
+ * `undefined` (field omitted) when there is nothing to advertise.
979
+ */
980
+ advertisedCapabilities() {
981
+ const declared = this.opts.capabilities;
982
+ if (this.opts.screenshotPage === void 0) return declared;
983
+ const list = [...declared ?? []];
984
+ if (!list.includes(SCREENSHOT_CAPABILITY)) list.push(SCREENSHOT_CAPABILITY);
985
+ return list;
986
+ }
850
987
  async buildContext() {
851
988
  return { kind: "live", commandId: null };
852
989
  }
@@ -1148,10 +1285,10 @@ async function loginDrive(page, { email, password, timeoutMs, log = () => {
1148
1285
  { timeout: timeoutMs }
1149
1286
  );
1150
1287
  } catch {
1151
- const errorText2 = await scrapeErrorCard(page);
1288
+ const errorText3 = await scrapeErrorCard(page);
1152
1289
  const finalUrl2 = page.url();
1153
1290
  log(`stalled ${finalUrl2}`);
1154
- return { ok: false, finalUrl: finalUrl2, landingPath: pathOf(finalUrl2), errorText: errorText2 };
1291
+ return { ok: false, finalUrl: finalUrl2, landingPath: pathOf(finalUrl2), errorText: errorText3 };
1155
1292
  }
1156
1293
  }
1157
1294
  await page.waitForLoadState("networkidle").catch(() => {
@@ -1165,8 +1302,8 @@ async function loginDrive(page, { email, password, timeoutMs, log = () => {
1165
1302
  const landingPath = await page.evaluate(() => location.pathname).catch(() => pathOf(finalUrl));
1166
1303
  const ok = !isAppLoginPath(finalUrl) && !isCognito(finalUrl) && !/\/auth\/callback/.test(pathOf(finalUrl));
1167
1304
  log(ok ? `authed ${finalUrl}` : `stalled ${finalUrl}`);
1168
- const errorText = ok ? null : await scrapeErrorCard(page);
1169
- return { ok, finalUrl, landingPath, errorText };
1305
+ const errorText2 = ok ? null : await scrapeErrorCard(page);
1306
+ return { ok, finalUrl, landingPath, errorText: errorText2 };
1170
1307
  }
1171
1308
 
1172
1309
  // src/action-outcome.ts