@qontinui/ui-bridge-wrapper 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -0
- package/dist/capture-specs-cli.cjs +135 -5
- package/dist/capture-specs-cli.cjs.map +1 -1
- package/dist/chunk-IKHDNIU4.js +478 -0
- package/dist/chunk-IKHDNIU4.js.map +1 -0
- package/dist/chunk-Z32CC665.cjs +488 -0
- package/dist/chunk-Z32CC665.cjs.map +1 -0
- package/dist/create-transport.d.ts.map +1 -1
- package/dist/frame-capture.d.ts +3 -28
- package/dist/frame-capture.d.ts.map +1 -1
- package/dist/frame-page.d.ts +35 -0
- package/dist/frame-page.d.ts.map +1 -0
- package/dist/index.cjs +64 -375
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -349
- package/dist/index.js.map +1 -1
- package/dist/inject-cli.cjs +131 -23
- package/dist/inject-cli.cjs.map +1 -1
- package/dist/live/index.cjs +40 -0
- package/dist/live/index.cjs.map +1 -0
- package/dist/live/index.d.ts +32 -0
- package/dist/live/index.d.ts.map +1 -0
- package/dist/live/index.js +3 -0
- package/dist/live/index.js.map +1 -0
- package/dist/login-web-cli.cjs +139 -9
- package/dist/login-web-cli.cjs.map +1 -1
- package/dist/screenshot-capture.d.ts +82 -0
- package/dist/screenshot-capture.d.ts.map +1 -0
- package/dist/timeline-capture.d.ts +6 -8
- package/dist/timeline-capture.d.ts.map +1 -1
- package/dist/transports/live.d.ts +27 -1
- package/dist/transports/live.d.ts.map +1 -1
- package/package.json +8 -3
package/README.md
CHANGED
|
@@ -39,6 +39,44 @@ await transport.dispatch('login', { email: 'a@b.c', password: '...' });
|
|
|
39
39
|
await transport.close();
|
|
40
40
|
```
|
|
41
41
|
|
|
42
|
+
## A live session the runner can capture
|
|
43
|
+
|
|
44
|
+
The runner's vision routes (`POST /ui-bridge/vision/capture` and the analyze,
|
|
45
|
+
OCR and audit routes beside it) frame a WebSocket-transport app by relaying
|
|
46
|
+
`getScreenshot` over its socket — and only to an app whose register frame
|
|
47
|
+
advertises the `screenshot` capability. A Node process that owns a Playwright
|
|
48
|
+
page opts in by handing that page to the live transport:
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
import WebSocket from 'ws';
|
|
52
|
+
import { createTransport } from '@qontinui/ui-bridge-wrapper';
|
|
53
|
+
|
|
54
|
+
const transport = createTransport({
|
|
55
|
+
kind: 'live',
|
|
56
|
+
runnerUrl: 'ws://127.0.0.1:9876/ui-bridge/ws',
|
|
57
|
+
appId: 'my-app',
|
|
58
|
+
options: { webSocketCtor: WebSocket, screenshotPage: page },
|
|
59
|
+
});
|
|
60
|
+
await transport.ready();
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
`screenshotPage` registers the `getScreenshot` handler and advertises
|
|
64
|
+
`screenshot` together, so the advertisement cannot drift from the behaviour.
|
|
65
|
+
The reply is `{ screenshot: <base64 png>, width, height }` (CSS pixels); a
|
|
66
|
+
capture is bounded at 10 s (`SCREENSHOT_TIMEOUT`, retryable) and a failed one
|
|
67
|
+
answers a typed `SCREENSHOT_FAILED`. Without `screenshotPage` the runner refuses
|
|
68
|
+
the app as a capture target by name (`FRAME_SOURCE_UNSUPPORTED`).
|
|
69
|
+
`registerScreenshotCapture(registrar, page)` registers the same handler on any
|
|
70
|
+
other transport that owns a page.
|
|
71
|
+
|
|
72
|
+
A browser bundle (an extension content script, a page) imports the
|
|
73
|
+
**browser-safe** entry `@qontinui/ui-bridge-wrapper/live` instead of the
|
|
74
|
+
package root: the root re-exports Node-only transports and cannot load where
|
|
75
|
+
there is no `require`. Both entries share one copy of `WrapperTransportError`,
|
|
76
|
+
`HandlerRegistry` and `LiveSessionTransport` within a module format, so they
|
|
77
|
+
may be mixed in one ESM (or one CJS) program — but not a `require` of one with
|
|
78
|
+
an `import` of the other, which load separate copies.
|
|
79
|
+
|
|
42
80
|
## Package layout
|
|
43
81
|
|
|
44
82
|
- `src/index.ts` — public surface (transports, helpers, types).
|
|
@@ -759,6 +759,113 @@ function isRegistrationMetadata(value) {
|
|
|
759
759
|
return typeof v.userId === "string" && typeof v.sessionId === "string";
|
|
760
760
|
}
|
|
761
761
|
|
|
762
|
+
// src/frame-page.ts
|
|
763
|
+
var PNG_SIGNATURE = [137, 80, 78, 71, 13, 10, 26, 10];
|
|
764
|
+
function pngDimensions(buf) {
|
|
765
|
+
if (buf.length < 24 || PNG_SIGNATURE.some((b, i) => buf[i] !== b)) {
|
|
766
|
+
throw new Error("not a PNG (signature mismatch or truncated header)");
|
|
767
|
+
}
|
|
768
|
+
const chunkType = String.fromCharCode(buf[12], buf[13], buf[14], buf[15]);
|
|
769
|
+
if (chunkType !== "IHDR") {
|
|
770
|
+
throw new Error(`PNG does not start with an IHDR chunk (found ${JSON.stringify(chunkType)})`);
|
|
771
|
+
}
|
|
772
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
773
|
+
return { width: view.getUint32(16), height: view.getUint32(20) };
|
|
774
|
+
}
|
|
775
|
+
async function readPageMetrics(page) {
|
|
776
|
+
return page.evaluate(() => ({
|
|
777
|
+
devicePixelRatio: window.devicePixelRatio,
|
|
778
|
+
innerWidth: window.innerWidth,
|
|
779
|
+
innerHeight: window.innerHeight,
|
|
780
|
+
colorScheme: window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light"
|
|
781
|
+
}));
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
// src/screenshot-capture.ts
|
|
785
|
+
var SCREENSHOT_CAPABILITY = "screenshot";
|
|
786
|
+
var SCREENSHOT_ACTION = "getScreenshot";
|
|
787
|
+
var SCREENSHOT_TIMEOUT_MS = 1e4;
|
|
788
|
+
var NO_COMPOSITOR_FRAME = /Unable to capture screenshot/;
|
|
789
|
+
var NO_COMPOSITOR_FRAME_RETRY_MS = 16;
|
|
790
|
+
function toBase64(bytes) {
|
|
791
|
+
const CHUNK = 32768;
|
|
792
|
+
let binary = "";
|
|
793
|
+
for (let i = 0; i < bytes.length; i += CHUNK) {
|
|
794
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
|
|
795
|
+
}
|
|
796
|
+
return btoa(binary);
|
|
797
|
+
}
|
|
798
|
+
function errorText(err) {
|
|
799
|
+
return err instanceof Error ? err.message : String(err);
|
|
800
|
+
}
|
|
801
|
+
async function captureScreenshotReply(page, options = {}) {
|
|
802
|
+
const timeoutMs = options.timeoutMs ?? SCREENSHOT_TIMEOUT_MS;
|
|
803
|
+
const retryMs = options.retryMs ?? NO_COMPOSITOR_FRAME_RETRY_MS;
|
|
804
|
+
const deadline = Date.now() + timeoutMs;
|
|
805
|
+
let timedOut = false;
|
|
806
|
+
let timer;
|
|
807
|
+
const attempt = async () => {
|
|
808
|
+
let metrics;
|
|
809
|
+
try {
|
|
810
|
+
metrics = await readPageMetrics(page);
|
|
811
|
+
} catch (err) {
|
|
812
|
+
throw new WrapperTransportError(
|
|
813
|
+
"SCREENSHOT_FAILED",
|
|
814
|
+
`could not read the page's viewport: ${errorText(err)}`,
|
|
815
|
+
{ cause: err }
|
|
816
|
+
);
|
|
817
|
+
}
|
|
818
|
+
for (; ; ) {
|
|
819
|
+
let png;
|
|
820
|
+
try {
|
|
821
|
+
png = await page.screenshot({ type: "png", scale: "device", fullPage: false });
|
|
822
|
+
} catch (err) {
|
|
823
|
+
const text = errorText(err);
|
|
824
|
+
const noFrameYet = NO_COMPOSITOR_FRAME.test(text);
|
|
825
|
+
if (noFrameYet && !timedOut && Date.now() + retryMs < deadline) {
|
|
826
|
+
await new Promise((r) => setTimeout(r, retryMs));
|
|
827
|
+
if (timedOut) throw err;
|
|
828
|
+
continue;
|
|
829
|
+
}
|
|
830
|
+
throw new WrapperTransportError("SCREENSHOT_FAILED", `screenshot failed: ${text}`, {
|
|
831
|
+
retryable: noFrameYet,
|
|
832
|
+
cause: err
|
|
833
|
+
});
|
|
834
|
+
}
|
|
835
|
+
try {
|
|
836
|
+
pngDimensions(png);
|
|
837
|
+
} catch (err) {
|
|
838
|
+
throw new WrapperTransportError(
|
|
839
|
+
"SCREENSHOT_FAILED",
|
|
840
|
+
`the screenshot is unreadable: ${errorText(err)}`,
|
|
841
|
+
{ cause: err }
|
|
842
|
+
);
|
|
843
|
+
}
|
|
844
|
+
return { screenshot: toBase64(png), width: metrics.innerWidth, height: metrics.innerHeight };
|
|
845
|
+
}
|
|
846
|
+
};
|
|
847
|
+
const timeout = new Promise((_, reject) => {
|
|
848
|
+
timer = setTimeout(() => {
|
|
849
|
+
timedOut = true;
|
|
850
|
+
reject(
|
|
851
|
+
new WrapperTransportError(
|
|
852
|
+
"SCREENSHOT_TIMEOUT",
|
|
853
|
+
`screenshot did not complete within ${timeoutMs}ms`,
|
|
854
|
+
{ retryable: true }
|
|
855
|
+
)
|
|
856
|
+
);
|
|
857
|
+
}, timeoutMs);
|
|
858
|
+
});
|
|
859
|
+
try {
|
|
860
|
+
return await Promise.race([attempt(), timeout]);
|
|
861
|
+
} finally {
|
|
862
|
+
clearTimeout(timer);
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
function registerScreenshotCapture(registrar, page, options = {}) {
|
|
866
|
+
registrar.register(SCREENSHOT_ACTION, () => captureScreenshotReply(page, options));
|
|
867
|
+
}
|
|
868
|
+
|
|
762
869
|
// src/transports/live.ts
|
|
763
870
|
var LiveSessionTransport = class extends BaseTransport {
|
|
764
871
|
kind = "live";
|
|
@@ -778,6 +885,16 @@ var LiveSessionTransport = class extends BaseTransport {
|
|
|
778
885
|
this.appId = config.appId;
|
|
779
886
|
this.appName = config.appName;
|
|
780
887
|
this.opts = config.options ?? {};
|
|
888
|
+
const page = this.opts.screenshotPage;
|
|
889
|
+
if (page !== void 0) {
|
|
890
|
+
if (page === null || typeof page.screenshot !== "function" || typeof page.evaluate !== "function") {
|
|
891
|
+
throw new WrapperTransportError(
|
|
892
|
+
"INVALID_CONFIG",
|
|
893
|
+
`options.screenshotPage must expose screenshot() and evaluate() (a Playwright Page does)`
|
|
894
|
+
);
|
|
895
|
+
}
|
|
896
|
+
registerScreenshotCapture(this.registry, page);
|
|
897
|
+
}
|
|
781
898
|
const fromInjection = injected?.webSocketCtor;
|
|
782
899
|
const fromGlobal = typeof globalThis.WebSocket === "function" ? globalThis.WebSocket : void 0;
|
|
783
900
|
const resolved = fromInjection ?? fromGlobal;
|
|
@@ -807,7 +924,8 @@ var LiveSessionTransport = class extends BaseTransport {
|
|
|
807
924
|
};
|
|
808
925
|
if (this.opts.framework !== void 0) frame.framework = this.opts.framework;
|
|
809
926
|
if (this.opts.version !== void 0) frame.version = this.opts.version;
|
|
810
|
-
|
|
927
|
+
const capabilities = this.advertisedCapabilities();
|
|
928
|
+
if (capabilities !== void 0) frame.capabilities = capabilities;
|
|
811
929
|
if (this.opts.origin !== void 0) frame.origin = this.opts.origin;
|
|
812
930
|
if (this.opts.pageUrl !== void 0) frame.pageUrl = this.opts.pageUrl;
|
|
813
931
|
ws.send(JSON.stringify(frame));
|
|
@@ -847,6 +965,18 @@ var LiveSessionTransport = class extends BaseTransport {
|
|
|
847
965
|
}
|
|
848
966
|
}
|
|
849
967
|
}
|
|
968
|
+
/**
|
|
969
|
+
* The register frame's `capabilities`: the caller's list, plus
|
|
970
|
+
* `"screenshot"` exactly when this transport registered the handler for it.
|
|
971
|
+
* `undefined` (field omitted) when there is nothing to advertise.
|
|
972
|
+
*/
|
|
973
|
+
advertisedCapabilities() {
|
|
974
|
+
const declared = this.opts.capabilities;
|
|
975
|
+
if (this.opts.screenshotPage === void 0) return declared;
|
|
976
|
+
const list = [...declared ?? []];
|
|
977
|
+
if (!list.includes(SCREENSHOT_CAPABILITY)) list.push(SCREENSHOT_CAPABILITY);
|
|
978
|
+
return list;
|
|
979
|
+
}
|
|
850
980
|
async buildContext() {
|
|
851
981
|
return { kind: "live", commandId: null };
|
|
852
982
|
}
|
|
@@ -1148,10 +1278,10 @@ async function loginDrive(page, { email, password, timeoutMs, log = () => {
|
|
|
1148
1278
|
{ timeout: timeoutMs }
|
|
1149
1279
|
);
|
|
1150
1280
|
} catch {
|
|
1151
|
-
const
|
|
1281
|
+
const errorText3 = await scrapeErrorCard(page);
|
|
1152
1282
|
const finalUrl2 = page.url();
|
|
1153
1283
|
log(`stalled ${finalUrl2}`);
|
|
1154
|
-
return { ok: false, finalUrl: finalUrl2, landingPath: pathOf(finalUrl2), errorText:
|
|
1284
|
+
return { ok: false, finalUrl: finalUrl2, landingPath: pathOf(finalUrl2), errorText: errorText3 };
|
|
1155
1285
|
}
|
|
1156
1286
|
}
|
|
1157
1287
|
await page.waitForLoadState("networkidle").catch(() => {
|
|
@@ -1165,8 +1295,8 @@ async function loginDrive(page, { email, password, timeoutMs, log = () => {
|
|
|
1165
1295
|
const landingPath = await page.evaluate(() => location.pathname).catch(() => pathOf(finalUrl));
|
|
1166
1296
|
const ok = !isAppLoginPath(finalUrl) && !isCognito(finalUrl) && !/\/auth\/callback/.test(pathOf(finalUrl));
|
|
1167
1297
|
log(ok ? `authed ${finalUrl}` : `stalled ${finalUrl}`);
|
|
1168
|
-
const
|
|
1169
|
-
return { ok, finalUrl, landingPath, errorText };
|
|
1298
|
+
const errorText2 = ok ? null : await scrapeErrorCard(page);
|
|
1299
|
+
return { ok, finalUrl, landingPath, errorText: errorText2 };
|
|
1170
1300
|
}
|
|
1171
1301
|
|
|
1172
1302
|
// src/action-outcome.ts
|