@qontinui/ui-bridge-wrapper 0.14.0 → 0.15.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +39 -0
- package/dist/capture-specs-cli.cjs +143 -6
- package/dist/capture-specs-cli.cjs.map +1 -1
- package/dist/chunk-3EIJDQJO.cjs +495 -0
- package/dist/chunk-3EIJDQJO.cjs.map +1 -0
- package/dist/chunk-WI4MTFWJ.js +485 -0
- package/dist/chunk-WI4MTFWJ.js.map +1 -0
- package/dist/create-transport.d.ts.map +1 -1
- package/dist/frame-capture.d.ts +3 -28
- package/dist/frame-capture.d.ts.map +1 -1
- package/dist/frame-page.d.ts +35 -0
- package/dist/frame-page.d.ts.map +1 -0
- package/dist/index.cjs +64 -375
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -349
- package/dist/index.js.map +1 -1
- package/dist/inject-cli.cjs +139 -24
- package/dist/inject-cli.cjs.map +1 -1
- package/dist/live/index.cjs +40 -0
- package/dist/live/index.cjs.map +1 -0
- package/dist/live/index.d.ts +32 -0
- package/dist/live/index.d.ts.map +1 -0
- package/dist/live/index.js +3 -0
- package/dist/live/index.js.map +1 -0
- package/dist/login-web-cli.cjs +147 -10
- package/dist/login-web-cli.cjs.map +1 -1
- package/dist/screenshot-capture.d.ts +82 -0
- package/dist/screenshot-capture.d.ts.map +1 -0
- package/dist/timeline-capture.d.ts +6 -8
- package/dist/timeline-capture.d.ts.map +1 -1
- package/dist/transports/live.d.ts +27 -1
- package/dist/transports/live.d.ts.map +1 -1
- package/package.json +8 -3
package/README.md
CHANGED
|
@@ -39,6 +39,45 @@ await transport.dispatch('login', { email: 'a@b.c', password: '...' });
|
|
|
39
39
|
await transport.close();
|
|
40
40
|
```
|
|
41
41
|
|
|
42
|
+
## A live session the runner can capture
|
|
43
|
+
|
|
44
|
+
The runner's vision routes (`POST /ui-bridge/vision/capture` and the analyze,
|
|
45
|
+
OCR and audit routes beside it) frame a WebSocket-transport app by relaying
|
|
46
|
+
`getScreenshot` over its socket — and only to an app whose register frame
|
|
47
|
+
advertises the `screenshot` capability. A Node process that owns a Playwright
|
|
48
|
+
page opts in by handing that page to the live transport:
|
|
49
|
+
|
|
50
|
+
```ts
|
|
51
|
+
import WebSocket from 'ws';
|
|
52
|
+
import { createTransport } from '@qontinui/ui-bridge-wrapper';
|
|
53
|
+
|
|
54
|
+
const transport = createTransport({
|
|
55
|
+
kind: 'live',
|
|
56
|
+
runnerUrl: 'ws://127.0.0.1:9876/ui-bridge/ws',
|
|
57
|
+
appId: 'my-app',
|
|
58
|
+
appName: 'My app', // optional; defaults to appId (the runner requires both)
|
|
59
|
+
options: { webSocketCtor: WebSocket, screenshotPage: page },
|
|
60
|
+
});
|
|
61
|
+
await transport.ready();
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
`screenshotPage` registers the `getScreenshot` handler and advertises
|
|
65
|
+
`screenshot` together, so the advertisement cannot drift from the behaviour.
|
|
66
|
+
The reply is `{ screenshot: <base64 png>, width, height }` (CSS pixels); a
|
|
67
|
+
capture is bounded at 10 s (`SCREENSHOT_TIMEOUT`, retryable) and a failed one
|
|
68
|
+
answers a typed `SCREENSHOT_FAILED`. Without `screenshotPage` the runner refuses
|
|
69
|
+
the app as a capture target by name (`FRAME_SOURCE_UNSUPPORTED`).
|
|
70
|
+
`registerScreenshotCapture(registrar, page)` registers the same handler on any
|
|
71
|
+
other transport that owns a page.
|
|
72
|
+
|
|
73
|
+
A browser bundle (an extension content script, a page) imports the
|
|
74
|
+
**browser-safe** entry `@qontinui/ui-bridge-wrapper/live` instead of the
|
|
75
|
+
package root: the root re-exports Node-only transports and cannot load where
|
|
76
|
+
there is no `require`. Both entries share one copy of `WrapperTransportError`,
|
|
77
|
+
`HandlerRegistry` and `LiveSessionTransport` within a module format, so they
|
|
78
|
+
may be mixed in one ESM (or one CJS) program — but not a `require` of one with
|
|
79
|
+
an `import` of the other, which load separate copies.
|
|
80
|
+
|
|
42
81
|
## Package layout
|
|
43
82
|
|
|
44
83
|
- `src/index.ts` — public surface (transports, helpers, types).
|
|
@@ -759,6 +759,113 @@ function isRegistrationMetadata(value) {
|
|
|
759
759
|
return typeof v.userId === "string" && typeof v.sessionId === "string";
|
|
760
760
|
}
|
|
761
761
|
|
|
762
|
+
// src/frame-page.ts
|
|
763
|
+
var PNG_SIGNATURE = [137, 80, 78, 71, 13, 10, 26, 10];
|
|
764
|
+
function pngDimensions(buf) {
|
|
765
|
+
if (buf.length < 24 || PNG_SIGNATURE.some((b, i) => buf[i] !== b)) {
|
|
766
|
+
throw new Error("not a PNG (signature mismatch or truncated header)");
|
|
767
|
+
}
|
|
768
|
+
const chunkType = String.fromCharCode(buf[12], buf[13], buf[14], buf[15]);
|
|
769
|
+
if (chunkType !== "IHDR") {
|
|
770
|
+
throw new Error(`PNG does not start with an IHDR chunk (found ${JSON.stringify(chunkType)})`);
|
|
771
|
+
}
|
|
772
|
+
const view = new DataView(buf.buffer, buf.byteOffset, buf.byteLength);
|
|
773
|
+
return { width: view.getUint32(16), height: view.getUint32(20) };
|
|
774
|
+
}
|
|
775
|
+
async function readPageMetrics(page) {
|
|
776
|
+
return page.evaluate(() => ({
|
|
777
|
+
devicePixelRatio: window.devicePixelRatio,
|
|
778
|
+
innerWidth: window.innerWidth,
|
|
779
|
+
innerHeight: window.innerHeight,
|
|
780
|
+
colorScheme: window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light"
|
|
781
|
+
}));
|
|
782
|
+
}
|
|
783
|
+
|
|
784
|
+
// src/screenshot-capture.ts
|
|
785
|
+
var SCREENSHOT_CAPABILITY = "screenshot";
|
|
786
|
+
var SCREENSHOT_ACTION = "getScreenshot";
|
|
787
|
+
var SCREENSHOT_TIMEOUT_MS = 1e4;
|
|
788
|
+
var NO_COMPOSITOR_FRAME = /Unable to capture screenshot/;
|
|
789
|
+
var NO_COMPOSITOR_FRAME_RETRY_MS = 16;
|
|
790
|
+
function toBase64(bytes) {
|
|
791
|
+
const CHUNK = 32768;
|
|
792
|
+
let binary = "";
|
|
793
|
+
for (let i = 0; i < bytes.length; i += CHUNK) {
|
|
794
|
+
binary += String.fromCharCode(...bytes.subarray(i, i + CHUNK));
|
|
795
|
+
}
|
|
796
|
+
return btoa(binary);
|
|
797
|
+
}
|
|
798
|
+
function errorText(err) {
|
|
799
|
+
return err instanceof Error ? err.message : String(err);
|
|
800
|
+
}
|
|
801
|
+
async function captureScreenshotReply(page, options = {}) {
|
|
802
|
+
const timeoutMs = options.timeoutMs ?? SCREENSHOT_TIMEOUT_MS;
|
|
803
|
+
const retryMs = options.retryMs ?? NO_COMPOSITOR_FRAME_RETRY_MS;
|
|
804
|
+
const deadline = Date.now() + timeoutMs;
|
|
805
|
+
let timedOut = false;
|
|
806
|
+
let timer;
|
|
807
|
+
const attempt = async () => {
|
|
808
|
+
let metrics;
|
|
809
|
+
try {
|
|
810
|
+
metrics = await readPageMetrics(page);
|
|
811
|
+
} catch (err) {
|
|
812
|
+
throw new WrapperTransportError(
|
|
813
|
+
"SCREENSHOT_FAILED",
|
|
814
|
+
`could not read the page's viewport: ${errorText(err)}`,
|
|
815
|
+
{ cause: err }
|
|
816
|
+
);
|
|
817
|
+
}
|
|
818
|
+
for (; ; ) {
|
|
819
|
+
let png;
|
|
820
|
+
try {
|
|
821
|
+
png = await page.screenshot({ type: "png", scale: "device", fullPage: false });
|
|
822
|
+
} catch (err) {
|
|
823
|
+
const text = errorText(err);
|
|
824
|
+
const noFrameYet = NO_COMPOSITOR_FRAME.test(text);
|
|
825
|
+
if (noFrameYet && !timedOut && Date.now() + retryMs < deadline) {
|
|
826
|
+
await new Promise((r) => setTimeout(r, retryMs));
|
|
827
|
+
if (timedOut) throw err;
|
|
828
|
+
continue;
|
|
829
|
+
}
|
|
830
|
+
throw new WrapperTransportError("SCREENSHOT_FAILED", `screenshot failed: ${text}`, {
|
|
831
|
+
retryable: noFrameYet,
|
|
832
|
+
cause: err
|
|
833
|
+
});
|
|
834
|
+
}
|
|
835
|
+
try {
|
|
836
|
+
pngDimensions(png);
|
|
837
|
+
} catch (err) {
|
|
838
|
+
throw new WrapperTransportError(
|
|
839
|
+
"SCREENSHOT_FAILED",
|
|
840
|
+
`the screenshot is unreadable: ${errorText(err)}`,
|
|
841
|
+
{ cause: err }
|
|
842
|
+
);
|
|
843
|
+
}
|
|
844
|
+
return { screenshot: toBase64(png), width: metrics.innerWidth, height: metrics.innerHeight };
|
|
845
|
+
}
|
|
846
|
+
};
|
|
847
|
+
const timeout = new Promise((_, reject) => {
|
|
848
|
+
timer = setTimeout(() => {
|
|
849
|
+
timedOut = true;
|
|
850
|
+
reject(
|
|
851
|
+
new WrapperTransportError(
|
|
852
|
+
"SCREENSHOT_TIMEOUT",
|
|
853
|
+
`screenshot did not complete within ${timeoutMs}ms`,
|
|
854
|
+
{ retryable: true }
|
|
855
|
+
)
|
|
856
|
+
);
|
|
857
|
+
}, timeoutMs);
|
|
858
|
+
});
|
|
859
|
+
try {
|
|
860
|
+
return await Promise.race([attempt(), timeout]);
|
|
861
|
+
} finally {
|
|
862
|
+
clearTimeout(timer);
|
|
863
|
+
}
|
|
864
|
+
}
|
|
865
|
+
function registerScreenshotCapture(registrar, page, options = {}) {
|
|
866
|
+
registrar.register(SCREENSHOT_ACTION, () => captureScreenshotReply(page, options));
|
|
867
|
+
}
|
|
868
|
+
|
|
762
869
|
// src/transports/live.ts
|
|
763
870
|
var LiveSessionTransport = class extends BaseTransport {
|
|
764
871
|
kind = "live";
|
|
@@ -774,10 +881,26 @@ var LiveSessionTransport = class extends BaseTransport {
|
|
|
774
881
|
if (!config.runnerUrl) {
|
|
775
882
|
throw new WrapperTransportError("INVALID_CONFIG", `Transport kind 'live' requires runnerUrl`);
|
|
776
883
|
}
|
|
884
|
+
if (!config.appId?.trim()) {
|
|
885
|
+
throw new WrapperTransportError(
|
|
886
|
+
"INVALID_CONFIG",
|
|
887
|
+
`Transport kind 'live' requires a non-blank appId (the runner refuses a register frame without one)`
|
|
888
|
+
);
|
|
889
|
+
}
|
|
777
890
|
this.url = config.runnerUrl;
|
|
778
891
|
this.appId = config.appId;
|
|
779
892
|
this.appName = config.appName;
|
|
780
893
|
this.opts = config.options ?? {};
|
|
894
|
+
const page = this.opts.screenshotPage;
|
|
895
|
+
if (page !== void 0) {
|
|
896
|
+
if (page === null || typeof page.screenshot !== "function" || typeof page.evaluate !== "function") {
|
|
897
|
+
throw new WrapperTransportError(
|
|
898
|
+
"INVALID_CONFIG",
|
|
899
|
+
`options.screenshotPage must expose screenshot() and evaluate() (a Playwright Page does)`
|
|
900
|
+
);
|
|
901
|
+
}
|
|
902
|
+
registerScreenshotCapture(this.registry, page);
|
|
903
|
+
}
|
|
781
904
|
const fromInjection = injected?.webSocketCtor;
|
|
782
905
|
const fromGlobal = typeof globalThis.WebSocket === "function" ? globalThis.WebSocket : void 0;
|
|
783
906
|
const resolved = fromInjection ?? fromGlobal;
|
|
@@ -802,12 +925,14 @@ var LiveSessionTransport = class extends BaseTransport {
|
|
|
802
925
|
type: "register",
|
|
803
926
|
transport: "websocket",
|
|
804
927
|
appId: this.appId,
|
|
805
|
-
appName:
|
|
928
|
+
// appName defaults to appId: the runner requires a non-blank one.
|
|
929
|
+
appName: this.appName?.trim() || this.appId,
|
|
806
930
|
appType: this.opts.appType ?? "wrapper"
|
|
807
931
|
};
|
|
808
932
|
if (this.opts.framework !== void 0) frame.framework = this.opts.framework;
|
|
809
933
|
if (this.opts.version !== void 0) frame.version = this.opts.version;
|
|
810
|
-
|
|
934
|
+
const capabilities = this.advertisedCapabilities();
|
|
935
|
+
if (capabilities !== void 0) frame.capabilities = capabilities;
|
|
811
936
|
if (this.opts.origin !== void 0) frame.origin = this.opts.origin;
|
|
812
937
|
if (this.opts.pageUrl !== void 0) frame.pageUrl = this.opts.pageUrl;
|
|
813
938
|
ws.send(JSON.stringify(frame));
|
|
@@ -847,6 +972,18 @@ var LiveSessionTransport = class extends BaseTransport {
|
|
|
847
972
|
}
|
|
848
973
|
}
|
|
849
974
|
}
|
|
975
|
+
/**
|
|
976
|
+
* The register frame's `capabilities`: the caller's list, plus
|
|
977
|
+
* `"screenshot"` exactly when this transport registered the handler for it.
|
|
978
|
+
* `undefined` (field omitted) when there is nothing to advertise.
|
|
979
|
+
*/
|
|
980
|
+
advertisedCapabilities() {
|
|
981
|
+
const declared = this.opts.capabilities;
|
|
982
|
+
if (this.opts.screenshotPage === void 0) return declared;
|
|
983
|
+
const list = [...declared ?? []];
|
|
984
|
+
if (!list.includes(SCREENSHOT_CAPABILITY)) list.push(SCREENSHOT_CAPABILITY);
|
|
985
|
+
return list;
|
|
986
|
+
}
|
|
850
987
|
async buildContext() {
|
|
851
988
|
return { kind: "live", commandId: null };
|
|
852
989
|
}
|
|
@@ -1148,10 +1285,10 @@ async function loginDrive(page, { email, password, timeoutMs, log = () => {
|
|
|
1148
1285
|
{ timeout: timeoutMs }
|
|
1149
1286
|
);
|
|
1150
1287
|
} catch {
|
|
1151
|
-
const
|
|
1288
|
+
const errorText3 = await scrapeErrorCard(page);
|
|
1152
1289
|
const finalUrl2 = page.url();
|
|
1153
1290
|
log(`stalled ${finalUrl2}`);
|
|
1154
|
-
return { ok: false, finalUrl: finalUrl2, landingPath: pathOf(finalUrl2), errorText:
|
|
1291
|
+
return { ok: false, finalUrl: finalUrl2, landingPath: pathOf(finalUrl2), errorText: errorText3 };
|
|
1155
1292
|
}
|
|
1156
1293
|
}
|
|
1157
1294
|
await page.waitForLoadState("networkidle").catch(() => {
|
|
@@ -1165,8 +1302,8 @@ async function loginDrive(page, { email, password, timeoutMs, log = () => {
|
|
|
1165
1302
|
const landingPath = await page.evaluate(() => location.pathname).catch(() => pathOf(finalUrl));
|
|
1166
1303
|
const ok = !isAppLoginPath(finalUrl) && !isCognito(finalUrl) && !/\/auth\/callback/.test(pathOf(finalUrl));
|
|
1167
1304
|
log(ok ? `authed ${finalUrl}` : `stalled ${finalUrl}`);
|
|
1168
|
-
const
|
|
1169
|
-
return { ok, finalUrl, landingPath, errorText };
|
|
1305
|
+
const errorText2 = ok ? null : await scrapeErrorCard(page);
|
|
1306
|
+
return { ok, finalUrl, landingPath, errorText: errorText2 };
|
|
1170
1307
|
}
|
|
1171
1308
|
|
|
1172
1309
|
// src/action-outcome.ts
|