@effect-agent/platform-cloudflare 0.1.0-beta.29 → 0.1.0-beta.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/browser-quick-action.d.mts +7 -3
- package/dist/browser-quick-action.mjs +138 -2
- package/dist/browser-quick-action.mjs.map +1 -1
- package/dist/browser-rest-capture.d.mts +19 -0
- package/dist/browser-rest-capture.mjs +226 -0
- package/dist/browser-rest-capture.mjs.map +1 -0
- package/dist/browser-rest-crawl.d.mts +16 -0
- package/dist/browser-rest-crawl.mjs +243 -0
- package/dist/browser-rest-crawl.mjs.map +1 -0
- package/dist/index.d.mts +40 -37
- package/dist/index.mjs +5 -2
- package/dist/index.mjs.map +1 -1
- package/dist/interactive-browser.d.mts +49 -0
- package/dist/interactive-browser.mjs +376 -0
- package/dist/interactive-browser.mjs.map +1 -0
- package/package.json +22 -9
- package/src/browser-quick-action.ts +218 -2
- package/src/browser-rest-capture.ts +420 -0
- package/src/browser-rest-crawl.ts +497 -0
- package/src/index.ts +3 -0
- package/src/interactive-browser.ts +748 -0
|
@@ -11,9 +11,16 @@ import {
|
|
|
11
11
|
PageCaptureResourceUse,
|
|
12
12
|
PageCaptureResult,
|
|
13
13
|
PageCaptureUnsupportedError,
|
|
14
|
+
PageScreenshot,
|
|
15
|
+
PageScreenshotOutputLimitError,
|
|
16
|
+
PageScreenshotResult,
|
|
17
|
+
type PageScreenshotCapture,
|
|
18
|
+
type PageScreenshotError,
|
|
19
|
+
type PageScreenshotRequest,
|
|
14
20
|
PageContentCaptured,
|
|
15
21
|
PageLinksCaptured,
|
|
16
22
|
PageMarkdownCaptured,
|
|
23
|
+
PageScrapeCaptured,
|
|
17
24
|
PageStructuredCaptured,
|
|
18
25
|
SandboxImplementation,
|
|
19
26
|
type PageCaptureAction,
|
|
@@ -47,6 +54,9 @@ export const browserQuickActionImplementation = SandboxImplementation.make({
|
|
|
47
54
|
* directly from the pinned Workers declarations rather than a local copy.
|
|
48
55
|
*/
|
|
49
56
|
export interface BrowserQuickActionClient {
|
|
57
|
+
readonly screenshot: (
|
|
58
|
+
options: BrowserRunScreenshotOptions,
|
|
59
|
+
) => Effect.Effect<Response, BrowserQuickActionRpcError>;
|
|
50
60
|
readonly content: (
|
|
51
61
|
options: BrowserRunContentOptions,
|
|
52
62
|
) => Effect.Effect<Response, BrowserQuickActionRpcError>;
|
|
@@ -56,6 +66,9 @@ export interface BrowserQuickActionClient {
|
|
|
56
66
|
readonly links: (
|
|
57
67
|
options: BrowserRunLinksOptions,
|
|
58
68
|
) => Effect.Effect<Response, BrowserQuickActionRpcError>;
|
|
69
|
+
readonly scrape: (
|
|
70
|
+
options: BrowserRunScrapeOptions,
|
|
71
|
+
) => Effect.Effect<Response, BrowserQuickActionRpcError>;
|
|
59
72
|
readonly json: (
|
|
60
73
|
options: BrowserRunJsonOptions,
|
|
61
74
|
) => Effect.Effect<Response, BrowserQuickActionRpcError>;
|
|
@@ -65,7 +78,7 @@ export interface BrowserQuickActionClient {
|
|
|
65
78
|
export class BrowserQuickActionRpcError extends Schema.TaggedError<BrowserQuickActionRpcError>()(
|
|
66
79
|
"BrowserQuickActionRpcError",
|
|
67
80
|
{
|
|
68
|
-
action: Schema.Literals(["content", "markdown", "links", "json"]),
|
|
81
|
+
action: Schema.Literals(["screenshot", "content", "markdown", "links", "scrape", "json"]),
|
|
69
82
|
cause: Schema.Defect(),
|
|
70
83
|
},
|
|
71
84
|
) {}
|
|
@@ -85,7 +98,7 @@ export class BrowserQuickActionBrowserBinding extends Context.Service<
|
|
|
85
98
|
): Layer.Layer<BrowserQuickActionBrowserBinding> {
|
|
86
99
|
const browser = options.browser;
|
|
87
100
|
const invoke = Effect.fn("BrowserQuickActionBrowserBinding.invoke")(function* (
|
|
88
|
-
action: "content" | "markdown" | "links" | "json",
|
|
101
|
+
action: "screenshot" | "content" | "markdown" | "links" | "scrape" | "json",
|
|
89
102
|
evaluate: () => Promise<Response>,
|
|
90
103
|
): Effect.fn.Return<Response, BrowserQuickActionRpcError> {
|
|
91
104
|
return yield* Effect.tryPromise({
|
|
@@ -94,9 +107,12 @@ export class BrowserQuickActionBrowserBinding extends Context.Service<
|
|
|
94
107
|
});
|
|
95
108
|
});
|
|
96
109
|
return Layer.succeed(BrowserQuickActionBrowserBinding)({
|
|
110
|
+
screenshot: (request) =>
|
|
111
|
+
invoke("screenshot", () => browser.quickAction("screenshot", request)),
|
|
97
112
|
content: (request) => invoke("content", () => browser.quickAction("content", request)),
|
|
98
113
|
markdown: (request) => invoke("markdown", () => browser.quickAction("markdown", request)),
|
|
99
114
|
links: (request) => invoke("links", () => browser.quickAction("links", request)),
|
|
115
|
+
scrape: (request) => invoke("scrape", () => browser.quickAction("scrape", request)),
|
|
100
116
|
json: (request) => invoke("json", () => browser.quickAction("json", request)),
|
|
101
117
|
});
|
|
102
118
|
}
|
|
@@ -210,6 +226,12 @@ const executeQuickAction = (
|
|
|
210
226
|
: { visibleLinksOnly: request.action.visibleLinksOnly }),
|
|
211
227
|
});
|
|
212
228
|
}
|
|
229
|
+
case "CapturePageScrape": {
|
|
230
|
+
return browser.scrape({
|
|
231
|
+
...options,
|
|
232
|
+
elements: request.action.selectors.map((selector) => ({ selector })),
|
|
233
|
+
});
|
|
234
|
+
}
|
|
213
235
|
case "CapturePageStructured": {
|
|
214
236
|
return browser.json({
|
|
215
237
|
...options,
|
|
@@ -378,6 +400,18 @@ const parseOutput = (
|
|
|
378
400
|
}
|
|
379
401
|
return decoded.value;
|
|
380
402
|
}
|
|
403
|
+
case "CapturePageScrape": {
|
|
404
|
+
const decoded = Schema.decodeUnknownOption(PageScrapeCaptured)({
|
|
405
|
+
_tag: "PageScrapeCaptured",
|
|
406
|
+
groups: envelope.value.result,
|
|
407
|
+
});
|
|
408
|
+
if (Option.isNone(decoded)) {
|
|
409
|
+
return protocolError(
|
|
410
|
+
"The scrape Quick Action did not return bounded grouped element records",
|
|
411
|
+
);
|
|
412
|
+
}
|
|
413
|
+
return decoded.value;
|
|
414
|
+
}
|
|
381
415
|
case "CapturePageStructured": {
|
|
382
416
|
return PageStructuredCaptured.make({ value: envelope.value.result });
|
|
383
417
|
}
|
|
@@ -512,3 +546,185 @@ export const browserQuickActionWorkersAiCaptureLayer = (): Layer.Layer<
|
|
|
512
546
|
return PageCapture.of({ capture: makeCapture(browser, workersAi) });
|
|
513
547
|
}),
|
|
514
548
|
);
|
|
549
|
+
|
|
550
|
+
const screenshotOptions = (request: PageScreenshotRequest): BrowserRunScreenshotOptions => {
|
|
551
|
+
const options: BrowserRunBaseOptions = {};
|
|
552
|
+
if (
|
|
553
|
+
request.navigation?.waitUntil !== undefined ||
|
|
554
|
+
request.navigation?.timeoutMillis !== undefined
|
|
555
|
+
) {
|
|
556
|
+
options.gotoOptions = {
|
|
557
|
+
...(request.navigation.waitUntil === undefined
|
|
558
|
+
? {}
|
|
559
|
+
: { waitUntil: request.navigation.waitUntil }),
|
|
560
|
+
...(request.navigation.timeoutMillis === undefined
|
|
561
|
+
? {}
|
|
562
|
+
: { timeout: request.navigation.timeoutMillis }),
|
|
563
|
+
};
|
|
564
|
+
}
|
|
565
|
+
if (request.navigation?.waitForSelector !== undefined) {
|
|
566
|
+
options.waitForSelector = {
|
|
567
|
+
selector: request.navigation.waitForSelector.selector,
|
|
568
|
+
...(request.navigation.waitForSelector.timeoutMillis === undefined
|
|
569
|
+
? {}
|
|
570
|
+
: { timeout: request.navigation.waitForSelector.timeoutMillis }),
|
|
571
|
+
};
|
|
572
|
+
}
|
|
573
|
+
if (request.viewport !== undefined) {
|
|
574
|
+
options.viewport = { width: request.viewport.width, height: request.viewport.height };
|
|
575
|
+
}
|
|
576
|
+
if (request.resourcePolicy?.rejectResourceTypes !== undefined) {
|
|
577
|
+
options.rejectResourceTypes = [...request.resourcePolicy.rejectResourceTypes];
|
|
578
|
+
}
|
|
579
|
+
if (request.resourcePolicy?.allowRequestPatterns !== undefined) {
|
|
580
|
+
options.allowRequestPattern = [...request.resourcePolicy.allowRequestPatterns];
|
|
581
|
+
}
|
|
582
|
+
return {
|
|
583
|
+
...options,
|
|
584
|
+
...(request.target._tag === "PageUrlTarget"
|
|
585
|
+
? { url: request.target.url }
|
|
586
|
+
: { html: request.target.html }),
|
|
587
|
+
screenshotOptions: { type: "png", encoding: "binary", fullPage: request.fullPage },
|
|
588
|
+
};
|
|
589
|
+
};
|
|
590
|
+
|
|
591
|
+
const cancelBody = (body: ReadableStream<Uint8Array>): Effect.Effect<void> =>
|
|
592
|
+
Effect.tryPromise({
|
|
593
|
+
try: () => body.cancel(),
|
|
594
|
+
catch: () => undefined,
|
|
595
|
+
}).pipe(Effect.catch(() => Effect.void));
|
|
596
|
+
|
|
597
|
+
const releaseScreenshotReader = (
|
|
598
|
+
reader: ReadableStreamDefaultReader<Uint8Array>,
|
|
599
|
+
): Effect.Effect<void> =>
|
|
600
|
+
Effect.tryPromise({ try: () => reader.cancel(), catch: () => undefined }).pipe(
|
|
601
|
+
Effect.catch(() => Effect.logWarning("Canceling the screenshot response failed")),
|
|
602
|
+
Effect.ensuring(
|
|
603
|
+
Effect.try({
|
|
604
|
+
try: () => reader.releaseLock(),
|
|
605
|
+
catch: () => undefined,
|
|
606
|
+
}).pipe(Effect.catch(() => Effect.logWarning("Releasing the screenshot response failed"))),
|
|
607
|
+
),
|
|
608
|
+
);
|
|
609
|
+
|
|
610
|
+
const pngResponse = (response: Response): boolean =>
|
|
611
|
+
response.headers.get("Content-Type")?.split(";", 1)[0]?.trim().toLowerCase() === "image/png";
|
|
612
|
+
|
|
613
|
+
const declaredLength = (response: Response): number | undefined => {
|
|
614
|
+
const raw = response.headers.get("Content-Length");
|
|
615
|
+
if (raw === null || !/^(0|[1-9][0-9]*)$/.test(raw)) return undefined;
|
|
616
|
+
const length = Number(raw);
|
|
617
|
+
return Number.isSafeInteger(length) ? length : undefined;
|
|
618
|
+
};
|
|
619
|
+
|
|
620
|
+
const readScreenshot = Effect.fn("BrowserQuickActionScreenshot.read")(function* (
|
|
621
|
+
response: Response,
|
|
622
|
+
request: PageScreenshotRequest,
|
|
623
|
+
) {
|
|
624
|
+
const body = response.body;
|
|
625
|
+
if (body === null) {
|
|
626
|
+
return yield* protocolError("The screenshot response had no body");
|
|
627
|
+
}
|
|
628
|
+
if (!pngResponse(response)) {
|
|
629
|
+
yield* cancelBody(body);
|
|
630
|
+
return yield* protocolError("The screenshot response was not image/png");
|
|
631
|
+
}
|
|
632
|
+
const length = declaredLength(response);
|
|
633
|
+
if (length !== undefined && length > request.limits.maxOutputBytes) {
|
|
634
|
+
yield* cancelBody(body);
|
|
635
|
+
return yield* PageScreenshotOutputLimitError.make({
|
|
636
|
+
implementation: browserQuickActionImplementation,
|
|
637
|
+
limit: request.limits.maxOutputBytes,
|
|
638
|
+
observed: length,
|
|
639
|
+
});
|
|
640
|
+
}
|
|
641
|
+
const reader = yield* Effect.acquireRelease(
|
|
642
|
+
Effect.try({
|
|
643
|
+
try: () => body.getReader(),
|
|
644
|
+
catch: (cause) => protocolError("Opening the screenshot response failed", cause),
|
|
645
|
+
}),
|
|
646
|
+
releaseScreenshotReader,
|
|
647
|
+
);
|
|
648
|
+
const chunks: Array<Uint8Array> = [];
|
|
649
|
+
let observed = 0;
|
|
650
|
+
while (true) {
|
|
651
|
+
const next = yield* Effect.tryPromise({
|
|
652
|
+
try: () => reader.read(),
|
|
653
|
+
catch: (cause) => protocolError("Reading the screenshot response failed", cause),
|
|
654
|
+
});
|
|
655
|
+
if (next.done) break;
|
|
656
|
+
observed += next.value.byteLength;
|
|
657
|
+
if (observed > request.limits.maxOutputBytes) {
|
|
658
|
+
return yield* PageScreenshotOutputLimitError.make({
|
|
659
|
+
implementation: browserQuickActionImplementation,
|
|
660
|
+
limit: request.limits.maxOutputBytes,
|
|
661
|
+
observed,
|
|
662
|
+
});
|
|
663
|
+
}
|
|
664
|
+
chunks.push(next.value);
|
|
665
|
+
}
|
|
666
|
+
const bytes = new Uint8Array(observed);
|
|
667
|
+
let offset = 0;
|
|
668
|
+
for (const chunk of chunks) {
|
|
669
|
+
bytes.set(chunk, offset);
|
|
670
|
+
offset += chunk.byteLength;
|
|
671
|
+
}
|
|
672
|
+
return bytes;
|
|
673
|
+
}, Effect.scoped);
|
|
674
|
+
|
|
675
|
+
const makeScreenshot = (browser: BrowserQuickActionClient): PageScreenshotCapture =>
|
|
676
|
+
Effect.fn("BrowserQuickActionScreenshot.capture")(function* (
|
|
677
|
+
request: PageScreenshotRequest,
|
|
678
|
+
): Effect.fn.Return<PageScreenshotResult, PageScreenshotError> {
|
|
679
|
+
if (request.engine !== "chromium") {
|
|
680
|
+
return yield* PageCaptureUnsupportedError.make({
|
|
681
|
+
implementation: browserQuickActionImplementation,
|
|
682
|
+
feature: "engine",
|
|
683
|
+
message: "The browser binding's screenshot action exposes no engine selector",
|
|
684
|
+
});
|
|
685
|
+
}
|
|
686
|
+
const response = yield* browser
|
|
687
|
+
.screenshot(screenshotOptions(request))
|
|
688
|
+
.pipe(
|
|
689
|
+
Effect.mapError((error) =>
|
|
690
|
+
protocolError("The browser binding rejected the screenshot", error.cause),
|
|
691
|
+
),
|
|
692
|
+
);
|
|
693
|
+
if (response.status === 429) {
|
|
694
|
+
const body = response.body;
|
|
695
|
+
if (body !== null) yield* cancelBody(body);
|
|
696
|
+
return yield* PageCaptureRateLimitedError.make({
|
|
697
|
+
implementation: browserQuickActionImplementation,
|
|
698
|
+
reason: "rate",
|
|
699
|
+
...(retryAfterMillis(response) === undefined
|
|
700
|
+
? {}
|
|
701
|
+
: { retryAfterMillis: retryAfterMillis(response) }),
|
|
702
|
+
message: "The screenshot Quick Action was rate limited",
|
|
703
|
+
});
|
|
704
|
+
}
|
|
705
|
+
if (!response.ok) {
|
|
706
|
+
const body = response.body;
|
|
707
|
+
if (body !== null) yield* cancelBody(body);
|
|
708
|
+
const message = `The screenshot Quick Action answered HTTP ${response.status}`;
|
|
709
|
+
return yield* response.status >= 500 ? protocolError(message) : navigationError(message);
|
|
710
|
+
}
|
|
711
|
+
const bytes = yield* readScreenshot(response, request);
|
|
712
|
+
return PageScreenshotResult.make({
|
|
713
|
+
implementation: browserQuickActionImplementation,
|
|
714
|
+
mediaType: "image/png",
|
|
715
|
+
bytes,
|
|
716
|
+
});
|
|
717
|
+
});
|
|
718
|
+
|
|
719
|
+
/** Native Browser Run screenshot adapter. It retains PNG bytes only for the caller's result. */
|
|
720
|
+
export const browserQuickActionScreenshotLayer = (): Layer.Layer<
|
|
721
|
+
PageScreenshot,
|
|
722
|
+
never,
|
|
723
|
+
BrowserQuickActionBrowserBinding
|
|
724
|
+
> =>
|
|
725
|
+
Layer.effect(
|
|
726
|
+
PageScreenshot,
|
|
727
|
+
Effect.map(BrowserQuickActionBrowserBinding, (browser) =>
|
|
728
|
+
PageScreenshot.of({ capture: makeScreenshot(browser) }),
|
|
729
|
+
),
|
|
730
|
+
);
|
|
@@ -0,0 +1,420 @@
|
|
|
1
|
+
import {
|
|
2
|
+
PageCapture,
|
|
3
|
+
PageCaptureInferencePolicyError,
|
|
4
|
+
PageCaptureInferenceUse,
|
|
5
|
+
PageCaptureNavigationError,
|
|
6
|
+
PageCaptureOutputLimitError,
|
|
7
|
+
PageCaptureProtocolError,
|
|
8
|
+
PageCaptureRateLimitedError,
|
|
9
|
+
PageCaptureResourceUse,
|
|
10
|
+
PageCaptureResult,
|
|
11
|
+
PageContentCaptured,
|
|
12
|
+
PageLinksCaptured,
|
|
13
|
+
PageMarkdownCaptured,
|
|
14
|
+
PageScrapeCaptured,
|
|
15
|
+
PageStructuredCaptured,
|
|
16
|
+
PageCaptureUnsupportedError,
|
|
17
|
+
SandboxImplementation,
|
|
18
|
+
type PageCaptureAction,
|
|
19
|
+
type PageCaptureCapture,
|
|
20
|
+
type PageCaptureError,
|
|
21
|
+
type PageCaptureOutput,
|
|
22
|
+
type PageCaptureRequest,
|
|
23
|
+
} from "@effect-agent/sandbox";
|
|
24
|
+
import { Effect, Layer, Option, Redacted, Schema, Stream } from "effect";
|
|
25
|
+
import { HttpClient, HttpClientRequest, type HttpClientError } from "effect/unstable/http";
|
|
26
|
+
|
|
27
|
+
import {
|
|
28
|
+
BrowserQuickActionWorkersAi,
|
|
29
|
+
type BrowserQuickActionWorkersAiPolicy,
|
|
30
|
+
} from "./browser-quick-action.ts";
|
|
31
|
+
|
|
32
|
+
/** Node-safe REST PageCapture implementation; it never imports Worker runtime modules. */
|
|
33
|
+
export const browserRestCaptureImplementation = SandboxImplementation.make({
|
|
34
|
+
isolation: "isolated",
|
|
35
|
+
identity: "cloudflare-browser-rest",
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
const API_ORIGIN = "https://api.cloudflare.com";
|
|
39
|
+
const MAX_DIAGNOSTIC_LENGTH = 8_000;
|
|
40
|
+
const boundedDiagnostic = (message: string): string => message.slice(0, MAX_DIAGNOSTIC_LENGTH);
|
|
41
|
+
|
|
42
|
+
/** Explicit construction inputs; credentials are only projected into the fixed-origin Authorization header. */
|
|
43
|
+
export interface BrowserRestCaptureOptions {
|
|
44
|
+
readonly accountId: string;
|
|
45
|
+
readonly apiToken: Redacted.Redacted<string>;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
const RestSuccessEnvelope = Schema.Struct({
|
|
49
|
+
success: Schema.Literal(true),
|
|
50
|
+
result: Schema.Json,
|
|
51
|
+
errors: Schema.optionalKey(Schema.Array(Schema.Unknown)),
|
|
52
|
+
meta: Schema.optionalKey(Schema.Unknown),
|
|
53
|
+
});
|
|
54
|
+
const RestErrorEnvelope = Schema.Struct({
|
|
55
|
+
success: Schema.Literal(false),
|
|
56
|
+
errors: Schema.Array(
|
|
57
|
+
Schema.Struct({
|
|
58
|
+
message: Schema.String,
|
|
59
|
+
code: Schema.optionalKey(Schema.Number),
|
|
60
|
+
}),
|
|
61
|
+
),
|
|
62
|
+
result: Schema.optionalKey(Schema.Json),
|
|
63
|
+
meta: Schema.optionalKey(Schema.Unknown),
|
|
64
|
+
});
|
|
65
|
+
const RestEnvelope = Schema.Union([RestSuccessEnvelope, RestErrorEnvelope]);
|
|
66
|
+
const decodeEnvelope = Schema.decodeUnknownOption(Schema.fromJsonString(RestEnvelope));
|
|
67
|
+
|
|
68
|
+
const protocolError = (message: string, cause?: unknown): PageCaptureProtocolError =>
|
|
69
|
+
PageCaptureProtocolError.make({
|
|
70
|
+
implementation: browserRestCaptureImplementation,
|
|
71
|
+
message: boundedDiagnostic(message),
|
|
72
|
+
...(cause === undefined ? {} : { cause }),
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
const navigationError = (message: string, cause?: unknown): PageCaptureNavigationError =>
|
|
76
|
+
PageCaptureNavigationError.make({
|
|
77
|
+
implementation: browserRestCaptureImplementation,
|
|
78
|
+
message: boundedDiagnostic(message),
|
|
79
|
+
...(cause === undefined ? {} : { cause }),
|
|
80
|
+
});
|
|
81
|
+
|
|
82
|
+
const privateResponseCause = (
|
|
83
|
+
bodyText: string,
|
|
84
|
+
apiToken: Redacted.Redacted<string>,
|
|
85
|
+
): Error | undefined => {
|
|
86
|
+
if (bodyText.length === 0) return undefined;
|
|
87
|
+
const token = Redacted.value(apiToken);
|
|
88
|
+
const diagnostic = boundedDiagnostic(bodyText);
|
|
89
|
+
return new Error(token.length === 0 ? diagnostic : diagnostic.replaceAll(token, "[REDACTED]"));
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
const requestBody = (request: PageCaptureRequest): Record<string, unknown> => {
|
|
93
|
+
const body: Record<string, unknown> =
|
|
94
|
+
request.target._tag === "PageUrlTarget"
|
|
95
|
+
? { url: request.target.url }
|
|
96
|
+
: { html: request.target.html };
|
|
97
|
+
if (request.navigation !== undefined) {
|
|
98
|
+
const goto: Record<string, unknown> = {};
|
|
99
|
+
if (request.navigation.waitUntil !== undefined) goto.waitUntil = request.navigation.waitUntil;
|
|
100
|
+
if (request.navigation.timeoutMillis !== undefined)
|
|
101
|
+
goto.timeout = request.navigation.timeoutMillis;
|
|
102
|
+
if (Object.keys(goto).length > 0) body.gotoOptions = goto;
|
|
103
|
+
if (request.navigation.waitForSelector !== undefined) {
|
|
104
|
+
body.waitForSelector = {
|
|
105
|
+
selector: request.navigation.waitForSelector.selector,
|
|
106
|
+
...(request.navigation.waitForSelector.timeoutMillis === undefined
|
|
107
|
+
? {}
|
|
108
|
+
: { timeout: request.navigation.waitForSelector.timeoutMillis }),
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
if (request.viewport !== undefined) body.viewport = request.viewport;
|
|
113
|
+
if (request.resourcePolicy?.rejectResourceTypes !== undefined) {
|
|
114
|
+
body.rejectResourceTypes = [...request.resourcePolicy.rejectResourceTypes];
|
|
115
|
+
}
|
|
116
|
+
if (request.resourcePolicy?.allowRequestPatterns !== undefined) {
|
|
117
|
+
body.allowRequestPattern = [...request.resourcePolicy.allowRequestPatterns];
|
|
118
|
+
}
|
|
119
|
+
return body;
|
|
120
|
+
};
|
|
121
|
+
|
|
122
|
+
const actionName = (
|
|
123
|
+
action: PageCaptureAction,
|
|
124
|
+
): "content" | "markdown" | "links" | "scrape" | "json" => {
|
|
125
|
+
switch (action._tag) {
|
|
126
|
+
case "CapturePageContent":
|
|
127
|
+
return "content";
|
|
128
|
+
case "CapturePageMarkdown":
|
|
129
|
+
return "markdown";
|
|
130
|
+
case "CapturePageLinks":
|
|
131
|
+
return "links";
|
|
132
|
+
case "CapturePageScrape":
|
|
133
|
+
return "scrape";
|
|
134
|
+
case "CapturePageStructured":
|
|
135
|
+
return "json";
|
|
136
|
+
}
|
|
137
|
+
};
|
|
138
|
+
|
|
139
|
+
const actionBody = (request: PageCaptureRequest): Record<string, unknown> => {
|
|
140
|
+
const body = requestBody(request);
|
|
141
|
+
switch (request.action._tag) {
|
|
142
|
+
case "CapturePageContent":
|
|
143
|
+
case "CapturePageMarkdown":
|
|
144
|
+
return body;
|
|
145
|
+
case "CapturePageLinks":
|
|
146
|
+
return {
|
|
147
|
+
...body,
|
|
148
|
+
...(request.action.visibleLinksOnly === undefined
|
|
149
|
+
? {}
|
|
150
|
+
: { visibleLinksOnly: request.action.visibleLinksOnly }),
|
|
151
|
+
};
|
|
152
|
+
case "CapturePageScrape":
|
|
153
|
+
return {
|
|
154
|
+
...body,
|
|
155
|
+
elements: request.action.selectors.map((selector) => ({ selector })),
|
|
156
|
+
};
|
|
157
|
+
case "CapturePageStructured":
|
|
158
|
+
return {
|
|
159
|
+
...body,
|
|
160
|
+
response_format: { type: "json_schema", json_schema: request.action.responseFormat },
|
|
161
|
+
...(request.action.prompt === undefined ? {} : { prompt: request.action.prompt }),
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
};
|
|
165
|
+
|
|
166
|
+
const retryAfterMillis = (
|
|
167
|
+
headers: Readonly<Record<string, string | undefined>>,
|
|
168
|
+
): number | undefined => {
|
|
169
|
+
const seconds = Number(headers["retry-after"]);
|
|
170
|
+
if (!Number.isSafeInteger(seconds) || seconds < 0) return undefined;
|
|
171
|
+
const millis = seconds * 1_000;
|
|
172
|
+
return Number.isSafeInteger(millis) ? millis : undefined;
|
|
173
|
+
};
|
|
174
|
+
|
|
175
|
+
const browserMillis = (
|
|
176
|
+
headers: Readonly<Record<string, string | undefined>>,
|
|
177
|
+
): number | undefined => {
|
|
178
|
+
const millis = Number(headers["x-browser-ms-used"]);
|
|
179
|
+
return Number.isSafeInteger(millis) && millis >= 0 ? millis : undefined;
|
|
180
|
+
};
|
|
181
|
+
|
|
182
|
+
/** Response framing is provider metadata, never content controlled by the rendered page. */
|
|
183
|
+
const isJsonResponse = (headers: Readonly<Record<string, string | undefined>>): boolean => {
|
|
184
|
+
const contentType = headers["content-type"];
|
|
185
|
+
if (contentType === undefined) return false;
|
|
186
|
+
const mediaType = contentType.split(";", 1)[0]?.trim().toLowerCase();
|
|
187
|
+
return mediaType === "application/json" || mediaType?.endsWith("+json") === true;
|
|
188
|
+
};
|
|
189
|
+
|
|
190
|
+
const readBoundedResponse = Effect.fn("BrowserRestCapture.readResponse")(function* (
|
|
191
|
+
response: { readonly stream: Stream.Stream<Uint8Array, HttpClientError.HttpClientError> },
|
|
192
|
+
request: PageCaptureRequest,
|
|
193
|
+
): Effect.fn.Return<string, PageCaptureError> {
|
|
194
|
+
const decoder = new TextDecoder("utf-8", { fatal: true, ignoreBOM: false });
|
|
195
|
+
const chunks = yield* Stream.runFoldEffect<
|
|
196
|
+
Uint8Array,
|
|
197
|
+
HttpClientError.HttpClientError,
|
|
198
|
+
never,
|
|
199
|
+
{ readonly observed: number; readonly text: string },
|
|
200
|
+
PageCaptureOutputLimitError | PageCaptureProtocolError,
|
|
201
|
+
never
|
|
202
|
+
>(
|
|
203
|
+
response.stream,
|
|
204
|
+
() => ({ observed: 0, text: "" }),
|
|
205
|
+
(state, chunk) => {
|
|
206
|
+
const observed = state.observed + chunk.byteLength;
|
|
207
|
+
if (observed > request.limits.maxOutputBytes) {
|
|
208
|
+
return Effect.fail(
|
|
209
|
+
PageCaptureOutputLimitError.make({
|
|
210
|
+
implementation: browserRestCaptureImplementation,
|
|
211
|
+
limit: request.limits.maxOutputBytes,
|
|
212
|
+
observed,
|
|
213
|
+
}),
|
|
214
|
+
);
|
|
215
|
+
}
|
|
216
|
+
return Effect.try({
|
|
217
|
+
try: () => ({ observed, text: state.text + decoder.decode(chunk, { stream: true }) }),
|
|
218
|
+
catch: (cause) => protocolError("Decoding the Browser Run response failed", cause),
|
|
219
|
+
});
|
|
220
|
+
},
|
|
221
|
+
).pipe(
|
|
222
|
+
Effect.mapError((error) =>
|
|
223
|
+
Schema.is(PageCaptureOutputLimitError)(error) || Schema.is(PageCaptureProtocolError)(error)
|
|
224
|
+
? error
|
|
225
|
+
: protocolError("Reading the Browser Run response failed", error),
|
|
226
|
+
),
|
|
227
|
+
);
|
|
228
|
+
return yield* Effect.try({
|
|
229
|
+
try: () => chunks.text + decoder.decode(),
|
|
230
|
+
catch: (cause) => protocolError("Decoding the Browser Run response failed", cause),
|
|
231
|
+
});
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
const parseOutput = (
|
|
235
|
+
action: PageCaptureAction,
|
|
236
|
+
bodyText: string,
|
|
237
|
+
apiToken: Redacted.Redacted<string>,
|
|
238
|
+
): PageCaptureOutput | PageCaptureNavigationError | PageCaptureProtocolError => {
|
|
239
|
+
const envelope = decodeEnvelope(bodyText);
|
|
240
|
+
if (Option.isNone(envelope)) {
|
|
241
|
+
return protocolError(
|
|
242
|
+
"The Browser Run response did not carry a valid response envelope",
|
|
243
|
+
privateResponseCause(bodyText, apiToken),
|
|
244
|
+
);
|
|
245
|
+
}
|
|
246
|
+
if (!envelope.value.success) {
|
|
247
|
+
return navigationError(
|
|
248
|
+
"Browser Run reported a navigation failure",
|
|
249
|
+
privateResponseCause(bodyText, apiToken),
|
|
250
|
+
);
|
|
251
|
+
}
|
|
252
|
+
switch (action._tag) {
|
|
253
|
+
case "CapturePageContent":
|
|
254
|
+
case "CapturePageMarkdown":
|
|
255
|
+
if (typeof envelope.value.result !== "string") {
|
|
256
|
+
return protocolError("The Browser Run response envelope carried a non-text result");
|
|
257
|
+
}
|
|
258
|
+
return action._tag === "CapturePageContent"
|
|
259
|
+
? PageContentCaptured.make({ html: envelope.value.result })
|
|
260
|
+
: PageMarkdownCaptured.make({ markdown: envelope.value.result });
|
|
261
|
+
case "CapturePageLinks": {
|
|
262
|
+
const decoded = Schema.decodeUnknownOption(PageLinksCaptured)({
|
|
263
|
+
_tag: "PageLinksCaptured",
|
|
264
|
+
links: envelope.value.result,
|
|
265
|
+
});
|
|
266
|
+
return Option.isSome(decoded)
|
|
267
|
+
? decoded.value
|
|
268
|
+
: protocolError("Browser Run links did not return a bounded array of valid URLs");
|
|
269
|
+
}
|
|
270
|
+
case "CapturePageScrape": {
|
|
271
|
+
const decoded = Schema.decodeUnknownOption(PageScrapeCaptured)({
|
|
272
|
+
_tag: "PageScrapeCaptured",
|
|
273
|
+
groups: envelope.value.result,
|
|
274
|
+
});
|
|
275
|
+
return Option.isSome(decoded)
|
|
276
|
+
? decoded.value
|
|
277
|
+
: protocolError("Browser Run scrape did not return bounded grouped element records");
|
|
278
|
+
}
|
|
279
|
+
case "CapturePageStructured":
|
|
280
|
+
return PageStructuredCaptured.make({ value: envelope.value.result });
|
|
281
|
+
}
|
|
282
|
+
};
|
|
283
|
+
|
|
284
|
+
const isQuotaMessage = (text: string): boolean => /time limit|daily|quota/i.test(text);
|
|
285
|
+
|
|
286
|
+
const makeCapture = (
|
|
287
|
+
client: HttpClient.HttpClient,
|
|
288
|
+
options: BrowserRestCaptureOptions,
|
|
289
|
+
workersAi?: BrowserQuickActionWorkersAiPolicy,
|
|
290
|
+
): PageCaptureCapture =>
|
|
291
|
+
Effect.fn("BrowserRestCapture.capture")(function* (
|
|
292
|
+
request: PageCaptureRequest,
|
|
293
|
+
): Effect.fn.Return<PageCaptureResult, PageCaptureError> {
|
|
294
|
+
const usesWorkersAi = request.action._tag === "CapturePageStructured";
|
|
295
|
+
if (usesWorkersAi) {
|
|
296
|
+
if (workersAi === undefined) {
|
|
297
|
+
return yield* PageCaptureUnsupportedError.make({
|
|
298
|
+
implementation: browserRestCaptureImplementation,
|
|
299
|
+
feature: "action",
|
|
300
|
+
message:
|
|
301
|
+
"Structured capture invokes separately billed Workers AI and requires an explicit authorization and accounting policy",
|
|
302
|
+
});
|
|
303
|
+
}
|
|
304
|
+
yield* workersAi.authorizeAndAccount(request).pipe(
|
|
305
|
+
Effect.mapError((cause) =>
|
|
306
|
+
PageCaptureInferencePolicyError.make({
|
|
307
|
+
implementation: browserRestCaptureImplementation,
|
|
308
|
+
provider: "cloudflare-workers-ai",
|
|
309
|
+
reason: cause.reason,
|
|
310
|
+
message:
|
|
311
|
+
cause.reason === "authorization"
|
|
312
|
+
? "Workers AI extraction was not authorized"
|
|
313
|
+
: "Workers AI extraction could not be accounted for",
|
|
314
|
+
cause,
|
|
315
|
+
}),
|
|
316
|
+
),
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
const action = actionName(request.action);
|
|
320
|
+
const path = `${API_ORIGIN}/client/v4/accounts/${encodeURIComponent(options.accountId)}/browser-rendering/${action}`;
|
|
321
|
+
const requestWithBody = yield* HttpClientRequest.post(path).pipe(
|
|
322
|
+
request.engine === "kitesurf"
|
|
323
|
+
? HttpClientRequest.setUrlParam("browser", "kitesurf")
|
|
324
|
+
: (value) => value,
|
|
325
|
+
HttpClientRequest.bearerToken(options.apiToken),
|
|
326
|
+
HttpClientRequest.acceptJson,
|
|
327
|
+
HttpClientRequest.bodyJson(actionBody(request)),
|
|
328
|
+
Effect.mapError((cause) => protocolError("Encoding the Browser Run request failed", cause)),
|
|
329
|
+
);
|
|
330
|
+
const response = yield* client
|
|
331
|
+
.execute(requestWithBody)
|
|
332
|
+
.pipe(
|
|
333
|
+
Effect.mapError((cause) => protocolError("Calling the Browser Run REST API failed", cause)),
|
|
334
|
+
);
|
|
335
|
+
const bodyText = yield* readBoundedResponse(response, request);
|
|
336
|
+
if (response.status === 429) {
|
|
337
|
+
const reason = isQuotaMessage(bodyText) ? "quota" : "rate";
|
|
338
|
+
return yield* PageCaptureRateLimitedError.make({
|
|
339
|
+
implementation: browserRestCaptureImplementation,
|
|
340
|
+
reason,
|
|
341
|
+
...(retryAfterMillis(response.headers) === undefined
|
|
342
|
+
? {}
|
|
343
|
+
: { retryAfterMillis: retryAfterMillis(response.headers) }),
|
|
344
|
+
...(privateResponseCause(bodyText, options.apiToken) === undefined
|
|
345
|
+
? {}
|
|
346
|
+
: { cause: privateResponseCause(bodyText, options.apiToken) }),
|
|
347
|
+
message:
|
|
348
|
+
reason === "quota"
|
|
349
|
+
? "Browser Run exceeded its browser quota"
|
|
350
|
+
: "Browser Run was rate limited",
|
|
351
|
+
});
|
|
352
|
+
}
|
|
353
|
+
if (response.status < 200 || response.status >= 300) {
|
|
354
|
+
const error =
|
|
355
|
+
response.status >= 500
|
|
356
|
+
? protocolError(
|
|
357
|
+
`Browser Run answered HTTP ${String(response.status)}`,
|
|
358
|
+
privateResponseCause(bodyText, options.apiToken),
|
|
359
|
+
)
|
|
360
|
+
: navigationError(
|
|
361
|
+
`Browser Run answered HTTP ${String(response.status)}`,
|
|
362
|
+
privateResponseCause(bodyText, options.apiToken),
|
|
363
|
+
);
|
|
364
|
+
return yield* error;
|
|
365
|
+
}
|
|
366
|
+
if (!isJsonResponse(response.headers)) {
|
|
367
|
+
return yield* protocolError(
|
|
368
|
+
"The Browser Run success response was not a JSON response envelope",
|
|
369
|
+
privateResponseCause(bodyText, options.apiToken),
|
|
370
|
+
);
|
|
371
|
+
}
|
|
372
|
+
const output = parseOutput(request.action, bodyText, options.apiToken);
|
|
373
|
+
if (
|
|
374
|
+
output._tag === "PageCaptureNavigationError" ||
|
|
375
|
+
output._tag === "PageCaptureProtocolError"
|
|
376
|
+
) {
|
|
377
|
+
return yield* output;
|
|
378
|
+
}
|
|
379
|
+
return PageCaptureResult.make({
|
|
380
|
+
implementation: browserRestCaptureImplementation,
|
|
381
|
+
output,
|
|
382
|
+
resourceUse: PageCaptureResourceUse.make({
|
|
383
|
+
...(browserMillis(response.headers) === undefined
|
|
384
|
+
? {}
|
|
385
|
+
: { browserMillis: browserMillis(response.headers) }),
|
|
386
|
+
...(usesWorkersAi
|
|
387
|
+
? {
|
|
388
|
+
inference: PageCaptureInferenceUse.make({
|
|
389
|
+
provider: "cloudflare-workers-ai",
|
|
390
|
+
modelCalls: 1,
|
|
391
|
+
}),
|
|
392
|
+
}
|
|
393
|
+
: {}),
|
|
394
|
+
}),
|
|
395
|
+
});
|
|
396
|
+
});
|
|
397
|
+
|
|
398
|
+
/** REST PageCapture layer for Node and other non-Worker composition roots. */
|
|
399
|
+
export const browserRestCaptureLayer = (
|
|
400
|
+
options: BrowserRestCaptureOptions,
|
|
401
|
+
): Layer.Layer<PageCapture, never, HttpClient.HttpClient> =>
|
|
402
|
+
Layer.effect(
|
|
403
|
+
PageCapture,
|
|
404
|
+
Effect.map(HttpClient.HttpClient, (client) =>
|
|
405
|
+
PageCapture.of({ capture: makeCapture(client, options) }),
|
|
406
|
+
),
|
|
407
|
+
);
|
|
408
|
+
|
|
409
|
+
/** REST structured extraction additionally requires explicit Workers AI authorization/accounting. */
|
|
410
|
+
export const browserRestWorkersAiCaptureLayer = (
|
|
411
|
+
options: BrowserRestCaptureOptions,
|
|
412
|
+
): Layer.Layer<PageCapture, never, HttpClient.HttpClient | BrowserQuickActionWorkersAi> =>
|
|
413
|
+
Layer.effect(
|
|
414
|
+
PageCapture,
|
|
415
|
+
Effect.gen(function* () {
|
|
416
|
+
const client = yield* HttpClient.HttpClient;
|
|
417
|
+
const workersAi = yield* BrowserQuickActionWorkersAi;
|
|
418
|
+
return PageCapture.of({ capture: makeCapture(client, options, workersAi) });
|
|
419
|
+
}),
|
|
420
|
+
);
|