@docsxai/engine 0.2.0 → 0.2.1-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +8 -1
- package/dist/auth/api-login.d.ts +3 -3
- package/dist/auth/api-login.js +6 -2
- package/dist/auth/email-otp.d.ts +60 -60
- package/dist/auth/jwt-injection.d.ts +45 -45
- package/dist/auth/pat-header.d.ts +2 -2
- package/dist/auth/types.d.ts +2 -2
- package/dist/auth/ui-form.d.ts +54 -54
- package/dist/auth/webauthn.d.ts +22 -22
- package/dist/backend-client-contracts.d.ts +2 -2
- package/dist/calibrate.d.ts +2 -2
- package/dist/cli-commands-session.js +1 -0
- package/dist/cli-usage.d.ts +1 -1
- package/dist/cli-usage.js +1 -1
- package/dist/doc-pack.d.ts +268 -179
- package/dist/doc-pack.js +19 -1
- package/dist/doctor-checks.js +4 -4
- package/dist/flow-file.d.ts +2 -2
- package/dist/flow-runtime.d.ts +18 -2
- package/dist/flow-runtime.js +16 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/obstacles.d.ts +33 -0
- package/dist/obstacles.js +72 -0
- package/dist/page-nearby-boxes.d.ts +9 -0
- package/dist/page-nearby-boxes.js +92 -0
- package/dist/playwright-driver.d.ts +2 -0
- package/dist/playwright-driver.js +25 -0
- package/dist/plugins/manifest.d.ts +9 -9
- package/dist/style.d.ts +2 -2
- package/dist/workspace.d.ts +8 -0
- package/dist/zip.d.ts +2 -2
- package/package.json +6 -6
package/dist/doc-pack.js
CHANGED
|
@@ -207,9 +207,17 @@ export const FlowFile = z
|
|
|
207
207
|
// ---------------------------------------------------------------------------
|
|
208
208
|
// Annotations (`<flow>/annotations.json`)
|
|
209
209
|
// ---------------------------------------------------------------------------
|
|
210
|
+
/** A rectangle in screenshot pixels: finite position, finite non-negative size. */
|
|
210
211
|
export const BoundingBox = z
|
|
211
|
-
.object({
|
|
212
|
+
.object({
|
|
213
|
+
x: z.number().finite(),
|
|
214
|
+
y: z.number().finite(),
|
|
215
|
+
width: z.number().finite().nonnegative(),
|
|
216
|
+
height: z.number().finite().nonnegative(),
|
|
217
|
+
})
|
|
212
218
|
.strict();
|
|
219
|
+
/** Most `obstacles` one annotation record may carry (the writer keeps the nearest this many). */
|
|
220
|
+
export const MAX_OBSTACLES = 40;
|
|
213
221
|
export const AnnotationRecord = z
|
|
214
222
|
.object({
|
|
215
223
|
step: z.string().min(1),
|
|
@@ -219,6 +227,16 @@ export const AnnotationRecord = z
|
|
|
219
227
|
arrow_style: ArrowStyle.optional(),
|
|
220
228
|
/** Optional pixel offset applied to the callout + arrow at render time — see {@link NudgeOffset}. */
|
|
221
229
|
nudge: NudgeOffset.optional(),
|
|
230
|
+
/**
|
|
231
|
+
* Optional boxes of page content (text, controls), in screenshot pixels, that the burner keeps this
|
|
232
|
+
* annotation's callout from covering. The target itself is not listed. `docsxai run` writes it when
|
|
233
|
+
* the workspace sets `annotations.obstacles`; a pipeline that knows the page layout may add it by
|
|
234
|
+
* hand. At most {@link MAX_OBSTACLES} boxes. The engine measures them right after the screenshot, so
|
|
235
|
+
* on a page that animates continuously they can differ slightly from what the image shows. Absent or
|
|
236
|
+
* empty: the callout goes next to the target. The structural mirror in `packages/viewer/src/annotations.ts`
|
|
237
|
+
* carries the same field.
|
|
238
|
+
*/
|
|
239
|
+
obstacles: z.array(BoundingBox).max(MAX_OBSTACLES).optional(),
|
|
222
240
|
/** 1-based index of this annotation *within its step's screenshot* — set only when the step has > 1 annotation, so the viewer can render a numbered badge. Absent → render as a plain (un-numbered) halo. */
|
|
223
241
|
index: z.number().int().positive().optional(),
|
|
224
242
|
})
|
package/dist/doctor-checks.js
CHANGED
|
@@ -42,14 +42,14 @@ async function readTextIfExists(p) {
|
|
|
42
42
|
// ---------------------------------------------------------------------------
|
|
43
43
|
export function checkNode(nodeVersion) {
|
|
44
44
|
const major = Number(nodeVersion.split(".")[0]);
|
|
45
|
-
if (Number.isFinite(major) && major >=
|
|
46
|
-
return { name: "node", ok: true, detail: `v${nodeVersion} (>=
|
|
45
|
+
if (Number.isFinite(major) && major >= 26) {
|
|
46
|
+
return { name: "node", ok: true, detail: `v${nodeVersion} (>= 26 required)` };
|
|
47
47
|
}
|
|
48
48
|
return {
|
|
49
49
|
name: "node",
|
|
50
50
|
ok: false,
|
|
51
|
-
detail: `v${nodeVersion} — the engine requires Node >=
|
|
52
|
-
fix: "upgrade Node (https://nodejs.org); the engine's `engines` field pins >=
|
|
51
|
+
detail: `v${nodeVersion} — the engine requires Node >= 26`,
|
|
52
|
+
fix: "upgrade Node (https://nodejs.org); the engine's `engines` field pins >= 26",
|
|
53
53
|
};
|
|
54
54
|
}
|
|
55
55
|
export async function checkChromium(probe) {
|
package/dist/flow-file.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { FlowFile } from "./doc-pack.js";
|
|
2
2
|
export declare class FlowFileError extends Error {
|
|
3
|
-
readonly cause?: unknown
|
|
4
|
-
constructor(message: string, cause?: unknown
|
|
3
|
+
readonly cause?: unknown;
|
|
4
|
+
constructor(message: string, cause?: unknown);
|
|
5
5
|
}
|
|
6
6
|
/** Parse + validate a flow-file from YAML text. Throws {@link FlowFileError} with a readable message on failure. */
|
|
7
7
|
export declare function parseFlowFile(yamlText: string, source?: string): FlowFile;
|
package/dist/flow-runtime.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { type AnnotationsFile, type BoundingBox, type FlowFile, type RedactionRegion, type RedactionStyle, type Step } from "./doc-pack.js";
|
|
2
|
+
import { type NearbyBoxes } from "./obstacles.js";
|
|
2
3
|
/**
|
|
3
4
|
* A redaction with its locator ref already resolved to a concrete selector. `selector` entries are
|
|
4
5
|
* turned into bounding boxes by the driver at capture time (absent/zero-box selectors are skipped
|
|
@@ -39,6 +40,15 @@ export interface BrowserDriver {
|
|
|
39
40
|
textOf(selector: string): Promise<string | null>;
|
|
40
41
|
/** Bounding box of an element, in page pixels. Pass `timeoutMs` (default = driver default) so this fails fast when the target has vanished. Returns `null` on miss. */
|
|
41
42
|
boundingBox(selector: string, timeoutMs?: number): Promise<BoundingBox | null>;
|
|
43
|
+
/**
|
|
44
|
+
* Boxes of the visible text and interactive elements within `radius` CSS px of `selector`, in
|
|
45
|
+
* the screenshot's pixel space (scaled by the device scale factor, like {@link boundingBox}).
|
|
46
|
+
* The target's own subtree and interactive elements containing it are left out. Order is the
|
|
47
|
+
* driver's; `selectObstacles` sorts, clips and caps. Returns `null` when the target isn't visible
|
|
48
|
+
* within `timeoutMs`; rejects if the scan itself runs past `timeoutMs`. The scan happens after the
|
|
49
|
+
* screenshot, so a continuously animating page can drift from the image. Only called when a workspace turns on `annotations.obstacles`.
|
|
50
|
+
*/
|
|
51
|
+
nearbyBoxes(selector: string, radius: number, timeoutMs?: number): Promise<NearbyBoxes | null>;
|
|
42
52
|
/** Capture a clean screenshot (no baked annotations), applying any `redactions` before it hits disk. */
|
|
43
53
|
screenshot(relPath: string, redactions?: ResolvedRedaction[]): Promise<void>;
|
|
44
54
|
/**
|
|
@@ -62,8 +72,8 @@ export interface BrowserDriver {
|
|
|
62
72
|
export type ActionableState = "actionable" | "not-found" | "multiple-matches" | "detached" | "not-visible" | "off-screen" | "covered" | "disabled";
|
|
63
73
|
export declare class FlowExecutionError extends Error {
|
|
64
74
|
readonly stepId: string;
|
|
65
|
-
readonly cause?: unknown
|
|
66
|
-
constructor(message: string, stepId: string, cause?: unknown
|
|
75
|
+
readonly cause?: unknown;
|
|
76
|
+
constructor(message: string, stepId: string, cause?: unknown);
|
|
67
77
|
}
|
|
68
78
|
/**
|
|
69
79
|
* Best-effort 1-line cause extracted from a Playwright actionability log (or similar driver
|
|
@@ -89,6 +99,12 @@ export interface RunFlowOptions {
|
|
|
89
99
|
* the caller is responsible for preserving the previous run's artifacts for them.
|
|
90
100
|
*/
|
|
91
101
|
startFrom?: string;
|
|
102
|
+
/**
|
|
103
|
+
* Record the visible text and controls around each annotation's target as `obstacles` on its
|
|
104
|
+
* record (screenshot pixels, target excluded) so the burner keeps callouts off them. Default
|
|
105
|
+
* false: records are then exactly what they were before the field existed.
|
|
106
|
+
*/
|
|
107
|
+
obstacles?: boolean;
|
|
92
108
|
}
|
|
93
109
|
export interface ExecutedStep {
|
|
94
110
|
id: string;
|
package/dist/flow-runtime.js
CHANGED
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
// The runtime is written against a thin {@link BrowserDriver} abstraction so it's testable
|
|
9
9
|
// without a real browser; the Playwright-backed driver lives in a separate module.
|
|
10
10
|
import { locatorRefName } from "./flow-file.js";
|
|
11
|
+
import { OBSTACLE_RADIUS, selectObstacles } from "./obstacles.js";
|
|
11
12
|
// ---------------------------------------------------------------------------
|
|
12
13
|
// Errors
|
|
13
14
|
// ---------------------------------------------------------------------------
|
|
@@ -72,6 +73,17 @@ export function resolveTarget(value, flow, resolver) {
|
|
|
72
73
|
}
|
|
73
74
|
return resolved;
|
|
74
75
|
}
|
|
76
|
+
/** Obstacles around `selector`, best-effort like the halo box: a failed scan leaves the field off. */
|
|
77
|
+
async function obstaclesAround(driver, selector, target, stepId) {
|
|
78
|
+
try {
|
|
79
|
+
const scan = await driver.nearbyBoxes(selector, OBSTACLE_RADIUS, 2000);
|
|
80
|
+
return scan ? selectObstacles(scan, target) : [];
|
|
81
|
+
}
|
|
82
|
+
catch (e) {
|
|
83
|
+
process.stderr.write(`runFlow: step "${stepId}" — obstacle scan skipped (${e.message})\n`);
|
|
84
|
+
return [];
|
|
85
|
+
}
|
|
86
|
+
}
|
|
75
87
|
async function applyWait(driver, wait, resolve) {
|
|
76
88
|
if (typeof wait === "string") {
|
|
77
89
|
if (wait === "network_idle")
|
|
@@ -246,10 +258,14 @@ export async function runFlow(flow, driver, opts = {}) {
|
|
|
246
258
|
const ann = anns[i];
|
|
247
259
|
const annSelector = ann.target ? resolve(ann.target) : selector;
|
|
248
260
|
const bbox = annSelector ? await driver.boundingBox(annSelector, 2000) : null;
|
|
261
|
+
const obstacles = opts.obstacles && annSelector && bbox
|
|
262
|
+
? await obstaclesAround(driver, annSelector, bbox, step.id)
|
|
263
|
+
: [];
|
|
249
264
|
annotations.push({
|
|
250
265
|
step: step.id,
|
|
251
266
|
selector: annSelector ?? "",
|
|
252
267
|
...(bbox ? { bounding_box: bbox } : {}),
|
|
268
|
+
...(obstacles.length > 0 ? { obstacles } : {}),
|
|
253
269
|
copy: ann.copy,
|
|
254
270
|
...(ann.arrow ? { arrow_style: ann.arrow } : {}),
|
|
255
271
|
...(ann.nudge ? { nudge: ann.nudge } : {}),
|
package/dist/index.d.ts
CHANGED
package/dist/index.js
CHANGED
|
@@ -7,6 +7,7 @@ export * from "./doc-pack.js";
|
|
|
7
7
|
export * from "./doc-pack-io.js";
|
|
8
8
|
export * from "./flow-file.js";
|
|
9
9
|
export * from "./flow-runtime.js";
|
|
10
|
+
export * from "./obstacles.js";
|
|
10
11
|
export * from "./flow-lint.js";
|
|
11
12
|
export * from "./flow-tree.js";
|
|
12
13
|
export * from "./auth.js";
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import { type BoundingBox } from "./doc-pack.js";
|
|
2
|
+
/** How far from the target (CSS px) page content still counts as an obstacle. */
|
|
3
|
+
export declare const OBSTACLE_RADIUS = 320;
|
|
4
|
+
/** The most obstacles one annotation records; the ones nearest the target win. */
|
|
5
|
+
export declare const OBSTACLE_LIMIT = 40;
|
|
6
|
+
/** What a driver reports about the content around a target. Every box is in screenshot pixels. */
|
|
7
|
+
export interface NearbyBoxes {
|
|
8
|
+
/** Screenshot size (viewport × device scale factor). */
|
|
9
|
+
image: {
|
|
10
|
+
width: number;
|
|
11
|
+
height: number;
|
|
12
|
+
};
|
|
13
|
+
/** Device scale factor: screenshot pixels per CSS pixel. */
|
|
14
|
+
scale: number;
|
|
15
|
+
/** Boxes of visible text and interactive elements, the target and its subtree already left out. */
|
|
16
|
+
boxes: BoundingBox[];
|
|
17
|
+
}
|
|
18
|
+
export interface SelectOptions {
|
|
19
|
+
/** Distance cut-off in CSS px. Default {@link OBSTACLE_RADIUS}. */
|
|
20
|
+
radius?: number;
|
|
21
|
+
/** Cap on the number of boxes. Default {@link OBSTACLE_LIMIT}. */
|
|
22
|
+
limit?: number;
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Pick the obstacles for one annotation. `target` is the annotation's own box (screenshot px).
|
|
26
|
+
*
|
|
27
|
+
* Boxes are rounded outward to whole pixels, clipped to the image, and dropped when empty, when
|
|
28
|
+
* farther than the radius from the target, when they sit entirely inside the target, or when
|
|
29
|
+
* another kept box already covers them (a label inside a button). Duplicates collapse to one. If
|
|
30
|
+
* more than the limit remain, the nearest to the target are kept. The result is sorted by y, x,
|
|
31
|
+
* width, height.
|
|
32
|
+
*/
|
|
33
|
+
export declare function selectObstacles(scan: NearbyBoxes, target: BoundingBox, options?: SelectOptions): BoundingBox[];
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
// Obstacle selection — the pure half of `annotations.obstacles`.
|
|
2
|
+
//
|
|
3
|
+
// A driver reports the boxes of visible text and controls near an annotation's target
|
|
4
|
+
// ({@link NearbyBoxes}). This module turns that raw scan into the `obstacles` array written on an
|
|
5
|
+
// annotation record: boxes in screenshot pixels that a burned callout must not cover. No IO, no
|
|
6
|
+
// browser: same scan in, same boxes out, so the doc pack stays byte-identical across runs.
|
|
7
|
+
import { MAX_OBSTACLES } from "./doc-pack.js";
|
|
8
|
+
/** How far from the target (CSS px) page content still counts as an obstacle. */
|
|
9
|
+
export const OBSTACLE_RADIUS = 320;
|
|
10
|
+
/** The most obstacles one annotation records; the ones nearest the target win. */
|
|
11
|
+
export const OBSTACLE_LIMIT = MAX_OBSTACLES;
|
|
12
|
+
const EPSILON = 1e-6;
|
|
13
|
+
const edges = (b) => ({
|
|
14
|
+
left: b.x,
|
|
15
|
+
top: b.y,
|
|
16
|
+
right: b.x + b.width,
|
|
17
|
+
bottom: b.y + b.height,
|
|
18
|
+
});
|
|
19
|
+
/** Integer box that contains `b`, clipped to the image; `null` when nothing of it is left. */
|
|
20
|
+
function clipOutward(b, image) {
|
|
21
|
+
const e = edges(b);
|
|
22
|
+
const left = Math.max(0, Math.floor(e.left + EPSILON));
|
|
23
|
+
const top = Math.max(0, Math.floor(e.top + EPSILON));
|
|
24
|
+
const right = Math.min(image.width, Math.ceil(e.right - EPSILON));
|
|
25
|
+
const bottom = Math.min(image.height, Math.ceil(e.bottom - EPSILON));
|
|
26
|
+
if (right <= left || bottom <= top)
|
|
27
|
+
return null;
|
|
28
|
+
return { x: left, y: top, width: right - left, height: bottom - top };
|
|
29
|
+
}
|
|
30
|
+
/** Straight-line gap between two boxes; 0 when they touch or overlap. */
|
|
31
|
+
function gap(a, b) {
|
|
32
|
+
const ea = edges(a);
|
|
33
|
+
const eb = edges(b);
|
|
34
|
+
const dx = Math.max(0, ea.left - eb.right, eb.left - ea.right);
|
|
35
|
+
const dy = Math.max(0, ea.top - eb.bottom, eb.top - ea.bottom);
|
|
36
|
+
return Math.hypot(dx, dy);
|
|
37
|
+
}
|
|
38
|
+
/** True when `inner` lies entirely inside `outer` (equal boxes count). */
|
|
39
|
+
function contains(outer, inner) {
|
|
40
|
+
const o = edges(outer);
|
|
41
|
+
const i = edges(inner);
|
|
42
|
+
return o.left <= i.left && o.top <= i.top && o.right >= i.right && o.bottom >= i.bottom;
|
|
43
|
+
}
|
|
44
|
+
const byPosition = (a, b) => a.y - b.y || a.x - b.x || a.width - b.width || a.height - b.height;
|
|
45
|
+
/**
|
|
46
|
+
* Pick the obstacles for one annotation. `target` is the annotation's own box (screenshot px).
|
|
47
|
+
*
|
|
48
|
+
* Boxes are rounded outward to whole pixels, clipped to the image, and dropped when empty, when
|
|
49
|
+
* farther than the radius from the target, when they sit entirely inside the target, or when
|
|
50
|
+
* another kept box already covers them (a label inside a button). Duplicates collapse to one. If
|
|
51
|
+
* more than the limit remain, the nearest to the target are kept. The result is sorted by y, x,
|
|
52
|
+
* width, height.
|
|
53
|
+
*/
|
|
54
|
+
export function selectObstacles(scan, target, options = {}) {
|
|
55
|
+
const radius = (options.radius ?? OBSTACLE_RADIUS) * scan.scale;
|
|
56
|
+
const limit = options.limit ?? OBSTACLE_LIMIT;
|
|
57
|
+
const unique = new Map();
|
|
58
|
+
for (const raw of scan.boxes) {
|
|
59
|
+
const box = clipOutward(raw, scan.image);
|
|
60
|
+
if (!box || contains(target, box) || gap(box, target) > radius)
|
|
61
|
+
continue;
|
|
62
|
+
unique.set(`${box.x},${box.y},${box.width},${box.height}`, box);
|
|
63
|
+
}
|
|
64
|
+
const candidates = [...unique.values()];
|
|
65
|
+
const exposed = candidates.filter((box) => !candidates.some((other) => other !== box && contains(other, box)));
|
|
66
|
+
return exposed
|
|
67
|
+
.map((box) => ({ box, distance: gap(box, target) }))
|
|
68
|
+
.sort((a, b) => a.distance - b.distance || byPosition(a.box, b.box))
|
|
69
|
+
.slice(0, limit)
|
|
70
|
+
.map(({ box }) => box)
|
|
71
|
+
.sort(byPosition);
|
|
72
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type NearbyBoxes } from "./obstacles.js";
|
|
2
|
+
/**
|
|
3
|
+
* Boxes of the visible text and interactive elements within `radius` CSS px of `targetArg`, in
|
|
4
|
+
* screenshot pixels. Text is measured per rendered line (the text itself, not its block), so a
|
|
5
|
+
* paragraph beside the target contributes the lines that are there. Left out: anything inside the
|
|
6
|
+
* target, interactive elements that contain it, hidden or fully clipped content, scripts and styles.
|
|
7
|
+
* The result is in document order; `selectObstacles` sorts, clips and caps it.
|
|
8
|
+
*/
|
|
9
|
+
export declare function collectNearbyBoxes(targetArg: unknown, radius: number): NearbyBoxes;
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
// In-page scan behind `BrowserDriver.nearbyBoxes`.
|
|
2
|
+
//
|
|
3
|
+
// `collectNearbyBoxes` runs inside the page (Playwright serializes it with `locator.evaluate`), so
|
|
4
|
+
// it must stay self-contained: no imports of values, no references to module scope. It imports no
|
|
5
|
+
// Playwright either; the driver is the only module that does. The DOM lib isn't in this package's
|
|
6
|
+
// TypeScript config, so the few DOM shapes it touches are declared locally.
|
|
7
|
+
/**
|
|
8
|
+
* Boxes of the visible text and interactive elements within `radius` CSS px of `targetArg`, in
|
|
9
|
+
* screenshot pixels. Text is measured per rendered line (the text itself, not its block), so a
|
|
10
|
+
* paragraph beside the target contributes the lines that are there. Left out: anything inside the
|
|
11
|
+
* target, interactive elements that contain it, hidden or fully clipped content, scripts and styles.
|
|
12
|
+
* The result is in document order; `selectObstacles` sorts, clips and caps it.
|
|
13
|
+
*/
|
|
14
|
+
export function collectNearbyBoxes(targetArg, radius) {
|
|
15
|
+
const target = targetArg;
|
|
16
|
+
const doc = target.ownerDocument;
|
|
17
|
+
const view = doc.defaultView;
|
|
18
|
+
const dpr = view.devicePixelRatio || 1;
|
|
19
|
+
const t = target.getBoundingClientRect();
|
|
20
|
+
const near = {
|
|
21
|
+
left: t.left - radius,
|
|
22
|
+
top: t.top - radius,
|
|
23
|
+
right: t.right + radius,
|
|
24
|
+
bottom: t.bottom + radius,
|
|
25
|
+
};
|
|
26
|
+
const boxes = [];
|
|
27
|
+
// Add `r` (viewport CSS px) when `owner` renders it: visible, not clipped away by an ancestor's
|
|
28
|
+
// overflow, and inside the viewport. `clipSelf` lets the owner's own overflow clip text it holds.
|
|
29
|
+
const add = (r, owner, clipSelf) => {
|
|
30
|
+
let { left, top, right, bottom } = r;
|
|
31
|
+
left = Math.max(left, 0);
|
|
32
|
+
top = Math.max(top, 0);
|
|
33
|
+
right = Math.min(right, view.innerWidth);
|
|
34
|
+
bottom = Math.min(bottom, view.innerHeight);
|
|
35
|
+
for (let cur = owner; cur; cur = cur.parentElement) {
|
|
36
|
+
const cs = view.getComputedStyle(cur);
|
|
37
|
+
if (cs.display === "none" || cs.opacity === "0")
|
|
38
|
+
return;
|
|
39
|
+
if (cur === owner && cs.visibility !== "visible")
|
|
40
|
+
return;
|
|
41
|
+
const clips = cs.overflow !== "visible" || cs.overflowX !== "visible" || cs.overflowY !== "visible";
|
|
42
|
+
if (clips && (cur !== owner || clipSelf)) {
|
|
43
|
+
const cr = cur.getBoundingClientRect();
|
|
44
|
+
left = Math.max(left, cr.left);
|
|
45
|
+
top = Math.max(top, cr.top);
|
|
46
|
+
right = Math.min(right, cr.right);
|
|
47
|
+
bottom = Math.min(bottom, cr.bottom);
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
if (right <= left || bottom <= top)
|
|
51
|
+
return;
|
|
52
|
+
if (right < near.left || left > near.right || bottom < near.top || top > near.bottom)
|
|
53
|
+
return;
|
|
54
|
+
boxes.push({
|
|
55
|
+
x: left * dpr,
|
|
56
|
+
y: top * dpr,
|
|
57
|
+
width: (right - left) * dpr,
|
|
58
|
+
height: (bottom - top) * dpr,
|
|
59
|
+
});
|
|
60
|
+
};
|
|
61
|
+
const controls = doc.querySelectorAll('a[href], button, input:not([type="hidden"]), select, textarea, summary, ' +
|
|
62
|
+
'[role="button"], [role="link"], [role="checkbox"], [role="radio"], [role="switch"], ' +
|
|
63
|
+
'[role="tab"], [role="menuitem"], [role="combobox"], [role="option"], ' +
|
|
64
|
+
'[contenteditable=""], [contenteditable="true"], [tabindex]:not([tabindex^="-"])');
|
|
65
|
+
for (let i = 0; i < controls.length; i++) {
|
|
66
|
+
const el = controls[i];
|
|
67
|
+
if (el === target || target.contains(el) || el.contains(target))
|
|
68
|
+
continue;
|
|
69
|
+
add(el.getBoundingClientRect(), el, false);
|
|
70
|
+
}
|
|
71
|
+
if (doc.body) {
|
|
72
|
+
const range = doc.createRange();
|
|
73
|
+
const walker = doc.createTreeWalker(doc.body, 4); // NodeFilter.SHOW_TEXT
|
|
74
|
+
for (let n = walker.nextNode(); n; n = walker.nextNode()) {
|
|
75
|
+
const text = n;
|
|
76
|
+
const owner = text.parentElement;
|
|
77
|
+
if (!owner || !/\S/.test(text.nodeValue ?? "") || target.contains(n))
|
|
78
|
+
continue;
|
|
79
|
+
if (/^(SCRIPT|STYLE|NOSCRIPT|TEMPLATE)$/.test(owner.tagName))
|
|
80
|
+
continue;
|
|
81
|
+
range.selectNodeContents(n);
|
|
82
|
+
const lines = range.getClientRects();
|
|
83
|
+
for (let i = 0; i < lines.length; i++)
|
|
84
|
+
add(lines[i], owner, true);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
return {
|
|
88
|
+
image: { width: Math.round(view.innerWidth * dpr), height: Math.round(view.innerHeight * dpr) },
|
|
89
|
+
scale: dpr,
|
|
90
|
+
boxes,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
@@ -2,6 +2,7 @@ import { type Browser, type BrowserContext, type Page } from "playwright-core";
|
|
|
2
2
|
import { type BoundingBox, type EnvironmentSpec, type ViewportSize } from "./doc-pack.js";
|
|
3
3
|
import { type ActionableState, type BrowserDriver, type ResolvedRedaction } from "./flow-runtime.js";
|
|
4
4
|
import { type StorageState } from "./auth.js";
|
|
5
|
+
import { type NearbyBoxes } from "./obstacles.js";
|
|
5
6
|
/**
|
|
6
7
|
* The cached Chromium binary path, or `undefined` when `playwright-core` has none installed.
|
|
7
8
|
*
|
|
@@ -93,6 +94,7 @@ export declare class PlaywrightDriver implements BrowserDriver {
|
|
|
93
94
|
count(selector: string): Promise<number>;
|
|
94
95
|
textOf(selector: string): Promise<string | null>;
|
|
95
96
|
boundingBox(selector: string, timeoutMs?: number): Promise<BoundingBox | null>;
|
|
97
|
+
nearbyBoxes(selector: string, radius: number, timeoutMs?: number): Promise<NearbyBoxes | null>;
|
|
96
98
|
screenshot(relPath: string, redactions?: ResolvedRedaction[]): Promise<void>;
|
|
97
99
|
/**
|
|
98
100
|
* Resolve redactions to pixel rects in the screenshot's device-pixel space: selector entries via
|
|
@@ -11,6 +11,7 @@ import { existsSync, promises as fs } from "node:fs";
|
|
|
11
11
|
import * as path from "node:path";
|
|
12
12
|
import { chromium, } from "playwright-core";
|
|
13
13
|
import { VIEWPORT_PRESETS, } from "./doc-pack.js";
|
|
14
|
+
import { collectNearbyBoxes } from "./page-nearby-boxes.js";
|
|
14
15
|
import { applyRedactions } from "./redact.js";
|
|
15
16
|
import { resolveWorkspacePathReal } from "./workspace.js";
|
|
16
17
|
/**
|
|
@@ -103,6 +104,16 @@ export async function launchPlaywrightSession(opts = {}) {
|
|
|
103
104
|
},
|
|
104
105
|
};
|
|
105
106
|
}
|
|
107
|
+
/** Rejects with a timeout error if `work` takes longer than `ms` (no limit when `ms` is undefined). */
|
|
108
|
+
function withTimeout(work, ms) {
|
|
109
|
+
if (ms === undefined)
|
|
110
|
+
return work;
|
|
111
|
+
let timer;
|
|
112
|
+
const timeout = new Promise((_, reject) => {
|
|
113
|
+
timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms`)), ms);
|
|
114
|
+
});
|
|
115
|
+
return Promise.race([work, timeout]).finally(() => clearTimeout(timer));
|
|
116
|
+
}
|
|
106
117
|
export class PlaywrightDriver {
|
|
107
118
|
page;
|
|
108
119
|
docPackRoot;
|
|
@@ -256,6 +267,20 @@ export class PlaywrightDriver {
|
|
|
256
267
|
})
|
|
257
268
|
.catch(() => null);
|
|
258
269
|
}
|
|
270
|
+
async nearbyBoxes(selector, radius, timeoutMs) {
|
|
271
|
+
const loc = this.page.locator(selector).first();
|
|
272
|
+
try {
|
|
273
|
+
await loc.waitFor({
|
|
274
|
+
state: "visible",
|
|
275
|
+
...(timeoutMs !== undefined ? { timeout: timeoutMs } : {}),
|
|
276
|
+
});
|
|
277
|
+
}
|
|
278
|
+
catch {
|
|
279
|
+
return null;
|
|
280
|
+
}
|
|
281
|
+
// `evaluate` has no timeout of its own; a hung page must not stall the run.
|
|
282
|
+
return withTimeout(loc.evaluate(collectNearbyBoxes, radius), timeoutMs);
|
|
283
|
+
}
|
|
259
284
|
async screenshot(relPath, redactions = []) {
|
|
260
285
|
// relPath segments carry flow names + step ids from the flow-file — containment-checked
|
|
261
286
|
// (symlink-aware) against the doc-pack root before writing.
|
|
@@ -20,35 +20,35 @@ declare const manifestSchema: z.ZodObject<{
|
|
|
20
20
|
plugin: z.ZodString;
|
|
21
21
|
version: z.ZodString;
|
|
22
22
|
}, "strict", z.ZodTypeAny, {
|
|
23
|
-
version: string;
|
|
24
23
|
plugin: string;
|
|
25
|
-
}, {
|
|
26
24
|
version: string;
|
|
25
|
+
}, {
|
|
27
26
|
plugin: string;
|
|
27
|
+
version: string;
|
|
28
28
|
}>, "many">>;
|
|
29
29
|
trust: z.ZodDefault<z.ZodEnum<["kalebtec", "community", "local"]>>;
|
|
30
30
|
}, "strict", z.ZodTypeAny, {
|
|
31
|
-
capabilities: string[];
|
|
32
31
|
apiVersion: string;
|
|
33
32
|
namespace: string;
|
|
34
33
|
register: string;
|
|
35
|
-
kinds: ("
|
|
34
|
+
kinds: ("auth-strategy" | "lint-rules" | "publisher" | "renderer")[];
|
|
35
|
+
capabilities: string[];
|
|
36
36
|
dependsOn: {
|
|
37
|
-
version: string;
|
|
38
37
|
plugin: string;
|
|
38
|
+
version: string;
|
|
39
39
|
}[];
|
|
40
|
-
trust: "
|
|
40
|
+
trust: "community" | "kalebtec" | "local";
|
|
41
41
|
}, {
|
|
42
42
|
apiVersion: string;
|
|
43
43
|
namespace: string;
|
|
44
44
|
register: string;
|
|
45
|
-
kinds: ("
|
|
45
|
+
kinds: ("auth-strategy" | "lint-rules" | "publisher" | "renderer")[];
|
|
46
46
|
capabilities?: string[] | undefined;
|
|
47
47
|
dependsOn?: {
|
|
48
|
-
version: string;
|
|
49
48
|
plugin: string;
|
|
49
|
+
version: string;
|
|
50
50
|
}[] | undefined;
|
|
51
|
-
trust?: "
|
|
51
|
+
trust?: "community" | "kalebtec" | "local" | undefined;
|
|
52
52
|
}>;
|
|
53
53
|
export type PluginManifest = z.infer<typeof manifestSchema>;
|
|
54
54
|
export declare class PluginManifestError extends Error {
|
package/dist/style.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { StyleArtifact } from "./doc-pack.js";
|
|
2
2
|
export declare class StyleError extends Error {
|
|
3
|
-
readonly cause?: unknown
|
|
4
|
-
constructor(message: string, cause?: unknown
|
|
3
|
+
readonly cause?: unknown;
|
|
4
|
+
constructor(message: string, cause?: unknown);
|
|
5
5
|
}
|
|
6
6
|
/** The seed style every workspace starts with — overwritable by the agent during calibration. */
|
|
7
7
|
export declare const DEFAULT_STYLE: StyleArtifact;
|
package/dist/workspace.d.ts
CHANGED
|
@@ -22,6 +22,14 @@ export interface WorkspaceConfig {
|
|
|
22
22
|
ignore_https_errors?: boolean;
|
|
23
23
|
/** Backend stub/service URL for `push`/`pull` (e.g. `http://localhost:4477`). Optional — workspaces operate fully locally without it. */
|
|
24
24
|
backend_url?: string;
|
|
25
|
+
/** How `run` records annotations. Absent keys keep today's output. */
|
|
26
|
+
annotations?: {
|
|
27
|
+
/**
|
|
28
|
+
* Write per-annotation `obstacles` (boxes of nearby text and controls, screenshot pixels, the
|
|
29
|
+
* target excluded) so the burner keeps callouts off page content. Default false.
|
|
30
|
+
*/
|
|
31
|
+
obstacles?: boolean;
|
|
32
|
+
};
|
|
25
33
|
/** Backend workspace ID, set by `push` after first round-trip. */
|
|
26
34
|
backend_workspace_id?: string;
|
|
27
35
|
/** Backend project ID, set by `push` after first round-trip. */
|
package/dist/zip.d.ts
CHANGED
|
@@ -11,7 +11,7 @@ export interface ZipResult {
|
|
|
11
11
|
entries: string[];
|
|
12
12
|
}
|
|
13
13
|
export declare class ZipError extends Error {
|
|
14
|
-
readonly cause?: unknown
|
|
15
|
-
constructor(message: string, cause?: unknown
|
|
14
|
+
readonly cause?: unknown;
|
|
15
|
+
constructor(message: string, cause?: unknown);
|
|
16
16
|
}
|
|
17
17
|
export declare function zipDocPack(opts: ZipOptions): Promise<ZipResult>;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@docsxai/engine",
|
|
3
|
-
"version": "0.2.
|
|
3
|
+
"version": "0.2.1-rc.1",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"publishConfig": {
|
|
6
6
|
"access": "public",
|
|
@@ -41,20 +41,20 @@
|
|
|
41
41
|
},
|
|
42
42
|
"dependencies": {
|
|
43
43
|
"fflate": "^0.8.3",
|
|
44
|
-
"playwright-core": "^1.
|
|
44
|
+
"playwright-core": "^1.63.0",
|
|
45
45
|
"pngjs": "^7.0.0",
|
|
46
|
-
"yaml": "^2.
|
|
46
|
+
"yaml": "^2.9.1",
|
|
47
47
|
"zod": "^3.23.0"
|
|
48
48
|
},
|
|
49
49
|
"devDependencies": {
|
|
50
|
-
"@types/node": "^20.
|
|
50
|
+
"@types/node": "^20.19.43",
|
|
51
51
|
"@types/pngjs": "^6.0.5",
|
|
52
52
|
"vitest": "^2.0.0",
|
|
53
|
-
"@docsxai/backend": "0.2.
|
|
53
|
+
"@docsxai/backend": "0.2.1-rc.1"
|
|
54
54
|
},
|
|
55
55
|
"author": "Kalebtec <security@kalebtec.com>",
|
|
56
56
|
"engines": {
|
|
57
|
-
"node": ">=
|
|
57
|
+
"node": ">=26"
|
|
58
58
|
},
|
|
59
59
|
"scripts": {
|
|
60
60
|
"build": "node ../../scripts/clean-dist.mjs && tsc -b tsconfig.build.json",
|