effect-browser 0.2.0-beta.8 → 0.3.0-beta.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (202) hide show
  1. package/README.md +705 -1166
  2. package/dist/Browser-CM65Uo7f.mjs +437 -0
  3. package/dist/Browser-CM65Uo7f.mjs.map +1 -0
  4. package/dist/Browser.d.mts +115 -3
  5. package/dist/Browser.mjs +2 -22
  6. package/dist/BrowserError-DXXIA6Rk.d.mts +135 -0
  7. package/dist/BrowserError.d.mts +2 -0
  8. package/dist/BrowserError.mjs +202 -0
  9. package/dist/BrowserError.mjs.map +1 -0
  10. package/dist/BrowserEvent-DzdNdvoh.d.mts +277 -0
  11. package/dist/BrowserEvent.d.mts +2 -0
  12. package/dist/BrowserEvent.mjs +295 -0
  13. package/dist/BrowserEvent.mjs.map +1 -0
  14. package/dist/Cdp.d.mts +32 -0
  15. package/dist/Cdp.mjs +61 -0
  16. package/dist/Cdp.mjs.map +1 -0
  17. package/dist/Change-DNvztNLu.mjs +138 -0
  18. package/dist/Change-DNvztNLu.mjs.map +1 -0
  19. package/dist/Change-DVP2IJWm.d.mts +71 -0
  20. package/dist/Change.d.mts +2 -0
  21. package/dist/Change.mjs +2 -0
  22. package/dist/Chromium.d.mts +29 -81
  23. package/dist/Chromium.mjs +57 -392
  24. package/dist/Chromium.mjs.map +1 -1
  25. package/dist/Expect.d.mts +62 -0
  26. package/dist/Expect.mjs +114 -0
  27. package/dist/Expect.mjs.map +1 -0
  28. package/dist/Frame-BERmf_gM.d.mts +117 -0
  29. package/dist/Frame.d.mts +2 -0
  30. package/dist/Frame.mjs +123 -0
  31. package/dist/Frame.mjs.map +1 -0
  32. package/dist/Moment.d.mts +2 -0
  33. package/dist/Moment.mjs +288 -0
  34. package/dist/Moment.mjs.map +1 -0
  35. package/dist/Motion.d.mts +40 -0
  36. package/dist/Motion.mjs +122 -0
  37. package/dist/Motion.mjs.map +1 -0
  38. package/dist/Page-D4281dIi.mjs +618 -0
  39. package/dist/Page-D4281dIi.mjs.map +1 -0
  40. package/dist/Page-DYTdVhYa.d.mts +593 -0
  41. package/dist/Page.d.mts +2 -0
  42. package/dist/Page.mjs +2 -0
  43. package/dist/Plan-CNNOU1Tx.d.mts +112 -0
  44. package/dist/Plan-Dtrb-8im.mjs +311 -0
  45. package/dist/Plan-Dtrb-8im.mjs.map +1 -0
  46. package/dist/Plan.d.mts +2 -3
  47. package/dist/Plan.mjs +2 -2
  48. package/dist/Policy.d.mts +76 -0
  49. package/dist/Policy.mjs +237 -0
  50. package/dist/Policy.mjs.map +1 -0
  51. package/dist/Presentation.d.mts +48 -0
  52. package/dist/Presentation.mjs +117 -0
  53. package/dist/Presentation.mjs.map +1 -0
  54. package/dist/Snapshot-B34hZ4Jl.d.mts +68 -0
  55. package/dist/Snapshot.d.mts +2 -0
  56. package/dist/Snapshot.mjs +79 -0
  57. package/dist/Snapshot.mjs.map +1 -0
  58. package/dist/Stage.d.mts +54 -0
  59. package/dist/Stage.mjs +133 -0
  60. package/dist/Stage.mjs.map +1 -0
  61. package/dist/Supervisor-BtEG93gR.mjs +420 -0
  62. package/dist/Supervisor-BtEG93gR.mjs.map +1 -0
  63. package/dist/Supervisor-Im6-wbg5.d.mts +230 -0
  64. package/dist/Supervisor.d.mts +2 -0
  65. package/dist/Supervisor.mjs +2 -0
  66. package/dist/context-DAYjATgC.mjs +137 -0
  67. package/dist/context-DAYjATgC.mjs.map +1 -0
  68. package/dist/index.d.mts +17 -14
  69. package/dist/index.mjs +18 -13
  70. package/dist/match.inpage-xOe9LNO0.mjs +48 -0
  71. package/dist/match.inpage-xOe9LNO0.mjs.map +1 -0
  72. package/dist/page-Banfm_zc.mjs +5097 -0
  73. package/dist/page-Banfm_zc.mjs.map +1 -0
  74. package/dist/rolldown-runtime-D7D4PA-g.mjs +13 -0
  75. package/dist/url-CcL0K7S0.mjs +262 -0
  76. package/dist/url-CcL0K7S0.mjs.map +1 -0
  77. package/package.json +96 -61
  78. package/src/Browser.ts +625 -0
  79. package/src/BrowserError.ts +225 -0
  80. package/src/BrowserEvent.ts +294 -0
  81. package/src/Cdp.ts +101 -0
  82. package/src/Change.ts +88 -0
  83. package/src/Chromium.ts +102 -0
  84. package/src/Expect.ts +180 -0
  85. package/src/Frame.ts +130 -0
  86. package/src/Moment.ts +479 -0
  87. package/src/Motion.ts +166 -0
  88. package/src/Page.ts +520 -0
  89. package/src/Plan.ts +350 -0
  90. package/src/Policy.ts +325 -0
  91. package/src/Presentation.ts +195 -0
  92. package/src/Snapshot.ts +89 -0
  93. package/src/Stage.ts +223 -0
  94. package/src/Supervisor.ts +353 -0
  95. package/src/index.ts +20 -0
  96. package/src/internal/input/actions.ts +607 -0
  97. package/src/internal/input/dispatch.ts +161 -0
  98. package/src/internal/input/edit.inpage.ts +162 -0
  99. package/src/internal/input/evidence.inpage.ts +164 -0
  100. package/src/internal/input/guard.inpage.ts +605 -0
  101. package/src/internal/input/guard.ts +183 -0
  102. package/src/internal/input/keyboard.ts +112 -0
  103. package/src/internal/input/keys.ts +127 -0
  104. package/src/internal/input/perform.ts +322 -0
  105. package/src/internal/input/pointer.ts +153 -0
  106. package/src/internal/input/replies.ts +159 -0
  107. package/src/internal/input/settle.ts +50 -0
  108. package/src/internal/input/style.ts +186 -0
  109. package/src/internal/input/targets.inpage.ts +293 -0
  110. package/src/internal/input/targets.ts +148 -0
  111. package/src/internal/page/bridge.ts +386 -0
  112. package/src/internal/page/context.ts +302 -0
  113. package/src/internal/page/lane.ts +301 -0
  114. package/src/internal/page/navigation.ts +183 -0
  115. package/src/internal/page/openings.ts +87 -0
  116. package/src/internal/page/page.ts +137 -0
  117. package/src/internal/page/url.ts +100 -0
  118. package/src/internal/page/viewport.ts +43 -0
  119. package/src/internal/pictures/capture.ts +778 -0
  120. package/src/internal/pictures/clock.ts +141 -0
  121. package/src/internal/pictures/pictures.ts +520 -0
  122. package/src/internal/pictures/transport.ts +242 -0
  123. package/src/internal/reading/choose.ts +100 -0
  124. package/src/internal/reading/context.inpage.ts +201 -0
  125. package/src/internal/reading/controls.inpage.ts +158 -0
  126. package/src/internal/reading/expect.inpage.ts +70 -0
  127. package/src/internal/reading/match.inpage.ts +92 -0
  128. package/src/internal/reading/names.inpage.ts +306 -0
  129. package/src/internal/reading/outline.inpage.ts +260 -0
  130. package/src/internal/reading/reading.ts +223 -0
  131. package/src/internal/reading/ready.inpage.ts +154 -0
  132. package/src/internal/reading/ready.ts +105 -0
  133. package/src/internal/reading/subjects.inpage.ts +188 -0
  134. package/src/internal/reading/text.inpage.ts +135 -0
  135. package/src/internal/reading/walk.inpage.ts +220 -0
  136. package/src/internal/supervisor/lifecycle.ts +265 -0
  137. package/src/internal/timeline/changes.inpage.ts +216 -0
  138. package/src/internal/timeline/changes.ts +100 -0
  139. package/src/internal/timeline/events.ts +93 -0
  140. package/src/internal/timeline/fold.inpage.ts +123 -0
  141. package/src/internal/timeline/history.inpage.ts +229 -0
  142. package/src/internal/timeline/marks.inpage.ts +207 -0
  143. package/src/internal/timeline/record.inpage.ts +133 -0
  144. package/src/internal/timeline/sight.inpage.ts +122 -0
  145. package/src/internal/timeline/window.ts +96 -0
  146. package/src/internal/timeline/words.ts +65 -0
  147. package/dist/Bootstrap-B9UUXsvF.d.mts +0 -183
  148. package/dist/Bootstrap-CeQOWBy6.mjs +0 -189
  149. package/dist/Bootstrap-CeQOWBy6.mjs.map +0 -1
  150. package/dist/Bootstrap.d.mts +0 -3
  151. package/dist/Bootstrap.mjs +0 -2
  152. package/dist/Browser-n-bKgBNN.d.mts +0 -372
  153. package/dist/Browser.mjs.map +0 -1
  154. package/dist/BrowserData-CPIIIHol.d.mts +0 -585
  155. package/dist/BrowserData.d.mts +0 -3
  156. package/dist/BrowserData.mjs +0 -695
  157. package/dist/BrowserData.mjs.map +0 -1
  158. package/dist/BrowserRuntime-CFe9uUTv.d.mts +0 -148
  159. package/dist/BrowserRuntime-HJ1yYlFS.mjs +0 -6741
  160. package/dist/BrowserRuntime-HJ1yYlFS.mjs.map +0 -1
  161. package/dist/BrowserRuntime.d.mts +0 -3
  162. package/dist/BrowserRuntime.mjs +0 -3
  163. package/dist/Capture.d.mts +0 -56
  164. package/dist/Capture.mjs +0 -678
  165. package/dist/Capture.mjs.map +0 -1
  166. package/dist/CaptureData-CYriZHX1.mjs +0 -201
  167. package/dist/CaptureData-CYriZHX1.mjs.map +0 -1
  168. package/dist/CaptureData.d.mts +0 -219
  169. package/dist/CaptureData.mjs +0 -2
  170. package/dist/Errors.d.mts +0 -158
  171. package/dist/Errors.mjs +0 -207
  172. package/dist/Errors.mjs.map +0 -1
  173. package/dist/Images-CaaPNeed.mjs +0 -76
  174. package/dist/Images-CaaPNeed.mjs.map +0 -1
  175. package/dist/OperationOptions-DAeRS6O9.mjs +0 -55
  176. package/dist/OperationOptions-DAeRS6O9.mjs.map +0 -1
  177. package/dist/PageControl.d.mts +0 -15
  178. package/dist/PageControl.mjs +0 -34
  179. package/dist/PageControl.mjs.map +0 -1
  180. package/dist/PageRegistry-B3q9DS3g.mjs +0 -164
  181. package/dist/PageRegistry-B3q9DS3g.mjs.map +0 -1
  182. package/dist/Plan-D8qQ48Ic.mjs +0 -512
  183. package/dist/Plan-D8qQ48Ic.mjs.map +0 -1
  184. package/dist/PlanData-B668RxbQ.d.mts +0 -4831
  185. package/dist/PlanData.d.mts +0 -3
  186. package/dist/PlanData.mjs +0 -268
  187. package/dist/PlanData.mjs.map +0 -1
  188. package/dist/PlanInput-BJcPPncs.mjs +0 -75
  189. package/dist/PlanInput-BJcPPncs.mjs.map +0 -1
  190. package/dist/PublicSession-Bs61Eal8.mjs +0 -5601
  191. package/dist/PublicSession-Bs61Eal8.mjs.map +0 -1
  192. package/dist/Testing.d.mts +0 -479
  193. package/dist/Testing.mjs +0 -2147
  194. package/dist/Testing.mjs.map +0 -1
  195. package/dist/Timeline-fV22Xk-S.d.mts +0 -29
  196. package/dist/Timeline.d.mts +0 -4
  197. package/dist/Timeline.mjs +0 -20
  198. package/dist/Timeline.mjs.map +0 -1
  199. package/dist/TimelineData-BsOn-YGZ.d.mts +0 -3818
  200. package/dist/TimelineData.d.mts +0 -3
  201. package/dist/TimelineData.mjs +0 -482
  202. package/dist/TimelineData.mjs.map +0 -1
package/src/Moment.ts ADDED
@@ -0,0 +1,479 @@
1
+ /**
2
+ * What a page showed, and what changed on it, over a window of time.
3
+ *
4
+ * A page's timeline has three tracks on its browser's host clock: its events, what visibly changed
5
+ * on it (`Change`), and its screencast frames. A `Window` holds the three from `since` to `until`,
6
+ * and says why a part it could not read is missing rather than fail. `Page.window` takes one, which
7
+ * can end in the past, at a frame a delayed consumer airs late, so that a line written now tells
8
+ * what its viewers will see. Its `account` says what was so at its end: how long the page had been
9
+ * still, which of its changes were still under way, and which had settled.
10
+ *
11
+ * A `Moment` is a window that ends at a picture of the page now: `capture` takes the picture, then
12
+ * the window up to it, and on request the page's outline. `toPrompt` turns a moment into one user
13
+ * message for any `effect/ai` call: a structured answer from `LanguageModel.generateObject`, a
14
+ * caption from `generateText`, or one turn of a `Chat` that follows moment after moment. It leads
15
+ * with what changed, news and what settled before what was still changing, as its account tells
16
+ * them apart, and names an action as what a change followed; as a step where changes followed it
17
+ * but none names it, or where its effect is drawn rather than written, as on a canvas, which only
18
+ * the screenshots show; and claims nothing of what changed where the record did not see, or of a
19
+ * part it could not read, which it names with why.
20
+ *
21
+ * Frames come from a running screencast, as far back as the browser's `frameHistory` keeps them.
22
+ * A moment's picture is the newest frame if it was painted after the page's latest input and at
23
+ * most 250 ms ago, else a new screenshot timed by its capture.
24
+ *
25
+ * @since 0.3.0
26
+ */
27
+ import { Duration, Effect, Option, Result, Schema } from "effect";
28
+ import { Prompt } from "effect/ai";
29
+
30
+ import { BrowserError, InvalidRequest } from "./BrowserError.ts";
31
+ import { type Action, BrowserEvent } from "./BrowserEvent.ts";
32
+ import { type Change, Changes } from "./Change.ts";
33
+ import { Frame } from "./Frame.ts";
34
+ import { undispatched } from "./internal/page/context.ts";
35
+ import { describe, named, quoted, where } from "./internal/timeline/words.ts";
36
+ import type * as Page from "./Page.ts";
37
+ import { Snapshot, type SnapshotOptions } from "./Snapshot.ts";
38
+
39
+ /** A page's events, changes and frames over one window of its browser's host clock. */
40
+ export class Window extends Schema.Class<Window>("effect-browser/Window")({
41
+ page: Schema.String,
42
+ /** Where the window starts, in host monotonic milliseconds on the owning browser’s clock. */
43
+ since: Schema.Finite,
44
+ /** Where it ends, never after it was taken. */
45
+ until: Schema.Finite,
46
+ /** The page's events after `since`, up to `until`, oldest first, without the input presentation track. */
47
+ events: Schema.Array(BrowserEvent),
48
+ /**
49
+ * What visibly changed after `since`, up to `until`; absent where `missing` says why. A page's
50
+ * first window starts its record, so the record sees none of it.
51
+ */
52
+ changes: Schema.optional(Changes),
53
+ /** Screencast frames painted from `since` to `until`, oldest first, as far back as kept. */
54
+ frames: Schema.Array(Frame),
55
+ /**
56
+ * Why each part that could not be read is missing: the read of its changes, by whatever
57
+ * operation failed there, or a moment's picture (`frame`) or outline (`snapshot`).
58
+ */
59
+ missing: Schema.Array(BrowserError),
60
+ }) {}
61
+
62
+ /**
63
+ * A window that ends at a picture of the page, its last frame, unless `missing` says why it has
64
+ * none. Its frames are a few, spread over the window, so that it still begins where it starts.
65
+ */
66
+ export class Moment extends Window.extend<Moment>("effect-browser/Moment")({
67
+ /** The page's outline at the end of the window, when the capture asked for it. */
68
+ snapshot: Schema.optional(Snapshot),
69
+ }) {}
70
+
71
+ export interface CaptureOptions {
72
+ /**
73
+ * Where the window starts, as for `Page.window`: a previous window or moment, so that consecutive
74
+ * ones neither repeat nor miss an event or a change, or a `Duration` back from the picture.
75
+ * Defaults to 5 seconds.
76
+ */
77
+ readonly since?: Page.WindowOptions["since"] | undefined;
78
+ /** Frames to keep, at least 1, spread over the window and ending with the picture. Defaults to 2. */
79
+ readonly frames?: number | undefined;
80
+ /**
81
+ * Whether to take the page's outline at the end, or its options; `true` means 4,000 characters.
82
+ * Defaults to `false`: the frames and the timeline carry most moments, while an outline costs a
83
+ * round trip to the page and can double a moment's tokens.
84
+ */
85
+ readonly snapshot?: SnapshotOptions | boolean | undefined;
86
+ /** Show what fields hold in the moment's changes; a secret field still reads `••••`. */
87
+ readonly unmask?: boolean | undefined;
88
+ }
89
+
90
+ /** Pick `count` frames spread evenly over `frames`, always including the last. */
91
+ const spread = (frames: ReadonlyArray<Frame>, count: number): ReadonlyArray<Frame> => {
92
+ if (frames.length <= count) return frames;
93
+ if (count <= 1) return frames.slice(-1);
94
+ const step = (frames.length - 1) / (count - 1);
95
+
96
+ return Array.from({ length: count }, (_, index) => frames[Math.round(index * step)]).filter(
97
+ (frame): frame is Frame => frame !== undefined,
98
+ );
99
+ };
100
+
101
+ export const capture = Effect.fn("Moment.capture")(function* (
102
+ page: Page.Page,
103
+ options: CaptureOptions = {},
104
+ ) {
105
+ const count = options.frames ?? 2;
106
+
107
+ if (!Number.isSafeInteger(count) || count < 1)
108
+ return yield* undispatched(
109
+ "moment",
110
+ new InvalidRequest({ detail: "frames must be a positive safe integer" }),
111
+ );
112
+ const outline = options.snapshot ?? false;
113
+
114
+ // The picture comes first, at no call with a running screencast, and the window ends at its
115
+ // paint, so its changes hold nothing it does not show; without one, the window ends now. The
116
+ // outline is taken alongside. What cannot be read is missing from the moment, not its end.
117
+ const { pictured, outlined } = yield* Effect.all(
118
+ {
119
+ pictured: Effect.result(page.frame({ after: "input" })).pipe(
120
+ Effect.flatMap((picture) =>
121
+ Effect.map(
122
+ page.window({
123
+ since: options.since ?? Duration.seconds(5),
124
+ until: Result.getOrUndefined(picture),
125
+ unmask: options.unmask,
126
+ }),
127
+ (window) => ({ window, picture }),
128
+ ),
129
+ ),
130
+ ),
131
+ outlined: Effect.result(
132
+ outline === false
133
+ ? Effect.void
134
+ : page.snapshot({ maxChars: 4000, ...(outline === true ? {} : outline) }),
135
+ ),
136
+ },
137
+ { concurrency: 2 },
138
+ );
139
+
140
+ const { window, picture } = pictured;
141
+ const shown = Result.getOrUndefined(picture);
142
+
143
+ return new Moment({
144
+ ...window,
145
+ frames: spread(
146
+ shown === undefined
147
+ ? window.frames
148
+ : [...window.frames.filter((frame) => frame !== shown), shown],
149
+ count,
150
+ ),
151
+ snapshot: Result.getOrUndefined(outlined) ?? undefined,
152
+ missing: [
153
+ ...window.missing,
154
+ ...Option.toArray(Result.getFailure(picture)),
155
+ ...Option.toArray(Result.getFailure(outlined)),
156
+ ],
157
+ });
158
+ });
159
+
160
+ /** A window's record of changes, if it saw any of the window: a page's first has none. */
161
+ const recordOf = ({ changes }: Window) =>
162
+ changes !== undefined && changes.from < changes.until ? changes : undefined;
163
+
164
+ /** What a window says of its page at its end: what was still in flux, and what had settled. */
165
+ export interface Account {
166
+ /**
167
+ * How long, in milliseconds, the page had been still at the end: since the last change in view
168
+ * its record shows, or the last paint its screencast frames show, or else since the window, or
169
+ * its record, began. Undefined where neither could tell, with no record of the window and no
170
+ * painted frame: a screenshot shows the page, not when it last changed. Frames show paint only
171
+ * while a capture runs, and the record does not see a canvas, so a canvas that keeps drawing
172
+ * with no capture running reads as still.
173
+ */
174
+ readonly stillFor: number | undefined;
175
+ /**
176
+ * The changes still under way at the end: what kept changing, more than once in the window or
177
+ * already before it, and changed again within twice its own pace before the end.
178
+ */
179
+ readonly changing: ReadonlyArray<Change>;
180
+ /** Every other change, which shows what it settled at. */
181
+ readonly settled: ReadonlyArray<Change>;
182
+ }
183
+
184
+ /** Whether a change was still under way at `until`, by its own pace. */
185
+ const underWay = (change: Change, until: number) => {
186
+ const pace =
187
+ change.count > 1
188
+ ? (change.at - change.startedAt) / (change.count - 1)
189
+ : change.earlier === undefined
190
+ ? undefined
191
+ : change.startedAt - change.earlier;
192
+
193
+ return pace !== undefined && until - change.at < 2 * pace;
194
+ };
195
+
196
+ /**
197
+ * An account of the window's page at its end: how long it had been still, and which of its
198
+ * changes were still under way and which had settled. A window whose record saw none of it has
199
+ * neither.
200
+ */
201
+ export const account = (window: Window): Account => {
202
+ const record = recordOf(window);
203
+ const changes = record?.changes ?? [];
204
+
205
+ const stirred = [
206
+ ...changes.flatMap((change) => (change.kind === "title" ? [] : [change.at])),
207
+ ...window.frames.flatMap((frame) =>
208
+ frame.timing._tag === "BrowserPaint" ? [frame.hostTime] : [],
209
+ ),
210
+ ];
211
+
212
+ return {
213
+ stillFor:
214
+ record === undefined && stirred.length === 0
215
+ ? undefined
216
+ : window.until - Math.max(window.since, record?.from ?? window.since, ...stirred),
217
+ changing: changes.filter((change) => underWay(change, window.until)),
218
+ settled: changes.filter((change) => !underWay(change, window.until)),
219
+ };
220
+ };
221
+
222
+ const seconds = (moment: Moment, at: number) => `${((at - moment.until) / 1000).toFixed(1)}s`;
223
+
224
+ const pointOf = (x: number | undefined, y: number | undefined) =>
225
+ x === undefined || y === undefined ? undefined : `(${Math.round(x)}, ${Math.round(y)})`;
226
+
227
+ /**
228
+ * What an action acted on. Without a subject the action failed before finding it, and a ref it was
229
+ * given is left out: refs are reused, so the page's outline may give it to something else by now.
230
+ */
231
+ const acted = (event: Action): string => {
232
+ const point = pointOf(event.x, event.y);
233
+
234
+ if (event.subject === undefined)
235
+ return event.target === undefined || /^e\d+$|"e\d+"/.test(event.target)
236
+ ? (point ?? "")
237
+ : event.target;
238
+ // A drag's point is where it ended.
239
+ if (event.to !== undefined)
240
+ return `${named(event.subject, undefined)} to ${named(event.to, point)}`;
241
+
242
+ return named(event.subject, point);
243
+ };
244
+
245
+ /**
246
+ * An action as a step: `click button "Pay"`. A failed one says only that: its error is advice for
247
+ * the caller that acted, which may name a ref, not a fact about the page.
248
+ */
249
+ const step = (event: Action): string => {
250
+ const target = acted(event);
251
+ const text = event.text === undefined ? "" : JSON.stringify(event.text);
252
+
253
+ const what =
254
+ event.name === "type" && target !== ""
255
+ ? `${text} into ${target}`
256
+ : event.name === "select" && target !== ""
257
+ ? `${text} in ${target}`
258
+ : `${target} ${text}`;
259
+
260
+ return `${event.name} ${what}${event.ok ? "" : " (failed)"}`.replace(/\s+/g, " ").trim();
261
+ };
262
+
263
+ /** What is drawn rather than written, whose changes only the screenshots show. */
264
+ const drawn = new Set(["canvas", "iframe", "video", "embed", "object", "svg"]);
265
+
266
+ const onDrawn = (event: Action) =>
267
+ event.ok && event.subject !== undefined && drawn.has(event.subject.role ?? event.subject.tag);
268
+
269
+ interface Line {
270
+ readonly at: number;
271
+ readonly rank: number;
272
+ readonly text: string;
273
+ }
274
+
275
+ /** At most this many lines of what happened; the least notable are counted, not listed. */
276
+ const maxLines = 24;
277
+
278
+ /**
279
+ * The changes as lines, news first, and the actions they name; cells of one column that changed
280
+ * together as one line. What kept changing says whether it still was at the moment, by its account.
281
+ */
282
+ const changed = (moment: Moment, record: Changes) => {
283
+ const actions = moment.events.filter((event): event is Action => event._tag === "Action");
284
+ const named = new Set<Action>();
285
+ const changing = new Set(account(moment).changing);
286
+
287
+ const state = (change: Change) =>
288
+ changing.has(change)
289
+ ? ", still changing at the moment"
290
+ : change.count > 1 || change.earlier !== undefined
291
+ ? `, settled since ${seconds(moment, change.at)}`
292
+ : "";
293
+
294
+ // The action whose input a change followed: one that ran while the input arrived, give or take
295
+ // the clock's mapping, so input the library did not send names no action.
296
+ const cause = (change: Change) => {
297
+ const input = change.cause;
298
+
299
+ const action =
300
+ input === undefined
301
+ ? undefined
302
+ : actions.findLast((event) => event.startedAt <= input + 100 && event.at >= input - 100);
303
+
304
+ if (action === undefined) return "";
305
+ named.add(action);
306
+
307
+ return `, after ${step(action)}`;
308
+ };
309
+
310
+ const columns = Map.groupBy(
311
+ record.changes.filter(
312
+ (change) => change.kind === "text" && change.subject.context.column !== undefined,
313
+ ),
314
+ ({ subject: { context } }) => JSON.stringify([context.heading, context.column]),
315
+ );
316
+
317
+ const lines: Array<Line> = [];
318
+ const collapsed = new Set<Change>();
319
+
320
+ for (const [first, ...cells] of columns.values())
321
+ if (first !== undefined && cells.length >= 2) {
322
+ const still = [first, ...cells].filter((cell) => changing.has(cell)).length;
323
+
324
+ for (const cell of [first, ...cells]) collapsed.add(cell);
325
+ lines.push({
326
+ at: first.startedAt,
327
+ rank: still > 0 ? 3 : 2,
328
+ text: `${cells.length + 1} cells in column ${quoted(first.subject.context.column)}${where({ heading: first.subject.context.heading })} changed, such as ${describe(first, true)}${still > 0 ? `, ${still} still changing at the moment` : ""}`,
329
+ });
330
+ }
331
+ for (const change of record.changes)
332
+ if (!collapsed.has(change))
333
+ lines.push({
334
+ at: change.startedAt,
335
+ rank: changing.has(change) ? 3 : 2,
336
+ text: describe(change) + cause(change) + state(change),
337
+ });
338
+
339
+ return { lines, named };
340
+ };
341
+
342
+ /** What a part a moment could not read was, by the operation that failed: a window's one read is its changes. */
343
+ const parts = new Map([
344
+ ["frame", "A picture of the moment"],
345
+ ["snapshot", "The page's outline"],
346
+ ]);
347
+
348
+ /** The moment's changes and events as lines, most notable first when its changes are recorded. */
349
+ const happened = (moment: Moment): ReadonlyArray<string> => {
350
+ const record = recordOf(moment);
351
+
352
+ const { lines, named } =
353
+ record === undefined
354
+ ? { lines: new Array<Line>(), named: new Set<Action>() }
355
+ : changed(moment, record);
356
+
357
+ // Whether anything but a field's own edit changed after an action began.
358
+ const followed = (event: Action) =>
359
+ record?.changes.some((change) => change.kind !== "value" && change.at > event.startedAt) ===
360
+ true;
361
+
362
+ // With a record, an action is told as a step where changes followed it and none names it, so a
363
+ // click whose effect a busy page leaves uncredited is not lost; where its effect is drawn; or
364
+ // where it started before the record began, as on the document a click opened. An attempt that
365
+ // nothing followed, a failed one, a hover and a scroll are left out.
366
+ const told = (event: Action) =>
367
+ !(event.ok && ["navigate", "back", "reload"].includes(event.name)) &&
368
+ (record === undefined ||
369
+ event.startedAt <= record.from ||
370
+ (event.ok &&
371
+ event.name !== "scroll" &&
372
+ (onDrawn(event) || (event.name !== "hover" && !named.has(event) && followed(event)))));
373
+
374
+ for (const event of moment.events)
375
+ if (event._tag === "Action" && event.page === moment.page && told(event))
376
+ lines.push({ at: event.at, rank: 4, text: step(event) });
377
+ else if (event._tag === "Navigated")
378
+ lines.push({
379
+ at: event.at,
380
+ rank: 0,
381
+ text: `${event.sameDocument ? "moved within the page" : "navigated"} to ${event.url}`,
382
+ });
383
+ else if (event._tag === "PageOpened")
384
+ lines.push({ at: event.at, rank: 0, text: `a tab opened at ${event.url}` });
385
+ else if (event._tag === "PageClosed")
386
+ lines.push({
387
+ at: event.at,
388
+ rank: 0,
389
+ text: `the tab ${event.cause === "crashed" ? "crashed" : "closed"}`,
390
+ });
391
+ else if (event._tag === "DialogShown")
392
+ lines.push({
393
+ at: event.at,
394
+ rank: 1,
395
+ text: `a dialog (${event.kind}) said ${JSON.stringify(event.message)}`,
396
+ });
397
+
398
+ const ordered = lines.toSorted((left, right) =>
399
+ record === undefined ? left.at - right.at : left.rank - right.rank || left.at - right.at,
400
+ );
401
+
402
+ const kept = ordered.slice(0, maxLines);
403
+ const drawing = moment.events.some((event) => event._tag === "Action" && onDrawn(event));
404
+ const begins = record === undefined ? moment.since : Math.max(moment.since, record.from);
405
+ const unread = moment.missing.some((error) => !parts.has(error.operation));
406
+
407
+ return [
408
+ ...kept.map((line) => `${seconds(moment, line.at)} ${line.text}`),
409
+ ...(ordered.length > maxLines ? [`…and ${ordered.length - maxLines} less notable`] : []),
410
+ ...(record === undefined
411
+ ? unread
412
+ ? []
413
+ : ["What changed on the page was not recorded."]
414
+ : record.changes.length > 0
415
+ ? []
416
+ : [
417
+ `No text in view changed${begins > moment.since ? ` after ${seconds(moment, begins)}` : ""}${drawing ? "; what is drawn, such as a canvas, shows only in the screenshots" : ""}.`,
418
+ ]),
419
+ ...(record !== undefined && record.dropped > 0
420
+ ? [
421
+ `The page changed too much to tell it all: at least ${record.dropped} changes are not told.`,
422
+ ]
423
+ : []),
424
+ ...(begins > moment.since
425
+ ? [`Changes before ${seconds(moment, begins)} are not all known.`]
426
+ : []),
427
+ ...moment.missing.map(
428
+ (error) =>
429
+ `${parts.get(error.operation) ?? "What changed on the page"} could not be read: ${error.reason.message}.`,
430
+ ),
431
+ ];
432
+ };
433
+
434
+ /**
435
+ * The moment as one user message: the page's outline when the moment has one, what changed, most
436
+ * notable first, what kept changing told as still changing at the moment or settled since when, by
437
+ * its account, with the actions that the changes followed, that changes followed though none
438
+ * names them, or that drew what only the screenshots show, what could not be read and why, and its
439
+ * frames captioned with their times, the last one "the moment" where it is the picture at its end.
440
+ * Give a model its task with `Prompt.setSystem`, and add what the caller knows about the page as
441
+ * more text.
442
+ */
443
+ export const toPrompt = (moment: Moment): Prompt.Prompt => {
444
+ const count = moment.frames.length;
445
+ const last = count - 1;
446
+ const pictured = moment.frames.at(-1)?.hostTime === moment.until;
447
+
448
+ const following =
449
+ count === 1 ? "One screenshot follows" : `${count} screenshots follow, oldest first`;
450
+
451
+ return Prompt.fromMessages([
452
+ Prompt.makeMessage("user", {
453
+ content: [
454
+ Prompt.makePart("text", {
455
+ text: [
456
+ "One moment of a browser session. Rely only on what this material shows, and prefer concrete details: numbers, names, colours, positions and motion.",
457
+ "",
458
+ ...(moment.snapshot === undefined ? [] : [moment.snapshot.rendered, ""]),
459
+ `${recordOf(moment) === undefined ? "What happened" : "What changed, most notable first"} (seconds before the moment, over ${((moment.until - moment.since) / 1000).toFixed(1)}s):`,
460
+ ...happened(moment),
461
+ ...(count === 0
462
+ ? []
463
+ : [
464
+ "",
465
+ `${following}${pictured ? (count === 1 ? ": the moment itself." : "; the last one is the moment itself.") : "."}`,
466
+ ]),
467
+ ].join("\n"),
468
+ }),
469
+ ...moment.frames.flatMap((frame, index) => [
470
+ Prompt.makePart("text", {
471
+ text:
472
+ index === last && pictured ? "The moment:" : `${seconds(moment, frame.hostTime)}:`,
473
+ }),
474
+ Prompt.makePart("file", { mediaType: "image/jpeg", data: frame.data }),
475
+ ]),
476
+ ],
477
+ }),
478
+ ]);
479
+ };
package/src/Motion.ts ADDED
@@ -0,0 +1,166 @@
1
+ /**
2
+ * Pointer motion as a complete, bounded schedule, independent of browser input and transport. A
3
+ * presenter takes a planner as a value: `lognormal` here, or another, such as recorded strokes.
4
+ *
5
+ * @since 0.3.0
6
+ */
7
+ import { Effect, Random, Schema } from "effect";
8
+
9
+ export const maximumSamples = 2048;
10
+ export const maximumDurationMillis = 5000;
11
+
12
+ export const Point = Schema.Struct({ x: Schema.Finite, y: Schema.Finite });
13
+ export type Point = typeof Point.Type;
14
+
15
+ export const Sample = Schema.Struct({
16
+ x: Schema.Finite,
17
+ y: Schema.Finite,
18
+ afterMillis: Schema.Finite.check(
19
+ Schema.isGreaterThanOrEqualTo(0),
20
+ Schema.isLessThanOrEqualTo(maximumDurationMillis),
21
+ ),
22
+ });
23
+
24
+ export type Sample = typeof Sample.Type;
25
+
26
+ /** A complete glide: 1 to 2,048 samples whose offsets never decrease. */
27
+ export const Plan = Schema.Array(Sample).check(
28
+ Schema.isBetweenLength(1, maximumSamples),
29
+ Schema.makeFilter(
30
+ (samples) =>
31
+ samples.every(
32
+ (sample, index) => sample.afterMillis >= (samples[index - 1]?.afterMillis ?? 0),
33
+ ),
34
+ { expected: "nondecreasing sample offsets" },
35
+ ),
36
+ );
37
+
38
+ export type Plan = typeof Plan.Type;
39
+
40
+ export interface Service {
41
+ /**
42
+ * Offsets start when the performer publishes the plan, before its first input. The browser
43
+ * decodes the result as a `Plan` whose last sample must be exactly `to`. Glide time has a budget
44
+ * of its own, outside the browser's `actionTimeout`, so a drag may glide twice for 5 seconds.
45
+ */
46
+ readonly plan: (from: Point, to: Point) => Effect.Effect<ReadonlyArray<Sample>>;
47
+ }
48
+
49
+ // These are the research's tuned two-stroke parameters. Width scaling was not evaluated, so the
50
+ // model keeps its 40px reference width rather than implying measured target-size behavior.
51
+ const tuning = {
52
+ intercept: 115.8,
53
+ slope: 291.8,
54
+ durationSigma: 0.407,
55
+ amplitude: 0.989,
56
+ amplitudeSigma: 0.138,
57
+ lateral: 0.0934,
58
+ primaryDuration: 0.512,
59
+ correctionDuration: 0.586,
60
+ overlap: 0.452,
61
+ sigmaLow: 0.155,
62
+ sigmaHigh: 0.53,
63
+ } as const;
64
+
65
+ const sampleMillis = 16.7;
66
+
67
+ const normal = Effect.gen(function* () {
68
+ const radius = Math.sqrt(-2 * Math.log(1 - (yield* Random.next)));
69
+
70
+ return radius * Math.cos(2 * Math.PI * (yield* Random.next));
71
+ });
72
+
73
+ // Abramowitz-Stegun 7.1.26 supplies erf without another runtime dependency (error < 1.5e-7).
74
+ const erf = (value: number) => {
75
+ const x = Math.abs(value);
76
+ const t = 1 / (1 + 0.3275911 * x);
77
+
78
+ const polynomial =
79
+ ((((1.061405429 * t - 1.453152027) * t + 1.421413741) * t - 0.284496736) * t + 0.254829592) * t;
80
+
81
+ return Math.sign(value) * (1 - polynomial * Math.exp(-x * x));
82
+ };
83
+
84
+ const cumulative = (time: number, onset: number, mu: number, sigma: number) =>
85
+ time <= onset ? 0 : (1 + erf((Math.log(time - onset) - mu) / (sigma * Math.SQRT2))) / 2;
86
+
87
+ const plan: Service["plan"] = Effect.fnUntraced(function* (from, to) {
88
+ // Work in normalized coordinates so finite extreme inputs cannot overflow the displacement.
89
+ const scale = Math.max(1, Math.abs(from.x), Math.abs(from.y), Math.abs(to.x), Math.abs(to.y));
90
+ const startX = from.x / scale;
91
+ const startY = from.y / scale;
92
+ const deltaX = to.x / scale - startX;
93
+ const deltaY = to.y / scale - startY;
94
+ const distance = Math.hypot(deltaX, deltaY);
95
+
96
+ if (distance * scale < 2) return [{ ...to, afterMillis: 0 }];
97
+ const unitX = deltaX / distance;
98
+ const unitY = deltaY / distance;
99
+
100
+ const drawnDuration =
101
+ (tuning.intercept + tuning.slope * Math.log2(1 + distance * (scale / 40))) *
102
+ Math.exp((yield* normal) * tuning.durationSigma);
103
+
104
+ const endFactor = Math.max(
105
+ tuning.primaryDuration,
106
+ tuning.overlap * tuning.primaryDuration + tuning.correctionDuration,
107
+ );
108
+
109
+ // Bound the duration before evaluating either stroke: dropping a sampled tail would remove
110
+ // its homing correction. The last regular 16.7ms sample remains inside the public time bound.
111
+ const lastRegularSample = Math.floor(maximumDurationMillis / sampleMillis) * sampleMillis;
112
+ const duration = Math.min(drawnDuration, lastRegularSample / endFactor);
113
+ const amplitude = distance * (tuning.amplitude + (yield* normal) * tuning.amplitudeSigma);
114
+ const angle = ((yield* normal) * Math.PI) / 60;
115
+ const primaryX = amplitude * (unitX * Math.cos(angle) - unitY * Math.sin(angle));
116
+ const primaryY = amplitude * (unitY * Math.cos(angle) + unitX * Math.sin(angle));
117
+ const sigma1 = yield* Random.nextBetween(tuning.sigmaLow, tuning.sigmaHigh);
118
+ const primaryDuration = tuning.primaryDuration * duration;
119
+ const mu1 = Math.log(primaryDuration / (Math.exp(3 * sigma1) - Math.exp(-3 * sigma1)));
120
+ const onset1 = -Math.exp(mu1 - 3 * sigma1);
121
+ const sigma2 = yield* Random.nextBetween(tuning.sigmaLow, tuning.sigmaHigh);
122
+ const correctionDuration = tuning.correctionDuration * duration;
123
+ const mu2 = Math.log(correctionDuration / (Math.exp(3 * sigma2) - Math.exp(-3 * sigma2)));
124
+ const onset2 = tuning.overlap * primaryDuration - Math.exp(mu2 - 3 * sigma2);
125
+ const end = Math.max(primaryDuration, onset2 + Math.exp(mu2 + 3 * sigma2));
126
+ const centered = Math.max(Number.EPSILON, yield* Random.next) - 0.5;
127
+
128
+ const lateral =
129
+ -tuning.lateral * Math.sign(centered) * Math.log(1 - 2 * Math.abs(centered)) * distance;
130
+
131
+ const count = Math.min(
132
+ Math.ceil(end / sampleMillis),
133
+ Math.floor(lastRegularSample / sampleMillis),
134
+ );
135
+
136
+ const samples: Array<Sample> = [];
137
+ let last: Point = from;
138
+
139
+ const coordinate = (value: number) =>
140
+ Math.round(Math.max(-Number.MAX_VALUE, Math.min(Number.MAX_VALUE, value * scale)));
141
+
142
+ for (let index = 0; index < count; index++) {
143
+ const afterMillis = index * sampleMillis;
144
+ const first = cumulative(afterMillis, onset1, mu1, sigma1);
145
+ const second = cumulative(afterMillis, onset2, mu2, sigma2);
146
+ const bend = lateral * Math.sin(Math.PI * first);
147
+ const x = coordinate(startX + primaryX * first + (deltaX - primaryX) * second - unitY * bend);
148
+ const y = coordinate(startY + primaryY * first + (deltaY - primaryY) * second + unitX * bend);
149
+
150
+ // A pointer that stays on its pixel reports nothing; the model's timing is unchanged.
151
+ if (x === last.x && y === last.y) continue;
152
+ last = { x, y };
153
+ samples.push({ x, y, afterMillis });
154
+ }
155
+
156
+ // The exact destination always lands at the model's end, even if a dwell precedes it.
157
+ samples.push({ ...to, afterMillis: count * sampleMillis });
158
+
159
+ return samples;
160
+ });
161
+
162
+ /**
163
+ * The research's tuned two-stroke sigma-lognormal model: a fast stroke toward the target and a
164
+ * slower one that homes in on it, with a little bend, drawn from `Random`.
165
+ */
166
+ export const lognormal: Service = { plan };