@opencraw/studio 0.0.7 → 0.0.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.esm.js +3239 -901
- package/dist/src/document-view/deck-view.mapper.d.ts +70 -0
- package/dist/src/document-view/index.d.ts +11 -0
- package/dist/src/document-view/pdf-view.mapper.d.ts +47 -0
- package/dist/src/document-view/table-preview.use-case.d.ts +152 -0
- package/dist/src/document-view/tree-view.mapper.d.ts +47 -0
- package/dist/src/document-view/workbook-view.mapper.d.ts +51 -0
- package/dist/src/flow-recording/action-to-step.mapper.d.ts +16 -0
- package/dist/src/flow-recording/index.d.ts +13 -0
- package/dist/src/flow-recording/next-link.policy.d.ts +27 -0
- package/dist/src/flow-recording/recorded-action.contract.d.ts +114 -0
- package/dist/src/flow-recording/recorder-script.client.d.ts +48 -0
- package/dist/src/flow-recording/recorder-session.use-case.d.ts +67 -0
- package/dist/src/flow-recording/secret-field.policy.d.ts +33 -0
- package/dist/src/flow-recording/wait-insertion.policy.d.ts +26 -0
- package/dist/src/index.d.ts +2 -2
- package/dist/src/page-inspector/dom-tree.mapper.d.ts +32 -0
- package/dist/src/page-inspector/index.d.ts +7 -0
- package/dist/src/page-inspector/page-data.algorithm.d.ts +39 -0
- package/dist/src/page-inspector/responses-seen.use-case.d.ts +36 -0
- package/dist/src/page-snapshot/index.d.ts +3 -0
- package/dist/src/page-snapshot/pdf-bytes.use-case.d.ts +16 -0
- package/dist/src/page-snapshot/reread-csv.use-case.d.ts +22 -0
- package/dist/src/page-snapshot/take-snapshot.use-case.d.ts +26 -1
- package/dist/src/studio-api/command.contract.d.ts +300 -0
- package/dist/src/studio-api/deck-preview-view.contract.d.ts +28 -0
- package/dist/src/studio-api/deck-view.contract.d.ts +127 -0
- package/dist/src/studio-api/document-tree-view.contract.d.ts +26 -0
- package/dist/src/studio-api/event.contract.d.ts +45 -0
- package/dist/src/studio-api/grid-preview-view.contract.d.ts +26 -0
- package/dist/src/studio-api/grid-view.contract.d.ts +75 -0
- package/dist/src/studio-api/index.d.ts +22 -4
- package/dist/src/studio-api/inspect-view.contract.d.ts +62 -0
- package/dist/src/studio-api/pdf-view.contract.d.ts +66 -0
- package/dist/src/studio-api/responses-seen-view.contract.d.ts +20 -0
- package/dist/src/studio-api/snapshot-view.contract.d.ts +22 -1
- package/dist/src/studio-api/table-preview-view.contract.d.ts +65 -0
- package/dist/src/studio-server/deck-preview.handler.d.ts +17 -0
- package/dist/src/studio-server/deck-view.handler.d.ts +15 -0
- package/dist/src/studio-server/document-tree.handler.d.ts +14 -0
- package/dist/src/studio-server/grid-preview.handler.d.ts +17 -0
- package/dist/src/studio-server/grid-view.handler.d.ts +20 -0
- package/dist/src/studio-server/inspect-page.handler.d.ts +18 -0
- package/dist/src/studio-server/pdf-bytes.handler.d.ts +17 -0
- package/dist/src/studio-server/pdf-view.handler.d.ts +14 -0
- package/dist/src/studio-server/recording.handler.d.ts +33 -0
- package/dist/src/studio-server/responses-seen.handler.d.ts +16 -0
- package/dist/src/studio-server/studio-http.use-case.d.ts +3 -1
- package/dist/src/studio-server/table-preview.handler.d.ts +16 -0
- package/dist/src/studio-server/workspace.store.d.ts +6 -2
- package/dist/ui/assets/index-DdiGdaTN.js +13 -0
- package/dist/ui/assets/pdf-lqXfRfkR.js +56 -0
- package/dist/ui/assets/pdf.worker.min-BmVo14Nb.mjs +28 -0
- package/dist/ui/assets/pdf.worker.min-D7i8o3wR.js +1 -0
- package/dist/ui/index.html +1 -1
- package/package.json +3 -2
- package/dist/ui/assets/index-DV8i4Zza.js +0 -13
package/dist/index.esm.js
CHANGED
|
@@ -1,11 +1,13 @@
|
|
|
1
|
-
import { normalize, sep,
|
|
1
|
+
import { join, normalize, sep, extname, dirname, resolve } from 'node:path';
|
|
2
2
|
import { createServer } from 'node:http';
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
+
import { BODY_KINDS, BrowserClient, HttpClient, NODE_ID_ATTRIBUTE, rowsOfCells, parseXml, findDeckTables, findGridTables, analyzeTables, isDeckDocument, readRecipeSource, parseInputRecipe, parseOutputRecipe, bindRecipeSet, memorySink, createCrawler, traceLine, resolveMissingPolicy, isWorkbookDocument, countMatches, matchesOf, tryParseJson, RecipeBindingError, RecipeValidationError, isPdfDocument, BrowserProfiles } from '@opencraw/core';
|
|
5
6
|
import { randomBytes, createHash } from 'node:crypto';
|
|
6
|
-
import { readRecipeSource, parseInputRecipe, parseOutputRecipe, bindRecipeSet, memorySink, createCrawler, traceLine, resolveMissingPolicy, BrowserClient, HttpClient, NODE_ID_ATTRIBUTE, countMatches, RecipeBindingError, RecipeValidationError } from '@opencraw/core';
|
|
7
7
|
import { load } from 'cheerio';
|
|
8
|
-
import { writeFile } from 'node:fs/promises';
|
|
8
|
+
import { readFile, writeFile } from 'node:fs/promises';
|
|
9
|
+
import { describeJson } from '@opencraw/probe';
|
|
10
|
+
import { tmpdir } from 'node:os';
|
|
9
11
|
import { existsSync, createReadStream } from 'node:fs';
|
|
10
12
|
|
|
11
13
|
const sentencePartSchema = z.object({
|
|
@@ -119,6 +121,151 @@ const inferSelectorCommandSchema = z.object({
|
|
|
119
121
|
path: z.string().min(1),
|
|
120
122
|
nodeIds: z.union([onePickedNodeSchema, twoPickedNodesSchema])
|
|
121
123
|
});
|
|
124
|
+
/**
|
|
125
|
+
* The Inspect panel's DOM tree and data-in-the-page findings, off the same
|
|
126
|
+
* cached snapshot `take-snapshot`/`infer-selector` already use (`page-inspector`,
|
|
127
|
+
* issue #93). Call `take-snapshot` first, same as `infer-selector`.
|
|
128
|
+
*/
|
|
129
|
+
const inspectPageCommandSchema = z.object({
|
|
130
|
+
type: z.literal('inspect-page'),
|
|
131
|
+
recipeId: z.string().min(1),
|
|
132
|
+
path: z.string().min(1)
|
|
133
|
+
});
|
|
134
|
+
/**
|
|
135
|
+
* The JSON responses seen while a recipe's start page rendered
|
|
136
|
+
* (`page-inspector`'s `responses-seen.use-case.ts`, issue #93), for the
|
|
137
|
+
* Inspect panel's "responses seen" list. Opens its own browser session
|
|
138
|
+
* (see that use-case's doc comment for why); slower than `inspect-page`.
|
|
139
|
+
*/
|
|
140
|
+
const responsesSeenCommandSchema = z.object({
|
|
141
|
+
type: z.literal('responses-seen'),
|
|
142
|
+
recipeId: z.string().min(1),
|
|
143
|
+
path: z.string().min(1)
|
|
144
|
+
});
|
|
145
|
+
/**
|
|
146
|
+
* The document tree for the content pane's tree canvas (`document-view`'s
|
|
147
|
+
* `tree-view.mapper.ts`, studio plan §3.4, issue #94's 5a), off the same
|
|
148
|
+
* cached snapshot `take-snapshot`/`inspect-page` already use — call
|
|
149
|
+
* `take-snapshot` first, same as `inspect-page`. Only a JSON/YAML/XML
|
|
150
|
+
* snapshot has one; PDF, workbook and deck documents get their own canvases
|
|
151
|
+
* (5b/5c/5d).
|
|
152
|
+
*/
|
|
153
|
+
const documentTreeCommandSchema = z.object({
|
|
154
|
+
type: z.literal('document-tree'),
|
|
155
|
+
recipeId: z.string().min(1),
|
|
156
|
+
path: z.string().min(1)
|
|
157
|
+
});
|
|
158
|
+
/**
|
|
159
|
+
* The PDF canvas's cells and rows (`document-view`'s `pdf-view.mapper.ts`,
|
|
160
|
+
* studio plan §3.4, issue #94's 5b), off the same cached snapshot
|
|
161
|
+
* `take-snapshot`/`document-tree` already use — call `take-snapshot` first.
|
|
162
|
+
* Only a PDF snapshot has one; JSON/YAML/XML get the tree canvas (5a), the
|
|
163
|
+
* workbook and deck their own canvases (5c/5d).
|
|
164
|
+
*/
|
|
165
|
+
const pdfViewCommandSchema = z.object({
|
|
166
|
+
type: z.literal('pdf-view'),
|
|
167
|
+
recipeId: z.string().min(1),
|
|
168
|
+
path: z.string().min(1)
|
|
169
|
+
});
|
|
170
|
+
/** A `table` extract's options, as the studio's PDF canvas picks build them — mirrors `document-view/table-preview.use-case.ts`'s `TablePreviewOptions`. */
|
|
171
|
+
const tablePreviewOptionsSchema = z.object({
|
|
172
|
+
header: z.string().min(1),
|
|
173
|
+
until: z.string().min(1).optional(),
|
|
174
|
+
columns: z.record(z.string(), z.string().min(1)).optional(),
|
|
175
|
+
align: z.enum(['auto', 'top', 'center', 'bottom']).optional()
|
|
176
|
+
});
|
|
177
|
+
/**
|
|
178
|
+
* The live preview of a `table` extract's options against a cached PDF
|
|
179
|
+
* snapshot (`document-view`'s `table-preview.use-case.ts`, studio plan §3.4,
|
|
180
|
+
* issue #94's 5b): the matched rows to highlight on the page, recomputed as
|
|
181
|
+
* the person edits the options — call `take-snapshot` first.
|
|
182
|
+
*/
|
|
183
|
+
const tablePreviewCommandSchema = z.object({
|
|
184
|
+
type: z.literal('table-preview'),
|
|
185
|
+
recipeId: z.string().min(1),
|
|
186
|
+
path: z.string().min(1),
|
|
187
|
+
options: tablePreviewOptionsSchema
|
|
188
|
+
});
|
|
189
|
+
/**
|
|
190
|
+
* The grid canvas's sheets and cells (`document-view`'s `workbook-view.mapper.ts`,
|
|
191
|
+
* studio plan §3.4, issue #94's 5c), off the same cached snapshot
|
|
192
|
+
* `take-snapshot`/`document-tree` already use — call `take-snapshot` first.
|
|
193
|
+
* Only a CSV/spreadsheet snapshot has one; JSON/YAML/XML get the tree canvas
|
|
194
|
+
* (5a), PDF the PDF canvas (5b), the deck its own canvas (5d).
|
|
195
|
+
*
|
|
196
|
+
* `delimiter`/`encoding` override a CSV's auto-detected format (issue #94's
|
|
197
|
+
* 5c: "the detected delimiter and encoding shown and overridable"); the
|
|
198
|
+
* server re-reads the file with the override applied and keeps it cached, so
|
|
199
|
+
* a later `grid-preview` sees the same reading. Both are rejected on a
|
|
200
|
+
* spreadsheet snapshot (there is nothing to detect).
|
|
201
|
+
*/
|
|
202
|
+
const gridViewCommandSchema = z.object({
|
|
203
|
+
type: z.literal('grid-view'),
|
|
204
|
+
recipeId: z.string().min(1),
|
|
205
|
+
path: z.string().min(1),
|
|
206
|
+
delimiter: z.string().min(1).optional(),
|
|
207
|
+
encoding: z.string().min(1).optional()
|
|
208
|
+
});
|
|
209
|
+
/** A `table` extract's options for a workbook, as the studio's grid canvas picks build them — mirrors `document-view/table-preview.use-case.ts`'s `GridTablePreviewOptions`. */
|
|
210
|
+
const gridPreviewOptionsSchema = z.object({
|
|
211
|
+
sheet: z.string().min(1).optional(),
|
|
212
|
+
header: z.string().min(1),
|
|
213
|
+
until: z.string().min(1).optional(),
|
|
214
|
+
columns: z.record(z.string(), z.string().min(1)).optional(),
|
|
215
|
+
headerRows: z.int().min(1).optional(),
|
|
216
|
+
fillDown: z.array(z.string().min(1)).min(1).optional(),
|
|
217
|
+
includeHidden: z.boolean().optional()
|
|
218
|
+
});
|
|
219
|
+
/**
|
|
220
|
+
* The live preview of a `table` extract's options against a cached CSV/
|
|
221
|
+
* spreadsheet snapshot (`document-view`'s `table-preview.use-case.ts`'s
|
|
222
|
+
* `previewGridTable`, studio plan §3.4, issue #94's 5c): the matched
|
|
223
|
+
* table(s), recomputed as the person edits the options — call `take-snapshot`
|
|
224
|
+
* and `grid-view` first.
|
|
225
|
+
*/
|
|
226
|
+
const gridPreviewCommandSchema = z.object({
|
|
227
|
+
type: z.literal('grid-preview'),
|
|
228
|
+
recipeId: z.string().min(1),
|
|
229
|
+
path: z.string().min(1),
|
|
230
|
+
options: gridPreviewOptionsSchema
|
|
231
|
+
});
|
|
232
|
+
/**
|
|
233
|
+
* The deck canvas's slides, shapes, tables, charts and notes for a deck
|
|
234
|
+
* (PowerPoint) snapshot (`document-view`'s `deck-view.mapper.ts`, studio plan
|
|
235
|
+
* §3.4, issue #94's 5d), off the same cached snapshot
|
|
236
|
+
* `take-snapshot`/`document-tree` already use — call `take-snapshot` first.
|
|
237
|
+
* Only a `.pptx` snapshot has one; JSON/YAML/XML get the tree canvas (5a),
|
|
238
|
+
* PDF the PDF canvas (5b), the workbook the grid canvas (5c).
|
|
239
|
+
*/
|
|
240
|
+
const deckViewCommandSchema = z.object({
|
|
241
|
+
type: z.literal('deck-view'),
|
|
242
|
+
recipeId: z.string().min(1),
|
|
243
|
+
path: z.string().min(1)
|
|
244
|
+
});
|
|
245
|
+
/** A `table` extract's options for a deck, as the studio's deck canvas picks build them — mirrors `document-view/table-preview.use-case.ts`'s `DeckTablePreviewOptions`. */
|
|
246
|
+
const deckPreviewOptionsSchema = z.object({
|
|
247
|
+
slide: z.string().min(1).optional(),
|
|
248
|
+
shapes: z.boolean().optional(),
|
|
249
|
+
align: z.enum(['auto', 'top', 'center', 'bottom']).optional(),
|
|
250
|
+
header: z.string().min(1),
|
|
251
|
+
until: z.string().min(1).optional(),
|
|
252
|
+
columns: z.record(z.string(), z.string().min(1)).optional(),
|
|
253
|
+
headerRows: z.int().min(1).optional(),
|
|
254
|
+
fillDown: z.array(z.string().min(1)).min(1).optional(),
|
|
255
|
+
includeHidden: z.boolean().optional()
|
|
256
|
+
});
|
|
257
|
+
/**
|
|
258
|
+
* The live preview of a `table` extract's options against a cached deck
|
|
259
|
+
* snapshot (`document-view`'s `table-preview.use-case.ts`'s `previewDeckTable`,
|
|
260
|
+
* studio plan §3.4, issue #94's 5d): the matched table(s), recomputed as the
|
|
261
|
+
* person edits the options — call `take-snapshot` and `deck-view` first.
|
|
262
|
+
*/
|
|
263
|
+
const deckPreviewCommandSchema = z.object({
|
|
264
|
+
type: z.literal('deck-preview'),
|
|
265
|
+
recipeId: z.string().min(1),
|
|
266
|
+
path: z.string().min(1),
|
|
267
|
+
options: deckPreviewOptionsSchema
|
|
268
|
+
});
|
|
122
269
|
/** A missing value in an emitted record, by its position in the last sample run's records (the order they streamed in, matching the UI's own `records` list). */
|
|
123
270
|
const missingWhyTargetSchema = z.object({
|
|
124
271
|
kind: z.literal('missing'),
|
|
@@ -144,13 +291,40 @@ const explainWhyCommandSchema = z.object({
|
|
|
144
291
|
type: z.literal('explain-why'),
|
|
145
292
|
target: whyTargetSchema
|
|
146
293
|
});
|
|
147
|
-
|
|
294
|
+
/**
|
|
295
|
+
* Opens the studio's own headed browser window and starts recording user
|
|
296
|
+
* actions on it (issue #95, phase 6): clicks, fills, selects, key presses
|
|
297
|
+
* and scrolls stream back as `recording-card` events, each already a
|
|
298
|
+
* verified step. `recipeId`'s start point is where the window opens.
|
|
299
|
+
*/
|
|
300
|
+
const startRecordingCommandSchema = z.object({
|
|
301
|
+
type: z.literal('start-recording'),
|
|
302
|
+
recipeId: z.string().min(1)
|
|
303
|
+
});
|
|
304
|
+
/**
|
|
305
|
+
* Closes the recording window, if one is open; a no-op otherwise. Answers
|
|
306
|
+
* with every step recorded, in order, so the caller can offer "make this
|
|
307
|
+
* the login" (writes them under `session.bootstrap`) or "keep as steps"
|
|
308
|
+
* (appends them to the recipe's own `steps`) without a second round trip.
|
|
309
|
+
*/
|
|
310
|
+
const stopRecordingCommandSchema = z.object({
|
|
311
|
+
type: z.literal('stop-recording')
|
|
312
|
+
});
|
|
313
|
+
const studioCommandSchema = z.discriminatedUnion('type', [openWorkspaceCommandSchema, runSampleCommandSchema, stopRunCommandSchema, saveRecipeCommandSchema, fetchStartPageCommandSchema, saveOutlineCommandSchema, takeSnapshotCommandSchema, verifySelectorCommandSchema, inferSelectorCommandSchema, explainWhyCommandSchema, inspectPageCommandSchema, responsesSeenCommandSchema, documentTreeCommandSchema, pdfViewCommandSchema, tablePreviewCommandSchema, gridViewCommandSchema, gridPreviewCommandSchema, deckViewCommandSchema, deckPreviewCommandSchema, startRecordingCommandSchema, stopRecordingCommandSchema]);
|
|
148
314
|
|
|
149
|
-
/**
|
|
315
|
+
/**
|
|
316
|
+
* What `take-snapshot` answers with: the rewritten document ready for the
|
|
317
|
+
* content pane's sandboxed iframe (studio plan §3.1, issue #91). `format`
|
|
318
|
+
* (api mode only; absent in web mode, whose document is always HTML) is
|
|
319
|
+
* what the content pane uses to choose a canvas — the snapshot iframe for
|
|
320
|
+
* `html` (and `docx`/`markdown`, already turned into HTML), else
|
|
321
|
+
* `document-view`'s tree/pdf/grid/deck canvas (studio plan §3.4, issue #94).
|
|
322
|
+
*/
|
|
150
323
|
const snapshotViewSchema = z.object({
|
|
151
324
|
html: z.string(),
|
|
152
325
|
nodeCount: z.number(),
|
|
153
|
-
baseUrl: z.string()
|
|
326
|
+
baseUrl: z.string(),
|
|
327
|
+
format: z.enum(BODY_KINDS).optional()
|
|
154
328
|
});
|
|
155
329
|
|
|
156
330
|
/**
|
|
@@ -253,7 +427,32 @@ const runFinishedEventSchema = z.object({
|
|
|
253
427
|
const workspaceChangedEventSchema = z.object({
|
|
254
428
|
type: z.literal('workspace-changed')
|
|
255
429
|
});
|
|
256
|
-
|
|
430
|
+
/**
|
|
431
|
+
* One recorded action landed as a step (issue #95): the card it becomes
|
|
432
|
+
* (`scope-outline`'s own `OutlineCard`/`OutlineBracket` shape — the wire
|
|
433
|
+
* form `outline-view.contract.ts` already defines, never a second card
|
|
434
|
+
* format) and whether its value is a secret placeholder, for the secret
|
|
435
|
+
* pill styling.
|
|
436
|
+
*/
|
|
437
|
+
const recordingCardEventSchema = z.object({
|
|
438
|
+
type: z.literal('recording-card'),
|
|
439
|
+
node: outlineNodeSchema,
|
|
440
|
+
secret: z.boolean()
|
|
441
|
+
});
|
|
442
|
+
/** A notice from the recording session that is not itself a step: a "next" link offer, or an unsupported iframe/shadow-DOM report. */
|
|
443
|
+
const recordingNoteEventSchema = z.object({
|
|
444
|
+
type: z.literal('recording-note'),
|
|
445
|
+
kind: z.enum(['next-link', 'unsupported', 'info']),
|
|
446
|
+
message: z.string()
|
|
447
|
+
});
|
|
448
|
+
/** One recorded step's raw JSON, exactly as `flow-recording` built it. */
|
|
449
|
+
const recordedStepSchema = z.record(z.string(), z.unknown());
|
|
450
|
+
/** The recording window closed — `stop-recording` completed, or the window was closed some other way. `steps` is every step recorded, in order, exactly as `stop-recording`'s own response carries them. */
|
|
451
|
+
const recordingStoppedEventSchema = z.object({
|
|
452
|
+
type: z.literal('recording-stopped'),
|
|
453
|
+
steps: z.array(recordedStepSchema)
|
|
454
|
+
});
|
|
455
|
+
const studioEventSchema = z.discriminatedUnion('type', [traceLineEventSchema, recordEventSchema, recordRejectedEventSchema, runFinishedEventSchema, workspaceChangedEventSchema, recordingCardEventSchema, recordingNoteEventSchema, recordingStoppedEventSchema]);
|
|
257
456
|
|
|
258
457
|
/** One problem found on a recipe file, with the JSON path it lives at. */
|
|
259
458
|
const recipeIssueSchema = z.object({
|
|
@@ -315,6 +514,236 @@ const whyViewSchema = z.object({
|
|
|
315
514
|
reason: z.string().optional()
|
|
316
515
|
});
|
|
317
516
|
|
|
517
|
+
const domTreeNodeViewSchema = z.lazy(() => z.object({
|
|
518
|
+
nodeId: z.string(),
|
|
519
|
+
tag: z.string(),
|
|
520
|
+
id: z.string().optional(),
|
|
521
|
+
classes: z.array(z.string()).optional(),
|
|
522
|
+
attributes: z.record(z.string(), z.string()),
|
|
523
|
+
text: z.string().optional(),
|
|
524
|
+
hidden: z.boolean(),
|
|
525
|
+
children: z.array(domTreeNodeViewSchema),
|
|
526
|
+
repeatCount: z.number().int().optional(),
|
|
527
|
+
repeatNodeIds: z.array(z.string()).optional()
|
|
528
|
+
}));
|
|
529
|
+
const pageDataKindSchema = z.enum(['ld-json', 'json-script', 'inline-state', 'meta', 'link']);
|
|
530
|
+
/** One "data in the page" entry — mirrors `page-inspector/page-data.algorithm.ts`'s `PageDataFinding`. */
|
|
531
|
+
const pageDataFindingSchema = z.object({
|
|
532
|
+
kind: pageDataKindSchema,
|
|
533
|
+
label: z.string(),
|
|
534
|
+
selector: z.string(),
|
|
535
|
+
selectorKind: z.literal('css'),
|
|
536
|
+
matches: z.number(),
|
|
537
|
+
keys: z.array(z.string()).optional(),
|
|
538
|
+
attribute: z.string().optional()
|
|
539
|
+
});
|
|
540
|
+
/** What `inspect-page` answers with (studio plan §3.3, issue #93): the DOM tree and the data-in-the-page findings, in one round trip — both come from the same cached snapshot/raw capture, so there is no reason to ask twice. */
|
|
541
|
+
const inspectViewSchema = z.object({
|
|
542
|
+
tree: domTreeNodeViewSchema,
|
|
543
|
+
pageData: z.array(pageDataFindingSchema)
|
|
544
|
+
});
|
|
545
|
+
|
|
546
|
+
/** One JSON response seen — mirrors `page-inspector/responses-seen.use-case.ts`'s `ObservedResponse`. */
|
|
547
|
+
const observedResponseSchema = z.object({
|
|
548
|
+
url: z.string(),
|
|
549
|
+
status: z.number(),
|
|
550
|
+
size: z.number(),
|
|
551
|
+
shape: z.string()
|
|
552
|
+
});
|
|
553
|
+
/** What `responses-seen` answers with (studio plan §3.3, issue #93): the JSON responses the page fetched while it rendered, for the panel's "responses seen" list. */
|
|
554
|
+
const responsesSeenViewSchema = z.object({
|
|
555
|
+
responses: z.array(observedResponseSchema)
|
|
556
|
+
});
|
|
557
|
+
|
|
558
|
+
const documentTreeNodeViewSchema = z.lazy(() => z.object({
|
|
559
|
+
id: z.string(),
|
|
560
|
+
label: z.string(),
|
|
561
|
+
valueType: z.enum(['object', 'array', 'string', 'number', 'boolean', 'null', 'element']),
|
|
562
|
+
preview: z.string().optional(),
|
|
563
|
+
jsonpath: z.string().optional(),
|
|
564
|
+
xpath: z.string().optional(),
|
|
565
|
+
listPath: z.string().optional(),
|
|
566
|
+
listCount: z.number().optional(),
|
|
567
|
+
attributes: z.record(z.string(), z.string()).optional(),
|
|
568
|
+
children: z.array(documentTreeNodeViewSchema)
|
|
569
|
+
}));
|
|
570
|
+
/** What `document-tree` answers with (studio plan §3.4, issue #94's 5a): the parsed JSON/XML document as a tree, and, for XML, the namespace prefixes a pick's `extract` can use in `namespaces`. */
|
|
571
|
+
const documentTreeViewSchema = z.object({
|
|
572
|
+
format: z.enum(['json', 'xml']),
|
|
573
|
+
root: documentTreeNodeViewSchema,
|
|
574
|
+
namespaces: z.record(z.string(), z.string()).optional()
|
|
575
|
+
});
|
|
576
|
+
|
|
577
|
+
/** One text cell, in PDF points — mirrors `document-view/pdf-view.mapper.ts`'s `PdfCellView`, the wire shape for the UI (studio plan §3.4, issue #94's 5b). */
|
|
578
|
+
const pdfCellViewSchema = z.object({
|
|
579
|
+
x: z.number(),
|
|
580
|
+
y: z.number(),
|
|
581
|
+
width: z.number(),
|
|
582
|
+
height: z.number(),
|
|
583
|
+
text: z.string()
|
|
584
|
+
});
|
|
585
|
+
const pdfRowViewSchema = z.object({
|
|
586
|
+
top: z.number(),
|
|
587
|
+
bottom: z.number(),
|
|
588
|
+
cells: z.array(pdfCellViewSchema)
|
|
589
|
+
});
|
|
590
|
+
const pdfPageViewSchema = z.object({
|
|
591
|
+
number: z.number().int(),
|
|
592
|
+
width: z.number(),
|
|
593
|
+
height: z.number(),
|
|
594
|
+
rows: z.array(pdfRowViewSchema),
|
|
595
|
+
rowCount: z.number().int(),
|
|
596
|
+
cellCount: z.number().int(),
|
|
597
|
+
hasTextLayer: z.boolean()
|
|
598
|
+
});
|
|
599
|
+
/** What `pdf-view` answers with (studio plan §3.4, issue #94's 5b): every page's cells and rows, for the PDF canvas to draw over its own `pdf.js` rendering. */
|
|
600
|
+
const pdfDocumentViewSchema = z.object({
|
|
601
|
+
pages: z.array(pdfPageViewSchema)
|
|
602
|
+
});
|
|
603
|
+
|
|
604
|
+
const tablePreviewRowSchema = z.record(z.string(), z.string());
|
|
605
|
+
/** One table matched by a `table` extract's options — mirrors `@opencraw/core`'s `PdfTable`. */
|
|
606
|
+
const tablePreviewTableSchema = z.object({
|
|
607
|
+
page: z.number().int(),
|
|
608
|
+
title: z.string(),
|
|
609
|
+
header: z.array(z.string()),
|
|
610
|
+
rows: z.array(tablePreviewRowSchema)
|
|
611
|
+
});
|
|
612
|
+
/** One column band a matched table's body clustered into — mirrors `@opencraw/core`'s `TableBandDiagnostics`. */
|
|
613
|
+
const tablePreviewBandSchema = z.object({
|
|
614
|
+
start: z.number(),
|
|
615
|
+
end: z.number(),
|
|
616
|
+
column: z.number().int(),
|
|
617
|
+
name: z.string()
|
|
618
|
+
});
|
|
619
|
+
/** One matched table, with the exact page rows to highlight — mirrors `document-view/table-preview.use-case.ts`'s `TablePreviewMatch`. */
|
|
620
|
+
const tablePreviewMatchSchema = z.object({
|
|
621
|
+
page: z.number().int(),
|
|
622
|
+
headerRowIndex: z.number().int(),
|
|
623
|
+
matchedRowIndices: z.array(z.number().int()),
|
|
624
|
+
bands: z.array(tablePreviewBandSchema),
|
|
625
|
+
bandTolerance: z.number(),
|
|
626
|
+
table: tablePreviewTableSchema
|
|
627
|
+
});
|
|
628
|
+
/**
|
|
629
|
+
* What `table-preview` answers with (studio plan §3.4, issue #94's 5b: the
|
|
630
|
+
* live preview): every match, or `error` when a pattern in the options does
|
|
631
|
+
* not compile — expected while the person is still typing it, not thrown.
|
|
632
|
+
*/
|
|
633
|
+
const tablePreviewViewSchema = z.object({
|
|
634
|
+
matches: z.array(tablePreviewMatchSchema),
|
|
635
|
+
error: z.string().optional()
|
|
636
|
+
});
|
|
637
|
+
|
|
638
|
+
/** One cell's value, and the type the grid shows it as — mirrors `document-view/workbook-view.mapper.ts`'s `GridCellView`, the wire shape for the UI (studio plan §3.4, issue #94's 5c). */
|
|
639
|
+
const gridCellViewSchema = z.object({
|
|
640
|
+
value: z.union([z.string(), z.number(), z.boolean()]),
|
|
641
|
+
type: z.enum(['string', 'number', 'boolean', 'date'])
|
|
642
|
+
});
|
|
643
|
+
/** One merged range, 0-based and inclusive. */
|
|
644
|
+
const gridMergeViewSchema = z.object({
|
|
645
|
+
ref: z.string(),
|
|
646
|
+
top: z.number().int(),
|
|
647
|
+
left: z.number().int(),
|
|
648
|
+
bottom: z.number().int(),
|
|
649
|
+
right: z.number().int()
|
|
650
|
+
});
|
|
651
|
+
const gridSheetViewSchema = z.object({
|
|
652
|
+
name: z.string(),
|
|
653
|
+
hidden: z.boolean(),
|
|
654
|
+
hiddenRows: z.array(z.number().int()),
|
|
655
|
+
columnCount: z.number().int(),
|
|
656
|
+
rows: z.array(z.array(gridCellViewSchema)),
|
|
657
|
+
merges: z.array(gridMergeViewSchema)
|
|
658
|
+
});
|
|
659
|
+
const csvFormatViewSchema = z.object({
|
|
660
|
+
encoding: z.string(),
|
|
661
|
+
delimiter: z.string()
|
|
662
|
+
});
|
|
663
|
+
/** What `grid-view` answers with (studio plan §3.4, issue #94's 5c): every sheet's cells, typed, with hidden sheets/rows marked and merged ranges resolved, for the grid canvas to draw. */
|
|
664
|
+
const workbookDocumentViewSchema = z.object({
|
|
665
|
+
sheets: z.array(gridSheetViewSchema),
|
|
666
|
+
csv: csvFormatViewSchema.optional()
|
|
667
|
+
});
|
|
668
|
+
|
|
669
|
+
const gridPreviewRowSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
|
|
670
|
+
/** One table matched by a `table` extract's options against a workbook — mirrors `@opencraw/core`'s `GridTable`. */
|
|
671
|
+
const gridTablePreviewMatchSchema = z.object({
|
|
672
|
+
sheet: z.string(),
|
|
673
|
+
title: z.string(),
|
|
674
|
+
header: z.array(z.string()),
|
|
675
|
+
rows: z.array(gridPreviewRowSchema)
|
|
676
|
+
});
|
|
677
|
+
/**
|
|
678
|
+
* What `grid-preview` answers with (studio plan §3.4, issue #94's 5c: the
|
|
679
|
+
* live preview): every matched table, or `error` when a pattern in the
|
|
680
|
+
* options does not compile — expected while the person is still typing it,
|
|
681
|
+
* not thrown.
|
|
682
|
+
*/
|
|
683
|
+
const gridTablePreviewViewSchema = z.object({
|
|
684
|
+
matches: z.array(gridTablePreviewMatchSchema),
|
|
685
|
+
error: z.string().optional()
|
|
686
|
+
});
|
|
687
|
+
|
|
688
|
+
/** One text box on a slide, in points — mirrors `document-view/deck-view.mapper.ts`'s `DeckShapeView`, the wire shape for the UI (studio plan §3.4, issue #94's 5d). */
|
|
689
|
+
const deckShapeViewSchema = z.object({
|
|
690
|
+
x: z.number(),
|
|
691
|
+
y: z.number(),
|
|
692
|
+
width: z.number(),
|
|
693
|
+
height: z.number(),
|
|
694
|
+
text: z.string(),
|
|
695
|
+
placeholder: z.string().optional()
|
|
696
|
+
});
|
|
697
|
+
const deckChartValueSchema = z.union([z.number(), z.null()]);
|
|
698
|
+
const deckChartSeriesViewSchema = z.object({
|
|
699
|
+
name: z.string(),
|
|
700
|
+
categories: z.array(z.string()),
|
|
701
|
+
values: z.array(deckChartValueSchema)
|
|
702
|
+
});
|
|
703
|
+
const deckChartViewSchema = z.object({
|
|
704
|
+
type: z.string(),
|
|
705
|
+
title: z.string().optional(),
|
|
706
|
+
series: z.array(deckChartSeriesViewSchema)
|
|
707
|
+
});
|
|
708
|
+
const deckShapeRowSchema = z.array(z.number().int());
|
|
709
|
+
const deckSlideViewSchema = z.object({
|
|
710
|
+
number: z.number().int(),
|
|
711
|
+
title: z.string().optional(),
|
|
712
|
+
hidden: z.boolean(),
|
|
713
|
+
shapes: z.array(deckShapeViewSchema),
|
|
714
|
+
/** Indices into `shapes`, grouped into visual rows — see `deck-view.mapper.ts`'s own doc comment. */
|
|
715
|
+
shapeRows: z.array(deckShapeRowSchema),
|
|
716
|
+
tables: z.array(gridSheetViewSchema),
|
|
717
|
+
charts: z.array(deckChartViewSchema),
|
|
718
|
+
notes: z.string()
|
|
719
|
+
});
|
|
720
|
+
/** What `deck-view` answers with (studio plan §3.4, issue #94's 5d): the slide size in points and every slide's shapes, tables, charts and notes, for the deck canvas to draw and pick from. */
|
|
721
|
+
const deckDocumentViewSchema = z.object({
|
|
722
|
+
width: z.number(),
|
|
723
|
+
height: z.number(),
|
|
724
|
+
slides: z.array(deckSlideViewSchema)
|
|
725
|
+
});
|
|
726
|
+
|
|
727
|
+
const deckPreviewRowSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
|
|
728
|
+
/** One table matched by a `table` extract's options against a deck — mirrors `@opencraw/core`'s `DeckTable`. */
|
|
729
|
+
const deckTablePreviewMatchSchema = z.object({
|
|
730
|
+
slide: z.number().int(),
|
|
731
|
+
slideTitle: z.string(),
|
|
732
|
+
title: z.string(),
|
|
733
|
+
header: z.array(z.string()),
|
|
734
|
+
rows: z.array(deckPreviewRowSchema)
|
|
735
|
+
});
|
|
736
|
+
/**
|
|
737
|
+
* What `deck-preview` answers with (studio plan §3.4, issue #94's 5d: the
|
|
738
|
+
* live preview): every matched table, or `error` when a pattern in the
|
|
739
|
+
* options does not compile — expected while the person is still typing it,
|
|
740
|
+
* not thrown.
|
|
741
|
+
*/
|
|
742
|
+
const deckTablePreviewViewSchema = z.object({
|
|
743
|
+
matches: z.array(deckTablePreviewMatchSchema),
|
|
744
|
+
error: z.string().optional()
|
|
745
|
+
});
|
|
746
|
+
|
|
318
747
|
/** The cookie set on the page's own document response, so its own asset and API requests carry the token too. */
|
|
319
748
|
const TOKEN_COOKIE = 'opencraw_token';
|
|
320
749
|
/** A random token, generated once at server startup and required on every request. */
|
|
@@ -358,779 +787,1481 @@ function cookieToken(header) {
|
|
|
358
787
|
}
|
|
359
788
|
|
|
360
789
|
/**
|
|
361
|
-
*
|
|
362
|
-
*
|
|
790
|
+
* Fetches an input recipe's start page as the engine would see it: a fresh
|
|
791
|
+
* Playwright page in web mode, core's HTTP client in api mode. Returns plain
|
|
792
|
+
* text: the page's HTML, or a readable rendering of a non-HTML api response.
|
|
793
|
+
* No rewriting, no script stripping, no node ids yet: that is phase 2's
|
|
794
|
+
* `page-snapshot` proper (studio plan, #91). Provisional for phase 0 in two
|
|
795
|
+
* ways, both worth knowing before relying on it: it opens a bare page
|
|
796
|
+
* without running the recipe's own steps first, so a start point behind a
|
|
797
|
+
* `session.bootstrap` login shows what an anonymous visitor sees; and it
|
|
798
|
+
* launches a fresh browser or HTTP context per call, one page then closes
|
|
799
|
+
* it, rather than reusing whatever the last sample run had open.
|
|
363
800
|
*
|
|
364
|
-
* @param
|
|
365
|
-
* @param
|
|
366
|
-
* @returns The
|
|
367
|
-
* @throws
|
|
368
|
-
* @throws Error when the input or its output is not in the folder.
|
|
801
|
+
* @param input - The input recipe; only `mode` and the first `start` point are used.
|
|
802
|
+
* @param browser - Browser launch settings for web mode (an executable path override, headless…).
|
|
803
|
+
* @returns The page's content as plain text.
|
|
804
|
+
* @throws Error when the recipe has no start point.
|
|
369
805
|
*/
|
|
370
|
-
async function
|
|
371
|
-
const
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
const input = parseInputRecipe(inputDocument.content, inputDocument.source);
|
|
375
|
-
const outputDocument = documents.find(document => kindOf(document.content) === 'output' && idOf(document.content) === input.output);
|
|
376
|
-
if (outputDocument === undefined) throw new Error(`no output recipe "${input.output}" in this workspace`);
|
|
377
|
-
return {
|
|
378
|
-
input,
|
|
379
|
-
output: parseOutputRecipe(outputDocument.content, outputDocument.source)
|
|
380
|
-
};
|
|
806
|
+
async function fetchStartPage(input, browser) {
|
|
807
|
+
const point = input.start[0];
|
|
808
|
+
if (point === undefined) throw new Error(`recipe "${input.id}" has no start point`);
|
|
809
|
+
return input.mode === 'web' ? fetchWeb(point.url, browser) : fetchApi(point.url);
|
|
381
810
|
}
|
|
382
|
-
function
|
|
383
|
-
|
|
811
|
+
async function fetchWeb(url, browser) {
|
|
812
|
+
const client = await BrowserClient.launch(browser);
|
|
813
|
+
try {
|
|
814
|
+
const session = await client.newSession();
|
|
815
|
+
try {
|
|
816
|
+
await session.page.goto(url);
|
|
817
|
+
return await session.page.content();
|
|
818
|
+
} finally {
|
|
819
|
+
await session.close();
|
|
820
|
+
}
|
|
821
|
+
} finally {
|
|
822
|
+
await client.close();
|
|
823
|
+
}
|
|
384
824
|
}
|
|
385
|
-
function
|
|
386
|
-
|
|
825
|
+
async function fetchApi(url) {
|
|
826
|
+
const client = await HttpClient.open({});
|
|
827
|
+
try {
|
|
828
|
+
const response = await client.send({
|
|
829
|
+
url
|
|
830
|
+
});
|
|
831
|
+
return textOf$2(response.body);
|
|
832
|
+
} finally {
|
|
833
|
+
await client.dispose();
|
|
834
|
+
}
|
|
835
|
+
}
|
|
836
|
+
/** Every body kind the studio can show as text today; others get a placeholder rather than a crash. */
|
|
837
|
+
function textOf$2(body) {
|
|
838
|
+
if (body.kind === 'html') return body.html;
|
|
839
|
+
if (body.kind === 'text') return body.text;
|
|
840
|
+
if (body.kind === 'json') return JSON.stringify(body.data, null, 2);
|
|
841
|
+
return `[${body.kind} document: the content pane shows HTML and JSON responses for now]`;
|
|
387
842
|
}
|
|
388
843
|
|
|
844
|
+
/** The attribute `hiddenMarksScript` stamps on an element hidden on the live page. */
|
|
845
|
+
const HIDDEN_ATTRIBUTE = 'data-oc-hidden';
|
|
389
846
|
/**
|
|
390
|
-
*
|
|
391
|
-
*
|
|
392
|
-
*
|
|
393
|
-
*
|
|
394
|
-
*
|
|
395
|
-
*
|
|
396
|
-
* it happens, not only in the final result. Step events (`step:start`,
|
|
397
|
-
* `step:skip`, `step:retry`) are aggregated by path into `result.steps`, so
|
|
398
|
-
* `explain-why` (#92) can later say whether the step that bound a mapping's
|
|
399
|
-
* source id ran, was skipped, or retried — without a second run.
|
|
847
|
+
* The in-page script `take-snapshot.use-case.ts` runs with `page.evaluate`
|
|
848
|
+
* *before* `page.content()`, so a `display: none`/`visibility: hidden`/
|
|
849
|
+
* zero-size element carries `data-oc-hidden="1"` into the captured markup
|
|
850
|
+
* (studio plan §3.1, issue #91) — the only way to know an element was
|
|
851
|
+
* hidden on the live page, since a static parse of the HTML afterwards has
|
|
852
|
+
* no computed style to ask.
|
|
400
853
|
*
|
|
401
|
-
*
|
|
402
|
-
*
|
|
403
|
-
*
|
|
404
|
-
*
|
|
405
|
-
*
|
|
406
|
-
*
|
|
407
|
-
*
|
|
854
|
+
* Kept as a plain string, not a TypeScript function passed to
|
|
855
|
+
* `page.evaluate`, because it references `window`/`document` as globals: `AGENTS.md`'s
|
|
856
|
+
* "things that bit before" calls out that in-page functions must not do
|
|
857
|
+
* that (the repo's `isolated-functions` lint rule), and every existing
|
|
858
|
+
* `page.evaluate` call with real DOM logic in this codebase (see
|
|
859
|
+
* `packages/core/src/captcha/resolve-captcha.use-case.ts`) already uses
|
|
860
|
+
* this same string-template shape.
|
|
861
|
+
*
|
|
862
|
+
* @param attribute - The attribute to stamp; defaults to `HIDDEN_ATTRIBUTE` (parameterised for tests and for a future caller that wants a different name).
|
|
863
|
+
* @returns A self-invoking script, ready for `page.evaluate(hiddenMarksScript())`.
|
|
408
864
|
*/
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
debug: true,
|
|
422
|
-
sample: budget,
|
|
423
|
-
browser,
|
|
424
|
-
onEvent: event => {
|
|
425
|
-
const line = traceLine(event);
|
|
426
|
-
if (line !== undefined) callbacks.onTraceLine(line);
|
|
427
|
-
trackStep(steps, event);
|
|
428
|
-
if (event.type === 'record:emit') {
|
|
429
|
-
const record = {
|
|
430
|
-
key: event.key,
|
|
431
|
-
data: event.data,
|
|
432
|
-
scope: event.scope,
|
|
433
|
-
mapping: event.mapping
|
|
434
|
-
};
|
|
435
|
-
records.push(record);
|
|
436
|
-
callbacks.onRecord(record);
|
|
437
|
-
} else if (event.type === 'record:reject') {
|
|
438
|
-
const rejected = {
|
|
439
|
-
field: event.field,
|
|
440
|
-
reason: event.reason,
|
|
441
|
-
url: event.url,
|
|
442
|
-
scope: event.scope
|
|
443
|
-
};
|
|
444
|
-
rejectedRecords.push(rejected);
|
|
445
|
-
callbacks.onRejected?.(rejected);
|
|
446
|
-
}
|
|
447
|
-
}
|
|
448
|
-
});
|
|
449
|
-
const result = runToResult(crawler, set, records, rejectedRecords, steps);
|
|
450
|
-
return {
|
|
451
|
-
stop: () => crawler.close(),
|
|
452
|
-
result
|
|
453
|
-
};
|
|
865
|
+
function hiddenMarksScript(attribute = HIDDEN_ATTRIBUTE) {
|
|
866
|
+
return `(() => {
|
|
867
|
+
var attribute = ${JSON.stringify(attribute)}
|
|
868
|
+
var elements = document.querySelectorAll('*')
|
|
869
|
+
for (var i = 0; i < elements.length; i++) {
|
|
870
|
+
var element = elements[i]
|
|
871
|
+
var style = window.getComputedStyle(element)
|
|
872
|
+
var rect = element.getBoundingClientRect()
|
|
873
|
+
var hidden = style.display === 'none' || style.visibility === 'hidden' || (rect.width === 0 && rect.height === 0)
|
|
874
|
+
if (hidden) element.setAttribute(attribute, '1')
|
|
875
|
+
}
|
|
876
|
+
})()`;
|
|
454
877
|
}
|
|
878
|
+
|
|
879
|
+
/** Attributes that carry a URL and are rewritten to absolute; `srcset` is handled separately (it carries several). */
|
|
880
|
+
const URL_ATTRIBUTES = ['href', 'src', 'action', 'poster'];
|
|
881
|
+
/** Schemes left untouched: they are not a location `new URL(value, base)` should resolve, or already carry their own meaning. */
|
|
882
|
+
const SKIP_SCHEME = /^(?:javascript|mailto|tel|data|about):/i;
|
|
455
883
|
/**
|
|
456
|
-
*
|
|
457
|
-
*
|
|
458
|
-
*
|
|
459
|
-
*
|
|
460
|
-
*
|
|
461
|
-
*
|
|
884
|
+
* Rewrites captured HTML into what the content pane's sandboxed iframe shows
|
|
885
|
+
* (studio plan §3.1, issue #91): every URL made absolute against `baseUrl`,
|
|
886
|
+
* a `<base>` injected, every `<script>` removed, forms disabled (`onsubmit`
|
|
887
|
+
* stripped, fields made read-only), `<meta http-equiv="refresh">` stripped
|
|
888
|
+
* (it would otherwise navigate the sandboxed frame on its own), and every
|
|
889
|
+
* element given a stable `data-oc-node` id, in document order, for the
|
|
890
|
+
* picker overlay and `selector-inference` to address specific nodes by.
|
|
891
|
+
*
|
|
892
|
+
* Purely a markup transform: it does not know whether an element is hidden
|
|
893
|
+
* on the live page (`display: none`, `visibility: hidden`, zero size) — in
|
|
894
|
+
* web mode that is `hidden-marks.algorithm.ts`'s job, run against the live
|
|
895
|
+
* page *before* this function ever sees the markup, so a `data-oc-hidden`
|
|
896
|
+
* attribute it wrote just survives serialisation untouched here.
|
|
897
|
+
*
|
|
898
|
+
* @param html - The captured document (`page.content()` in web mode, the fetched body in api mode).
|
|
899
|
+
* @param baseUrl - The page's own URL, for resolving relative links and as the `<base>`.
|
|
900
|
+
* @returns The rewritten document and how many elements it stamped.
|
|
462
901
|
*/
|
|
463
|
-
function
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
902
|
+
function rewriteDocument(html, baseUrl) {
|
|
903
|
+
const $ = load(html);
|
|
904
|
+
if ($('head').length === 0) $('html').prepend('<head></head>');
|
|
905
|
+
const head = $('head').first();
|
|
906
|
+
head.find('base').remove();
|
|
907
|
+
head.prepend(`<base href="${escapeAttribute(baseUrl)}">`);
|
|
908
|
+
$('meta').each((_index, element) => {
|
|
909
|
+
if (($(element).attr('http-equiv') ?? '').toLowerCase() === 'refresh') $(element).remove();
|
|
910
|
+
});
|
|
911
|
+
$('script').remove();
|
|
912
|
+
$('form').removeAttr('onsubmit');
|
|
913
|
+
$('input, textarea').attr('readonly', 'readonly');
|
|
914
|
+
$('select, button, input[type="submit"], input[type="button"]').attr('disabled', 'disabled');
|
|
915
|
+
$('*').each((_index, element) => {
|
|
916
|
+
const node = $(element);
|
|
917
|
+
for (const attribute of URL_ATTRIBUTES) {
|
|
918
|
+
const value = node.attr(attribute);
|
|
919
|
+
if (value !== undefined) node.attr(attribute, resolveUrl(value, baseUrl) ?? value);
|
|
920
|
+
}
|
|
921
|
+
const srcset = node.attr('srcset');
|
|
922
|
+
if (srcset !== undefined) node.attr('srcset', rewriteSrcset(srcset, baseUrl));
|
|
923
|
+
});
|
|
924
|
+
let nodeCount = 0;
|
|
925
|
+
$('*').each((_index, element) => {
|
|
926
|
+
$(element).attr(NODE_ID_ATTRIBUTE, `n${nodeCount}`);
|
|
927
|
+
nodeCount += 1;
|
|
928
|
+
});
|
|
929
|
+
return {
|
|
930
|
+
html: $.html(),
|
|
931
|
+
nodeCount
|
|
932
|
+
};
|
|
933
|
+
}
|
|
934
|
+
/** Resolves one URL against `baseUrl`; a scheme that is not a location (`javascript:`, `data:`…), or a value `new URL` cannot parse, is left as it was. */
|
|
935
|
+
function resolveUrl(value, baseUrl) {
|
|
936
|
+
const trimmed = value.trim();
|
|
937
|
+
if (trimmed === '' || SKIP_SCHEME.test(trimmed)) return undefined;
|
|
938
|
+
try {
|
|
939
|
+
return new URL(trimmed, baseUrl).href;
|
|
940
|
+
} catch {
|
|
941
|
+
return undefined;
|
|
482
942
|
}
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
943
|
+
}
|
|
944
|
+
/** `srcset`'s own micro-syntax: comma-separated `url descriptor?` pairs. Malformed entries are passed through unresolved rather than dropped. */
|
|
945
|
+
function rewriteSrcset(value, baseUrl) {
|
|
946
|
+
return value.split(',').map(candidate => {
|
|
947
|
+
const trimmed = candidate.trim();
|
|
948
|
+
const spaceAt = trimmed.indexOf(' ');
|
|
949
|
+
const url = spaceAt === -1 ? trimmed : trimmed.slice(0, spaceAt);
|
|
950
|
+
const descriptor = spaceAt === -1 ? '' : trimmed.slice(spaceAt);
|
|
951
|
+
return `${resolveUrl(url, baseUrl) ?? url}${descriptor}`;
|
|
952
|
+
}).join(', ');
|
|
953
|
+
}
|
|
954
|
+
function escapeAttribute(value) {
|
|
955
|
+
return value.replaceAll('&', '&').replaceAll('"', '"');
|
|
956
|
+
}
|
|
957
|
+
|
|
958
|
+
/**
|
|
959
|
+
* Captures the snapshot the content pane shows (studio plan §3.1, issue
|
|
960
|
+
* #91): runs the recipe up to `stepPath`, serialises the document, marks
|
|
961
|
+
* hidden elements in web mode, and hands it to `rewrite-document.mapper.ts`
|
|
962
|
+
* for display.
|
|
963
|
+
*
|
|
964
|
+
* `stepPath` is accepted (and cached against, by `snapshot.store.ts`) for
|
|
965
|
+
* the API the UI needs — one snapshot per selected step — but v1 only
|
|
966
|
+
* actually *captures* `"start"`: the first `start` point, before any step
|
|
967
|
+
* of the recipe has run, the same reach `fetch-start-page.use-case.ts` (phase
|
|
968
|
+
* 0) already had. Snapshotting mid-recipe (after an arbitrary later step)
|
|
969
|
+
* needs a "stop the crawl after step X and hand back the live page/document"
|
|
970
|
+
* hook in `crawl-execution` that does not exist yet; building one was out of
|
|
971
|
+
* this phase's budget (see this phase's final report for the tradeoff this
|
|
972
|
+
* was weighed against). Any other `stepPath` throws, clearly, rather than
|
|
973
|
+
* silently returning the start page under a misleading label.
|
|
974
|
+
*
|
|
975
|
+
* @param input - The input recipe; only `mode` and the first `start` point are used.
|
|
976
|
+
* @param stepPath - The step to snapshot after, e.g. `"start"` or `"steps.2"` (`scope-outline`'s path format).
|
|
977
|
+
* @param browser - Browser launch settings for web mode.
|
|
978
|
+
* @returns The rewritten snapshot.
|
|
979
|
+
* @throws Error when the recipe has no start point, or `stepPath` is not `"start"`.
|
|
980
|
+
*/
|
|
981
|
+
async function takeSnapshot(input, stepPath, browser) {
|
|
982
|
+
if (stepPath !== 'start') throw new Error(`takeSnapshot: "${stepPath}" is not supported yet — only "start" (the first start point, before any step runs) is; see this function's doc comment`);
|
|
983
|
+
const point = input.start[0];
|
|
984
|
+
if (point === undefined) throw new Error(`recipe "${input.id}" has no start point`);
|
|
985
|
+
return input.mode === 'web' ? snapshotWeb(point.url, browser) : snapshotApi(point.url);
|
|
986
|
+
}
|
|
987
|
+
async function snapshotWeb(url, browser) {
|
|
988
|
+
const client = await BrowserClient.launch(browser);
|
|
989
|
+
try {
|
|
990
|
+
const session = await client.newSession();
|
|
991
|
+
try {
|
|
992
|
+
await session.page.goto(url);
|
|
993
|
+
await session.page.evaluate(hiddenMarksScript(HIDDEN_ATTRIBUTE));
|
|
994
|
+
const html = await session.page.content();
|
|
995
|
+
const baseUrl = session.page.url();
|
|
996
|
+
const rewritten = rewriteDocument(html, baseUrl);
|
|
997
|
+
return {
|
|
998
|
+
...rewritten,
|
|
999
|
+
baseUrl,
|
|
1000
|
+
rawHtml: html
|
|
1001
|
+
};
|
|
1002
|
+
} finally {
|
|
1003
|
+
await session.close();
|
|
1004
|
+
}
|
|
1005
|
+
} finally {
|
|
1006
|
+
await client.close();
|
|
491
1007
|
}
|
|
492
1008
|
}
|
|
493
|
-
async function
|
|
1009
|
+
async function snapshotApi(url) {
|
|
1010
|
+
const client = await HttpClient.open({});
|
|
494
1011
|
try {
|
|
495
|
-
const
|
|
496
|
-
|
|
1012
|
+
const response = await client.send({
|
|
1013
|
+
url
|
|
1014
|
+
});
|
|
1015
|
+
const text = textOf$1(response.body);
|
|
1016
|
+
const rewritten = rewriteDocument(text, url);
|
|
497
1017
|
return {
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
error: recipe.error,
|
|
504
|
-
stoppedBy: recipe.stoppedBy,
|
|
505
|
-
records,
|
|
506
|
-
rejectedRecords,
|
|
507
|
-
// eslint-disable-next-line unicorn/prefer-iterator-to-array -- `.toArray()` needs a `lib` newer than this repo's `es2022` (tsconfig.base.json)
|
|
508
|
-
steps: [...steps.values()]
|
|
1018
|
+
...rewritten,
|
|
1019
|
+
baseUrl: url,
|
|
1020
|
+
rawHtml: text,
|
|
1021
|
+
format: response.format,
|
|
1022
|
+
body: response.body
|
|
509
1023
|
};
|
|
510
1024
|
} finally {
|
|
511
|
-
await
|
|
1025
|
+
await client.dispose();
|
|
512
1026
|
}
|
|
513
1027
|
}
|
|
1028
|
+
/** Every body kind the studio can show as text today; others get a placeholder rather than a crash (mirrors `fetch-start-page.use-case.ts`'s own `textOf`). */
|
|
1029
|
+
function textOf$1(body) {
|
|
1030
|
+
if (body.kind === 'html') return body.html;
|
|
1031
|
+
if (body.kind === 'text') return body.text;
|
|
1032
|
+
if (body.kind === 'json') return `<pre>${JSON.stringify(body.data, null, 2)}</pre>`;
|
|
1033
|
+
return `<p>[${body.kind} document: the content pane shows HTML and JSON responses for now]</p>`;
|
|
1034
|
+
}
|
|
514
1035
|
|
|
515
|
-
/** @returns
|
|
516
|
-
function
|
|
1036
|
+
/** @returns An empty cache, for a freshly opened workspace. */
|
|
1037
|
+
function createSnapshotCache() {
|
|
517
1038
|
return {
|
|
518
1039
|
entries: new Map()
|
|
519
1040
|
};
|
|
520
1041
|
}
|
|
521
|
-
/**
|
|
522
|
-
function
|
|
523
|
-
cache.entries.
|
|
524
|
-
recipeId,
|
|
525
|
-
result
|
|
526
|
-
});
|
|
1042
|
+
/** @returns The cached snapshot for `recipeId`/`path`, or `undefined` when none was taken yet. */
|
|
1043
|
+
function cachedSnapshot(cache, recipeId, path) {
|
|
1044
|
+
return cache.entries.get(keyOf(recipeId, path));
|
|
527
1045
|
}
|
|
528
|
-
/**
|
|
529
|
-
function
|
|
530
|
-
|
|
1046
|
+
/** Records a freshly taken snapshot, replacing whatever was cached for the same recipe and step path. */
|
|
1047
|
+
function putSnapshot(cache, recipeId, path, snapshot) {
|
|
1048
|
+
cache.entries.set(keyOf(recipeId, path), snapshot);
|
|
1049
|
+
}
|
|
1050
|
+
function keyOf(recipeId, path) {
|
|
1051
|
+
return `${recipeId}:${path}`;
|
|
531
1052
|
}
|
|
532
1053
|
|
|
533
1054
|
/**
|
|
534
|
-
*
|
|
535
|
-
*
|
|
536
|
-
*
|
|
537
|
-
*
|
|
538
|
-
*
|
|
1055
|
+
* Fetches a PDF's raw bytes for the PDF canvas's `pdf.js` to render in the
|
|
1056
|
+
* browser (studio plan §3.4, issue #94's 5b: "the PDF bytes served by the
|
|
1057
|
+
* studio"). `take-snapshot` already read this same document with
|
|
1058
|
+
* `@opencraw/core`'s `readPdf` for the cells/rows `document-view` maps, but
|
|
1059
|
+
* `HttpClient`/`readPdf` parse the bytes and keep only the result — pdf.js
|
|
1060
|
+
* needs the file itself, so this re-fetches it from the same URL a cached
|
|
1061
|
+
* snapshot's own `SnapshotResult.baseUrl` gives, the way `HttpClient`'s own
|
|
1062
|
+
* `file:` branch (`readLocalFile`) and its HTTP fetch do.
|
|
539
1063
|
*
|
|
540
|
-
* @param
|
|
541
|
-
* @
|
|
542
|
-
* @
|
|
1064
|
+
* @param url - The document's URL — a `file:` URL or an `http(s):` URL.
|
|
1065
|
+
* @returns The raw bytes.
|
|
1066
|
+
* @throws Error when a `file:` URL cannot be read, or the fetch does not succeed.
|
|
543
1067
|
*/
|
|
544
|
-
function
|
|
545
|
-
if (
|
|
546
|
-
const
|
|
547
|
-
if (
|
|
548
|
-
|
|
549
|
-
if (typeof value === 'object' && value !== null) {
|
|
550
|
-
const keys = Object.keys(value);
|
|
551
|
-
return keys.length === 0 ? undefined : `"${boundId}" has: ${keys.slice(0, 5).join(', ')}`;
|
|
552
|
-
}
|
|
553
|
-
return undefined;
|
|
1068
|
+
async function pdfBytes(url) {
|
|
1069
|
+
if (url.startsWith('file:')) return new Uint8Array(await readFile(fileURLToPath(url)));
|
|
1070
|
+
const response = await fetch(url);
|
|
1071
|
+
if (!response.ok) throw new Error(`pdfBytes: ${String(response.status)} fetching ${url}`);
|
|
1072
|
+
return new Uint8Array(await response.arrayBuffer());
|
|
554
1073
|
}
|
|
555
1074
|
|
|
556
1075
|
/**
|
|
557
|
-
*
|
|
558
|
-
*
|
|
559
|
-
*
|
|
560
|
-
* it
|
|
561
|
-
*
|
|
562
|
-
*
|
|
563
|
-
*
|
|
564
|
-
* This is the scope-snapshot-only half of the design: phase 2's per-item DOM
|
|
565
|
-
* snapshot (the CSS-selector "the item has `.price_new`" hint the issue's
|
|
566
|
-
* example shows) is not wired in here — `sample-run` keeps the mapping's
|
|
567
|
-
* scope snapshot (bound ids and their values), not a per-`forEach`-item DOM
|
|
568
|
-
* capture, and building that link is a bigger integration than this phase's
|
|
569
|
-
* budget covers (see this package's `page-snapshot` slice, which captures a
|
|
570
|
-
* step's page for the content pane, not per-iteration). `hint`, when given,
|
|
571
|
-
* is instead built from the scope snapshot itself (`item-hint.algorithm.ts`).
|
|
1076
|
+
* Re-reads a CSV document with an overridden delimiter and/or encoding
|
|
1077
|
+
* (studio plan §3.4, issue #94's 5c). `take-snapshot` already read this same
|
|
1078
|
+
* document once with auto-detection (`http.client.ts`'s own `csvWorkbook`
|
|
1079
|
+
* call); it keeps only the parsed result, not the raw bytes, so re-reading
|
|
1080
|
+
* the file/URL is the only way to apply a different choice — the same
|
|
1081
|
+
* "the studio re-fetches, the reader keeps no bytes around" tradeoff
|
|
1082
|
+
* `pdf-bytes.use-case.ts` documents for pdf.js.
|
|
572
1083
|
*
|
|
573
|
-
* @param
|
|
574
|
-
* @
|
|
1084
|
+
* @param url - The document's URL — a `file:` URL or an `http(s):` URL (`page-snapshot`'s cached `SnapshotResult.baseUrl`).
|
|
1085
|
+
* @param override - The delimiter and/or encoding to force; `HttpClient` still detects whichever one is left out.
|
|
1086
|
+
* @returns The freshly read workbook.
|
|
1087
|
+
* @throws Error when the URL cannot be read, or it no longer reads back as a workbook.
|
|
575
1088
|
*/
|
|
576
|
-
function
|
|
577
|
-
|
|
1089
|
+
async function rereadCsv(url, override) {
|
|
1090
|
+
const client = await HttpClient.open({});
|
|
1091
|
+
try {
|
|
1092
|
+
const response = await client.send({
|
|
1093
|
+
url,
|
|
1094
|
+
as: 'csv',
|
|
1095
|
+
delimiter: override.delimiter,
|
|
1096
|
+
encoding: override.encoding
|
|
1097
|
+
});
|
|
1098
|
+
if (response.body.kind !== 'workbook') throw new Error(`rereadCsv: ${url} did not read back as a workbook`);
|
|
1099
|
+
return response.body;
|
|
1100
|
+
} finally {
|
|
1101
|
+
await client.dispose();
|
|
1102
|
+
}
|
|
578
1103
|
}
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
1104
|
+
|
|
1105
|
+
/** A cell reads as an ISO date (`2026-06-01`), an ISO moment (`2026-06-01T09:30:00`) or a bare time of day (`09:30:00`) — the three shapes `workbook-document`'s `read-xlsx.client.ts` writes for a spreadsheet date. */
|
|
1106
|
+
const ISO_DATE_OR_TIME = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2})?$|^\d{2}:\d{2}:\d{2}$/;
|
|
1107
|
+
/**
|
|
1108
|
+
* Builds the grid canvas's view model (studio plan §3.4, issue #94's 5c) from
|
|
1109
|
+
* an already-read `WorkbookDocument` (`take-snapshot`'s cached body — no
|
|
1110
|
+
* second read of the file): every sheet's cells, typed, with hidden sheets
|
|
1111
|
+
* and rows marked, and merged ranges resolved to bounds so the grid draws
|
|
1112
|
+
* them as one cell. Table-specific diagnostics (the matched rows of the
|
|
1113
|
+
* current `table` extract's options) are not here — they depend on the
|
|
1114
|
+
* picks so far, so they come from `table-preview.use-case.ts`'s
|
|
1115
|
+
* `previewGridTable` instead, computed live as those options change.
|
|
1116
|
+
*
|
|
1117
|
+
* @param document - The read workbook (a spreadsheet or a CSV).
|
|
1118
|
+
* @returns The view.
|
|
1119
|
+
*/
|
|
1120
|
+
function workbookDocumentView(document) {
|
|
1121
|
+
return {
|
|
1122
|
+
sheets: document.sheets.map(sheet => sheetView(sheet)),
|
|
1123
|
+
...(document.csv !== undefined && {
|
|
1124
|
+
csv: document.csv
|
|
1125
|
+
})
|
|
1126
|
+
};
|
|
583
1127
|
}
|
|
584
|
-
function
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
1128
|
+
function sheetView(sheet) {
|
|
1129
|
+
return {
|
|
1130
|
+
name: sheet.name,
|
|
1131
|
+
hidden: sheet.hidden === true,
|
|
1132
|
+
hiddenRows: sheet.hiddenRows === undefined ? [] : [...sheet.hiddenRows],
|
|
1133
|
+
columnCount: Math.max(0, ...sheet.rows.map(row => row.length)),
|
|
1134
|
+
rows: sheet.rows.map(row => row.map(cell => cellView(cell))),
|
|
1135
|
+
merges: (sheet.merges ?? []).flatMap(reference => {
|
|
1136
|
+
const range = rangeOf(reference);
|
|
1137
|
+
return range === undefined ? [] : [{
|
|
1138
|
+
ref: reference,
|
|
1139
|
+
...range
|
|
1140
|
+
}];
|
|
1141
|
+
})
|
|
1142
|
+
};
|
|
589
1143
|
}
|
|
590
|
-
function
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
if (step.outcome === 'not-found') return `: no step in this recipe binds "${boundId}"`;
|
|
596
|
-
if (step.outcome === 'skipped') return `: ${label} (${step.path}), which binds "${boundId}", was skipped${step.error === undefined ? '' : ` (${step.error})`}`;
|
|
597
|
-
if (step.outcome === 'retried') return `: ${label} (${step.path}), which binds "${boundId}", had to retry${step.error === undefined ? '' : ` (${step.error})`} and still found nothing for "${facts.field}"`;
|
|
598
|
-
return `: ${label} (${step.path}) ran and bound "${boundId}", but "${facts.field}" reads ${sourceOf(facts)} from it, which was empty`;
|
|
1144
|
+
function cellView(cell) {
|
|
1145
|
+
return {
|
|
1146
|
+
value: cell,
|
|
1147
|
+
type: cellType(cell)
|
|
1148
|
+
};
|
|
599
1149
|
}
|
|
600
|
-
function
|
|
601
|
-
|
|
1150
|
+
function cellType(cell) {
|
|
1151
|
+
if (typeof cell === 'number') return 'number';
|
|
1152
|
+
if (typeof cell === 'boolean') return 'boolean';
|
|
1153
|
+
return ISO_DATE_OR_TIME.test(cell) ? 'date' : 'string';
|
|
1154
|
+
}
|
|
1155
|
+
/** `B10:B13` (or a single cell, `A1`) as 0-based inclusive bounds; `undefined` for anything else — mirrors `@opencraw/core`'s own (unexported) `rangeOf`/`cellOf` in `grid-table.algorithm.ts`: a small pure parse, kept here rather than reached for across the package boundary (a sibling slice's non-`index.ts` file is never imported, and this is `@opencraw/core`'s own private helper, not even on its barrel). */
|
|
1156
|
+
function rangeOf(reference) {
|
|
1157
|
+
const [from, to = from] = reference.split(':', 2);
|
|
1158
|
+
const start = cellOf(from);
|
|
1159
|
+
const end = cellOf(to);
|
|
1160
|
+
if (start === undefined || end === undefined) return undefined;
|
|
1161
|
+
return {
|
|
1162
|
+
top: Math.min(start.row, end.row),
|
|
1163
|
+
left: Math.min(start.column, end.column),
|
|
1164
|
+
bottom: Math.max(start.row, end.row),
|
|
1165
|
+
right: Math.max(start.column, end.column)
|
|
1166
|
+
};
|
|
602
1167
|
}
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
}
|
|
614
|
-
case 'forEach':
|
|
615
|
-
{
|
|
616
|
-
return 'the loop';
|
|
617
|
-
}
|
|
618
|
-
case 'paginate':
|
|
619
|
-
{
|
|
620
|
-
return 'the pagination';
|
|
621
|
-
}
|
|
622
|
-
case 'goto':
|
|
623
|
-
{
|
|
624
|
-
return 'the Go to step';
|
|
625
|
-
}
|
|
626
|
-
default:
|
|
627
|
-
{
|
|
628
|
-
return `the ${stepType} step`;
|
|
629
|
-
}
|
|
630
|
-
}
|
|
1168
|
+
function cellOf(reference) {
|
|
1169
|
+
const match = /^\$?([A-Z]+)\$?(\d+)$/i.exec(reference.trim());
|
|
1170
|
+
if (match === null) return undefined;
|
|
1171
|
+
const letters = match[1].toUpperCase();
|
|
1172
|
+
let column = 0;
|
|
1173
|
+
for (const char of letters) column = column * 26 + (char.codePointAt(0) ?? 64) - 64;
|
|
1174
|
+
return {
|
|
1175
|
+
row: Number(match[2]) - 1,
|
|
1176
|
+
column: column - 1
|
|
1177
|
+
};
|
|
631
1178
|
}
|
|
632
1179
|
|
|
633
1180
|
/**
|
|
634
|
-
*
|
|
635
|
-
*
|
|
636
|
-
*
|
|
637
|
-
*
|
|
638
|
-
*
|
|
1181
|
+
* Builds the deck canvas's view model (studio plan §3.4, issue #94's 5d)
|
|
1182
|
+
* from an already-read `DeckDocument` (`take-snapshot`'s cached body — no
|
|
1183
|
+
* second read of the file): every slide's shapes (with their visual-row
|
|
1184
|
+
* grouping, for the text-box-grid pick), native tables (through
|
|
1185
|
+
* `workbook-view.mapper.ts`'s own sheet view — a deck table *is* a sheet),
|
|
1186
|
+
* charts and notes. Table-specific diagnostics for a `table` extract's own
|
|
1187
|
+
* options (the live preview) are not here — they come from
|
|
1188
|
+
* `table-preview.use-case.ts`'s `previewDeckTable` instead, computed live as
|
|
1189
|
+
* those options change, the same split `pdf-view.mapper.ts`/`workbook-view.mapper.ts`
|
|
1190
|
+
* already use.
|
|
639
1191
|
*
|
|
640
|
-
* @param
|
|
641
|
-
*
|
|
642
|
-
* have happened since).
|
|
643
|
-
* @param cache - The server's last-run cache (`sample-run`'s `LastRunCache`).
|
|
644
|
-
* @param loadPair - Loads the input/output recipe pair; injected so this
|
|
645
|
-
* stays testable without a real workspace folder (`sample-run`'s
|
|
646
|
-
* `loadRecipePair` in production).
|
|
647
|
-
* @param target - What was clicked.
|
|
648
|
-
* @returns The explanation.
|
|
649
|
-
* @throws Error when no sample run has finished for this recipe yet, or the target does not name a real record/field.
|
|
1192
|
+
* @param document - The read deck.
|
|
1193
|
+
* @returns The view.
|
|
650
1194
|
*/
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
} = await loadPair(folder, target.recipeId);
|
|
658
|
-
return target.kind === 'rejected' ? explainRejected(target.recipeId, input, output, entry.result.rejectedRecords, entry.result.steps, target.rejectedIndex) : explainMissing(target.recipeId, input, output, entry.result.records, entry.result.steps, target.recordIndex, target.field);
|
|
1195
|
+
function deckDocumentView(document) {
|
|
1196
|
+
return {
|
|
1197
|
+
width: document.width,
|
|
1198
|
+
height: document.height,
|
|
1199
|
+
slides: document.slides.map(slide => slideView(slide, document.height))
|
|
1200
|
+
};
|
|
659
1201
|
}
|
|
660
|
-
function
|
|
661
|
-
const rejected = rejectedRecords[index];
|
|
662
|
-
if (rejected === undefined) throw new Error(`no rejected record at index ${index}`);
|
|
663
|
-
const rule = ruleFor(input.mapping, rejected.field);
|
|
664
|
-
const field = fieldFor(output.fields, rejected.field);
|
|
665
|
-
const policy = field === undefined ? undefined : resolveMissingPolicy(field, output, rule);
|
|
666
|
-
const boundId = boundIdOf(rule);
|
|
667
|
-
const step = boundId === undefined ? undefined : findStep(input.steps, boundId, steps);
|
|
668
|
-
const hint = hintFor(rejected.scope, boundId);
|
|
669
|
-
const sentence = buildWhySentence({
|
|
670
|
-
field: rejected.field,
|
|
671
|
-
outcome: 'rejected',
|
|
672
|
-
reason: rejected.reason,
|
|
673
|
-
policy,
|
|
674
|
-
boundId,
|
|
675
|
-
step,
|
|
676
|
-
hint
|
|
677
|
-
});
|
|
1202
|
+
function slideView(slide, slideHeight) {
|
|
678
1203
|
return {
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
1204
|
+
number: slide.number,
|
|
1205
|
+
...(slide.title !== undefined && {
|
|
1206
|
+
title: slide.title
|
|
1207
|
+
}),
|
|
1208
|
+
hidden: slide.hidden,
|
|
1209
|
+
shapes: slide.shapes.map(shape => shapeView(shape)),
|
|
1210
|
+
shapeRows: shapeRowsOf(slide.shapes, slideHeight),
|
|
1211
|
+
tables: workbookDocumentView({
|
|
1212
|
+
sheets: slide.tables
|
|
1213
|
+
}).sheets,
|
|
1214
|
+
charts: slide.charts.map(chart => chartView(chart)),
|
|
1215
|
+
notes: slide.notes
|
|
686
1216
|
};
|
|
687
1217
|
}
|
|
688
|
-
function
|
|
689
|
-
const record = records[index];
|
|
690
|
-
if (record === undefined) throw new Error(`no record at index ${index}`);
|
|
691
|
-
const trace = record.mapping?.[field];
|
|
692
|
-
const rule = ruleFor(input.mapping, field);
|
|
693
|
-
const fieldSpec = fieldFor(output.fields, field);
|
|
694
|
-
const policy = fieldSpec === undefined ? undefined : resolveMissingPolicy(fieldSpec, output, rule);
|
|
695
|
-
const boundId = boundIdOf(rule);
|
|
696
|
-
const step = boundId === undefined ? undefined : findStep(input.steps, boundId, steps);
|
|
697
|
-
const hint = hintFor(record.scope, boundId);
|
|
698
|
-
const sentence = buildWhySentence({
|
|
699
|
-
field,
|
|
700
|
-
outcome: 'missing',
|
|
701
|
-
policy,
|
|
702
|
-
trace,
|
|
703
|
-
boundId,
|
|
704
|
-
step,
|
|
705
|
-
hint
|
|
706
|
-
});
|
|
1218
|
+
function shapeView(shape) {
|
|
707
1219
|
return {
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
1220
|
+
x: shape.x,
|
|
1221
|
+
y: shape.y,
|
|
1222
|
+
width: shape.width,
|
|
1223
|
+
height: shape.height,
|
|
1224
|
+
text: shape.text,
|
|
1225
|
+
...(shape.placeholder !== undefined && {
|
|
1226
|
+
placeholder: shape.placeholder
|
|
1227
|
+
})
|
|
1228
|
+
};
|
|
1229
|
+
}
|
|
1230
|
+
function chartView(chart) {
|
|
1231
|
+
return {
|
|
1232
|
+
type: chart.type,
|
|
1233
|
+
...(chart.title !== undefined && {
|
|
1234
|
+
title: chart.title
|
|
1235
|
+
}),
|
|
1236
|
+
series: chart.series.map(series => ({
|
|
1237
|
+
name: series.name,
|
|
1238
|
+
categories: [...series.categories],
|
|
1239
|
+
values: [...series.values]
|
|
1240
|
+
}))
|
|
714
1241
|
};
|
|
715
1242
|
}
|
|
716
1243
|
/**
|
|
717
|
-
*
|
|
718
|
-
* (
|
|
719
|
-
*
|
|
720
|
-
*
|
|
721
|
-
*
|
|
722
|
-
*
|
|
723
|
-
*
|
|
1244
|
+
* Groups a slide's shapes into visual rows the same way `deck-table.algorithm.ts`'s
|
|
1245
|
+
* own (core-internal) `shapeTables` does: each shape is one cell, `y`
|
|
1246
|
+
* flipped to the PDF-style bottom-up convention `rowsOfCells` (and the whole
|
|
1247
|
+
* table-reading pipeline it feeds) works in, so the grouping this returns is
|
|
1248
|
+
* byte-for-byte what `findDeckTables(document, { shapes: true })` groups at
|
|
1249
|
+
* run time — a pick built from it matches for real, not just visually.
|
|
1250
|
+
*
|
|
1251
|
+
* @param shapes - The slide's shapes, in reading order.
|
|
1252
|
+
* @param slideHeight - The deck's slide height, in points (`DeckDocument.height`).
|
|
1253
|
+
* @returns Each row's shape indices, left to right; a blank shape (whitespace-only text) is never part of a row.
|
|
724
1254
|
*/
|
|
725
|
-
function
|
|
726
|
-
const
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
if (field === undefined) return undefined;
|
|
742
|
-
current = match[2] === undefined ? field.fields : field.items?.fields;
|
|
743
|
-
}
|
|
744
|
-
return field;
|
|
745
|
-
}
|
|
746
|
-
/** The leading id a `from` rule reads (`item` of `item.description`, `price` of `price`); `undefined` for an `each` rule, a `.` (self), or a rule this run found nothing to say about. */
|
|
747
|
-
function boundIdOf(rule) {
|
|
748
|
-
if (rule === undefined || !('from' in rule)) return undefined;
|
|
749
|
-
const first = Array.isArray(rule.from) ? rule.from[0] : rule.from;
|
|
750
|
-
const match = /^([A-Z_][\w-]*)/i.exec(first);
|
|
751
|
-
return match?.[1];
|
|
1255
|
+
function shapeRowsOf(shapes, slideHeight) {
|
|
1256
|
+
const indexOf = new Map();
|
|
1257
|
+
const cells = shapes.flatMap((shape, index) => {
|
|
1258
|
+
const text = shape.text.replaceAll(/\s+/g, ' ').trim();
|
|
1259
|
+
if (text === '') return [];
|
|
1260
|
+
const cell = {
|
|
1261
|
+
x: shape.x,
|
|
1262
|
+
y: slideHeight - shape.y - shape.height,
|
|
1263
|
+
width: shape.width,
|
|
1264
|
+
height: shape.height,
|
|
1265
|
+
text
|
|
1266
|
+
};
|
|
1267
|
+
indexOf.set(cell, index);
|
|
1268
|
+
return [cell];
|
|
1269
|
+
});
|
|
1270
|
+
return rowsOfCells(cells).map(row => row.cells.map(cell => indexOf.get(cell)));
|
|
752
1271
|
}
|
|
1272
|
+
|
|
1273
|
+
const TEXT_LIMIT$1 = 120;
|
|
753
1274
|
/**
|
|
754
|
-
*
|
|
755
|
-
*
|
|
756
|
-
*
|
|
757
|
-
* whole sample run from `stepSummaries` (`sample-run`'s per-path summary).
|
|
1275
|
+
* Builds the document tree for `document-tree` (studio plan §3.4, issue
|
|
1276
|
+
* #94's 5a) from the body `page-snapshot`'s `take-snapshot.use-case.ts`
|
|
1277
|
+
* already fetched — no second read of the document.
|
|
758
1278
|
*
|
|
759
|
-
* @
|
|
1279
|
+
* @param body - The parsed body (`HttpBody`); YAML and JSON Lines both
|
|
1280
|
+
* arrive here already normalised to `{ kind: 'json' }` by `http.client.ts`.
|
|
1281
|
+
* @returns The tree.
|
|
1282
|
+
* @throws Error when `body`'s kind is not `json` or `xml` (PDF, workbook and
|
|
1283
|
+
* deck documents get their own canvases — 5b/5c/5d, not this mapper).
|
|
760
1284
|
*/
|
|
761
|
-
function
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
stepType: '',
|
|
766
|
-
outcome: 'not-found'
|
|
1285
|
+
function documentTreeView(body) {
|
|
1286
|
+
if (body.kind === 'json') return {
|
|
1287
|
+
format: 'json',
|
|
1288
|
+
root: jsonNode(body.data, '$', 'root')
|
|
767
1289
|
};
|
|
768
|
-
|
|
1290
|
+
if (body.kind === 'xml') return xmlTreeView(body.xml);
|
|
1291
|
+
throw new Error(`document-tree: "${body.kind}" has no tree canvas (only "json" and "xml" bodies do)`);
|
|
1292
|
+
}
|
|
1293
|
+
function listFields(path, context) {
|
|
1294
|
+
if (context === undefined) return {};
|
|
769
1295
|
return {
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
stepId: found.step.id,
|
|
773
|
-
outcome: summary?.outcome ?? 'ran',
|
|
774
|
-
error: summary?.error
|
|
1296
|
+
listPath: context.generalPrefix + path.slice(context.itemPrefix.length),
|
|
1297
|
+
listCount: context.count
|
|
775
1298
|
};
|
|
776
1299
|
}
|
|
777
|
-
function
|
|
778
|
-
|
|
779
|
-
const
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
1300
|
+
function jsonNode(value, path, label, context) {
|
|
1301
|
+
if (Array.isArray(value)) {
|
|
1302
|
+
const children = value.map((item, index) => {
|
|
1303
|
+
const itemPath = `${path}[${index}]`;
|
|
1304
|
+
return jsonNode(item, itemPath, String(index), {
|
|
1305
|
+
itemPrefix: itemPath,
|
|
1306
|
+
generalPrefix: `${path}[*]`,
|
|
1307
|
+
count: value.length
|
|
1308
|
+
});
|
|
1309
|
+
});
|
|
1310
|
+
return {
|
|
1311
|
+
id: path,
|
|
1312
|
+
label,
|
|
1313
|
+
valueType: 'array',
|
|
1314
|
+
jsonpath: path,
|
|
1315
|
+
...listFields(path, context),
|
|
1316
|
+
children
|
|
783
1317
|
};
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
1318
|
+
}
|
|
1319
|
+
if (value !== null && typeof value === 'object') {
|
|
1320
|
+
const entries = Object.entries(value);
|
|
1321
|
+
// A plain object passes its own context through unchanged: only an array (or, in XML, a repeated element) starts a new one.
|
|
1322
|
+
const children = entries.map(([key, item]) => jsonNode(item, `${path}${propertyAccess(key)}`, key, context));
|
|
1323
|
+
return {
|
|
1324
|
+
id: path,
|
|
1325
|
+
label,
|
|
1326
|
+
valueType: 'object',
|
|
1327
|
+
jsonpath: path,
|
|
1328
|
+
...listFields(path, context),
|
|
1329
|
+
children
|
|
1330
|
+
};
|
|
1331
|
+
}
|
|
1332
|
+
return {
|
|
1333
|
+
id: path,
|
|
1334
|
+
label,
|
|
1335
|
+
valueType: leafType(value),
|
|
1336
|
+
jsonpath: path,
|
|
1337
|
+
preview: previewOf(value),
|
|
1338
|
+
...listFields(path, context),
|
|
1339
|
+
children: []
|
|
1340
|
+
};
|
|
1341
|
+
}
|
|
1342
|
+
function leafType(value) {
|
|
1343
|
+
if (value === null) return 'null';
|
|
1344
|
+
if (typeof value === 'string') return 'string';
|
|
1345
|
+
if (typeof value === 'number') return 'number';
|
|
1346
|
+
if (typeof value === 'boolean') return 'boolean';
|
|
1347
|
+
return 'string';
|
|
1348
|
+
}
|
|
1349
|
+
function previewOf(value) {
|
|
1350
|
+
const text = value === null ? 'null' : typeof value === 'string' ? value : JSON.stringify(value);
|
|
1351
|
+
return text.length > TEXT_LIMIT$1 ? `${text.slice(0, TEXT_LIMIT$1)}…` : text;
|
|
1352
|
+
}
|
|
1353
|
+
/** A JSONPath property step: dot notation for a plain identifier, bracket notation (quoted) otherwise — jsonpath-plus accepts both, but only dot notation round-trips through a person reading the card. */
|
|
1354
|
+
function propertyAccess(key) {
|
|
1355
|
+
return /^[A-Z_]\w*$/i.test(key) ? `.${key}` : `[${JSON.stringify(key)}]`;
|
|
1356
|
+
}
|
|
1357
|
+
// --- XML ---
|
|
1358
|
+
const DEFAULT_NS_PREFIX = 'ns';
|
|
1359
|
+
function xmlTreeView(xml) {
|
|
1360
|
+
const document = parseXml(xml, 'document.xml');
|
|
1361
|
+
const root = document.documentElement;
|
|
1362
|
+
if (root === null) throw new Error('document-tree: the XML document has no root element');
|
|
1363
|
+
const namespaces = declaredNamespaces(root);
|
|
1364
|
+
const tree = xmlNode(root, '', namespaces, undefined);
|
|
1365
|
+
return {
|
|
1366
|
+
format: 'xml',
|
|
1367
|
+
root: tree,
|
|
1368
|
+
...(Object.keys(namespaces).length > 0 && {
|
|
1369
|
+
namespaces
|
|
1370
|
+
})
|
|
1371
|
+
};
|
|
1372
|
+
}
|
|
1373
|
+
/** Every `xmlns:*` the root declares, plus a synthesised prefix (`ns`, `ns2`, …) for a bare default `xmlns="…"` — XPath 1.0 has no default namespace, so a query needs a prefix even when the markup does not use one (mirrors `xpath.algorithm.ts`'s own `declaredPrefixes`, extended for the default case that function leaves out on purpose — it is resolved automatically there, but a *picked* card still needs to name it explicitly in `namespaces`). */
|
|
1374
|
+
function declaredNamespaces(root) {
|
|
1375
|
+
const namespaces = {};
|
|
1376
|
+
let synthetic = 0;
|
|
1377
|
+
for (const attribute of root.attributes) {
|
|
1378
|
+
if (attribute.prefix === 'xmlns') namespaces[attribute.localName] = attribute.value;else if (attribute.name === 'xmlns') {
|
|
1379
|
+
synthetic += 1;
|
|
1380
|
+
namespaces[synthetic === 1 ? DEFAULT_NS_PREFIX : `${DEFAULT_NS_PREFIX}${synthetic}`] = attribute.value;
|
|
791
1381
|
}
|
|
792
1382
|
}
|
|
793
|
-
return
|
|
1383
|
+
return namespaces;
|
|
1384
|
+
}
|
|
1385
|
+
function prefixFor(element, namespaces) {
|
|
1386
|
+
if (element.namespaceURI === null || element.namespaceURI === undefined) return undefined;
|
|
1387
|
+
if (element.prefix !== null && element.prefix !== undefined && element.prefix !== '') return element.prefix;
|
|
1388
|
+
return Object.entries(namespaces).find(([, uri]) => uri === element.namespaceURI)?.[0];
|
|
1389
|
+
}
|
|
1390
|
+
function qualifiedName(element, namespaces) {
|
|
1391
|
+
const prefix = prefixFor(element, namespaces);
|
|
1392
|
+
return prefix === undefined ? element.localName ?? element.tagName : `${prefix}:${element.localName ?? element.tagName}`;
|
|
1393
|
+
}
|
|
1394
|
+
function xmlNode(element, parentPath, namespaces, context) {
|
|
1395
|
+
const name = qualifiedName(element, namespaces);
|
|
1396
|
+
const siblingIndex = elementSiblingIndex(element);
|
|
1397
|
+
const siblingCount = elementSiblingCount(element);
|
|
1398
|
+
const step = siblingCount > 1 ? `${name}[${siblingIndex}]` : name;
|
|
1399
|
+
const path = `${parentPath}/${step}`;
|
|
1400
|
+
// I am one of several same-named siblings: my own listPath generalises myself, and my children inherit *my* context (innermost repetition wins) instead of whatever ancestor context I received.
|
|
1401
|
+
const selfContext = siblingCount > 1 ? {
|
|
1402
|
+
itemPrefix: path,
|
|
1403
|
+
generalPrefix: `${parentPath}/${name}`,
|
|
1404
|
+
count: siblingCount
|
|
1405
|
+
} : context;
|
|
1406
|
+
const childContext = siblingCount > 1 ? selfContext : context;
|
|
1407
|
+
const attributes = elementAttributes(element);
|
|
1408
|
+
const childElements = [...element.childNodes].filter(node => node.nodeType === 1);
|
|
1409
|
+
const children = [...Object.entries(attributes).map(([key, value]) => attributeNode(key, value, path, childContext)), ...childElements.map(child => xmlNode(child, path, namespaces, childContext))];
|
|
1410
|
+
const text = childElements.length === 0 ? textOf(element) : undefined;
|
|
1411
|
+
return {
|
|
1412
|
+
id: path,
|
|
1413
|
+
label: name,
|
|
1414
|
+
valueType: 'element',
|
|
1415
|
+
xpath: path,
|
|
1416
|
+
...listFields(path, selfContext),
|
|
1417
|
+
...(Object.keys(attributes).length > 0 && {
|
|
1418
|
+
attributes
|
|
1419
|
+
}),
|
|
1420
|
+
...(text !== undefined && {
|
|
1421
|
+
preview: text,
|
|
1422
|
+
valueType: childElements.length === 0 ? 'string' : 'element'
|
|
1423
|
+
}),
|
|
1424
|
+
children
|
|
1425
|
+
};
|
|
1426
|
+
}
|
|
1427
|
+
function attributeNode(name, value, elementPath, context) {
|
|
1428
|
+
const path = `${elementPath}/@${name}`;
|
|
1429
|
+
return {
|
|
1430
|
+
id: path,
|
|
1431
|
+
label: `@${name}`,
|
|
1432
|
+
valueType: 'string',
|
|
1433
|
+
xpath: path,
|
|
1434
|
+
preview: previewOf(value),
|
|
1435
|
+
...listFields(path, context),
|
|
1436
|
+
children: []
|
|
1437
|
+
};
|
|
1438
|
+
}
|
|
1439
|
+
function elementAttributes(element) {
|
|
1440
|
+
return Object.fromEntries([...element.attributes].filter(attribute => attribute.prefix !== 'xmlns' && attribute.name !== 'xmlns').map(attribute => [attribute.name, attribute.value]));
|
|
1441
|
+
}
|
|
1442
|
+
function elementSiblingIndex(element) {
|
|
1443
|
+
let index = 1;
|
|
1444
|
+
let sibling = element.previousSibling;
|
|
1445
|
+
while (sibling !== null) {
|
|
1446
|
+
if (sibling.nodeType === 1 && sibling.tagName === element.tagName) index += 1;
|
|
1447
|
+
sibling = sibling.previousSibling;
|
|
1448
|
+
}
|
|
1449
|
+
return index;
|
|
1450
|
+
}
|
|
1451
|
+
function elementSiblingCount(element) {
|
|
1452
|
+
const parent = element.parentNode;
|
|
1453
|
+
if (parent === null) return 1;
|
|
1454
|
+
return [...parent.childNodes].filter(node => node.nodeType === 1 && node.tagName === element.tagName).length;
|
|
1455
|
+
}
|
|
1456
|
+
function textOf(element) {
|
|
1457
|
+
const text = (element.textContent ?? '').trim().replaceAll(/\s+/g, ' ');
|
|
1458
|
+
return text === '' ? undefined : previewOf(text);
|
|
794
1459
|
}
|
|
795
1460
|
|
|
796
1461
|
/**
|
|
797
|
-
*
|
|
798
|
-
*
|
|
799
|
-
* `
|
|
1462
|
+
* Builds the PDF canvas's view model (studio plan §3.4, issue #94's 5b) from
|
|
1463
|
+
* an already-read `PdfDocument` (`take-snapshot`'s cached body — no second
|
|
1464
|
+
* read of the file): every cell and row `readPdf` found, in PDF points, plus
|
|
1465
|
+
* per-page counts and whether the page has a text layer at all. Table-specific
|
|
1466
|
+
* diagnostics (the header row, the column bands, the wrapped-row regrouping)
|
|
1467
|
+
* are not here — they depend on a `table` extract's own options, so they come
|
|
1468
|
+
* from `table-preview.use-case.ts` instead, computed live as those options
|
|
1469
|
+
* change.
|
|
800
1470
|
*
|
|
801
|
-
* @param
|
|
802
|
-
* @
|
|
803
|
-
* @returns The explanation.
|
|
804
|
-
* @throws Error when no workspace is open, no sample run has finished for the target's recipe, or the target does not name a real record/field.
|
|
1471
|
+
* @param document - The read PDF.
|
|
1472
|
+
* @returns The view.
|
|
805
1473
|
*/
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
1474
|
+
function pdfDocumentView(document) {
|
|
1475
|
+
return {
|
|
1476
|
+
pages: document.pages.map(page => ({
|
|
1477
|
+
number: page.number,
|
|
1478
|
+
width: page.width,
|
|
1479
|
+
height: page.height,
|
|
1480
|
+
rows: page.rows.map(row => ({
|
|
1481
|
+
top: row.top,
|
|
1482
|
+
bottom: row.bottom,
|
|
1483
|
+
cells: row.cells.map(cell => ({
|
|
1484
|
+
x: cell.x,
|
|
1485
|
+
y: cell.y,
|
|
1486
|
+
width: cell.width,
|
|
1487
|
+
height: cell.height,
|
|
1488
|
+
text: cell.text
|
|
1489
|
+
}))
|
|
1490
|
+
})),
|
|
1491
|
+
rowCount: page.rows.length,
|
|
1492
|
+
cellCount: page.rows.reduce((total, row) => total + row.cells.length, 0),
|
|
1493
|
+
hasTextLayer: page.rows.length > 0
|
|
1494
|
+
}))
|
|
1495
|
+
};
|
|
809
1496
|
}
|
|
810
1497
|
|
|
811
1498
|
/**
|
|
812
|
-
*
|
|
813
|
-
*
|
|
814
|
-
*
|
|
815
|
-
*
|
|
816
|
-
* `
|
|
817
|
-
* ways, both worth knowing before relying on it: it opens a bare page
|
|
818
|
-
* without running the recipe's own steps first, so a start point behind a
|
|
819
|
-
* `session.bootstrap` login shows what an anonymous visitor sees; and it
|
|
820
|
-
* launches a fresh browser or HTTP context per call, one page then closes
|
|
821
|
-
* it, rather than reusing whatever the last sample run had open.
|
|
1499
|
+
* Runs a `table` extract's options against an already-read PDF (studio plan
|
|
1500
|
+
* §3.4, issue #94's 5b: the live preview) — a pure function of the document
|
|
1501
|
+
* and the options, so re-running it on every keystroke as the person edits
|
|
1502
|
+
* `selector`/`until`/`columns` costs nothing: no re-fetch, no re-parse
|
|
1503
|
+
* (`readPdf` already ran once, when `take-snapshot` captured the document).
|
|
822
1504
|
*
|
|
823
|
-
*
|
|
824
|
-
*
|
|
825
|
-
*
|
|
826
|
-
*
|
|
1505
|
+
* The workbook grid's live preview (issue #94's 5c) is `previewGridTable`,
|
|
1506
|
+
* below: the same idea (a pure re-run of the options against the cached
|
|
1507
|
+
* document), but its own result shape — a workbook table has no page, no
|
|
1508
|
+
* column bands, no wrapped-row regrouping, so folding it into
|
|
1509
|
+
* `TablePreviewMatch`/`TablePreviewResult` would leave most of those fields
|
|
1510
|
+
* meaningless for a grid pick; a `GridTablePreviewMatch`/`GridTablePreviewResult`
|
|
1511
|
+
* sibling reads honestly instead.
|
|
1512
|
+
*
|
|
1513
|
+
* @param document - The already-read PDF.
|
|
1514
|
+
* @param options - The `table` extract's options, not yet compiled to `RegExp`.
|
|
1515
|
+
* @returns Every match, each with the exact row indices to highlight — or `error` (and no matches) when a pattern is not a valid regular expression, expected while the person is still typing it.
|
|
827
1516
|
*/
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
if (point === undefined) throw new Error(`recipe "${input.id}" has no start point`);
|
|
831
|
-
return input.mode === 'web' ? fetchWeb(point.url, browser) : fetchApi(point.url);
|
|
832
|
-
}
|
|
833
|
-
async function fetchWeb(url, browser) {
|
|
834
|
-
const client = await BrowserClient.launch(browser);
|
|
1517
|
+
function previewPdfTable(document, options) {
|
|
1518
|
+
let query;
|
|
835
1519
|
try {
|
|
836
|
-
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
1520
|
+
query = {
|
|
1521
|
+
header: compile(options.header, 'selector'),
|
|
1522
|
+
until: options.until === undefined ? undefined : compile(options.until, 'until'),
|
|
1523
|
+
columns: options.columns === undefined ? undefined : Object.fromEntries(Object.entries(options.columns).map(([key, pattern]) => [key, compile(pattern, `columns.${key}`)])),
|
|
1524
|
+
align: options.align
|
|
1525
|
+
};
|
|
1526
|
+
} catch (error) {
|
|
1527
|
+
return {
|
|
1528
|
+
matches: [],
|
|
1529
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1530
|
+
};
|
|
845
1531
|
}
|
|
1532
|
+
const analyses = analyzeTables(document, query);
|
|
1533
|
+
return {
|
|
1534
|
+
matches: analyses.map(analysis => ({
|
|
1535
|
+
page: analysis.page,
|
|
1536
|
+
headerRowIndex: analysis.headerRowIndex,
|
|
1537
|
+
matchedRowIndices: rowIndicesOf(analysis),
|
|
1538
|
+
bands: analysis.bands,
|
|
1539
|
+
bandTolerance: analysis.bandTolerance,
|
|
1540
|
+
table: analysis.table
|
|
1541
|
+
}))
|
|
1542
|
+
};
|
|
846
1543
|
}
|
|
847
|
-
|
|
848
|
-
const
|
|
1544
|
+
function rowIndicesOf(analysis) {
|
|
1545
|
+
const indices = new Set([analysis.headerRowIndex, ...analysis.rowGroups.flat()]);
|
|
1546
|
+
if (analysis.untilRowIndex !== undefined) indices.add(analysis.untilRowIndex);
|
|
1547
|
+
return [...indices].sort((a, b) => a - b);
|
|
1548
|
+
}
|
|
1549
|
+
function compile(source, where) {
|
|
849
1550
|
try {
|
|
850
|
-
|
|
851
|
-
|
|
1551
|
+
return new RegExp(source, 'i');
|
|
1552
|
+
} catch (error) {
|
|
1553
|
+
throw new Error(`${where}: invalid pattern ${source} (${error.message})`, {
|
|
1554
|
+
cause: error
|
|
852
1555
|
});
|
|
853
|
-
return textOf$1(response.body);
|
|
854
|
-
} finally {
|
|
855
|
-
await client.dispose();
|
|
856
1556
|
}
|
|
857
1557
|
}
|
|
858
|
-
/**
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
1558
|
+
/**
|
|
1559
|
+
* Runs a `table` extract's options against an already-read workbook (studio
|
|
1560
|
+
* plan §3.4, issue #94's 5c: the live preview) — a pure function of the
|
|
1561
|
+
* document and the options, exactly like `previewPdfTable` above: no
|
|
1562
|
+
* re-fetch, no re-parse (`csvWorkbook`/`readXlsxWorkbook` already ran once,
|
|
1563
|
+
* when `take-snapshot` captured the document).
|
|
1564
|
+
*
|
|
1565
|
+
* @param document - The already-read workbook.
|
|
1566
|
+
* @param options - The `table` extract's options, not yet compiled to `RegExp`.
|
|
1567
|
+
* @returns Every matched table — or `error` (and no matches) when a pattern is not a valid regular expression, expected while the person is still typing it.
|
|
1568
|
+
*/
|
|
1569
|
+
function previewGridTable(document, options) {
|
|
1570
|
+
let query;
|
|
1571
|
+
try {
|
|
1572
|
+
query = {
|
|
1573
|
+
header: compile(options.header, 'selector'),
|
|
1574
|
+
until: options.until === undefined ? undefined : compile(options.until, 'until'),
|
|
1575
|
+
columns: options.columns === undefined ? undefined : Object.fromEntries(Object.entries(options.columns).map(([key, pattern]) => [key, compile(pattern, `columns.${key}`)])),
|
|
1576
|
+
sheet: options.sheet === undefined ? undefined : compile(options.sheet, 'sheet'),
|
|
1577
|
+
headerRows: options.headerRows,
|
|
1578
|
+
fillDown: options.fillDown,
|
|
1579
|
+
includeHidden: options.includeHidden
|
|
1580
|
+
};
|
|
1581
|
+
} catch (error) {
|
|
1582
|
+
return {
|
|
1583
|
+
matches: [],
|
|
1584
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1585
|
+
};
|
|
1586
|
+
}
|
|
1587
|
+
const tables = findGridTables(document, query);
|
|
1588
|
+
return {
|
|
1589
|
+
matches: tables.map(table => ({
|
|
1590
|
+
sheet: table.sheet,
|
|
1591
|
+
title: table.title,
|
|
1592
|
+
header: table.header,
|
|
1593
|
+
rows: table.rows
|
|
1594
|
+
}))
|
|
1595
|
+
};
|
|
1596
|
+
}
|
|
1597
|
+
/**
|
|
1598
|
+
* Runs a `table` extract's options against an already-read deck (studio plan
|
|
1599
|
+
* §3.4, issue #94's 5d: the live preview) — a pure function of the document
|
|
1600
|
+
* and the options, exactly like `previewPdfTable`/`previewGridTable` above:
|
|
1601
|
+
* no re-fetch, no re-parse (`readPptxDeck` already ran once, when
|
|
1602
|
+
* `take-snapshot` captured the document).
|
|
1603
|
+
*
|
|
1604
|
+
* @param document - The already-read deck.
|
|
1605
|
+
* @param options - The `table` extract's options, not yet compiled to `RegExp`.
|
|
1606
|
+
* @returns Every matched table — or `error` (and no matches) when a pattern is not a valid regular expression, expected while the person is still typing it.
|
|
1607
|
+
*/
|
|
1608
|
+
function previewDeckTable(document, options) {
|
|
1609
|
+
let query;
|
|
1610
|
+
try {
|
|
1611
|
+
query = {
|
|
1612
|
+
slide: options.slide === undefined ? undefined : compile(options.slide, 'slide'),
|
|
1613
|
+
shapes: options.shapes,
|
|
1614
|
+
align: options.align,
|
|
1615
|
+
header: compile(options.header, 'selector'),
|
|
1616
|
+
until: options.until === undefined ? undefined : compile(options.until, 'until'),
|
|
1617
|
+
columns: options.columns === undefined ? undefined : Object.fromEntries(Object.entries(options.columns).map(([key, pattern]) => [key, compile(pattern, `columns.${key}`)])),
|
|
1618
|
+
headerRows: options.headerRows,
|
|
1619
|
+
fillDown: options.fillDown,
|
|
1620
|
+
includeHidden: options.includeHidden
|
|
1621
|
+
};
|
|
1622
|
+
} catch (error) {
|
|
1623
|
+
return {
|
|
1624
|
+
matches: [],
|
|
1625
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1626
|
+
};
|
|
1627
|
+
}
|
|
1628
|
+
const tables = findDeckTables(document, query);
|
|
1629
|
+
return {
|
|
1630
|
+
matches: tables.map(table => ({
|
|
1631
|
+
slide: table.slide,
|
|
1632
|
+
slideTitle: table.slideTitle,
|
|
1633
|
+
title: table.title,
|
|
1634
|
+
header: table.header,
|
|
1635
|
+
rows: table.rows
|
|
1636
|
+
}))
|
|
1637
|
+
};
|
|
864
1638
|
}
|
|
865
1639
|
|
|
866
|
-
/** The attribute `hiddenMarksScript` stamps on an element hidden on the live page. */
|
|
867
|
-
const HIDDEN_ATTRIBUTE = 'data-oc-hidden';
|
|
868
1640
|
/**
|
|
869
|
-
*
|
|
870
|
-
*
|
|
871
|
-
*
|
|
872
|
-
*
|
|
873
|
-
*
|
|
874
|
-
*
|
|
1641
|
+
* Handles `deck-preview`: the deck canvas's live preview of a `table`
|
|
1642
|
+
* extract's options (`document-view`'s `table-preview.use-case.ts`'s
|
|
1643
|
+
* `previewDeckTable`, studio plan §3.4, issue #94's 5d), off the same cached
|
|
1644
|
+
* snapshot `take-snapshot`/`deck-view` already captured — a pure computation
|
|
1645
|
+
* over it, so this answers instantly on every option change, no re-fetch or
|
|
1646
|
+
* re-parse.
|
|
875
1647
|
*
|
|
876
|
-
*
|
|
877
|
-
*
|
|
878
|
-
*
|
|
879
|
-
*
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
1648
|
+
* @param state - The server state.
|
|
1649
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
1650
|
+
* @returns The matched tables, or `error` when an option's pattern does not compile.
|
|
1651
|
+
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
1652
|
+
*/
|
|
1653
|
+
function handleDeckPreview(state, command) {
|
|
1654
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
1655
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
1656
|
+
if (!isDeckDocument(snapshot.body)) throw new Error(`"${command.recipeId}" at "${command.path}" is not a deck snapshot: deck-preview only reads a read PowerPoint document`);
|
|
1657
|
+
return previewDeckTable(snapshot.body, command.options);
|
|
1658
|
+
}
|
|
1659
|
+
|
|
1660
|
+
/**
|
|
1661
|
+
* Handles `deck-view`: the deck canvas's slides, shapes, tables, charts and
|
|
1662
|
+
* notes (`document-view`'s `deck-view.mapper.ts`, studio plan §3.4, issue
|
|
1663
|
+
* #94's 5d), off the same cached snapshot `take-snapshot`/`document-tree`
|
|
1664
|
+
* already use.
|
|
883
1665
|
*
|
|
884
|
-
* @param
|
|
885
|
-
* @
|
|
1666
|
+
* @param state - The server state.
|
|
1667
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
1668
|
+
* @returns The document's view.
|
|
1669
|
+
* @throws Error when no workspace is open, `command.path` has no cached snapshot yet, or the snapshot's document is not a read deck.
|
|
886
1670
|
*/
|
|
887
|
-
function
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
|
|
892
|
-
var element = elements[i]
|
|
893
|
-
var style = window.getComputedStyle(element)
|
|
894
|
-
var rect = element.getBoundingClientRect()
|
|
895
|
-
var hidden = style.display === 'none' || style.visibility === 'hidden' || (rect.width === 0 && rect.height === 0)
|
|
896
|
-
if (hidden) element.setAttribute(attribute, '1')
|
|
897
|
-
}
|
|
898
|
-
})()`;
|
|
1671
|
+
function handleDeckView(state, command) {
|
|
1672
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
1673
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
1674
|
+
if (!isDeckDocument(snapshot.body)) throw new Error(`"${command.recipeId}" at "${command.path}" is not a deck snapshot: the deck canvas only reads a read PowerPoint document`);
|
|
1675
|
+
return deckDocumentView(snapshot.body);
|
|
899
1676
|
}
|
|
900
1677
|
|
|
901
|
-
/** Attributes that carry a URL and are rewritten to absolute; `srcset` is handled separately (it carries several). */
|
|
902
|
-
const URL_ATTRIBUTES = ['href', 'src', 'action', 'poster'];
|
|
903
|
-
/** Schemes left untouched: they are not a location `new URL(value, base)` should resolve, or already carry their own meaning. */
|
|
904
|
-
const SKIP_SCHEME = /^(?:javascript|mailto|tel|data|about):/i;
|
|
905
1678
|
/**
|
|
906
|
-
*
|
|
907
|
-
* (studio plan §3.
|
|
908
|
-
*
|
|
909
|
-
* stripped, fields made read-only), `<meta http-equiv="refresh">` stripped
|
|
910
|
-
* (it would otherwise navigate the sandboxed frame on its own), and every
|
|
911
|
-
* element given a stable `data-oc-node` id, in document order, for the
|
|
912
|
-
* picker overlay and `selector-inference` to address specific nodes by.
|
|
1679
|
+
* Handles `document-tree`: the content pane's tree canvas for a JSON, YAML
|
|
1680
|
+
* or XML document (`document-view`, studio plan §3.4, issue #94's 5a), off
|
|
1681
|
+
* the same cached snapshot `take-snapshot`/`inspect-page` already use.
|
|
913
1682
|
*
|
|
914
|
-
*
|
|
915
|
-
*
|
|
916
|
-
*
|
|
917
|
-
*
|
|
918
|
-
|
|
1683
|
+
* @param state - The server state.
|
|
1684
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
1685
|
+
* @returns The tree.
|
|
1686
|
+
* @throws Error when no workspace is open, `command.path` has no cached snapshot yet, or the snapshot's document is not JSON/YAML/XML (no parsed body at all in web mode; PDF/workbook/deck in api mode — see `documentTreeView`'s own doc comment).
|
|
1687
|
+
*/
|
|
1688
|
+
function handleDocumentTree(state, command) {
|
|
1689
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
1690
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
1691
|
+
if (snapshot.body === undefined) throw new Error(`"${command.recipeId}" at "${command.path}" is a web-mode (HTML) snapshot: the tree canvas only reads a JSON/YAML/XML document`);
|
|
1692
|
+
return documentTreeView(snapshot.body);
|
|
1693
|
+
}
|
|
1694
|
+
|
|
1695
|
+
/**
|
|
1696
|
+
* Finds and parses one input recipe of a workspace folder, and the output
|
|
1697
|
+
* recipe it names.
|
|
919
1698
|
*
|
|
920
|
-
* @param
|
|
921
|
-
* @param
|
|
922
|
-
* @returns The
|
|
1699
|
+
* @param folder - The workspace folder.
|
|
1700
|
+
* @param recipeId - The input recipe's id.
|
|
1701
|
+
* @returns The parsed pair.
|
|
1702
|
+
* @throws RecipeValidationError when either recipe does not parse.
|
|
1703
|
+
* @throws Error when the input or its output is not in the folder.
|
|
923
1704
|
*/
|
|
924
|
-
function
|
|
925
|
-
const
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
if (($(element).attr('http-equiv') ?? '').toLowerCase() === 'refresh') $(element).remove();
|
|
932
|
-
});
|
|
933
|
-
$('script').remove();
|
|
934
|
-
$('form').removeAttr('onsubmit');
|
|
935
|
-
$('input, textarea').attr('readonly', 'readonly');
|
|
936
|
-
$('select, button, input[type="submit"], input[type="button"]').attr('disabled', 'disabled');
|
|
937
|
-
$('*').each((_index, element) => {
|
|
938
|
-
const node = $(element);
|
|
939
|
-
for (const attribute of URL_ATTRIBUTES) {
|
|
940
|
-
const value = node.attr(attribute);
|
|
941
|
-
if (value !== undefined) node.attr(attribute, resolveUrl(value, baseUrl) ?? value);
|
|
942
|
-
}
|
|
943
|
-
const srcset = node.attr('srcset');
|
|
944
|
-
if (srcset !== undefined) node.attr('srcset', rewriteSrcset(srcset, baseUrl));
|
|
945
|
-
});
|
|
946
|
-
let nodeCount = 0;
|
|
947
|
-
$('*').each((_index, element) => {
|
|
948
|
-
$(element).attr(NODE_ID_ATTRIBUTE, `n${nodeCount}`);
|
|
949
|
-
nodeCount += 1;
|
|
950
|
-
});
|
|
1705
|
+
async function loadRecipePair(folder, recipeId) {
|
|
1706
|
+
const documents = await readRecipeSource(folder);
|
|
1707
|
+
const inputDocument = documents.find(document => kindOf(document.content) === 'input' && idOf(document.content) === recipeId);
|
|
1708
|
+
if (inputDocument === undefined) throw new Error(`no input recipe "${recipeId}" in this workspace`);
|
|
1709
|
+
const input = parseInputRecipe(inputDocument.content, inputDocument.source);
|
|
1710
|
+
const outputDocument = documents.find(document => kindOf(document.content) === 'output' && idOf(document.content) === input.output);
|
|
1711
|
+
if (outputDocument === undefined) throw new Error(`no output recipe "${input.output}" in this workspace`);
|
|
951
1712
|
return {
|
|
952
|
-
|
|
953
|
-
|
|
1713
|
+
input,
|
|
1714
|
+
output: parseOutputRecipe(outputDocument.content, outputDocument.source)
|
|
954
1715
|
};
|
|
955
1716
|
}
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
const trimmed = value.trim();
|
|
959
|
-
if (trimmed === '' || SKIP_SCHEME.test(trimmed)) return undefined;
|
|
960
|
-
try {
|
|
961
|
-
return new URL(trimmed, baseUrl).href;
|
|
962
|
-
} catch {
|
|
963
|
-
return undefined;
|
|
964
|
-
}
|
|
965
|
-
}
|
|
966
|
-
/** `srcset`'s own micro-syntax: comma-separated `url descriptor?` pairs. Malformed entries are passed through unresolved rather than dropped. */
|
|
967
|
-
function rewriteSrcset(value, baseUrl) {
|
|
968
|
-
return value.split(',').map(candidate => {
|
|
969
|
-
const trimmed = candidate.trim();
|
|
970
|
-
const spaceAt = trimmed.indexOf(' ');
|
|
971
|
-
const url = spaceAt === -1 ? trimmed : trimmed.slice(0, spaceAt);
|
|
972
|
-
const descriptor = spaceAt === -1 ? '' : trimmed.slice(spaceAt);
|
|
973
|
-
return `${resolveUrl(url, baseUrl) ?? url}${descriptor}`;
|
|
974
|
-
}).join(', ');
|
|
1717
|
+
function kindOf(content) {
|
|
1718
|
+
return content?.kind;
|
|
975
1719
|
}
|
|
976
|
-
function
|
|
977
|
-
return
|
|
1720
|
+
function idOf(content) {
|
|
1721
|
+
return content?.id;
|
|
978
1722
|
}
|
|
979
1723
|
|
|
980
1724
|
/**
|
|
981
|
-
*
|
|
982
|
-
*
|
|
983
|
-
*
|
|
984
|
-
*
|
|
985
|
-
*
|
|
986
|
-
* `
|
|
987
|
-
*
|
|
988
|
-
*
|
|
989
|
-
*
|
|
990
|
-
*
|
|
991
|
-
* needs a "stop the crawl after step X and hand back the live page/document"
|
|
992
|
-
* hook in `crawl-execution` that does not exist yet; building one was out of
|
|
993
|
-
* this phase's budget (see this phase's final report for the tradeoff this
|
|
994
|
-
* was weighed against). Any other `stepPath` throws, clearly, rather than
|
|
995
|
-
* silently returning the start page under a misleading label.
|
|
1725
|
+
* Runs one input recipe of a workspace folder as a sample: `debug: true` (so
|
|
1726
|
+
* each `record:emit` carries its scope snapshot and mapping trace, and each
|
|
1727
|
+
* `record:reject` its scope), an in-memory sink, and `budget` as the
|
|
1728
|
+
* crawler's `sample` (`@opencraw/core`'s `CrawlOptions.sample`, #89). Every
|
|
1729
|
+
* engine event becomes a trace line with the same `traceLine` formatter the
|
|
1730
|
+
* cli's `--trace` uses; every emitted record reaches `callbacks.onRecord` as
|
|
1731
|
+
* it happens, not only in the final result. Step events (`step:start`,
|
|
1732
|
+
* `step:skip`, `step:retry`) are aggregated by path into `result.steps`, so
|
|
1733
|
+
* `explain-why` (#92) can later say whether the step that bound a mapping's
|
|
1734
|
+
* source id ran, was skipped, or retried — without a second run.
|
|
996
1735
|
*
|
|
997
|
-
* @param
|
|
998
|
-
* @param
|
|
999
|
-
* @param
|
|
1000
|
-
* @
|
|
1001
|
-
* @
|
|
1736
|
+
* @param folder - The workspace folder the recipe lives in.
|
|
1737
|
+
* @param recipeId - The input recipe to run.
|
|
1738
|
+
* @param budget - The sample budget; unset runs the recipe to completion (still capped by its own `limits`).
|
|
1739
|
+
* @param callbacks - Where trace lines and records go as the run proceeds.
|
|
1740
|
+
* @param browser - Browser launch settings for a web-mode recipe (an executable path override, headless…); the server-wide setting `studio-http.use-case.ts`'s `StudioServerOptions.browser` carries in, e.g. `OPENCRAW_CHROMIUM` in a sandbox with no full Playwright install.
|
|
1741
|
+
* @returns A handle: `stop` closes the crawler cleanly, `result` resolves when the run ends.
|
|
1742
|
+
* @throws RecipeValidationError, RecipeBindingError, Error when the recipe or its output cannot be loaded or bound.
|
|
1002
1743
|
*/
|
|
1003
|
-
async function
|
|
1004
|
-
|
|
1005
|
-
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
|
|
1010
|
-
const
|
|
1011
|
-
|
|
1012
|
-
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1022
|
-
|
|
1023
|
-
|
|
1024
|
-
|
|
1744
|
+
async function runSample(folder, recipeId, budget, callbacks, browser) {
|
|
1745
|
+
const {
|
|
1746
|
+
input,
|
|
1747
|
+
output
|
|
1748
|
+
} = await loadRecipePair(folder, recipeId);
|
|
1749
|
+
const set = bindRecipeSet(output, [input]);
|
|
1750
|
+
const sink = memorySink();
|
|
1751
|
+
const records = [];
|
|
1752
|
+
const rejectedRecords = [];
|
|
1753
|
+
const steps = new Map();
|
|
1754
|
+
const crawler = createCrawler({
|
|
1755
|
+
sink,
|
|
1756
|
+
debug: true,
|
|
1757
|
+
sample: budget,
|
|
1758
|
+
browser,
|
|
1759
|
+
onEvent: event => {
|
|
1760
|
+
const line = traceLine(event);
|
|
1761
|
+
if (line !== undefined) callbacks.onTraceLine(line);
|
|
1762
|
+
trackStep(steps, event);
|
|
1763
|
+
if (event.type === 'record:emit') {
|
|
1764
|
+
const record = {
|
|
1765
|
+
key: event.key,
|
|
1766
|
+
data: event.data,
|
|
1767
|
+
scope: event.scope,
|
|
1768
|
+
mapping: event.mapping
|
|
1769
|
+
};
|
|
1770
|
+
records.push(record);
|
|
1771
|
+
callbacks.onRecord(record);
|
|
1772
|
+
} else if (event.type === 'record:reject') {
|
|
1773
|
+
const rejected = {
|
|
1774
|
+
field: event.field,
|
|
1775
|
+
reason: event.reason,
|
|
1776
|
+
url: event.url,
|
|
1777
|
+
scope: event.scope
|
|
1778
|
+
};
|
|
1779
|
+
rejectedRecords.push(rejected);
|
|
1780
|
+
callbacks.onRejected?.(rejected);
|
|
1781
|
+
}
|
|
1025
1782
|
}
|
|
1026
|
-
}
|
|
1027
|
-
|
|
1783
|
+
});
|
|
1784
|
+
const result = runToResult(crawler, set, records, rejectedRecords, steps);
|
|
1785
|
+
return {
|
|
1786
|
+
stop: () => crawler.close(),
|
|
1787
|
+
result
|
|
1788
|
+
};
|
|
1789
|
+
}
|
|
1790
|
+
/**
|
|
1791
|
+
* Folds one `step:*` event into `steps`, by path: `skip` always wins over an
|
|
1792
|
+
* earlier `ran`/`retried` outcome for the same path (a step skipped even
|
|
1793
|
+
* once is worth flagging), `retry` wins over a plain `ran`, and a path seen
|
|
1794
|
+
* only as `finish`/`start` (no skip, no retry) settles on `ran`. A `forEach`
|
|
1795
|
+
* runs its body once per item, so the same path folds repeatedly; this is
|
|
1796
|
+
* deliberately a summary across the whole sample, not a per-record trace.
|
|
1797
|
+
*/
|
|
1798
|
+
function trackStep(steps, event) {
|
|
1799
|
+
if (event.type === 'step:start') {
|
|
1800
|
+
if (!steps.has(event.path)) steps.set(event.path, {
|
|
1801
|
+
path: event.path,
|
|
1802
|
+
stepType: event.stepType,
|
|
1803
|
+
stepId: event.stepId,
|
|
1804
|
+
outcome: 'ran'
|
|
1805
|
+
});
|
|
1806
|
+
return;
|
|
1807
|
+
}
|
|
1808
|
+
if (event.type === 'step:retry') {
|
|
1809
|
+
steps.set(event.path, {
|
|
1810
|
+
path: event.path,
|
|
1811
|
+
stepType: event.stepType,
|
|
1812
|
+
stepId: event.stepId,
|
|
1813
|
+
outcome: 'retried',
|
|
1814
|
+
error: event.error
|
|
1815
|
+
});
|
|
1816
|
+
return;
|
|
1817
|
+
}
|
|
1818
|
+
if (event.type === 'step:skip') {
|
|
1819
|
+
steps.set(event.path, {
|
|
1820
|
+
path: event.path,
|
|
1821
|
+
stepType: event.stepType,
|
|
1822
|
+
stepId: event.stepId,
|
|
1823
|
+
outcome: 'skipped',
|
|
1824
|
+
error: event.error
|
|
1825
|
+
});
|
|
1028
1826
|
}
|
|
1029
1827
|
}
|
|
1030
|
-
async function
|
|
1031
|
-
const client = await HttpClient.open({});
|
|
1828
|
+
async function runToResult(crawler, set, records, rejectedRecords, steps) {
|
|
1032
1829
|
try {
|
|
1033
|
-
const
|
|
1034
|
-
|
|
1035
|
-
});
|
|
1036
|
-
const rewritten = rewriteDocument(textOf(response.body), url);
|
|
1830
|
+
const report = await crawler.run(set);
|
|
1831
|
+
const recipe = report.recipes[0];
|
|
1037
1832
|
return {
|
|
1038
|
-
|
|
1039
|
-
|
|
1833
|
+
recipeId: recipe.recipeId,
|
|
1834
|
+
emitted: recipe.emitted,
|
|
1835
|
+
rejected: recipe.rejected,
|
|
1836
|
+
duplicates: recipe.duplicates,
|
|
1837
|
+
durationMs: recipe.durationMs,
|
|
1838
|
+
error: recipe.error,
|
|
1839
|
+
stoppedBy: recipe.stoppedBy,
|
|
1840
|
+
records,
|
|
1841
|
+
rejectedRecords,
|
|
1842
|
+
// eslint-disable-next-line unicorn/prefer-iterator-to-array -- `.toArray()` needs a `lib` newer than this repo's `es2022` (tsconfig.base.json)
|
|
1843
|
+
steps: [...steps.values()]
|
|
1040
1844
|
};
|
|
1041
1845
|
} finally {
|
|
1042
|
-
await
|
|
1846
|
+
await crawler.close();
|
|
1043
1847
|
}
|
|
1044
1848
|
}
|
|
1045
|
-
/** Every body kind the studio can show as text today; others get a placeholder rather than a crash (mirrors `fetch-start-page.use-case.ts`'s own `textOf`). */
|
|
1046
|
-
function textOf(body) {
|
|
1047
|
-
if (body.kind === 'html') return body.html;
|
|
1048
|
-
if (body.kind === 'text') return body.text;
|
|
1049
|
-
if (body.kind === 'json') return `<pre>${JSON.stringify(body.data, null, 2)}</pre>`;
|
|
1050
|
-
return `<p>[${body.kind} document: the content pane shows HTML and JSON responses for now]</p>`;
|
|
1051
|
-
}
|
|
1052
1849
|
|
|
1053
|
-
/** @returns
|
|
1054
|
-
function
|
|
1850
|
+
/** @returns A fresh, empty cache. */
|
|
1851
|
+
function createLastRunCache() {
|
|
1055
1852
|
return {
|
|
1056
1853
|
entries: new Map()
|
|
1057
1854
|
};
|
|
1058
1855
|
}
|
|
1059
|
-
/**
|
|
1060
|
-
function
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
cache.entries.set(keyOf(recipeId, path), snapshot);
|
|
1856
|
+
/** Records a finished sample run, replacing whatever this recipe's cache entry held before. */
|
|
1857
|
+
function recordLastRun(cache, recipeId, result) {
|
|
1858
|
+
cache.entries.set(recipeId, {
|
|
1859
|
+
recipeId,
|
|
1860
|
+
result
|
|
1861
|
+
});
|
|
1066
1862
|
}
|
|
1067
|
-
|
|
1068
|
-
|
|
1863
|
+
/** @returns The recipe's last completed sample run, or `undefined` when none has finished yet. */
|
|
1864
|
+
function lastRunOf(cache, recipeId) {
|
|
1865
|
+
return cache.entries.get(recipeId);
|
|
1069
1866
|
}
|
|
1070
1867
|
|
|
1071
1868
|
/**
|
|
1072
|
-
*
|
|
1073
|
-
*
|
|
1074
|
-
*
|
|
1869
|
+
* A best-effort hint about what a missing value's bound id actually held in
|
|
1870
|
+
* the scope snapshot: the keys of an object, or the length of a list — the
|
|
1871
|
+
* fallback when there is no per-item DOM snapshot to compare a selector
|
|
1872
|
+
* against (see `why-sentence.mapper.ts`'s doc comment for why the richer,
|
|
1873
|
+
* "the item has `.price_new`" hint is out of this phase's scope).
|
|
1075
1874
|
*
|
|
1076
|
-
* @param
|
|
1077
|
-
* @param
|
|
1078
|
-
* @returns
|
|
1079
|
-
* @throws Error when no workspace is open, or the recipe cannot be found or loaded.
|
|
1875
|
+
* @param scope - The record's scope snapshot (`SampleRunRecord.scope`/`SampleRunRejected.scope`).
|
|
1876
|
+
* @param boundId - The leading id the rule's `from` reads.
|
|
1877
|
+
* @returns A short clause (no leading punctuation), or `undefined` when the id is not in scope or nothing useful can be said about it.
|
|
1080
1878
|
*/
|
|
1081
|
-
|
|
1082
|
-
if (
|
|
1083
|
-
const
|
|
1084
|
-
|
|
1085
|
-
}
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
}
|
|
1879
|
+
function hintFor(scope, boundId) {
|
|
1880
|
+
if (scope === undefined || boundId === undefined) return undefined;
|
|
1881
|
+
const value = scope[boundId];
|
|
1882
|
+
if (value === undefined) return undefined;
|
|
1883
|
+
if (Array.isArray(value)) return value.length === 0 ? undefined : `"${boundId}" is a list of ${value.length}, not a single value`;
|
|
1884
|
+
if (typeof value === 'object' && value !== null) {
|
|
1885
|
+
const keys = Object.keys(value);
|
|
1886
|
+
return keys.length === 0 ? undefined : `"${boundId}" has: ${keys.slice(0, 5).join(', ')}`;
|
|
1887
|
+
}
|
|
1888
|
+
return undefined;
|
|
1090
1889
|
}
|
|
1091
1890
|
|
|
1092
|
-
const GENERATED_ID = /^(?:react-|radix-|:r|ember|css-|mui-|headlessui-)/i;
|
|
1093
1891
|
/**
|
|
1094
|
-
*
|
|
1095
|
-
*
|
|
1096
|
-
*
|
|
1097
|
-
*
|
|
1098
|
-
*
|
|
1892
|
+
* Builds the Why? tab's sentence (studio plan §4.4, issue #92) from the
|
|
1893
|
+
* facts `why.use-case.ts` gathers: the value the mapping read (and each
|
|
1894
|
+
* transform's effect on it), the step that bound its source id and whether
|
|
1895
|
+
* it ran, was skipped, or bound nothing this recipe knows of, the
|
|
1896
|
+
* missing-value policy that applied, and a hint about what was actually
|
|
1897
|
+
* there instead.
|
|
1099
1898
|
*
|
|
1100
|
-
*
|
|
1101
|
-
*
|
|
1102
|
-
*
|
|
1899
|
+
* This is the scope-snapshot-only half of the design: phase 2's per-item DOM
|
|
1900
|
+
* snapshot (the CSS-selector "the item has `.price_new`" hint the issue's
|
|
1901
|
+
* example shows) is not wired in here — `sample-run` keeps the mapping's
|
|
1902
|
+
* scope snapshot (bound ids and their values), not a per-`forEach`-item DOM
|
|
1903
|
+
* capture, and building that link is a bigger integration than this phase's
|
|
1904
|
+
* budget covers (see this package's `page-snapshot` slice, which captures a
|
|
1905
|
+
* step's page for the content pane, not per-iteration). `hint`, when given,
|
|
1906
|
+
* is instead built from the scope snapshot itself (`item-hint.algorithm.ts`).
|
|
1907
|
+
*
|
|
1908
|
+
* @param facts - What is known about the value.
|
|
1909
|
+
* @returns One sentence, always ending in a period.
|
|
1103
1910
|
*/
|
|
1104
|
-
function
|
|
1105
|
-
|
|
1106
|
-
let current = node;
|
|
1107
|
-
while (current !== undefined && current.length > 0 && isElement(current.get(0))) {
|
|
1108
|
-
chain.unshift(current);
|
|
1109
|
-
const parent = current.parent();
|
|
1110
|
-
current = parent.length > 0 ? parent : undefined;
|
|
1111
|
-
}
|
|
1112
|
-
return chain.map(element => levelOf($, element));
|
|
1113
|
-
}
|
|
1114
|
-
function levelOf($, element) {
|
|
1115
|
-
const node = element.get(0);
|
|
1116
|
-
const tag = node.tagName.toLowerCase();
|
|
1117
|
-
const attrs = {};
|
|
1118
|
-
const entries = Object.entries(node.attribs ?? {});
|
|
1119
|
-
for (const [name, value] of entries) attrs[name] = value;
|
|
1120
|
-
const classes = (attrs.class ?? '').split(/\s+/).filter(token => token.length > 0);
|
|
1121
|
-
const siblings = element.parent().length > 0 ? element.parent().children(tag) : $.root().children(tag);
|
|
1122
|
-
const index = siblings.toArray().indexOf(node) + 1;
|
|
1123
|
-
return {
|
|
1124
|
-
tag,
|
|
1125
|
-
id: attrs.id,
|
|
1126
|
-
classes,
|
|
1127
|
-
attrs,
|
|
1128
|
-
index: Math.max(index, 1),
|
|
1129
|
-
siblingCount: Math.max(siblings.length, 1)
|
|
1130
|
-
};
|
|
1911
|
+
function buildWhySentence(facts) {
|
|
1912
|
+
return facts.outcome === 'rejected' ? rejectedSentence(facts) : missingSentence(facts);
|
|
1131
1913
|
}
|
|
1132
|
-
function
|
|
1133
|
-
|
|
1914
|
+
function rejectedSentence(facts) {
|
|
1915
|
+
const parts = [`"${facts.field}" was rejected: ${facts.reason ?? 'the engine gave no reason'}`];
|
|
1916
|
+
if (facts.policy !== undefined) parts.push(` (policy: ${facts.policy})`);
|
|
1917
|
+
return `${parts.join('')}.`;
|
|
1918
|
+
}
|
|
1919
|
+
function missingSentence(facts) {
|
|
1920
|
+
const parts = [`"${facts.field}" is missing`, stepClause(facts)];
|
|
1921
|
+
if (facts.policy !== undefined) parts.push(`; left ${facts.policy === 'null' ? 'null' : facts.policy} by policy`);
|
|
1922
|
+
if (facts.hint !== undefined) parts.push(`; ${facts.hint}`);
|
|
1923
|
+
return `${parts.join('')}.`;
|
|
1924
|
+
}
|
|
1925
|
+
function stepClause(facts) {
|
|
1926
|
+
const boundId = facts.boundId ?? facts.field;
|
|
1927
|
+
const step = facts.step;
|
|
1928
|
+
if (step === undefined) return ': no step in this recipe binds its source';
|
|
1929
|
+
const label = step.stepId === undefined ? describeStepType(step.stepType) : `${describeStepType(step.stepType)} "${step.stepId}"`;
|
|
1930
|
+
if (step.outcome === 'not-found') return `: no step in this recipe binds "${boundId}"`;
|
|
1931
|
+
if (step.outcome === 'skipped') return `: ${label} (${step.path}), which binds "${boundId}", was skipped${step.error === undefined ? '' : ` (${step.error})`}`;
|
|
1932
|
+
if (step.outcome === 'retried') return `: ${label} (${step.path}), which binds "${boundId}", had to retry${step.error === undefined ? '' : ` (${step.error})`} and still found nothing for "${facts.field}"`;
|
|
1933
|
+
return `: ${label} (${step.path}) ran and bound "${boundId}", but "${facts.field}" reads ${sourceOf(facts)} from it, which was empty`;
|
|
1934
|
+
}
|
|
1935
|
+
function sourceOf(facts) {
|
|
1936
|
+
return facts.trace?.from === undefined ? 'a path' : JSON.stringify(facts.trace.from);
|
|
1937
|
+
}
|
|
1938
|
+
/** A step's type read as a short phrase, matching the Steps outline's own vocabulary (studio plan §4.1: "Go to…", "Read… from…"). */
|
|
1939
|
+
function describeStepType(stepType) {
|
|
1940
|
+
switch (stepType) {
|
|
1941
|
+
case 'extract':
|
|
1942
|
+
{
|
|
1943
|
+
return 'the Read step';
|
|
1944
|
+
}
|
|
1945
|
+
case 'request':
|
|
1946
|
+
{
|
|
1947
|
+
return 'the request step';
|
|
1948
|
+
}
|
|
1949
|
+
case 'forEach':
|
|
1950
|
+
{
|
|
1951
|
+
return 'the loop';
|
|
1952
|
+
}
|
|
1953
|
+
case 'paginate':
|
|
1954
|
+
{
|
|
1955
|
+
return 'the pagination';
|
|
1956
|
+
}
|
|
1957
|
+
case 'goto':
|
|
1958
|
+
{
|
|
1959
|
+
return 'the Go to step';
|
|
1960
|
+
}
|
|
1961
|
+
default:
|
|
1962
|
+
{
|
|
1963
|
+
return `the ${stepType} step`;
|
|
1964
|
+
}
|
|
1965
|
+
}
|
|
1966
|
+
}
|
|
1967
|
+
|
|
1968
|
+
/**
|
|
1969
|
+
* Explains one missing or rejected value from a recipe's last sample run
|
|
1970
|
+
* (issue #92's Why? tab, studio plan §4.4): the mapping trace and scope
|
|
1971
|
+
* snapshot `run-sample` kept, the recipe's own rule and field for the
|
|
1972
|
+
* missing-value policy, and the step that bound the value's source id — did
|
|
1973
|
+
* it run, was it skipped, or does no step in this recipe bind it at all.
|
|
1974
|
+
*
|
|
1975
|
+
* @param folder - The workspace folder (to re-read the recipe pair: the last
|
|
1976
|
+
* run only kept the run's own data, not the recipe text, and a save may
|
|
1977
|
+
* have happened since).
|
|
1978
|
+
* @param cache - The server's last-run cache (`sample-run`'s `LastRunCache`).
|
|
1979
|
+
* @param loadPair - Loads the input/output recipe pair; injected so this
|
|
1980
|
+
* stays testable without a real workspace folder (`sample-run`'s
|
|
1981
|
+
* `loadRecipePair` in production).
|
|
1982
|
+
* @param target - What was clicked.
|
|
1983
|
+
* @returns The explanation.
|
|
1984
|
+
* @throws Error when no sample run has finished for this recipe yet, or the target does not name a real record/field.
|
|
1985
|
+
*/
|
|
1986
|
+
async function explainWhy(folder, cache, loadPair, target) {
|
|
1987
|
+
const entry = lastRunOf(cache, target.recipeId);
|
|
1988
|
+
if (entry === undefined) throw new Error(`no finished sample run for "${target.recipeId}" yet — run a sample first`);
|
|
1989
|
+
const {
|
|
1990
|
+
input,
|
|
1991
|
+
output
|
|
1992
|
+
} = await loadPair(folder, target.recipeId);
|
|
1993
|
+
return target.kind === 'rejected' ? explainRejected(target.recipeId, input, output, entry.result.rejectedRecords, entry.result.steps, target.rejectedIndex) : explainMissing(target.recipeId, input, output, entry.result.records, entry.result.steps, target.recordIndex, target.field);
|
|
1994
|
+
}
|
|
1995
|
+
function explainRejected(recipeId, input, output, rejectedRecords, steps, index) {
|
|
1996
|
+
const rejected = rejectedRecords[index];
|
|
1997
|
+
if (rejected === undefined) throw new Error(`no rejected record at index ${index}`);
|
|
1998
|
+
const rule = ruleFor(input.mapping, rejected.field);
|
|
1999
|
+
const field = fieldFor(output.fields, rejected.field);
|
|
2000
|
+
const policy = field === undefined ? undefined : resolveMissingPolicy(field, output, rule);
|
|
2001
|
+
const boundId = boundIdOf(rule);
|
|
2002
|
+
const step = boundId === undefined ? undefined : findStep(input.steps, boundId, steps);
|
|
2003
|
+
const hint = hintFor(rejected.scope, boundId);
|
|
2004
|
+
const sentence = buildWhySentence({
|
|
2005
|
+
field: rejected.field,
|
|
2006
|
+
outcome: 'rejected',
|
|
2007
|
+
reason: rejected.reason,
|
|
2008
|
+
policy,
|
|
2009
|
+
boundId,
|
|
2010
|
+
step,
|
|
2011
|
+
hint
|
|
2012
|
+
});
|
|
2013
|
+
return {
|
|
2014
|
+
sentence,
|
|
2015
|
+
field: rejected.field,
|
|
2016
|
+
recipeId,
|
|
2017
|
+
outcome: 'rejected',
|
|
2018
|
+
reason: rejected.reason,
|
|
2019
|
+
policy,
|
|
2020
|
+
stepPath: step?.path
|
|
2021
|
+
};
|
|
2022
|
+
}
|
|
2023
|
+
function explainMissing(recipeId, input, output, records, steps, index, field) {
|
|
2024
|
+
const record = records[index];
|
|
2025
|
+
if (record === undefined) throw new Error(`no record at index ${index}`);
|
|
2026
|
+
const trace = record.mapping?.[field];
|
|
2027
|
+
const rule = ruleFor(input.mapping, field);
|
|
2028
|
+
const fieldSpec = fieldFor(output.fields, field);
|
|
2029
|
+
const policy = fieldSpec === undefined ? undefined : resolveMissingPolicy(fieldSpec, output, rule);
|
|
2030
|
+
const boundId = boundIdOf(rule);
|
|
2031
|
+
const step = boundId === undefined ? undefined : findStep(input.steps, boundId, steps);
|
|
2032
|
+
const hint = hintFor(record.scope, boundId);
|
|
2033
|
+
const sentence = buildWhySentence({
|
|
2034
|
+
field,
|
|
2035
|
+
outcome: 'missing',
|
|
2036
|
+
policy,
|
|
2037
|
+
trace,
|
|
2038
|
+
boundId,
|
|
2039
|
+
step,
|
|
2040
|
+
hint
|
|
2041
|
+
});
|
|
2042
|
+
return {
|
|
2043
|
+
sentence,
|
|
2044
|
+
field,
|
|
2045
|
+
recipeId,
|
|
2046
|
+
outcome: 'missing',
|
|
2047
|
+
policy,
|
|
2048
|
+
stepPath: step?.path
|
|
2049
|
+
};
|
|
2050
|
+
}
|
|
2051
|
+
/**
|
|
2052
|
+
* The rule bound to one output field path: a direct match on `input.mapping`
|
|
2053
|
+
* (its keys are themselves dotted target paths — `stock.inStock`, `tax` —
|
|
2054
|
+
* `map-record.use-case.ts` writes each with `setPath`), or, for a path
|
|
2055
|
+
* inside an `each` list's items (`variants[1].size`), the item's own rule
|
|
2056
|
+
* inside the list rule's `fields`. Only one level of `each` nesting is
|
|
2057
|
+
* resolved; a field inside a nested `each` (an `each` of `each`) is not —
|
|
2058
|
+
* out of this phase's scope.
|
|
2059
|
+
*/
|
|
2060
|
+
function ruleFor(mapping, path) {
|
|
2061
|
+
const direct = mapping[path];
|
|
2062
|
+
if (direct !== undefined) return direct;
|
|
2063
|
+
const match = /^([^[]+)\[\d+\]\.(.+)$/.exec(path);
|
|
2064
|
+
if (match === null) return undefined;
|
|
2065
|
+
const parent = mapping[match[1]];
|
|
2066
|
+
return parent !== undefined && 'each' in parent ? parent.fields[match[2]] : undefined;
|
|
2067
|
+
}
|
|
2068
|
+
/** The output field spec at one dotted path (`stock.inStock`, `variants[1].size`), walking `object`/`array` members the same way `map-record.use-case.ts` builds them. */
|
|
2069
|
+
function fieldFor(fields, path) {
|
|
2070
|
+
let current = fields;
|
|
2071
|
+
let field;
|
|
2072
|
+
for (const token of path.split('.')) {
|
|
2073
|
+
const match = /^([^[]+)(\[\d+\])?$/.exec(token);
|
|
2074
|
+
if (match === null || current === undefined) return undefined;
|
|
2075
|
+
field = current[match[1]];
|
|
2076
|
+
if (field === undefined) return undefined;
|
|
2077
|
+
current = match[2] === undefined ? field.fields : field.items?.fields;
|
|
2078
|
+
}
|
|
2079
|
+
return field;
|
|
2080
|
+
}
|
|
2081
|
+
/** The leading id a `from` rule reads (`item` of `item.description`, `price` of `price`); `undefined` for an `each` rule, a `.` (self), or a rule this run found nothing to say about. */
|
|
2082
|
+
function boundIdOf(rule) {
|
|
2083
|
+
if (rule === undefined || !('from' in rule)) return undefined;
|
|
2084
|
+
const first = Array.isArray(rule.from) ? rule.from[0] : rule.from;
|
|
2085
|
+
const match = /^([A-Z_][\w-]*)/i.exec(first);
|
|
2086
|
+
return match?.[1];
|
|
2087
|
+
}
|
|
2088
|
+
/**
|
|
2089
|
+
* Finds the step of the recipe whose `id` or `as` (a `forEach`'s loop
|
|
2090
|
+
* variable) is `boundId`, walking into `forEach`/`paginate`/`if` bodies the
|
|
2091
|
+
* same way `scope-outline`'s outline does, and reports how it went over the
|
|
2092
|
+
* whole sample run from `stepSummaries` (`sample-run`'s per-path summary).
|
|
2093
|
+
*
|
|
2094
|
+
* @returns `undefined` only when nothing in `steps` looks like a step at all; a `boundId` that names no real step still gets a fact, with `outcome: 'not-found'`.
|
|
2095
|
+
*/
|
|
2096
|
+
function findStep(steps, boundId, stepSummaries) {
|
|
2097
|
+
const found = search(steps, boundId, 'steps');
|
|
2098
|
+
if (found === undefined) return {
|
|
2099
|
+
path: '',
|
|
2100
|
+
stepType: '',
|
|
2101
|
+
outcome: 'not-found'
|
|
2102
|
+
};
|
|
2103
|
+
const summary = stepSummaries.find(entry => entry.path === found.path);
|
|
2104
|
+
return {
|
|
2105
|
+
path: found.path,
|
|
2106
|
+
stepType: found.step.type ?? 'unknown',
|
|
2107
|
+
stepId: found.step.id,
|
|
2108
|
+
outcome: summary?.outcome ?? 'ran',
|
|
2109
|
+
error: summary?.error
|
|
2110
|
+
};
|
|
2111
|
+
}
|
|
2112
|
+
function search(steps, boundId, prefix) {
|
|
2113
|
+
for (const [index, step] of steps.entries()) {
|
|
2114
|
+
const path = `${prefix}.${index}`;
|
|
2115
|
+
if (step.id === boundId || step.as === boundId) return {
|
|
2116
|
+
path,
|
|
2117
|
+
step
|
|
2118
|
+
};
|
|
2119
|
+
if (step.steps !== undefined) {
|
|
2120
|
+
const nested = search(step.steps, boundId, `${path}.steps`);
|
|
2121
|
+
if (nested !== undefined) return nested;
|
|
2122
|
+
}
|
|
2123
|
+
if (step.else !== undefined) {
|
|
2124
|
+
const nested = search(step.else, boundId, `${path}.else`);
|
|
2125
|
+
if (nested !== undefined) return nested;
|
|
2126
|
+
}
|
|
2127
|
+
}
|
|
2128
|
+
return undefined;
|
|
2129
|
+
}
|
|
2130
|
+
|
|
2131
|
+
/**
|
|
2132
|
+
* Handles `explain-why` (issue #92's Why? tab): builds the sentence for a
|
|
2133
|
+
* missing or rejected value from the recipe's last sample run, kept in
|
|
2134
|
+
* `state.lastRuns` by `handleRunSample`.
|
|
2135
|
+
*
|
|
2136
|
+
* @param state - The server state: `folder` must already be set (`open-workspace` first).
|
|
2137
|
+
* @param command - The command.
|
|
2138
|
+
* @returns The explanation.
|
|
2139
|
+
* @throws Error when no workspace is open, no sample run has finished for the target's recipe, or the target does not name a real record/field.
|
|
2140
|
+
*/
|
|
2141
|
+
async function handleExplainWhy(state, command) {
|
|
2142
|
+
if (state.folder === undefined) throw new Error('open a workspace first');
|
|
2143
|
+
return explainWhy(state.folder, state.lastRuns, loadRecipePair, command.target);
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
/**
|
|
2147
|
+
* Handles `fetch-start-page`: loads the named input recipe from the current
|
|
2148
|
+
* workspace and fetches its start point (see `page-snapshot`'s own caveats:
|
|
2149
|
+
* no session bootstrap, a fresh browser or HTTP context per call).
|
|
2150
|
+
*
|
|
2151
|
+
* @param state - The server state: `folder` must already be set.
|
|
2152
|
+
* @param command - The command.
|
|
2153
|
+
* @returns The page's content as plain text.
|
|
2154
|
+
* @throws Error when no workspace is open, or the recipe cannot be found or loaded.
|
|
2155
|
+
*/
|
|
2156
|
+
async function handleFetchStartPage(state, command) {
|
|
2157
|
+
if (state.folder === undefined) throw new Error('open a workspace first');
|
|
2158
|
+
const {
|
|
2159
|
+
input
|
|
2160
|
+
} = await loadRecipePair(state.folder, command.recipeId);
|
|
2161
|
+
const html = await fetchStartPage(input, state.browser);
|
|
2162
|
+
return {
|
|
2163
|
+
html
|
|
2164
|
+
};
|
|
2165
|
+
}
|
|
2166
|
+
|
|
2167
|
+
/**
|
|
2168
|
+
* Handles `grid-preview`: the grid canvas's live preview of a `table`
|
|
2169
|
+
* extract's options (`document-view`'s `table-preview.use-case.ts`'s
|
|
2170
|
+
* `previewGridTable`, studio plan §3.4, issue #94's 5c), off the same
|
|
2171
|
+
* cached snapshot `take-snapshot`/`grid-view` already captured — a pure
|
|
2172
|
+
* computation over it, so this answers instantly on every option change, no
|
|
2173
|
+
* re-fetch or re-parse.
|
|
2174
|
+
*
|
|
2175
|
+
* @param state - The server state.
|
|
2176
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
2177
|
+
* @returns The matched tables, or `error` when an option's pattern does not compile.
|
|
2178
|
+
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
2179
|
+
*/
|
|
2180
|
+
function handleGridPreview(state, command) {
|
|
2181
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
2182
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
2183
|
+
if (!isWorkbookDocument(snapshot.body)) throw new Error(`"${command.recipeId}" at "${command.path}" is not a workbook snapshot: grid-preview only reads a read CSV/spreadsheet document`);
|
|
2184
|
+
return previewGridTable(snapshot.body, command.options);
|
|
2185
|
+
}
|
|
2186
|
+
|
|
2187
|
+
/**
|
|
2188
|
+
* Handles `grid-view`: the grid canvas's sheets and cells (`document-view`'s
|
|
2189
|
+
* `workbook-view.mapper.ts`, studio plan §3.4, issue #94's 5c), off the same
|
|
2190
|
+
* cached snapshot `take-snapshot`/`document-tree`/`pdf-view` already use.
|
|
2191
|
+
*
|
|
2192
|
+
* `delimiter`/`encoding` re-read a CSV with the override applied
|
|
2193
|
+
* (`page-snapshot`'s `rereadCsv`) and replace the cached snapshot's body with
|
|
2194
|
+
* the fresh reading, so a following `grid-preview` (and another `grid-view`
|
|
2195
|
+
* with no override) sees the same one — the override sticks until
|
|
2196
|
+
* `take-snapshot` recaptures.
|
|
2197
|
+
*
|
|
2198
|
+
* @param state - The server state.
|
|
2199
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
2200
|
+
* @returns The document's view.
|
|
2201
|
+
* @throws Error when no workspace is open, `command.path` has no cached snapshot yet, an override is given for a snapshot that is not a CSV, or the snapshot's document is not a read workbook.
|
|
2202
|
+
*/
|
|
2203
|
+
async function handleGridView(state, command) {
|
|
2204
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
2205
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
2206
|
+
const overriding = command.delimiter !== undefined || command.encoding !== undefined;
|
|
2207
|
+
if (!overriding) {
|
|
2208
|
+
if (!isWorkbookDocument(snapshot.body)) throw new Error(`"${command.recipeId}" at "${command.path}" is not a workbook snapshot: the grid canvas only reads a read CSV/spreadsheet document`);
|
|
2209
|
+
return workbookDocumentView(snapshot.body);
|
|
2210
|
+
}
|
|
2211
|
+
if (snapshot.format !== 'csv') throw new Error(`"${command.recipeId}" at "${command.path}" is not a CSV snapshot: "delimiter"/"encoding" only override a CSV reading`);
|
|
2212
|
+
const body = await rereadCsv(snapshot.baseUrl, {
|
|
2213
|
+
delimiter: command.delimiter,
|
|
2214
|
+
encoding: command.encoding
|
|
2215
|
+
});
|
|
2216
|
+
putSnapshot(state.snapshots, command.recipeId, command.path, {
|
|
2217
|
+
...snapshot,
|
|
2218
|
+
body
|
|
2219
|
+
});
|
|
2220
|
+
return workbookDocumentView(body);
|
|
2221
|
+
}
|
|
2222
|
+
|
|
2223
|
+
const GENERATED_ID = /^(?:react-|radix-|:r|ember|css-|mui-|headlessui-)/i;
|
|
2224
|
+
/**
|
|
2225
|
+
* The path from the document root to `node`, one level per ancestor. Two
|
|
2226
|
+
* elements that share an ancestor produce identical levels up to that
|
|
2227
|
+
* ancestor and diverge exactly where their parents stop being the same node
|
|
2228
|
+
* — `list-inference.algorithm.ts` leans on that to find the nearest pair of
|
|
2229
|
+
* siblings two picks belong to, without ever touching cheerio again.
|
|
2230
|
+
*
|
|
2231
|
+
* @param $ - The parsed document.
|
|
2232
|
+
* @param node - The element to describe.
|
|
2233
|
+
* @returns The path, root first.
|
|
2234
|
+
*/
|
|
2235
|
+
function domPathOf($, node) {
|
|
2236
|
+
const chain = [];
|
|
2237
|
+
let current = node;
|
|
2238
|
+
while (current !== undefined && current.length > 0 && isElement(current.get(0))) {
|
|
2239
|
+
chain.unshift(current);
|
|
2240
|
+
const parent = current.parent();
|
|
2241
|
+
current = parent.length > 0 ? parent : undefined;
|
|
2242
|
+
}
|
|
2243
|
+
return chain.map(element => levelOf($, element));
|
|
2244
|
+
}
|
|
2245
|
+
function levelOf($, element) {
|
|
2246
|
+
const node = element.get(0);
|
|
2247
|
+
const tag = node.tagName.toLowerCase();
|
|
2248
|
+
const attrs = {};
|
|
2249
|
+
const entries = Object.entries(node.attribs ?? {});
|
|
2250
|
+
for (const [name, value] of entries) attrs[name] = value;
|
|
2251
|
+
const classes = (attrs.class ?? '').split(/\s+/).filter(token => token.length > 0);
|
|
2252
|
+
const siblings = element.parent().length > 0 ? element.parent().children(tag) : $.root().children(tag);
|
|
2253
|
+
const index = siblings.toArray().indexOf(node) + 1;
|
|
2254
|
+
return {
|
|
2255
|
+
tag,
|
|
2256
|
+
id: attrs.id,
|
|
2257
|
+
classes,
|
|
2258
|
+
attrs,
|
|
2259
|
+
index: Math.max(index, 1),
|
|
2260
|
+
siblingCount: Math.max(siblings.length, 1)
|
|
2261
|
+
};
|
|
2262
|
+
}
|
|
2263
|
+
function isElement(node) {
|
|
2264
|
+
return node !== undefined && node.type === 'tag';
|
|
1134
2265
|
}
|
|
1135
2266
|
/**
|
|
1136
2267
|
* Whether `id` looks handed by a framework (`react-`, a CSS-module or a
|
|
@@ -1402,7 +2533,7 @@ const TIER_ORDER = {
|
|
|
1402
2533
|
function rankCandidates(html, candidates, targetNodeId) {
|
|
1403
2534
|
const ranked = candidates.map(candidate => {
|
|
1404
2535
|
const matches = countMatches(html, candidate.selector);
|
|
1405
|
-
const hitsTarget = targetNodeId === undefined ;
|
|
2536
|
+
const hitsTarget = targetNodeId === undefined || matches === 0 ? targetNodeId === undefined : matchesOf(html, candidate.selector).includes(targetNodeId);
|
|
1406
2537
|
return {
|
|
1407
2538
|
...candidate,
|
|
1408
2539
|
matches,
|
|
@@ -1581,6 +2712,333 @@ function fieldPick(html, clicked, rankSource = clicked) {
|
|
|
1581
2712
|
};
|
|
1582
2713
|
}
|
|
1583
2714
|
|
|
2715
|
+
/** A leaf element's own text longer than this is cut, with an ellipsis. */
|
|
2716
|
+
const TEXT_LIMIT = 80;
|
|
2717
|
+
/** Consecutive siblings whose subtree is identical (ids aside) collapse into one row once there are at least this many — a real page can have tens of thousands of `<li>`s, and showing all of them is not "inspecting" it (issue #93). */
|
|
2718
|
+
const MIN_REPEAT_RUN = 3;
|
|
2719
|
+
/**
|
|
2720
|
+
* Builds the Inspect panel's DOM tree from a captured snapshot (studio plan
|
|
2721
|
+
* §3.3, issue #93): the same rewritten, marked-up document the content pane's
|
|
2722
|
+
* iframe shows (`page-snapshot`'s `take-snapshot.use-case.ts`), so a tree
|
|
2723
|
+
* row's `nodeId` addresses exactly the element the iframe would highlight —
|
|
2724
|
+
* no separate capture, per issue #93's scope note to reuse phase 2's
|
|
2725
|
+
* snapshot rather than recapture.
|
|
2726
|
+
*
|
|
2727
|
+
* @param snapshotHtml - The cached, rewritten snapshot (`SnapshotResult.html`).
|
|
2728
|
+
* @returns The tree's root (the document's `<html>` element).
|
|
2729
|
+
* @throws Error when the document has no `<html>` element (should not happen: `rewriteDocument` always produces one).
|
|
2730
|
+
*/
|
|
2731
|
+
function domTree(snapshotHtml) {
|
|
2732
|
+
const $ = load(snapshotHtml);
|
|
2733
|
+
const html = $('html').get(0);
|
|
2734
|
+
if (html === undefined) throw new Error('domTree: the snapshot has no <html> element');
|
|
2735
|
+
return buildNode($, html);
|
|
2736
|
+
}
|
|
2737
|
+
function buildNode($, element) {
|
|
2738
|
+
const $el = $(element);
|
|
2739
|
+
const attributes = {
|
|
2740
|
+
...element.attribs
|
|
2741
|
+
};
|
|
2742
|
+
const nodeId = attributes[NODE_ID_ATTRIBUTE] ?? '';
|
|
2743
|
+
const hidden = attributes[HIDDEN_ATTRIBUTE] === '1';
|
|
2744
|
+
delete attributes[NODE_ID_ATTRIBUTE];
|
|
2745
|
+
delete attributes[HIDDEN_ATTRIBUTE];
|
|
2746
|
+
const id = attributes.id;
|
|
2747
|
+
const classes = (attributes.class ?? '').split(/\s+/).filter(token => token !== '');
|
|
2748
|
+
const childElements = $el.children().toArray();
|
|
2749
|
+
const children = collapseRepeats(childElements.map(child => buildNode($, child)));
|
|
2750
|
+
const text = childElements.length === 0 ? abbreviate($el.text()) : undefined;
|
|
2751
|
+
return {
|
|
2752
|
+
nodeId,
|
|
2753
|
+
tag: element.tagName.toLowerCase(),
|
|
2754
|
+
...(id !== undefined && id !== '' && {
|
|
2755
|
+
id
|
|
2756
|
+
}),
|
|
2757
|
+
...(classes.length > 0 && {
|
|
2758
|
+
classes
|
|
2759
|
+
}),
|
|
2760
|
+
attributes,
|
|
2761
|
+
...(text !== undefined && text !== '' && {
|
|
2762
|
+
text
|
|
2763
|
+
}),
|
|
2764
|
+
hidden,
|
|
2765
|
+
children
|
|
2766
|
+
};
|
|
2767
|
+
}
|
|
2768
|
+
/**
|
|
2769
|
+
* Merges runs of `MIN_REPEAT_RUN` or more consecutive siblings whose subtree
|
|
2770
|
+
* is identical (their own `nodeId`s aside) into one representative row, the
|
|
2771
|
+
* issue's `×20` — real pages have tens of thousands of nodes, and this is
|
|
2772
|
+
* what makes the tree usable rather than merely correct.
|
|
2773
|
+
*
|
|
2774
|
+
* @param nodes - Already-built sibling rows, in document order.
|
|
2775
|
+
* @returns The same rows, with each long-enough identical run replaced by one.
|
|
2776
|
+
*/
|
|
2777
|
+
function collapseRepeats(nodes) {
|
|
2778
|
+
const result = [];
|
|
2779
|
+
let at = 0;
|
|
2780
|
+
while (at < nodes.length) {
|
|
2781
|
+
const signature = signatureOf(nodes[at]);
|
|
2782
|
+
let end = at + 1;
|
|
2783
|
+
while (end < nodes.length && signatureOf(nodes[end]) === signature) end += 1;
|
|
2784
|
+
const run = nodes.slice(at, end);
|
|
2785
|
+
if (run.length >= MIN_REPEAT_RUN) {
|
|
2786
|
+
result.push({
|
|
2787
|
+
...run[0],
|
|
2788
|
+
repeatCount: run.length,
|
|
2789
|
+
repeatNodeIds: run.map(node => node.nodeId)
|
|
2790
|
+
});
|
|
2791
|
+
} else {
|
|
2792
|
+
result.push(...run);
|
|
2793
|
+
}
|
|
2794
|
+
at = end;
|
|
2795
|
+
}
|
|
2796
|
+
return result;
|
|
2797
|
+
}
|
|
2798
|
+
/** A run member's identity for collapsing: everything but its own `nodeId` (every instance has a different one by construction). */
|
|
2799
|
+
function signatureOf(node) {
|
|
2800
|
+
return JSON.stringify({
|
|
2801
|
+
...node,
|
|
2802
|
+
nodeId: undefined,
|
|
2803
|
+
children: node.children.map(child => signatureOf(child))
|
|
2804
|
+
});
|
|
2805
|
+
}
|
|
2806
|
+
function abbreviate(text) {
|
|
2807
|
+
const trimmed = text.trim().replaceAll(/\s+/g, ' ');
|
|
2808
|
+
return trimmed.length > TEXT_LIMIT ? `${trimmed.slice(0, TEXT_LIMIT)}…` : trimmed;
|
|
2809
|
+
}
|
|
2810
|
+
|
|
2811
|
+
/** The global names sites hydrate state into, that `describeHtml`'s own JSON-LD/table scan does not cover (issue #93's "inline state assignments"). */
|
|
2812
|
+
const INLINE_STATE_NAMES = ['__NEXT_DATA__', '__NUXT__', '__INITIAL_STATE__', '__APOLLO_STATE__', '__PRELOADED_STATE__', '__STATE__'];
|
|
2813
|
+
const INLINE_STATE_PATTERN = new RegExp(String.raw`(?:window\.)?(${INLINE_STATE_NAMES.join('|')})\s*=`);
|
|
2814
|
+
/**
|
|
2815
|
+
* Finds the data-in-the-page findings the Inspect panel lists apart from the
|
|
2816
|
+
* DOM tree (studio plan §3.3, issue #93): JSON-LD and `application/json`
|
|
2817
|
+
* script blocks, inline state assignments, `<meta>` name/property values and
|
|
2818
|
+
* `<link rel>`s — reusing `@opencraw/probe`'s `describeJson` for a found
|
|
2819
|
+
* value's shape (its own `find-data.algorithm.ts`/`html-findings.mapper.ts`
|
|
2820
|
+
* scan a page's raw text for *reporting*, with no element to point a pick at;
|
|
2821
|
+
* this function's own job — locating each finding's element so a pick can
|
|
2822
|
+
* reach it — is new).
|
|
2823
|
+
*
|
|
2824
|
+
* Takes the **raw** captured document, not the display snapshot
|
|
2825
|
+
* (`page-snapshot`'s `SnapshotResult.html`): `rewriteDocument` strips every
|
|
2826
|
+
* `<script>` for the sandboxed iframe, which would make JSON-LD, inline
|
|
2827
|
+
* state and other script-tag data invisible here. `<meta>`/`<link>` survive
|
|
2828
|
+
* rewriting untouched, so using the raw document for every kind (rather than
|
|
2829
|
+
* mixing two documents) keeps this function simple and its selectors
|
|
2830
|
+
* verified against what the engine will actually crawl.
|
|
2831
|
+
*
|
|
2832
|
+
* @param rawHtml - The document as fetched/rendered, before `rewriteDocument` strips scripts.
|
|
2833
|
+
* @returns The findings, in document order.
|
|
2834
|
+
*/
|
|
2835
|
+
function pageData(rawHtml) {
|
|
2836
|
+
const $ = load(rawHtml);
|
|
2837
|
+
const findings = [];
|
|
2838
|
+
$('script[type="application/ld+json"]').each((_index, element) => {
|
|
2839
|
+
findings.push(...scriptFinding($, element, rawHtml, 'ld-json', ldJsonLabel));
|
|
2840
|
+
});
|
|
2841
|
+
$('script[type="application/json"]').each((_index, element) => {
|
|
2842
|
+
findings.push(...scriptFinding($, element, rawHtml, 'json-script', () => `json: ${structuralSelector($, element)}`));
|
|
2843
|
+
});
|
|
2844
|
+
$('script').each((_index, element) => {
|
|
2845
|
+
const type = $(element).attr('type');
|
|
2846
|
+
if (type !== undefined && type !== 'text/javascript' && type !== 'module') return;
|
|
2847
|
+
const text = $(element).text();
|
|
2848
|
+
const match = INLINE_STATE_PATTERN.exec(text);
|
|
2849
|
+
if (match === null) return;
|
|
2850
|
+
const value = tryParseJson(text);
|
|
2851
|
+
const selector = structuralSelector($, element);
|
|
2852
|
+
findings.push({
|
|
2853
|
+
kind: 'inline-state',
|
|
2854
|
+
label: match[1],
|
|
2855
|
+
selector,
|
|
2856
|
+
selectorKind: 'css',
|
|
2857
|
+
matches: countMatches(rawHtml, selector),
|
|
2858
|
+
keys: keysOf(value)
|
|
2859
|
+
});
|
|
2860
|
+
});
|
|
2861
|
+
$('meta[name], meta[property]').each((_index, element) => {
|
|
2862
|
+
const name = $(element).attr('name') ?? $(element).attr('property') ?? '';
|
|
2863
|
+
const selector = structuralSelector($, element);
|
|
2864
|
+
findings.push({
|
|
2865
|
+
kind: 'meta',
|
|
2866
|
+
label: `meta: ${name}`,
|
|
2867
|
+
selector,
|
|
2868
|
+
selectorKind: 'css',
|
|
2869
|
+
matches: countMatches(rawHtml, selector),
|
|
2870
|
+
attribute: 'content'
|
|
2871
|
+
});
|
|
2872
|
+
});
|
|
2873
|
+
$('link[rel]').each((_index, element) => {
|
|
2874
|
+
const rel = $(element).attr('rel') ?? '';
|
|
2875
|
+
const selector = structuralSelector($, element);
|
|
2876
|
+
findings.push({
|
|
2877
|
+
kind: 'link',
|
|
2878
|
+
label: `link: ${rel}`,
|
|
2879
|
+
selector,
|
|
2880
|
+
selectorKind: 'css',
|
|
2881
|
+
matches: countMatches(rawHtml, selector),
|
|
2882
|
+
attribute: 'href'
|
|
2883
|
+
});
|
|
2884
|
+
});
|
|
2885
|
+
return findings;
|
|
2886
|
+
}
|
|
2887
|
+
function scriptFinding($, element, rawHtml, kind, labelOf) {
|
|
2888
|
+
const value = tryParseJson($(element).text());
|
|
2889
|
+
if (value === undefined) return [];
|
|
2890
|
+
const selector = structuralSelector($, element);
|
|
2891
|
+
return [{
|
|
2892
|
+
kind,
|
|
2893
|
+
label: labelOf(value),
|
|
2894
|
+
selector,
|
|
2895
|
+
selectorKind: 'css',
|
|
2896
|
+
matches: countMatches(rawHtml, selector),
|
|
2897
|
+
keys: keysOf(value)
|
|
2898
|
+
}];
|
|
2899
|
+
}
|
|
2900
|
+
function ldJsonLabel(value) {
|
|
2901
|
+
if (Array.isArray(value)) return `ld+json: array (${value.length})`;
|
|
2902
|
+
if (typeof value === 'object' && value !== null) {
|
|
2903
|
+
const type = value['@type'];
|
|
2904
|
+
if (typeof type === 'string') return `ld+json: ${type}`;
|
|
2905
|
+
}
|
|
2906
|
+
return `ld+json: ${describeJson(value, 'json').type}`;
|
|
2907
|
+
}
|
|
2908
|
+
function keysOf(value) {
|
|
2909
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value) ? Object.keys(value) : undefined;
|
|
2910
|
+
}
|
|
2911
|
+
/**
|
|
2912
|
+
* A unique CSS path for an element with no stable attribute of its own
|
|
2913
|
+
* (a `<script>` tag has no id or class to speak of): `<ancestor> > tag`
|
|
2914
|
+
* from `<html>` down, `:nth-of-type` added only where a tag has siblings of
|
|
2915
|
+
* the same name under the same parent. Always resolves to exactly one
|
|
2916
|
+
* element, on the raw document this ran against and on the real page (a
|
|
2917
|
+
* purely structural selector, unaffected by the snapshot's own rewriting).
|
|
2918
|
+
*
|
|
2919
|
+
* @param $ - The document.
|
|
2920
|
+
* @param element - The element to address.
|
|
2921
|
+
* @returns The selector.
|
|
2922
|
+
*/
|
|
2923
|
+
function structuralSelector($, element) {
|
|
2924
|
+
const segments = [];
|
|
2925
|
+
let $current = $(element);
|
|
2926
|
+
while ($current.length > 0) {
|
|
2927
|
+
const tag = $current.prop('tagName')?.toLowerCase();
|
|
2928
|
+
if (tag === undefined) break;
|
|
2929
|
+
const $parent = $current.parent();
|
|
2930
|
+
const siblingsOfType = $parent.length > 0 ? $parent.children(tag) : $current;
|
|
2931
|
+
const index = siblingsOfType.toArray().indexOf($current.get(0)) + 1;
|
|
2932
|
+
segments.unshift(siblingsOfType.length > 1 ? `${tag}:nth-of-type(${index})` : tag);
|
|
2933
|
+
if (tag === 'html' || $parent.length === 0) break;
|
|
2934
|
+
$current = $parent;
|
|
2935
|
+
}
|
|
2936
|
+
return segments.join(' > ');
|
|
2937
|
+
}
|
|
2938
|
+
|
|
2939
|
+
/** How long the browser is given to settle after `goto`, watching for JSON responses (mirrors `packages/cli`'s `probe --browser`, `observeBrowserJson`). */
|
|
2940
|
+
const OBSERVE_MS = 4000;
|
|
2941
|
+
/** Responses kept, largest/most-recent-first ordering left to the caller; just a sane cap on what the panel lists. */
|
|
2942
|
+
const RESPONSE_LIMIT = 15;
|
|
2943
|
+
/**
|
|
2944
|
+
* Records the JSON responses seen while an input recipe's start page
|
|
2945
|
+
* rendered (studio plan §3.3, issue #93): the same technique `packages/cli`'s
|
|
2946
|
+
* `probe --browser` uses (`observeBrowserJson`) — render the page, listen for
|
|
2947
|
+
* `response` events whose content type is JSON, give it a few seconds to
|
|
2948
|
+
* settle. Kept as its own capture, independent of `page-snapshot`'s
|
|
2949
|
+
* `take-snapshot.use-case.ts`: that capture reads `page.content()` right
|
|
2950
|
+
* after `goto` resolves, with no observation window, and changing its shape
|
|
2951
|
+
* to also collect responses was a larger, riskier change to already-shipped,
|
|
2952
|
+
* tested code than this phase's budget allowed (see this phase's final
|
|
2953
|
+
* report) — so this opens its own short-lived browser session instead of
|
|
2954
|
+
* piggy-backing on the cached snapshot.
|
|
2955
|
+
*
|
|
2956
|
+
* Only `"start"` is supported, mirroring `take-snapshot.use-case.ts`'s own
|
|
2957
|
+
* reach; api-mode recipes have no page load to observe, so they always
|
|
2958
|
+
* answer with an empty list rather than an error (there is nothing wrong
|
|
2959
|
+
* with asking, the answer is just "none").
|
|
2960
|
+
*
|
|
2961
|
+
* @param input - The input recipe; only `mode` and the first `start` point are used.
|
|
2962
|
+
* @param stepPath - The step to observe after, e.g. `"start"`.
|
|
2963
|
+
* @param browser - Browser launch settings, shared with `take-snapshot`/`verify-selector`/`run-sample`.
|
|
2964
|
+
* @returns The distinct JSON responses observed, at most `RESPONSE_LIMIT`.
|
|
2965
|
+
* @throws Error when the recipe has no start point, or `stepPath` is not `"start"`.
|
|
2966
|
+
*/
|
|
2967
|
+
async function responsesSeen(input, stepPath, browser) {
|
|
2968
|
+
if (stepPath !== 'start') throw new Error(`responsesSeen: "${stepPath}" is not supported yet — only "start" (the first start point, before any step ran) is`);
|
|
2969
|
+
if (input.mode !== 'web') return [];
|
|
2970
|
+
const point = input.start[0];
|
|
2971
|
+
if (point === undefined) throw new Error(`recipe "${input.id}" has no start point`);
|
|
2972
|
+
const client = await BrowserClient.launch(browser);
|
|
2973
|
+
try {
|
|
2974
|
+
const session = await client.newSession();
|
|
2975
|
+
try {
|
|
2976
|
+
const responses = [];
|
|
2977
|
+
const seen = new Set();
|
|
2978
|
+
const pending = [];
|
|
2979
|
+
session.page.on('response', response => {
|
|
2980
|
+
if (!(response.headers()['content-type'] ?? '').includes('json')) return;
|
|
2981
|
+
const key = `${response.status()} ${response.url()}`;
|
|
2982
|
+
if (seen.has(key)) return;
|
|
2983
|
+
seen.add(key);
|
|
2984
|
+
pending.push(recordResponse(response, responses));
|
|
2985
|
+
});
|
|
2986
|
+
try {
|
|
2987
|
+
await session.page.goto(point.url, {
|
|
2988
|
+
waitUntil: 'networkidle'
|
|
2989
|
+
});
|
|
2990
|
+
} catch {
|
|
2991
|
+
// a page that never idles (long-poll, websocket) still yields whatever was observed by the timeout below
|
|
2992
|
+
}
|
|
2993
|
+
await session.page.waitForTimeout(OBSERVE_MS);
|
|
2994
|
+
await Promise.all(pending);
|
|
2995
|
+
return responses.slice(0, RESPONSE_LIMIT);
|
|
2996
|
+
} finally {
|
|
2997
|
+
await session.close();
|
|
2998
|
+
}
|
|
2999
|
+
} finally {
|
|
3000
|
+
await client.close();
|
|
3001
|
+
}
|
|
3002
|
+
}
|
|
3003
|
+
/** Reads one response's body, when it still can be (not aborted, not redirected away); a response that cannot be read is silently left out rather than failing the whole capture. */
|
|
3004
|
+
async function recordResponse(response, into) {
|
|
3005
|
+
try {
|
|
3006
|
+
const body = await response.text();
|
|
3007
|
+
const value = tryParseJson(body);
|
|
3008
|
+
into.push({
|
|
3009
|
+
url: response.url(),
|
|
3010
|
+
status: response.status(),
|
|
3011
|
+
size: body.length,
|
|
3012
|
+
shape: value === undefined ? 'text' : describeJson(value, 'json').type
|
|
3013
|
+
});
|
|
3014
|
+
} catch {
|
|
3015
|
+
// response body unavailable (aborted, redirected, the page navigated away) — skip it
|
|
3016
|
+
}
|
|
3017
|
+
}
|
|
3018
|
+
|
|
3019
|
+
/**
|
|
3020
|
+
* Handles `inspect-page`: the Inspect panel's DOM tree and data-in-the-page
|
|
3021
|
+
* findings, off the same cached snapshot `take-snapshot`/`infer-selector`
|
|
3022
|
+
* already use (`page-inspector`, issue #93). `domTree` reads the cached,
|
|
3023
|
+
* rewritten snapshot (the same markup the content pane's iframe shows, so a
|
|
3024
|
+
* tree row's `nodeId` lines up with it exactly); `pageData` reads the raw
|
|
3025
|
+
* capture (`SnapshotResult.rawHtml`), since the display snapshot has had its
|
|
3026
|
+
* `<script>` tags stripped for the sandboxed iframe.
|
|
3027
|
+
*
|
|
3028
|
+
* @param state - The server state.
|
|
3029
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
3030
|
+
* @returns The tree and the data-in-the-page findings.
|
|
3031
|
+
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
3032
|
+
*/
|
|
3033
|
+
function handleInspectPage(state, command) {
|
|
3034
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
3035
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
3036
|
+
return {
|
|
3037
|
+
tree: domTree(snapshot.html),
|
|
3038
|
+
pageData: pageData(snapshot.rawHtml)
|
|
3039
|
+
};
|
|
3040
|
+
}
|
|
3041
|
+
|
|
1584
3042
|
/**
|
|
1585
3043
|
* The outline model: an `InputRecipe`'s `steps` (and only `steps`; the rest
|
|
1586
3044
|
* of the recipe passes through untouched, see `recipe-to-outline.mapper.ts`)
|
|
@@ -1705,306 +3163,1078 @@ function selectChoice(step) {
|
|
|
1705
3163
|
if (Array.isArray(step.values)) return code(step.values.join(', '));
|
|
1706
3164
|
return code('');
|
|
1707
3165
|
}
|
|
1708
|
-
function waitFor(step) {
|
|
1709
|
-
if (typeof step.selector === 'string') return code(step.selector);
|
|
1710
|
-
if (typeof step.ms === 'number') return code(`${step.ms}ms`);
|
|
1711
|
-
if (typeof step.state === 'string') return code(step.state);
|
|
1712
|
-
return code('');
|
|
3166
|
+
function waitFor(step) {
|
|
3167
|
+
if (typeof step.selector === 'string') return code(step.selector);
|
|
3168
|
+
if (typeof step.ms === 'number') return code(`${step.ms}ms`);
|
|
3169
|
+
if (typeof step.state === 'string') return code(step.state);
|
|
3170
|
+
return code('');
|
|
3171
|
+
}
|
|
3172
|
+
function forEachSentence(step) {
|
|
3173
|
+
const parts = [word('For each'), pillField(step, 'as', '(item)'), word('in'), typeof step.over === 'string' ? pill(step.over) : targetField(step)];
|
|
3174
|
+
if (step.emit !== undefined) parts.push(word('→'), word('one record'));
|
|
3175
|
+
return parts;
|
|
3176
|
+
}
|
|
3177
|
+
function paginateSentence(step) {
|
|
3178
|
+
const parts = [word('For every page')];
|
|
3179
|
+
const next = step.next;
|
|
3180
|
+
if (next !== null && typeof next === 'object') {
|
|
3181
|
+
const record = next;
|
|
3182
|
+
if (typeof record.selector === 'string') parts.push(word('via'), code(record.selector));else if (typeof record.url === 'string') parts.push(word('via'), code(record.url));else if (typeof record.jsonpath === 'string') parts.push(word('via'), code(record.jsonpath));
|
|
3183
|
+
}
|
|
3184
|
+
if (typeof step.until === 'string') parts.push(word('until'), code(step.until));
|
|
3185
|
+
return parts;
|
|
3186
|
+
}
|
|
3187
|
+
function emitSentence(step) {
|
|
3188
|
+
if (typeof step.output === 'string') return [word('Emit'), word('to'), pill(step.output)];
|
|
3189
|
+
return [word('Emit')];
|
|
3190
|
+
}
|
|
3191
|
+
|
|
3192
|
+
/**
|
|
3193
|
+
* Reads a recipe's `steps` as an outline. Works on whatever JSON parsed
|
|
3194
|
+
* (not a validated `InputRecipe`): a recipe mid-edit in the JSON tab may be
|
|
3195
|
+
* missing fields or have the wrong shape, and the outline still has to
|
|
3196
|
+
* render something rather than crash the tab. Anything that is not a
|
|
3197
|
+
* well-formed step becomes a custom card holding what was there; anything
|
|
3198
|
+
* that is not an object at the top becomes an outline with no steps.
|
|
3199
|
+
*
|
|
3200
|
+
* @param content - The recipe's parsed JSON (the JSON tab's own `JSON.parse` of its text).
|
|
3201
|
+
* @returns The outline: the untouched top-level record plus the step tree.
|
|
3202
|
+
*/
|
|
3203
|
+
function recipeToOutline(content) {
|
|
3204
|
+
const recipe = isRecord(content) ? content : {};
|
|
3205
|
+
const steps = Array.isArray(recipe.steps) ? recipe.steps : [];
|
|
3206
|
+
return {
|
|
3207
|
+
recipe,
|
|
3208
|
+
steps: stepsToNodes(steps, 'steps')
|
|
3209
|
+
};
|
|
3210
|
+
}
|
|
3211
|
+
function stepsToNodes(steps, pathPrefix) {
|
|
3212
|
+
return steps.map((step, index) => stepToNode(step, `${pathPrefix}.${index}`));
|
|
3213
|
+
}
|
|
3214
|
+
function stepToNode(step, path) {
|
|
3215
|
+
if (!isRecord(step)) {
|
|
3216
|
+
return {
|
|
3217
|
+
kind: 'card',
|
|
3218
|
+
path,
|
|
3219
|
+
stepType: 'unknown',
|
|
3220
|
+
sentence: [{
|
|
3221
|
+
kind: 'word',
|
|
3222
|
+
text: 'custom step'
|
|
3223
|
+
}],
|
|
3224
|
+
step: {},
|
|
3225
|
+
custom: true
|
|
3226
|
+
};
|
|
3227
|
+
}
|
|
3228
|
+
const stepType = typeof step.type === 'string' ? step.type : 'unknown';
|
|
3229
|
+
const {
|
|
3230
|
+
parts,
|
|
3231
|
+
custom
|
|
3232
|
+
} = cardSentence(step);
|
|
3233
|
+
if (!isBracketStepType(stepType)) return {
|
|
3234
|
+
kind: 'card',
|
|
3235
|
+
path,
|
|
3236
|
+
stepType,
|
|
3237
|
+
sentence: parts,
|
|
3238
|
+
step,
|
|
3239
|
+
custom
|
|
3240
|
+
};
|
|
3241
|
+
const children = Array.isArray(step.steps) ? stepsToNodes(step.steps, `${path}.steps`) : [];
|
|
3242
|
+
const elseChildren = stepType === 'if' && Array.isArray(step.else) ? stepsToNodes(step.else, `${path}.else`) : undefined;
|
|
3243
|
+
return {
|
|
3244
|
+
kind: 'bracket',
|
|
3245
|
+
path,
|
|
3246
|
+
stepType,
|
|
3247
|
+
sentence: parts,
|
|
3248
|
+
step,
|
|
3249
|
+
children,
|
|
3250
|
+
...(elseChildren !== undefined && {
|
|
3251
|
+
elseChildren
|
|
3252
|
+
})
|
|
3253
|
+
};
|
|
3254
|
+
}
|
|
3255
|
+
function isRecord(value) {
|
|
3256
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
3257
|
+
}
|
|
3258
|
+
|
|
3259
|
+
/**
|
|
3260
|
+
* Rebuilds a recipe from its outline: `recipe`'s keys, in their original
|
|
3261
|
+
* order, with `steps` (when the recipe had one) replaced by the outline's
|
|
3262
|
+
* (possibly edited) step tree. A card's own raw `step` is used as-is; a
|
|
3263
|
+
* bracket's `step` supplies every field but `steps`/`else`, which come from
|
|
3264
|
+
* `children`/`elseChildren` instead, put back at the position `step` itself
|
|
3265
|
+
* held that key at (or appended, for a bracket a `+` menu just added, whose
|
|
3266
|
+
* `step` never had one) — this, together with `recipe-to-outline.mapper.ts`
|
|
3267
|
+
* always keeping the step's whole original object, is what makes
|
|
3268
|
+
* `recipe → outline → recipe` preserve every key and its order.
|
|
3269
|
+
*
|
|
3270
|
+
* @param outline - An outline, from `recipe-to-outline.mapper.ts` or edited by the UI.
|
|
3271
|
+
* @returns The recipe's plain JSON, ready for `save-recipe`.
|
|
3272
|
+
*/
|
|
3273
|
+
function outlineToRecipe(outline) {
|
|
3274
|
+
if (!('steps' in outline.recipe) && outline.steps.length === 0) return {
|
|
3275
|
+
...outline.recipe
|
|
3276
|
+
};
|
|
3277
|
+
return {
|
|
3278
|
+
...outline.recipe,
|
|
3279
|
+
steps: nodesToSteps(outline.steps)
|
|
3280
|
+
};
|
|
3281
|
+
}
|
|
3282
|
+
function nodesToSteps(nodes) {
|
|
3283
|
+
return nodes.map(node => nodeToStep(node));
|
|
3284
|
+
}
|
|
3285
|
+
function nodeToStep(node) {
|
|
3286
|
+
if (!isBracket(node)) return node.step;
|
|
3287
|
+
const keys = Object.keys(node.step);
|
|
3288
|
+
const step = {};
|
|
3289
|
+
for (const key of keys) {
|
|
3290
|
+
if (key === 'steps') step.steps = nodesToSteps(node.children);else if (key === 'else' && node.stepType === 'if') step.else = nodesToSteps(node.elseChildren ?? []);else step[key] = node.step[key];
|
|
3291
|
+
}
|
|
3292
|
+
if (!keys.includes('steps')) step.steps = nodesToSteps(node.children);
|
|
3293
|
+
if (node.stepType === 'if' && node.elseChildren !== undefined && !keys.includes('else')) step.else = nodesToSteps(node.elseChildren);
|
|
3294
|
+
return step;
|
|
3295
|
+
}
|
|
3296
|
+
|
|
3297
|
+
/**
|
|
3298
|
+
* Opens a recipe folder: every `.json`/`.jsonl` recipe directly inside it
|
|
3299
|
+
* (core's own directory reading, so relative `file:` URLs resolve the same
|
|
3300
|
+
* way a real run would), parsed and, for each output recipe found, bound to
|
|
3301
|
+
* the input recipes that name it. Nothing throws for a bad recipe or a
|
|
3302
|
+
* recipe that does not bind: every problem is reported on its file with the
|
|
3303
|
+
* JSON path core already computes, exactly as `opencraw validate` prints it.
|
|
3304
|
+
*
|
|
3305
|
+
* @param folder - The workspace folder.
|
|
3306
|
+
* @returns Every recipe file found, each with its validation and binding issues.
|
|
3307
|
+
* @throws Error when the folder does not exist or cannot be read.
|
|
3308
|
+
*/
|
|
3309
|
+
async function openWorkspace(folder) {
|
|
3310
|
+
const documents = await readRecipeSource(folder);
|
|
3311
|
+
const entries = documents.map(document => parseDocument(document));
|
|
3312
|
+
const outputs = entries.filter(isParsed('output'));
|
|
3313
|
+
const inputs = entries.filter(isParsed('input'));
|
|
3314
|
+
for (const output of outputs) bindOutput(output, inputs);
|
|
3315
|
+
for (const input of inputs) {
|
|
3316
|
+
if (outputs.every(output => output.recipe.id !== input.recipe.output)) {
|
|
3317
|
+
input.issues.push({
|
|
3318
|
+
path: 'output',
|
|
3319
|
+
message: `no output recipe "${input.recipe.output}" in this workspace`,
|
|
3320
|
+
kind: 'binding'
|
|
3321
|
+
});
|
|
3322
|
+
}
|
|
3323
|
+
}
|
|
3324
|
+
return {
|
|
3325
|
+
folder,
|
|
3326
|
+
recipes: entries.map(({
|
|
3327
|
+
file,
|
|
3328
|
+
kind,
|
|
3329
|
+
id,
|
|
3330
|
+
issues,
|
|
3331
|
+
content
|
|
3332
|
+
}) => ({
|
|
3333
|
+
file,
|
|
3334
|
+
kind,
|
|
3335
|
+
id,
|
|
3336
|
+
issues,
|
|
3337
|
+
text: `${JSON.stringify(content, null, 2)}\n`,
|
|
3338
|
+
// The Steps tab needs steps to render: an output/unknown file has none. recipeToOutline never
|
|
3339
|
+
// throws, so this covers a file that fails validation too (its outline just has "custom" cards).
|
|
3340
|
+
outline: kind === 'input' ? recipeToOutline(content) : undefined
|
|
3341
|
+
}))
|
|
3342
|
+
};
|
|
3343
|
+
}
|
|
3344
|
+
/** Binds one output recipe to the inputs that name it, filing every binding issue back on the recipe it names. */
|
|
3345
|
+
function bindOutput(output, inputs) {
|
|
3346
|
+
const matching = inputs.filter(input => input.recipe.output === output.recipe.id);
|
|
3347
|
+
try {
|
|
3348
|
+
bindRecipeSet(output.recipe, matching.map(input => input.recipe));
|
|
3349
|
+
} catch (error) {
|
|
3350
|
+
if (!(error instanceof RecipeBindingError)) throw error;
|
|
3351
|
+
for (const issue of error.issues) {
|
|
3352
|
+
const target = issue.recipeId === output.recipe.id ? output : matching.find(input => input.recipe.id === issue.recipeId);
|
|
3353
|
+
target?.issues.push({
|
|
3354
|
+
path: issue.path,
|
|
3355
|
+
message: issue.message,
|
|
3356
|
+
kind: 'binding'
|
|
3357
|
+
});
|
|
3358
|
+
}
|
|
3359
|
+
}
|
|
3360
|
+
}
|
|
3361
|
+
function isParsed(kind) {
|
|
3362
|
+
return entry => entry.kind === kind && entry.recipe !== undefined;
|
|
3363
|
+
}
|
|
3364
|
+
function parseDocument(document) {
|
|
3365
|
+
const kind = document.content?.kind;
|
|
3366
|
+
if (kind === 'output') return parseOne(document.source, document.content, 'output', parseOutputRecipe);
|
|
3367
|
+
if (kind === 'input') return parseOne(document.source, document.content, 'input', parseInputRecipe);
|
|
3368
|
+
return {
|
|
3369
|
+
file: document.source,
|
|
3370
|
+
kind: 'unknown',
|
|
3371
|
+
content: document.content,
|
|
3372
|
+
issues: [{
|
|
3373
|
+
path: 'kind',
|
|
3374
|
+
message: '"kind" is missing or not "input"/"output"',
|
|
3375
|
+
kind: 'validation'
|
|
3376
|
+
}]
|
|
3377
|
+
};
|
|
3378
|
+
}
|
|
3379
|
+
function parseOne(file, content, kind, parse) {
|
|
3380
|
+
try {
|
|
3381
|
+
const recipe = parse(content, file);
|
|
3382
|
+
return {
|
|
3383
|
+
file,
|
|
3384
|
+
kind,
|
|
3385
|
+
content,
|
|
3386
|
+
id: recipe.id,
|
|
3387
|
+
issues: [],
|
|
3388
|
+
recipe
|
|
3389
|
+
};
|
|
3390
|
+
} catch (error) {
|
|
3391
|
+
if (!(error instanceof RecipeValidationError)) throw error;
|
|
3392
|
+
return {
|
|
3393
|
+
file,
|
|
3394
|
+
kind,
|
|
3395
|
+
content,
|
|
3396
|
+
issues: error.issues.map(issue => ({
|
|
3397
|
+
path: issue.path === '' ? '(root)' : issue.path,
|
|
3398
|
+
message: issue.message,
|
|
3399
|
+
kind: 'validation'
|
|
3400
|
+
}))
|
|
3401
|
+
};
|
|
3402
|
+
}
|
|
3403
|
+
}
|
|
3404
|
+
|
|
3405
|
+
/**
|
|
3406
|
+
* Writes a recipe back to its file: pretty JSON, a 2-space indent, a
|
|
3407
|
+
* trailing newline. Phase 0 takes the JSON tab's whole parsed object rather
|
|
3408
|
+
* than a patch, so this is a full replacement, not a splice: loading a file
|
|
3409
|
+
* the studio has not touched and saving it again keeps every key, value and
|
|
3410
|
+
* key order (`JSON.parse` preserves insertion order, and `JSON.stringify`
|
|
3411
|
+
* writes it back), but not necessarily the file's exact original formatting
|
|
3412
|
+
* (alignment, how arrays wrap, a trailing comment). Byte-identical
|
|
3413
|
+
* round-trip for a hand-formatted file is future work; see this file's test
|
|
3414
|
+
* for what is guaranteed today.
|
|
3415
|
+
*
|
|
3416
|
+
* @param path - The recipe file to write.
|
|
3417
|
+
* @param recipe - The whole recipe object.
|
|
3418
|
+
*/
|
|
3419
|
+
async function saveRecipe(path, recipe) {
|
|
3420
|
+
await writeFile(path, `${JSON.stringify(recipe, null, 2)}\n`, 'utf8');
|
|
3421
|
+
}
|
|
3422
|
+
|
|
3423
|
+
/** Where the studio's own recorder profile lives when `StudioServerOptions.recorderProfileDir` is not given: never a crawl's own `session.browserProfile` directory, never the person's default browser profile. */
|
|
3424
|
+
function defaultRecorderProfileDir() {
|
|
3425
|
+
return join(tmpdir(), 'opencraw-studio', 'recorder-profile');
|
|
1713
3426
|
}
|
|
1714
|
-
|
|
1715
|
-
|
|
1716
|
-
|
|
1717
|
-
|
|
3427
|
+
/** @returns Fresh state for a new server instance: no workspace open, no run, no recording, no sockets, an empty snapshot cache, no last runs, Playwright's own default browser, the default recorder profile directory. */
|
|
3428
|
+
function createStudioState() {
|
|
3429
|
+
return {
|
|
3430
|
+
sockets: new Set(),
|
|
3431
|
+
snapshots: createSnapshotCache(),
|
|
3432
|
+
lastRuns: createLastRunCache(),
|
|
3433
|
+
recorderProfileDir: defaultRecorderProfileDir()
|
|
3434
|
+
};
|
|
1718
3435
|
}
|
|
1719
|
-
|
|
1720
|
-
|
|
1721
|
-
|
|
1722
|
-
|
|
1723
|
-
|
|
1724
|
-
|
|
3436
|
+
/**
|
|
3437
|
+
* Sends one event to every connected client. A socket that errors while
|
|
3438
|
+
* sending (gone away without a clean close yet reaching us) is dropped
|
|
3439
|
+
* rather than left to fail every later broadcast too.
|
|
3440
|
+
*
|
|
3441
|
+
* @param state - The server state holding the connected sockets.
|
|
3442
|
+
* @param event - The event to send.
|
|
3443
|
+
*/
|
|
3444
|
+
function broadcast(state, event) {
|
|
3445
|
+
const text = JSON.stringify(event);
|
|
3446
|
+
for (const socket of state.sockets) {
|
|
3447
|
+
try {
|
|
3448
|
+
socket.send(text);
|
|
3449
|
+
} catch {
|
|
3450
|
+
state.sockets.delete(socket);
|
|
3451
|
+
}
|
|
1725
3452
|
}
|
|
1726
|
-
if (typeof step.until === 'string') parts.push(word('until'), code(step.until));
|
|
1727
|
-
return parts;
|
|
1728
3453
|
}
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
3454
|
+
|
|
3455
|
+
/**
|
|
3456
|
+
* Handles `open-workspace`: reads the folder, remembers it as the server's
|
|
3457
|
+
* current workspace (every later command that names a recipe resolves it
|
|
3458
|
+
* against this folder), and tells every connected client the workspace changed.
|
|
3459
|
+
*
|
|
3460
|
+
* @param state - The server state to update.
|
|
3461
|
+
* @param command - The command.
|
|
3462
|
+
* @returns The workspace's recipes and their issues.
|
|
3463
|
+
*/
|
|
3464
|
+
async function handleOpenWorkspace(state, command) {
|
|
3465
|
+
const view = await openWorkspace(command.folder);
|
|
3466
|
+
state.folder = command.folder;
|
|
3467
|
+
broadcast(state, {
|
|
3468
|
+
type: 'workspace-changed'
|
|
3469
|
+
});
|
|
3470
|
+
return view;
|
|
1732
3471
|
}
|
|
1733
3472
|
|
|
1734
3473
|
/**
|
|
1735
|
-
*
|
|
1736
|
-
*
|
|
1737
|
-
*
|
|
1738
|
-
*
|
|
1739
|
-
*
|
|
1740
|
-
*
|
|
3474
|
+
* Handles `GET /api/pdf-bytes`: the raw bytes of a cached PDF snapshot, for
|
|
3475
|
+
* the PDF canvas's `pdf.js` to render in the browser (studio plan §3.4,
|
|
3476
|
+
* issue #94's 5b). Not a `studio-api` command — the other commands all
|
|
3477
|
+
* answer JSON, and pdf.js wants the file itself — so this is its own route,
|
|
3478
|
+
* read directly by `studio-http.use-case.ts` (mirrors `static-file.handler.ts`'s
|
|
3479
|
+
* own reach outside the command dispatch).
|
|
1741
3480
|
*
|
|
1742
|
-
* @param
|
|
1743
|
-
* @
|
|
3481
|
+
* @param state - The server state.
|
|
3482
|
+
* @param recipeId - The recipe.
|
|
3483
|
+
* @param path - The step path; must already have a cached PDF snapshot (call `take-snapshot` first).
|
|
3484
|
+
* @returns The raw PDF bytes.
|
|
3485
|
+
* @throws Error when `path` has no cached snapshot yet, or the snapshot's document is not a PDF.
|
|
1744
3486
|
*/
|
|
1745
|
-
function
|
|
1746
|
-
const
|
|
1747
|
-
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
steps: stepsToNodes(steps, 'steps')
|
|
1751
|
-
};
|
|
3487
|
+
async function handlePdfBytes(state, recipeId, path) {
|
|
3488
|
+
const snapshot = cachedSnapshot(state.snapshots, recipeId, path);
|
|
3489
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${recipeId}" at "${path}": call take-snapshot first`);
|
|
3490
|
+
if (snapshot.format !== 'pdf') throw new Error(`"${recipeId}" at "${path}" is not a PDF snapshot`);
|
|
3491
|
+
return pdfBytes(snapshot.baseUrl);
|
|
1752
3492
|
}
|
|
1753
|
-
|
|
1754
|
-
|
|
3493
|
+
|
|
3494
|
+
/**
|
|
3495
|
+
* Handles `pdf-view`: the PDF canvas's cells and rows (`document-view`'s
|
|
3496
|
+
* `pdf-view.mapper.ts`, studio plan §3.4, issue #94's 5b), off the same
|
|
3497
|
+
* cached snapshot `take-snapshot`/`document-tree` already use.
|
|
3498
|
+
*
|
|
3499
|
+
* @param state - The server state.
|
|
3500
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
3501
|
+
* @returns The document's view.
|
|
3502
|
+
* @throws Error when no workspace is open, `command.path` has no cached snapshot yet, or the snapshot's document is not a read PDF.
|
|
3503
|
+
*/
|
|
3504
|
+
function handlePdfView(state, command) {
|
|
3505
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
3506
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
3507
|
+
if (!isPdfDocument(snapshot.body)) throw new Error(`"${command.recipeId}" at "${command.path}" is not a PDF snapshot: the PDF canvas only reads a read PDF document`);
|
|
3508
|
+
return pdfDocumentView(snapshot.body);
|
|
1755
3509
|
}
|
|
1756
|
-
|
|
1757
|
-
|
|
1758
|
-
|
|
1759
|
-
|
|
1760
|
-
|
|
1761
|
-
|
|
1762
|
-
|
|
1763
|
-
|
|
1764
|
-
|
|
1765
|
-
|
|
1766
|
-
|
|
1767
|
-
|
|
1768
|
-
|
|
3510
|
+
|
|
3511
|
+
/**
|
|
3512
|
+
* Turns one resolved recorded action into the recipe step it reads as
|
|
3513
|
+
* (issue #95's card sentences): `Click <selector>`, `Fill <selector> with
|
|
3514
|
+
* <value>`, `Select <selector> = <value>`, `Press <key>`, `Scroll to
|
|
3515
|
+
* bottom`. Pure: `recorder-session.use-case.ts` has already resolved the
|
|
3516
|
+
* selector (`selector-inference`) and, for a secret field, already replaced
|
|
3517
|
+
* `value` with its `{{env.NAME}}` placeholder (`secret-field.policy.ts`) —
|
|
3518
|
+
* this mapper never sees, and so can never leak, a real secret value.
|
|
3519
|
+
*
|
|
3520
|
+
* @param action - The resolved action.
|
|
3521
|
+
* @returns The step it becomes, or `undefined` for an action this mapper cannot place (a click/fill/select with no verified selector).
|
|
3522
|
+
*/
|
|
3523
|
+
function actionToStep(action) {
|
|
3524
|
+
switch (action.kind) {
|
|
3525
|
+
case 'click':
|
|
3526
|
+
{
|
|
3527
|
+
if (action.selector === undefined) return undefined;
|
|
3528
|
+
const step = {
|
|
3529
|
+
type: 'click',
|
|
3530
|
+
selector: action.selector
|
|
3531
|
+
};
|
|
3532
|
+
return step;
|
|
3533
|
+
}
|
|
3534
|
+
case 'fill':
|
|
3535
|
+
{
|
|
3536
|
+
if (action.selector === undefined) return undefined;
|
|
3537
|
+
const step = {
|
|
3538
|
+
type: 'fill',
|
|
3539
|
+
selector: action.selector,
|
|
3540
|
+
value: action.value ?? ''
|
|
3541
|
+
};
|
|
3542
|
+
return step;
|
|
3543
|
+
}
|
|
3544
|
+
case 'select':
|
|
3545
|
+
{
|
|
3546
|
+
if (action.selector === undefined) return undefined;
|
|
3547
|
+
const step = {
|
|
3548
|
+
type: 'select',
|
|
3549
|
+
selector: action.selector,
|
|
3550
|
+
value: action.value
|
|
3551
|
+
};
|
|
3552
|
+
return step;
|
|
3553
|
+
}
|
|
3554
|
+
case 'keypress':
|
|
3555
|
+
{
|
|
3556
|
+
const key = action.key ?? 'Enter';
|
|
3557
|
+
const step = action.selector === undefined ? {
|
|
3558
|
+
type: 'press',
|
|
3559
|
+
key
|
|
3560
|
+
} : {
|
|
3561
|
+
type: 'press',
|
|
3562
|
+
selector: action.selector,
|
|
3563
|
+
key
|
|
3564
|
+
};
|
|
3565
|
+
return step;
|
|
3566
|
+
}
|
|
3567
|
+
case 'scroll':
|
|
3568
|
+
{
|
|
3569
|
+
const step = {
|
|
3570
|
+
type: 'scroll',
|
|
3571
|
+
to: 'bottom'
|
|
3572
|
+
};
|
|
3573
|
+
return step;
|
|
3574
|
+
}
|
|
1769
3575
|
}
|
|
1770
|
-
const stepType = typeof step.type === 'string' ? step.type : 'unknown';
|
|
1771
|
-
const {
|
|
1772
|
-
parts,
|
|
1773
|
-
custom
|
|
1774
|
-
} = cardSentence(step);
|
|
1775
|
-
if (!isBracketStepType(stepType)) return {
|
|
1776
|
-
kind: 'card',
|
|
1777
|
-
path,
|
|
1778
|
-
stepType,
|
|
1779
|
-
sentence: parts,
|
|
1780
|
-
step,
|
|
1781
|
-
custom
|
|
1782
|
-
};
|
|
1783
|
-
const children = Array.isArray(step.steps) ? stepsToNodes(step.steps, `${path}.steps`) : [];
|
|
1784
|
-
const elseChildren = stepType === 'if' && Array.isArray(step.else) ? stepsToNodes(step.else, `${path}.else`) : undefined;
|
|
1785
|
-
return {
|
|
1786
|
-
kind: 'bracket',
|
|
1787
|
-
path,
|
|
1788
|
-
stepType,
|
|
1789
|
-
sentence: parts,
|
|
1790
|
-
step,
|
|
1791
|
-
children,
|
|
1792
|
-
...(elseChildren !== undefined && {
|
|
1793
|
-
elseChildren
|
|
1794
|
-
})
|
|
1795
|
-
};
|
|
1796
3576
|
}
|
|
1797
|
-
|
|
1798
|
-
|
|
3577
|
+
|
|
3578
|
+
/** A `rel` token that means "the next page" on an `<a>`/`<link>`. */
|
|
3579
|
+
const REL_NEXT = /(?:^|\s)next(?:\s|$)/i;
|
|
3580
|
+
/** Link text that reads as "next", loosely: "Next", "Next »", "Next page", punctuation and arrows ignored. */
|
|
3581
|
+
const TEXT_NEXT = /^\s*next(?:\s*page)?\s*(?:[›»>]\s*)?$/i;
|
|
3582
|
+
/** The selector itself names the pattern (`li.next a`, `a[aria-label=Next]`) even when the reported text/rel do not. */
|
|
3583
|
+
const SELECTOR_NEXT = /(?:^|[\s>.])li\.next\b|\[aria-label\s*=\s*['"]?next['"]?\]/i;
|
|
3584
|
+
/**
|
|
3585
|
+
* Whether a clicked, navigating element looks like a "next" link (issue
|
|
3586
|
+
* #95's exact heuristics): `rel="next"`, link text matching "next", or the
|
|
3587
|
+
* resolved selector itself carrying `li.next a`/`a[aria-label=Next]`. Only
|
|
3588
|
+
* meaningful for a click that navigated — the caller checks that first.
|
|
3589
|
+
*
|
|
3590
|
+
* @param link - The anchor's own `rel`/text/`aria-label`, as reported by `recorder-script.client.ts`.
|
|
3591
|
+
* @param selector - The action's resolved selector (`selector-inference`'s own candidate).
|
|
3592
|
+
* @returns Whether the studio should offer to turn this click into a `paginate` step.
|
|
3593
|
+
*/
|
|
3594
|
+
function looksLikeNextLink(link, selector) {
|
|
3595
|
+
if (link?.rel !== undefined && REL_NEXT.test(link.rel)) return true;
|
|
3596
|
+
if (link?.ariaLabel !== undefined && /^next$/i.test(link.ariaLabel.trim())) return true;
|
|
3597
|
+
if (link?.text !== undefined && TEXT_NEXT.test(link.text)) return true;
|
|
3598
|
+
if (selector !== undefined && SELECTOR_NEXT.test(selector)) return true;
|
|
3599
|
+
return false;
|
|
1799
3600
|
}
|
|
1800
3601
|
|
|
3602
|
+
/** Why a report was refused before it became an action: out of this phase's scope (issue #95). */
|
|
3603
|
+
const UNSUPPORTED_REASONS = ['iframe', 'shadow-dom'];
|
|
1801
3604
|
/**
|
|
1802
|
-
*
|
|
1803
|
-
*
|
|
1804
|
-
*
|
|
1805
|
-
|
|
1806
|
-
|
|
1807
|
-
|
|
1808
|
-
|
|
1809
|
-
|
|
1810
|
-
|
|
3605
|
+
* The type/name/id of the form field a `fill`/`select` landed on, exactly as
|
|
3606
|
+
* the live DOM reports them — `secret-field.policy.ts`'s only input. Never
|
|
3607
|
+
* carries the field's value: that travels separately, on the action itself.
|
|
3608
|
+
*/
|
|
3609
|
+
const recordedFieldSchema = z.object({
|
|
3610
|
+
type: z.string().optional(),
|
|
3611
|
+
name: z.string().optional(),
|
|
3612
|
+
id: z.string().optional()
|
|
3613
|
+
});
|
|
3614
|
+
/** What `next-link.policy.ts` needs off a clicked anchor: never the whole element. */
|
|
3615
|
+
const recordedLinkSchema = z.object({
|
|
3616
|
+
rel: z.string().optional(),
|
|
3617
|
+
text: z.string().optional(),
|
|
3618
|
+
ariaLabel: z.string().optional()
|
|
3619
|
+
});
|
|
3620
|
+
/**
|
|
3621
|
+
* The wire shape of one action as `recorder-script.client.ts` reports it,
|
|
3622
|
+
* over the Playwright binding `recorder-session.use-case.ts` exposes: the
|
|
3623
|
+
* `data-oc-node` id of the element involved (`selector-inference`'s own
|
|
3624
|
+
* marker attribute, `@opencraw/core`'s `NODE_ID_ATTRIBUTE`), never a
|
|
3625
|
+
* selector — the studio, not the page, decides that (studio plan §3.2,
|
|
3626
|
+
* issue #95). Never the field's real value when it might be a secret: the
|
|
3627
|
+
* script itself does not know which fields are secret (`secret-field.policy.ts`
|
|
3628
|
+
* runs server-side, once a `RecordedField` is known), but nothing here
|
|
3629
|
+
* *requires* a value to be typed out either way — `recorder-session.use-case.ts`
|
|
3630
|
+
* discards it the moment a secret is detected.
|
|
1811
3631
|
*
|
|
1812
|
-
*
|
|
1813
|
-
*
|
|
3632
|
+
* `html` is the document's own markup, captured by the script itself at the
|
|
3633
|
+
* moment of the event (`document.documentElement.outerHTML`), not fetched
|
|
3634
|
+
* again afterwards: a click that navigates can leave the page before a
|
|
3635
|
+
* later `page.content()` call would resolve, reading the *next* page's
|
|
3636
|
+
* markup instead of the one the click actually happened on. Capturing it
|
|
3637
|
+
* synchronously, in the same event the node id is stamped in, means the
|
|
3638
|
+
* selector is always resolved against the exact document the action
|
|
3639
|
+
* happened on — still "verified against the page" (issue #95), just without
|
|
3640
|
+
* the race a second round trip would risk.
|
|
1814
3641
|
*/
|
|
1815
|
-
|
|
1816
|
-
|
|
1817
|
-
|
|
1818
|
-
|
|
1819
|
-
|
|
1820
|
-
|
|
1821
|
-
|
|
1822
|
-
|
|
3642
|
+
const rawActionSchema = z.discriminatedUnion('kind', [z.object({
|
|
3643
|
+
kind: z.literal('click'),
|
|
3644
|
+
nodeId: z.string().min(1),
|
|
3645
|
+
html: z.string().min(1),
|
|
3646
|
+
link: recordedLinkSchema.optional()
|
|
3647
|
+
}), z.object({
|
|
3648
|
+
kind: z.literal('fill'),
|
|
3649
|
+
nodeId: z.string().min(1),
|
|
3650
|
+
html: z.string().min(1),
|
|
3651
|
+
value: z.string(),
|
|
3652
|
+
field: recordedFieldSchema.optional()
|
|
3653
|
+
}), z.object({
|
|
3654
|
+
kind: z.literal('select'),
|
|
3655
|
+
nodeId: z.string().min(1),
|
|
3656
|
+
html: z.string().min(1),
|
|
3657
|
+
value: z.string(),
|
|
3658
|
+
field: recordedFieldSchema.optional()
|
|
3659
|
+
}), z.object({
|
|
3660
|
+
kind: z.literal('keypress'),
|
|
3661
|
+
nodeId: z.string().min(1).optional(),
|
|
3662
|
+
html: z.string().optional(),
|
|
3663
|
+
key: z.string().min(1)
|
|
3664
|
+
}), z.object({
|
|
3665
|
+
kind: z.literal('scroll'),
|
|
3666
|
+
to: z.literal('bottom')
|
|
3667
|
+
}), z.object({
|
|
3668
|
+
kind: z.literal('unsupported'),
|
|
3669
|
+
reason: z.enum(UNSUPPORTED_REASONS)
|
|
3670
|
+
})]);
|
|
3671
|
+
|
|
3672
|
+
/**
|
|
3673
|
+
* The in-page script `recorder-session.use-case.ts` installs on every page
|
|
3674
|
+
* of the studio's own headed browser window (`context.addInitScript`),
|
|
3675
|
+
* reporting the person's actions to the binding it also exposes on the
|
|
3676
|
+
* context (`context.exposeBinding`) — issue #95's "a script injected into
|
|
3677
|
+
* every page ... reports the user's actions: click, type, select, key
|
|
3678
|
+
* press, scroll".
|
|
3679
|
+
*
|
|
3680
|
+
* Kept as a plain string, not a TypeScript function passed to
|
|
3681
|
+
* `page.evaluate`/`addInitScript`, for the same reason as
|
|
3682
|
+
* `page-snapshot/hidden-marks.algorithm.ts`'s `hiddenMarksScript`: it
|
|
3683
|
+
* references `window`/`document` as globals, which `AGENTS.md`'s "things
|
|
3684
|
+
* that bit before" and the repo's `isolated-functions` lint rule forbid for
|
|
3685
|
+
* an in-page TypeScript function.
|
|
3686
|
+
*
|
|
3687
|
+
* What it reports, and what it deliberately does not:
|
|
3688
|
+
*
|
|
3689
|
+
* - **click**: the nearest clickable ancestor (`a`, `button`,
|
|
3690
|
+
* `input[type=submit|button]`) of the element clicked, or the element
|
|
3691
|
+
* itself; for an `<a>`, also its `rel`, text and `aria-label` —
|
|
3692
|
+
* `next-link.policy.ts`'s only input.
|
|
3693
|
+
* - **fill**: on `blur` of a text-like input/textarea whose value changed
|
|
3694
|
+
* since it was focused (never on every keystroke) — the field's `type`,
|
|
3695
|
+
* `name` and `id`, and the value *as typed*; whether that value is a
|
|
3696
|
+
* secret, and what replaces it, is `secret-field.policy.ts`'s call,
|
|
3697
|
+
* server-side, not this script's.
|
|
3698
|
+
* - **select**: on `change` of a `<select>`.
|
|
3699
|
+
* - **keypress**: `Enter` only (issue #95's own example, "Press Enter");
|
|
3700
|
+
* every other key is noise a recipe step has no shape for.
|
|
3701
|
+
* - **scroll**: once per "scroll session" (debounced), when the page has
|
|
3702
|
+
* been scrolled within `NEAR_BOTTOM_PX` of its bottom.
|
|
3703
|
+
* - **unsupported**: once, the first time an event's target sits inside a
|
|
3704
|
+
* shadow root, or this script runs in a frame that is not the top window
|
|
3705
|
+
* (an iframe) — issue #95's explicit out-of-scope cases, reported instead
|
|
3706
|
+
* of attempted.
|
|
3707
|
+
*
|
|
3708
|
+
* Every reported element is marked with `attribute` (`data-oc-node` by
|
|
3709
|
+
* default, `@opencraw/core`'s `NODE_ID_ATTRIBUTE`) if it does not already
|
|
3710
|
+
* carry one, the same attribute `selector-inference`'s `pathToNode` reads —
|
|
3711
|
+
* so `recorder-session.use-case.ts` can turn a report straight into a
|
|
3712
|
+
* verified selector with the phase 2 generator, never inventing its own.
|
|
3713
|
+
*
|
|
3714
|
+
* @param bindingName - The name Playwright's `exposeBinding` gave the reporting function (`window[bindingName]`).
|
|
3715
|
+
* @param attribute - The marker attribute; defaults to `NODE_ID_ATTRIBUTE`.
|
|
3716
|
+
* @returns A self-invoking script, ready for `context.addInitScript({ content: recorderScript(...) })`.
|
|
3717
|
+
*/
|
|
3718
|
+
function recorderScript(bindingName, attribute = NODE_ID_ATTRIBUTE) {
|
|
3719
|
+
return `(() => {
|
|
3720
|
+
if (window.top !== window.self) {
|
|
3721
|
+
try { window.${bindingName}({ kind: 'unsupported', reason: 'iframe' }) } catch (e) {}
|
|
3722
|
+
return
|
|
3723
|
+
}
|
|
3724
|
+
var attribute = ${JSON.stringify(attribute)}
|
|
3725
|
+
var seq = 0
|
|
3726
|
+
var unsupportedReported = false
|
|
3727
|
+
var scrollTimer = null
|
|
3728
|
+
var focusValues = new WeakMap()
|
|
3729
|
+
|
|
3730
|
+
function report (payload) {
|
|
3731
|
+
try { window.${bindingName}(payload) } catch (e) {}
|
|
3732
|
+
}
|
|
3733
|
+
|
|
3734
|
+
function inShadowDom (element) {
|
|
3735
|
+
var root = element.getRootNode ? element.getRootNode() : document
|
|
3736
|
+
return root !== document
|
|
3737
|
+
}
|
|
3738
|
+
|
|
3739
|
+
function reportUnsupportedOnce (reason) {
|
|
3740
|
+
if (unsupportedReported) return
|
|
3741
|
+
unsupportedReported = true
|
|
3742
|
+
report({ kind: 'unsupported', reason: reason })
|
|
3743
|
+
}
|
|
3744
|
+
|
|
3745
|
+
function markNode (element) {
|
|
3746
|
+
var id = element.getAttribute(attribute)
|
|
3747
|
+
if (id) return id
|
|
3748
|
+
seq += 1
|
|
3749
|
+
id = 'rec-' + Date.now().toString(36) + '-' + seq
|
|
3750
|
+
element.setAttribute(attribute, id)
|
|
3751
|
+
return id
|
|
3752
|
+
}
|
|
3753
|
+
|
|
3754
|
+
function clickable (element) {
|
|
3755
|
+
return element.closest('a, button, input[type="submit"], input[type="button"]') || element
|
|
3756
|
+
}
|
|
3757
|
+
|
|
3758
|
+
function isTextEntry (element) {
|
|
3759
|
+
var tag = element.tagName
|
|
3760
|
+
if (tag === 'TEXTAREA' || tag === 'SELECT') return true
|
|
3761
|
+
if (tag !== 'INPUT') return false
|
|
3762
|
+
var type = (element.getAttribute('type') || 'text').toLowerCase()
|
|
3763
|
+
return type !== 'submit' && type !== 'button' && type !== 'reset'
|
|
3764
|
+
}
|
|
3765
|
+
|
|
3766
|
+
function linkInfo (element) {
|
|
3767
|
+
if (element.tagName !== 'A') return undefined
|
|
3768
|
+
var aria = element.getAttribute('aria-label')
|
|
3769
|
+
return {
|
|
3770
|
+
rel: element.getAttribute('rel') || undefined,
|
|
3771
|
+
text: (element.textContent || '').trim() || undefined,
|
|
3772
|
+
ariaLabel: aria || undefined,
|
|
3773
|
+
}
|
|
3774
|
+
}
|
|
3775
|
+
|
|
3776
|
+
document.addEventListener('click', function (event) {
|
|
3777
|
+
var target = event.target
|
|
3778
|
+
if (!(target instanceof Element)) return
|
|
3779
|
+
if (inShadowDom(target)) { reportUnsupportedOnce('shadow-dom'); return }
|
|
3780
|
+
// Clicking into a text field, a textarea or a select to focus it is not itself a recordable
|
|
3781
|
+
// action — the fill/select the person then does is what becomes a step, on blur/change.
|
|
3782
|
+
if (isTextEntry(target)) return
|
|
3783
|
+
var element = clickable(target)
|
|
3784
|
+
var id = markNode(element)
|
|
3785
|
+
report({ kind: 'click', nodeId: id, html: document.documentElement.outerHTML, link: linkInfo(element) })
|
|
3786
|
+
}, true)
|
|
3787
|
+
|
|
3788
|
+
function flushFill (target) {
|
|
3789
|
+
if (!(target instanceof HTMLInputElement) && !(target instanceof HTMLTextAreaElement)) return
|
|
3790
|
+
if (!focusValues.has(target)) return
|
|
3791
|
+
var before = focusValues.get(target)
|
|
3792
|
+
focusValues.delete(target)
|
|
3793
|
+
if (before === target.value) return
|
|
3794
|
+
if (inShadowDom(target)) { reportUnsupportedOnce('shadow-dom'); return }
|
|
3795
|
+
var id = markNode(target)
|
|
3796
|
+
report({
|
|
3797
|
+
kind: 'fill',
|
|
3798
|
+
nodeId: id,
|
|
3799
|
+
html: document.documentElement.outerHTML,
|
|
3800
|
+
value: target.value,
|
|
3801
|
+
field: { type: target.type || undefined, name: target.name || undefined, id: target.id || undefined },
|
|
3802
|
+
})
|
|
3803
|
+
}
|
|
3804
|
+
|
|
3805
|
+
document.addEventListener('focusin', function (event) {
|
|
3806
|
+
var target = event.target
|
|
3807
|
+
if (!(target instanceof HTMLInputElement) && !(target instanceof HTMLTextAreaElement)) return
|
|
3808
|
+
focusValues.set(target, target.value)
|
|
3809
|
+
}, true)
|
|
3810
|
+
|
|
3811
|
+
// Losing focus (Tab, clicking elsewhere) is the ordinary way a fill is reported.
|
|
3812
|
+
document.addEventListener('focusout', function (event) { flushFill(event.target) }, true)
|
|
3813
|
+
|
|
3814
|
+
document.addEventListener('change', function (event) {
|
|
3815
|
+
var target = event.target
|
|
3816
|
+
if (!(target instanceof HTMLSelectElement)) return
|
|
3817
|
+
if (inShadowDom(target)) { reportUnsupportedOnce('shadow-dom'); return }
|
|
3818
|
+
var id = markNode(target)
|
|
3819
|
+
report({
|
|
3820
|
+
kind: 'select',
|
|
3821
|
+
nodeId: id,
|
|
3822
|
+
html: document.documentElement.outerHTML,
|
|
3823
|
+
value: target.value,
|
|
3824
|
+
field: { type: 'select', name: target.name || undefined, id: target.id || undefined },
|
|
3825
|
+
})
|
|
3826
|
+
}, true)
|
|
3827
|
+
|
|
3828
|
+
document.addEventListener('keydown', function (event) {
|
|
3829
|
+
if (event.key !== 'Enter') return
|
|
3830
|
+
var target = event.target
|
|
3831
|
+
// Enter often submits the very field it was pressed in, which never blurs first — flush its
|
|
3832
|
+
// fill now, so typing a query then pressing Enter still reports the fill before the press.
|
|
3833
|
+
flushFill(target)
|
|
3834
|
+
var id = target instanceof Element && target !== document.body ? markNode(target) : undefined
|
|
3835
|
+
report({ kind: 'keypress', nodeId: id, html: id === undefined ? undefined : document.documentElement.outerHTML, key: 'Enter' })
|
|
3836
|
+
}, true)
|
|
3837
|
+
|
|
3838
|
+
var NEAR_BOTTOM_PX = 40
|
|
3839
|
+
window.addEventListener('scroll', function () {
|
|
3840
|
+
if (scrollTimer !== null) clearTimeout(scrollTimer)
|
|
3841
|
+
scrollTimer = setTimeout(function () {
|
|
3842
|
+
scrollTimer = null
|
|
3843
|
+
var atBottom = window.innerHeight + window.scrollY >= document.documentElement.scrollHeight - NEAR_BOTTOM_PX
|
|
3844
|
+
if (atBottom) report({ kind: 'scroll', to: 'bottom' })
|
|
3845
|
+
}, 300)
|
|
3846
|
+
}, true)
|
|
3847
|
+
})()`;
|
|
1823
3848
|
}
|
|
1824
|
-
|
|
1825
|
-
|
|
3849
|
+
|
|
3850
|
+
/** Field names that mark a value as a secret even outside a `type="password"` input (issue #95). */
|
|
3851
|
+
const SECRET_NAME = /token|secret|api[-_]?key/i;
|
|
3852
|
+
/**
|
|
3853
|
+
* Whether a recorded `fill`/`select` field's typed value is a secret: a
|
|
3854
|
+
* `type="password"` input, or a field the person explicitly marked (`marked`,
|
|
3855
|
+
* a future "mark as secret" UI action — not built by this phase, but the
|
|
3856
|
+
* flag is honoured wherever it comes from), or a name/id matching `token`,
|
|
3857
|
+
* `secret` or `api[-_]?key` (issue #95's exact wording).
|
|
3858
|
+
*
|
|
3859
|
+
* @param field - The field's type/name/id, as reported by `recorder-script.client.ts`.
|
|
3860
|
+
* @param marked - The person explicitly marked this field as secret; default `false`.
|
|
3861
|
+
* @returns Whether the field's value must become `{{env.NAME}}` instead of the real text.
|
|
3862
|
+
*/
|
|
3863
|
+
function isSecretField(field, marked = false) {
|
|
3864
|
+
if (marked) return true;
|
|
3865
|
+
if (field.type === 'password') return true;
|
|
3866
|
+
const name = field.name ?? field.id ?? '';
|
|
3867
|
+
return name.length > 0 && SECRET_NAME.test(name);
|
|
1826
3868
|
}
|
|
1827
|
-
|
|
1828
|
-
|
|
1829
|
-
|
|
1830
|
-
|
|
1831
|
-
|
|
1832
|
-
|
|
1833
|
-
|
|
1834
|
-
|
|
1835
|
-
|
|
1836
|
-
|
|
3869
|
+
/**
|
|
3870
|
+
* The `{{env.NAME}}` variable name a secret field suggests, from its own
|
|
3871
|
+
* name/id (falling back to `PASSWORD` for a bare `type="password"` field
|
|
3872
|
+
* with neither): `camelCase`/`kebab-case`/`snake_case` all become
|
|
3873
|
+
* `SCREAMING_SNAKE_CASE`, punctuation dropped, so `user-pass`, `userPass`
|
|
3874
|
+
* and `user_pass` all suggest `USER_PASS`.
|
|
3875
|
+
*
|
|
3876
|
+
* @param field - The field's type/name/id.
|
|
3877
|
+
* @returns A valid env-var-shaped name (letters, digits, underscores, not starting with a digit).
|
|
3878
|
+
*/
|
|
3879
|
+
function secretEnvName(field) {
|
|
3880
|
+
const source = field.name ?? field.id ?? (field.type === 'password' ? 'password' : 'secret');
|
|
3881
|
+
const withBoundaries = source.replaceAll(/([a-z0-9])([A-Z])/g, '$1_$2');
|
|
3882
|
+
const screaming = withBoundaries.replaceAll(/[^A-Z0-9]+/gi, '_').toUpperCase().replace(/^_+/, '').replace(/_+$/, '');
|
|
3883
|
+
const named = screaming.length === 0 ? 'SECRET' : screaming;
|
|
3884
|
+
return /^\d/.test(named) ? `_${named}` : named;
|
|
3885
|
+
}
|
|
3886
|
+
/**
|
|
3887
|
+
* The placeholder a secret field's value becomes in the recorded recipe:
|
|
3888
|
+
* never the real text, in the recipe JSON or anywhere the studio logs.
|
|
3889
|
+
*
|
|
3890
|
+
* @param field - The field's type/name/id.
|
|
3891
|
+
* @returns `{{env.NAME}}`.
|
|
3892
|
+
*/
|
|
3893
|
+
function secretPlaceholder(field) {
|
|
3894
|
+
return `{{env.${secretEnvName(field)}}}`;
|
|
1837
3895
|
}
|
|
1838
3896
|
|
|
1839
3897
|
/**
|
|
1840
|
-
*
|
|
1841
|
-
* (
|
|
1842
|
-
*
|
|
1843
|
-
*
|
|
1844
|
-
*
|
|
1845
|
-
*
|
|
3898
|
+
* Whether a `Wait for <selector>` step belongs between the action that just
|
|
3899
|
+
* happened and the next one (issue #95): after a navigation or an XHR, the
|
|
3900
|
+
* recorder inserts a wait for the element the person interacts with next, so
|
|
3901
|
+
* a replayed recipe does not race the page's own load. Nothing to wait for
|
|
3902
|
+
* without a next selector (a page-level key press, a scroll) — and no
|
|
3903
|
+
* navigation or fetch either: a recording of a page that never moves should
|
|
3904
|
+
* not fill up with waits nobody asked for.
|
|
3905
|
+
*
|
|
3906
|
+
* @param context - What happened since the previous action.
|
|
3907
|
+
* @param nextSelector - The next action's own resolved selector, when it has one.
|
|
3908
|
+
* @returns The wait step to insert before the next action's own step, or `undefined`.
|
|
3909
|
+
*/
|
|
3910
|
+
function insertWaitStep(context, nextSelector) {
|
|
3911
|
+
if (nextSelector === undefined || !context.navigated && !context.xhr) return undefined;
|
|
3912
|
+
return {
|
|
3913
|
+
type: 'wait',
|
|
3914
|
+
selector: nextSelector
|
|
3915
|
+
};
|
|
3916
|
+
}
|
|
3917
|
+
|
|
3918
|
+
/** The name Playwright's `context.exposeBinding` gives the function `recorder-script.client.ts` calls. */
|
|
3919
|
+
const REPORT_BINDING = '__opencraw_recorder_report__';
|
|
3920
|
+
/** The studio's own recorder profile: never a crawl's own profile, never the person's default browser (issue #95). */
|
|
3921
|
+
const RECORDER_PROFILE_NAME = 'studio-recorder';
|
|
3922
|
+
/** How long a click/key press is given to start a navigation before it counts as not having navigated. */
|
|
3923
|
+
const NAVIGATION_GRACE_MS = 600;
|
|
3924
|
+
/**
|
|
3925
|
+
* Opens the headed browser window a flow is recorded in (issue #95, studio
|
|
3926
|
+
* plan §3.1's last paragraph / phase 6): a Playwright persistent context, in
|
|
3927
|
+
* the studio's own profile (`BrowserProfiles`, `headless: false` — the only
|
|
3928
|
+
* precedent this codebase already has for a headed launch,
|
|
3929
|
+
* `packages/core/src/browser-session/browser-profile.store.ts`), with
|
|
3930
|
+
* `recorder-script.client.ts` installed on every page
|
|
3931
|
+
* (`context.addInitScript`) and reporting through a context-wide binding
|
|
3932
|
+
* (`context.exposeBinding`, so a page opened mid-recording — a link with
|
|
3933
|
+
* `target=_blank`, say — is covered too, not only the first page).
|
|
3934
|
+
*
|
|
3935
|
+
* Each reported action is resolved into a verified selector
|
|
3936
|
+
* (`selector-inference`, against the page's own current markup — issue
|
|
3937
|
+
* #95's "verified against the page"), turned into a step
|
|
3938
|
+
* (`action-to-step.mapper.ts`), and broadcast as an outline card built by
|
|
3939
|
+
* `scope-outline`'s own `recipeToOutline` — the same sentence/bracket logic
|
|
3940
|
+
* every other phase's cards already go through, never a second card format.
|
|
3941
|
+
* A secret field's value is replaced by its `{{env.NAME}}` placeholder
|
|
3942
|
+
* before it reaches a step, a card, or this function's return value: the
|
|
3943
|
+
* real value is discarded the instant `secret-field.policy.ts` flags the
|
|
3944
|
+
* field, never logged, never kept.
|
|
3945
|
+
*
|
|
3946
|
+
* Recording inside an iframe or shadow DOM is out of this phase's scope
|
|
3947
|
+
* (issue #95): `recorder-script.client.ts` detects both and reports them as
|
|
3948
|
+
* `unsupported` notes instead of guessing at a selector that would not
|
|
3949
|
+
* verify.
|
|
1846
3950
|
*
|
|
1847
|
-
* @param
|
|
1848
|
-
* @
|
|
1849
|
-
* @
|
|
3951
|
+
* @param options - The start URL, the profile directory, and browser launch settings.
|
|
3952
|
+
* @param callbacks - Where cards and notices go as they happen.
|
|
3953
|
+
* @returns A handle: `stop` closes the window and returns every step recorded.
|
|
1850
3954
|
*/
|
|
1851
|
-
async function
|
|
1852
|
-
const
|
|
1853
|
-
|
|
1854
|
-
|
|
1855
|
-
|
|
1856
|
-
|
|
1857
|
-
|
|
1858
|
-
|
|
1859
|
-
|
|
1860
|
-
|
|
1861
|
-
|
|
1862
|
-
|
|
3955
|
+
async function openRecorderSession(options, callbacks) {
|
|
3956
|
+
const profiles = new BrowserProfiles(options.profileDir, {
|
|
3957
|
+
...options.browser,
|
|
3958
|
+
headless: false
|
|
3959
|
+
});
|
|
3960
|
+
const session = await profiles.open(RECORDER_PROFILE_NAME, {}, {});
|
|
3961
|
+
const context = session.context;
|
|
3962
|
+
const steps = [];
|
|
3963
|
+
let closed = false;
|
|
3964
|
+
let waitPending = {
|
|
3965
|
+
navigated: false,
|
|
3966
|
+
xhr: false
|
|
3967
|
+
};
|
|
3968
|
+
let xhrSeen = false;
|
|
3969
|
+
context.on('response', response => {
|
|
3970
|
+
const type = response.request().resourceType();
|
|
3971
|
+
if (type === 'xhr' || type === 'fetch') xhrSeen = true;
|
|
3972
|
+
});
|
|
3973
|
+
await context.exposeBinding(REPORT_BINDING, (_source, raw) => {
|
|
3974
|
+
void handleRaw(raw);
|
|
3975
|
+
});
|
|
3976
|
+
await context.addInitScript({
|
|
3977
|
+
content: recorderScript(REPORT_BINDING)
|
|
3978
|
+
});
|
|
3979
|
+
await session.page.goto(options.startUrl);
|
|
3980
|
+
async function handleRaw(raw) {
|
|
3981
|
+
if (closed) return;
|
|
3982
|
+
const parsed = rawActionSchema.safeParse(raw);
|
|
3983
|
+
if (!parsed.success) return;
|
|
3984
|
+
const action = parsed.data;
|
|
3985
|
+
if (action.kind === 'unsupported') {
|
|
3986
|
+
callbacks.onNote({
|
|
3987
|
+
kind: 'unsupported',
|
|
3988
|
+
message: `recording inside ${action.reason === 'iframe' ? 'an iframe' : 'shadow DOM'} is not supported; continue outside it, or add this step by hand`
|
|
1863
3989
|
});
|
|
3990
|
+
return;
|
|
1864
3991
|
}
|
|
3992
|
+
await handleAction(action);
|
|
1865
3993
|
}
|
|
1866
|
-
|
|
1867
|
-
|
|
1868
|
-
|
|
1869
|
-
|
|
1870
|
-
|
|
1871
|
-
|
|
1872
|
-
|
|
1873
|
-
|
|
1874
|
-
|
|
1875
|
-
|
|
1876
|
-
|
|
1877
|
-
|
|
1878
|
-
|
|
1879
|
-
|
|
1880
|
-
|
|
1881
|
-
|
|
1882
|
-
|
|
1883
|
-
|
|
1884
|
-
|
|
1885
|
-
|
|
1886
|
-
|
|
1887
|
-
function bindOutput(output, inputs) {
|
|
1888
|
-
const matching = inputs.filter(input => input.recipe.output === output.recipe.id);
|
|
1889
|
-
try {
|
|
1890
|
-
bindRecipeSet(output.recipe, matching.map(input => input.recipe));
|
|
1891
|
-
} catch (error) {
|
|
1892
|
-
if (!(error instanceof RecipeBindingError)) throw error;
|
|
1893
|
-
for (const issue of error.issues) {
|
|
1894
|
-
const target = issue.recipeId === output.recipe.id ? output : matching.find(input => input.recipe.id === issue.recipeId);
|
|
1895
|
-
target?.issues.push({
|
|
1896
|
-
path: issue.path,
|
|
1897
|
-
message: issue.message,
|
|
1898
|
-
kind: 'binding'
|
|
3994
|
+
async function handleAction(action) {
|
|
3995
|
+
const page = session.page;
|
|
3996
|
+
const urlBefore = page.url();
|
|
3997
|
+
xhrSeen = false;
|
|
3998
|
+
const resolved = resolveAction(action);
|
|
3999
|
+
if (resolved === undefined) return;
|
|
4000
|
+
if (action.kind === 'click' || action.kind === 'keypress') resolved.navigated = await detectNavigation(page, urlBefore);
|
|
4001
|
+
if (resolved.field !== undefined && resolved.value !== undefined && isSecretField(resolved.field)) resolved.value = secretPlaceholder(resolved.field);
|
|
4002
|
+
const waitStep = insertWaitStep(waitPending, resolved.selector);
|
|
4003
|
+
if (waitStep !== undefined) pushStep(waitStep, false);
|
|
4004
|
+
waitPending = {
|
|
4005
|
+
navigated: resolved.navigated !== undefined,
|
|
4006
|
+
xhr: xhrSeen
|
|
4007
|
+
};
|
|
4008
|
+
const step = actionToStep(resolved);
|
|
4009
|
+
if (step === undefined) return;
|
|
4010
|
+
pushStep(step, resolved.field !== undefined && isSecretField(resolved.field));
|
|
4011
|
+
if (resolved.navigated !== undefined && action.kind === 'click' && looksLikeNextLink(resolved.link, resolved.selector)) {
|
|
4012
|
+
callbacks.onNote({
|
|
4013
|
+
kind: 'next-link',
|
|
4014
|
+
message: `this click looks like a "next" link — turn it into "For every page — click ${resolved.selector ?? ''}"?`
|
|
1899
4015
|
});
|
|
1900
4016
|
}
|
|
1901
4017
|
}
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
4018
|
+
function pushStep(step, secret) {
|
|
4019
|
+
steps.push(step);
|
|
4020
|
+
const outline = recipeToOutline({
|
|
4021
|
+
steps
|
|
4022
|
+
});
|
|
4023
|
+
const node = outline.steps.at(-1);
|
|
4024
|
+
if (node !== undefined) callbacks.onCard(node, secret);
|
|
4025
|
+
}
|
|
1910
4026
|
return {
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
|
|
4027
|
+
page: session.page,
|
|
4028
|
+
stop: async () => {
|
|
4029
|
+
if (!closed) {
|
|
4030
|
+
closed = true;
|
|
4031
|
+
await session.close();
|
|
4032
|
+
}
|
|
4033
|
+
return {
|
|
4034
|
+
steps: steps
|
|
4035
|
+
};
|
|
4036
|
+
}
|
|
1919
4037
|
};
|
|
1920
4038
|
}
|
|
1921
|
-
|
|
1922
|
-
|
|
1923
|
-
|
|
1924
|
-
|
|
1925
|
-
|
|
1926
|
-
|
|
1927
|
-
|
|
1928
|
-
|
|
1929
|
-
|
|
1930
|
-
|
|
1931
|
-
|
|
1932
|
-
|
|
1933
|
-
|
|
1934
|
-
|
|
1935
|
-
|
|
1936
|
-
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
|
|
1943
|
-
|
|
4039
|
+
/**
|
|
4040
|
+
* Resolves one raw action's `nodeId` (when it has one — `scroll` and a
|
|
4041
|
+
* page-level `keypress` do not) into a verified selector, the same way
|
|
4042
|
+
* `infer-selector.handler.ts` does for a pick: `pathToNode` off the
|
|
4043
|
+
* document the action itself carries (`action.html`, captured by
|
|
4044
|
+
* `recorder-script.client.ts` at the moment of the event — see
|
|
4045
|
+
* `recorded-action.contract.ts` for why that beats a later `page.content()`),
|
|
4046
|
+
* `candidatesFor`/`bestCandidate` off that path.
|
|
4047
|
+
*
|
|
4048
|
+
* @param action - The raw action, `unsupported` already ruled out by the caller.
|
|
4049
|
+
* @returns The resolved action (selector filled in where relevant), or `undefined` when no candidate verified — the action is dropped rather than recorded with a guessed, unverified selector.
|
|
4050
|
+
*/
|
|
4051
|
+
function resolveAction(action) {
|
|
4052
|
+
if (action.kind === 'scroll') return {
|
|
4053
|
+
kind: 'scroll'
|
|
4054
|
+
};
|
|
4055
|
+
if (action.kind === 'keypress' && action.nodeId === undefined) return {
|
|
4056
|
+
kind: 'keypress',
|
|
4057
|
+
key: action.key
|
|
4058
|
+
};
|
|
4059
|
+
const nodeId = action.nodeId;
|
|
4060
|
+
const html = action.html;
|
|
4061
|
+
if (nodeId === undefined || html === undefined) return undefined;
|
|
4062
|
+
const path = pathToNode(html, nodeId);
|
|
4063
|
+
if (path === undefined || crossesShadowRoot(path)) return undefined;
|
|
4064
|
+
const best = bestCandidate(html, candidatesFor(path), nodeId);
|
|
4065
|
+
if (best === undefined) return undefined;
|
|
4066
|
+
switch (action.kind) {
|
|
4067
|
+
case 'click':
|
|
4068
|
+
{
|
|
4069
|
+
return {
|
|
4070
|
+
kind: 'click',
|
|
4071
|
+
selector: best.selector,
|
|
4072
|
+
link: action.link
|
|
4073
|
+
};
|
|
4074
|
+
}
|
|
4075
|
+
case 'fill':
|
|
4076
|
+
{
|
|
4077
|
+
return {
|
|
4078
|
+
kind: 'fill',
|
|
4079
|
+
selector: best.selector,
|
|
4080
|
+
value: action.value,
|
|
4081
|
+
field: action.field
|
|
4082
|
+
};
|
|
4083
|
+
}
|
|
4084
|
+
case 'select':
|
|
4085
|
+
{
|
|
4086
|
+
return {
|
|
4087
|
+
kind: 'select',
|
|
4088
|
+
selector: best.selector,
|
|
4089
|
+
value: action.value,
|
|
4090
|
+
field: action.field
|
|
4091
|
+
};
|
|
4092
|
+
}
|
|
4093
|
+
case 'keypress':
|
|
4094
|
+
{
|
|
4095
|
+
return {
|
|
4096
|
+
kind: 'keypress',
|
|
4097
|
+
selector: best.selector,
|
|
4098
|
+
key: action.key
|
|
4099
|
+
};
|
|
4100
|
+
}
|
|
1944
4101
|
}
|
|
1945
4102
|
}
|
|
1946
|
-
|
|
1947
4103
|
/**
|
|
1948
|
-
*
|
|
1949
|
-
*
|
|
1950
|
-
*
|
|
1951
|
-
* the
|
|
1952
|
-
*
|
|
1953
|
-
* writes it back), but not necessarily the file's exact original formatting
|
|
1954
|
-
* (alignment, how arrays wrap, a trailing comment). Byte-identical
|
|
1955
|
-
* round-trip for a hand-formatted file is future work; see this file's test
|
|
1956
|
-
* for what is guaranteed today.
|
|
4104
|
+
* Whether, and where to, the page navigated within `NAVIGATION_GRACE_MS` of
|
|
4105
|
+
* a click or key press: polled rather than driven (the navigation, if any,
|
|
4106
|
+
* was started by the person in the live window, not by this function), with
|
|
4107
|
+
* the main frame's own navigation response (for its status) picked up
|
|
4108
|
+
* alongside.
|
|
1957
4109
|
*
|
|
1958
|
-
* @param
|
|
1959
|
-
* @param
|
|
4110
|
+
* @param page - The recorder's own page.
|
|
4111
|
+
* @param urlBefore - `page.url()` right before the action was reported.
|
|
4112
|
+
* @returns The page reached and its status, or `undefined` when the URL never changed.
|
|
1960
4113
|
*/
|
|
1961
|
-
async function
|
|
1962
|
-
|
|
4114
|
+
async function detectNavigation(page, urlBefore) {
|
|
4115
|
+
let response;
|
|
4116
|
+
const onResponse = candidate => {
|
|
4117
|
+
if (candidate.request().isNavigationRequest() && candidate.frame() === page.mainFrame()) response = candidate;
|
|
4118
|
+
};
|
|
4119
|
+
page.on('response', onResponse);
|
|
4120
|
+
const deadline = Date.now() + NAVIGATION_GRACE_MS;
|
|
4121
|
+
try {
|
|
4122
|
+
while (Date.now() < deadline && page.url() === urlBefore) {
|
|
4123
|
+
await new Promise(resolve => setTimeout(resolve, 25));
|
|
4124
|
+
}
|
|
4125
|
+
} finally {
|
|
4126
|
+
page.off('response', onResponse);
|
|
4127
|
+
}
|
|
4128
|
+
if (page.url() === urlBefore) return undefined;
|
|
4129
|
+
try {
|
|
4130
|
+
await page.waitForLoadState('load', {
|
|
4131
|
+
timeout: NAVIGATION_GRACE_MS
|
|
4132
|
+
});
|
|
4133
|
+
} catch {
|
|
4134
|
+
// The load event never fired within the grace period; the page still moved, so report it anyway.
|
|
4135
|
+
}
|
|
4136
|
+
return {
|
|
4137
|
+
url: page.url(),
|
|
4138
|
+
status: response?.status()
|
|
4139
|
+
};
|
|
1963
4140
|
}
|
|
1964
4141
|
|
|
1965
|
-
/**
|
|
1966
|
-
|
|
4142
|
+
/**
|
|
4143
|
+
* Handles `start-recording`: opens the studio's own headed browser window
|
|
4144
|
+
* on the named recipe's start point (issue #95, phase 6) and streams
|
|
4145
|
+
* `recording-card`/`recording-note` events as the person drives it. One
|
|
4146
|
+
* recording at a time, like one sample run at a time: a second
|
|
4147
|
+
* `start-recording` stops whatever was already open first.
|
|
4148
|
+
*
|
|
4149
|
+
* @param state - The server state: `folder` must already be set, `recorderProfileDir` names the studio's own recorder profile (never a crawl's own, never the person's default browser).
|
|
4150
|
+
* @param command - The command.
|
|
4151
|
+
* @returns `{ started: true }` once the window is open.
|
|
4152
|
+
* @throws Error when no workspace is open, or the recipe cannot be found or loaded.
|
|
4153
|
+
*/
|
|
4154
|
+
async function handleStartRecording(state, command) {
|
|
4155
|
+
if (state.folder === undefined) throw new Error('open a workspace first');
|
|
4156
|
+
await state.activeRecording?.stop();
|
|
4157
|
+
const {
|
|
4158
|
+
input
|
|
4159
|
+
} = await loadRecipePair(state.folder, command.recipeId);
|
|
4160
|
+
const point = input.start[0];
|
|
4161
|
+
if (point === undefined) throw new Error(`recipe "${command.recipeId}" has no start point`);
|
|
4162
|
+
const handle = await openRecorderSession({
|
|
4163
|
+
startUrl: point.url,
|
|
4164
|
+
profileDir: state.recorderProfileDir,
|
|
4165
|
+
browser: state.browser
|
|
4166
|
+
}, {
|
|
4167
|
+
onCard: (node, secret) => {
|
|
4168
|
+
broadcast(state, {
|
|
4169
|
+
type: 'recording-card',
|
|
4170
|
+
node,
|
|
4171
|
+
secret
|
|
4172
|
+
});
|
|
4173
|
+
},
|
|
4174
|
+
onNote: note => {
|
|
4175
|
+
broadcast(state, {
|
|
4176
|
+
type: 'recording-note',
|
|
4177
|
+
kind: note.kind,
|
|
4178
|
+
message: note.message
|
|
4179
|
+
});
|
|
4180
|
+
}
|
|
4181
|
+
});
|
|
4182
|
+
state.activeRecording = handle;
|
|
1967
4183
|
return {
|
|
1968
|
-
|
|
1969
|
-
snapshots: createSnapshotCache(),
|
|
1970
|
-
lastRuns: createLastRunCache()
|
|
4184
|
+
started: true
|
|
1971
4185
|
};
|
|
1972
4186
|
}
|
|
1973
4187
|
/**
|
|
1974
|
-
*
|
|
1975
|
-
*
|
|
1976
|
-
*
|
|
4188
|
+
* Handles `stop-recording`: closes the recording window, if one is open,
|
|
4189
|
+
* and answers with every step recorded, in order — enough for the caller to
|
|
4190
|
+
* build "make this the login" (`session.bootstrap`) or "keep as steps"
|
|
4191
|
+
* without asking the server again. Also broadcasts `recording-stopped`, so
|
|
4192
|
+
* every connected client (not only the one that asked) knows the window
|
|
4193
|
+
* closed.
|
|
1977
4194
|
*
|
|
1978
|
-
* @param state - The server state holding the
|
|
1979
|
-
* @param
|
|
4195
|
+
* @param state - The server state holding the active recording, if any.
|
|
4196
|
+
* @param _command - The command (carries nothing beyond its type).
|
|
4197
|
+
* @returns Every step recorded, in order; empty when nothing was recording.
|
|
1980
4198
|
*/
|
|
1981
|
-
function
|
|
1982
|
-
const
|
|
1983
|
-
|
|
1984
|
-
|
|
1985
|
-
|
|
1986
|
-
|
|
1987
|
-
|
|
1988
|
-
|
|
1989
|
-
}
|
|
4199
|
+
async function handleStopRecording(state, _command) {
|
|
4200
|
+
const recording = state.activeRecording;
|
|
4201
|
+
state.activeRecording = undefined;
|
|
4202
|
+
if (recording === undefined) return {
|
|
4203
|
+
steps: []
|
|
4204
|
+
};
|
|
4205
|
+
const {
|
|
4206
|
+
steps
|
|
4207
|
+
} = await recording.stop();
|
|
4208
|
+
broadcast(state, {
|
|
4209
|
+
type: 'recording-stopped',
|
|
4210
|
+
steps
|
|
4211
|
+
});
|
|
4212
|
+
return {
|
|
4213
|
+
steps
|
|
4214
|
+
};
|
|
1990
4215
|
}
|
|
1991
4216
|
|
|
1992
4217
|
/**
|
|
1993
|
-
* Handles `
|
|
1994
|
-
*
|
|
1995
|
-
*
|
|
4218
|
+
* Handles `responses-seen`: the JSON responses observed while a recipe's
|
|
4219
|
+
* start page rendered (`page-inspector`'s `responses-seen.use-case.ts`,
|
|
4220
|
+
* issue #93), for the Inspect panel's "responses seen" list — picking one
|
|
4221
|
+
* is the seam the UI uses to switch the recipe to api mode (issue #93's
|
|
4222
|
+
* acceptance; the JSON tree canvas that follows is phase 5's).
|
|
1996
4223
|
*
|
|
1997
|
-
* @param state - The server state
|
|
4224
|
+
* @param state - The server state: `folder` must already be set.
|
|
1998
4225
|
* @param command - The command.
|
|
1999
|
-
* @returns The
|
|
4226
|
+
* @returns The observed responses.
|
|
4227
|
+
* @throws Error when no workspace is open, or the recipe cannot be found or loaded.
|
|
2000
4228
|
*/
|
|
2001
|
-
async function
|
|
2002
|
-
|
|
2003
|
-
|
|
2004
|
-
|
|
2005
|
-
|
|
2006
|
-
|
|
2007
|
-
return
|
|
4229
|
+
async function handleResponsesSeen(state, command) {
|
|
4230
|
+
if (state.folder === undefined) throw new Error('open a workspace first');
|
|
4231
|
+
const {
|
|
4232
|
+
input
|
|
4233
|
+
} = await loadRecipePair(state.folder, command.recipeId);
|
|
4234
|
+
const responses = await responsesSeen(input, command.path, state.browser);
|
|
4235
|
+
return {
|
|
4236
|
+
responses
|
|
4237
|
+
};
|
|
2008
4238
|
}
|
|
2009
4239
|
|
|
2010
4240
|
/**
|
|
@@ -2167,6 +4397,25 @@ function serveStatic(root, pathname, response) {
|
|
|
2167
4397
|
createReadStream(file).pipe(response);
|
|
2168
4398
|
}
|
|
2169
4399
|
|
|
4400
|
+
/**
|
|
4401
|
+
* Handles `table-preview`: the PDF canvas's live preview of a `table`
|
|
4402
|
+
* extract's options (`document-view`'s `table-preview.use-case.ts`, studio
|
|
4403
|
+
* plan §3.4, issue #94's 5b), off the same cached snapshot `take-snapshot`
|
|
4404
|
+
* already captured — a pure computation over it, so this answers instantly
|
|
4405
|
+
* on every option change, no re-fetch or re-parse.
|
|
4406
|
+
*
|
|
4407
|
+
* @param state - The server state.
|
|
4408
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
4409
|
+
* @returns The matches, or `error` when an option's pattern does not compile.
|
|
4410
|
+
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
4411
|
+
*/
|
|
4412
|
+
function handleTablePreview(state, command) {
|
|
4413
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
4414
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
4415
|
+
if (!isPdfDocument(snapshot.body)) throw new Error(`"${command.recipeId}" at "${command.path}" is not a PDF snapshot: table-preview only reads a read PDF document`);
|
|
4416
|
+
return previewPdfTable(snapshot.body, command.options);
|
|
4417
|
+
}
|
|
4418
|
+
|
|
2170
4419
|
/**
|
|
2171
4420
|
* Handles `take-snapshot`: loads the named input recipe, captures its
|
|
2172
4421
|
* snapshot at `command.path` (only `"start"` is captured for now — see
|
|
@@ -2182,13 +4431,24 @@ function serveStatic(root, pathname, response) {
|
|
|
2182
4431
|
async function handleTakeSnapshot(state, command) {
|
|
2183
4432
|
if (state.folder === undefined) throw new Error('open a workspace first');
|
|
2184
4433
|
const cached = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
2185
|
-
if (cached !== undefined) return cached;
|
|
4434
|
+
if (cached !== undefined) return snapshotView(cached);
|
|
2186
4435
|
const {
|
|
2187
4436
|
input
|
|
2188
4437
|
} = await loadRecipePair(state.folder, command.recipeId);
|
|
2189
4438
|
const snapshot = await takeSnapshot(input, command.path, state.browser);
|
|
2190
4439
|
putSnapshot(state.snapshots, command.recipeId, command.path, snapshot);
|
|
2191
|
-
return snapshot;
|
|
4440
|
+
return snapshotView(snapshot);
|
|
4441
|
+
}
|
|
4442
|
+
/** The wire view of a cached snapshot: `html`/`nodeCount`/`baseUrl`/`format`, never `rawHtml` or the parsed `body` (server-side only — see `SnapshotResult`'s own doc comments). */
|
|
4443
|
+
function snapshotView(snapshot) {
|
|
4444
|
+
return {
|
|
4445
|
+
html: snapshot.html,
|
|
4446
|
+
nodeCount: snapshot.nodeCount,
|
|
4447
|
+
baseUrl: snapshot.baseUrl,
|
|
4448
|
+
...(snapshot.format !== undefined && {
|
|
4449
|
+
format: snapshot.format
|
|
4450
|
+
})
|
|
4451
|
+
};
|
|
2192
4452
|
}
|
|
2193
4453
|
|
|
2194
4454
|
/**
|
|
@@ -2354,6 +4614,7 @@ async function startStudioServer(options = {}) {
|
|
|
2354
4614
|
const state = createStudioState();
|
|
2355
4615
|
if (options.initialFolder !== undefined) state.folder = options.initialFolder;
|
|
2356
4616
|
state.browser = options.browser;
|
|
4617
|
+
if (options.recorderProfileDir !== undefined) state.recorderProfileDir = options.recorderProfileDir;
|
|
2357
4618
|
const server = createServer((request, response) => {
|
|
2358
4619
|
void handleRequest(request, response, token, state, uiRoot);
|
|
2359
4620
|
});
|
|
@@ -2375,6 +4636,7 @@ async function startStudioServer(options = {}) {
|
|
|
2375
4636
|
}
|
|
2376
4637
|
async function stop(server, state) {
|
|
2377
4638
|
await state.activeRun?.stop();
|
|
4639
|
+
await state.activeRecording?.stop();
|
|
2378
4640
|
server.closeAllConnections();
|
|
2379
4641
|
await new Promise((resolve, reject) => {
|
|
2380
4642
|
server.close(error => error === undefined ? resolve() : reject(error));
|
|
@@ -2395,9 +4657,41 @@ async function handleRequest(request, response, token, state, uiRoot) {
|
|
|
2395
4657
|
await handleCommand(request, response, state);
|
|
2396
4658
|
return;
|
|
2397
4659
|
}
|
|
2398
|
-
if (url.pathname === '/' &&
|
|
4660
|
+
if (url.pathname === '/api/pdf-bytes' && request.method === 'GET') {
|
|
4661
|
+
await handlePdfBytesRequest(response, state, url.searchParams);
|
|
4662
|
+
return;
|
|
4663
|
+
}
|
|
4664
|
+
if (url.pathname === '/' && url.searchParams.has('token')) response.setHeader('set-cookie', `${TOKEN_COOKIE}=${token}; HttpOnly; SameSite=Strict; Path=/`);
|
|
2399
4665
|
serveStatic(uiRoot, url.pathname, response);
|
|
2400
4666
|
}
|
|
4667
|
+
/**
|
|
4668
|
+
* `GET /api/pdf-bytes?recipeId=…&path=…`: the raw bytes of a cached PDF
|
|
4669
|
+
* snapshot (`pdf-bytes.handler.ts`, studio plan §3.4, issue #94's 5b) — the
|
|
4670
|
+
* PDF canvas's `pdf.js` fetches this URL directly, so it answers bytes, not
|
|
4671
|
+
* the JSON envelope every `/api/command` response uses.
|
|
4672
|
+
*/
|
|
4673
|
+
async function handlePdfBytesRequest(response, state, query) {
|
|
4674
|
+
const recipeId = query.get('recipeId');
|
|
4675
|
+
const path = query.get('path');
|
|
4676
|
+
if (recipeId === null || path === null) {
|
|
4677
|
+
respondJson(response, 400, {
|
|
4678
|
+
error: '"recipeId" and "path" are required query params'
|
|
4679
|
+
});
|
|
4680
|
+
return;
|
|
4681
|
+
}
|
|
4682
|
+
try {
|
|
4683
|
+
const bytes = await handlePdfBytes(state, recipeId, path);
|
|
4684
|
+
response.writeHead(200, {
|
|
4685
|
+
'content-type': 'application/pdf',
|
|
4686
|
+
'content-length': bytes.byteLength
|
|
4687
|
+
});
|
|
4688
|
+
response.end(Buffer.from(bytes));
|
|
4689
|
+
} catch (error) {
|
|
4690
|
+
respondJson(response, 500, {
|
|
4691
|
+
error: error instanceof Error ? error.message : String(error)
|
|
4692
|
+
});
|
|
4693
|
+
}
|
|
4694
|
+
}
|
|
2401
4695
|
async function handleCommand(request, response, state) {
|
|
2402
4696
|
let command;
|
|
2403
4697
|
try {
|
|
@@ -2466,6 +4760,50 @@ function dispatch(command, state) {
|
|
|
2466
4760
|
{
|
|
2467
4761
|
return handleExplainWhy(state, command);
|
|
2468
4762
|
}
|
|
4763
|
+
case 'inspect-page':
|
|
4764
|
+
{
|
|
4765
|
+
return Promise.resolve(handleInspectPage(state, command));
|
|
4766
|
+
}
|
|
4767
|
+
case 'responses-seen':
|
|
4768
|
+
{
|
|
4769
|
+
return handleResponsesSeen(state, command);
|
|
4770
|
+
}
|
|
4771
|
+
case 'document-tree':
|
|
4772
|
+
{
|
|
4773
|
+
return Promise.resolve(handleDocumentTree(state, command));
|
|
4774
|
+
}
|
|
4775
|
+
case 'pdf-view':
|
|
4776
|
+
{
|
|
4777
|
+
return Promise.resolve(handlePdfView(state, command));
|
|
4778
|
+
}
|
|
4779
|
+
case 'table-preview':
|
|
4780
|
+
{
|
|
4781
|
+
return Promise.resolve(handleTablePreview(state, command));
|
|
4782
|
+
}
|
|
4783
|
+
case 'grid-view':
|
|
4784
|
+
{
|
|
4785
|
+
return handleGridView(state, command);
|
|
4786
|
+
}
|
|
4787
|
+
case 'grid-preview':
|
|
4788
|
+
{
|
|
4789
|
+
return Promise.resolve(handleGridPreview(state, command));
|
|
4790
|
+
}
|
|
4791
|
+
case 'deck-view':
|
|
4792
|
+
{
|
|
4793
|
+
return Promise.resolve(handleDeckView(state, command));
|
|
4794
|
+
}
|
|
4795
|
+
case 'deck-preview':
|
|
4796
|
+
{
|
|
4797
|
+
return Promise.resolve(handleDeckPreview(state, command));
|
|
4798
|
+
}
|
|
4799
|
+
case 'start-recording':
|
|
4800
|
+
{
|
|
4801
|
+
return handleStartRecording(state, command);
|
|
4802
|
+
}
|
|
4803
|
+
case 'stop-recording':
|
|
4804
|
+
{
|
|
4805
|
+
return handleStopRecording(state);
|
|
4806
|
+
}
|
|
2469
4807
|
}
|
|
2470
4808
|
}
|
|
2471
4809
|
function respondJson(response, status, body) {
|
|
@@ -2522,5 +4860,5 @@ async function main(argv) {
|
|
|
2522
4860
|
process.on('SIGTERM', shutdown);
|
|
2523
4861
|
}
|
|
2524
4862
|
|
|
2525
|
-
export { explainWhyCommandSchema, fetchStartPage, fetchStartPageCommandSchema, fieldPickSchema, fieldTraceSchema, inferSelectorCommandSchema, inferSelectorViewSchema, loadRecipePair, main, missingWhyTargetSchema, openWorkspace, openWorkspaceCommandSchema, outlineBracketSchema, outlineCardSchema, outlineNodeSchema, outlineViewSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, recordRejectedEventSchema, rejectedWhyTargetSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveOutlineCommandSchema, saveRecipe, saveRecipeCommandSchema, sentencePartSchema, snapshotViewSchema, startPageViewSchema, startStudioServer, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, takeSnapshotCommandSchema, traceLineEventSchema, verifySelectorCommandSchema, verifySelectorViewSchema, whyTargetSchema, whyViewSchema, workspaceChangedEventSchema, workspaceViewSchema };
|
|
4863
|
+
export { deckChartSeriesViewSchema, deckChartViewSchema, deckDocumentViewSchema, deckPreviewCommandSchema, deckPreviewOptionsSchema, deckShapeViewSchema, deckSlideViewSchema, deckTablePreviewMatchSchema, deckTablePreviewViewSchema, deckViewCommandSchema, documentTreeCommandSchema, documentTreeNodeViewSchema, documentTreeViewSchema, domTreeNodeViewSchema, explainWhyCommandSchema, fetchStartPage, fetchStartPageCommandSchema, fieldPickSchema, fieldTraceSchema, gridCellViewSchema, gridMergeViewSchema, gridPreviewCommandSchema, gridPreviewOptionsSchema, gridSheetViewSchema, gridTablePreviewMatchSchema, gridTablePreviewViewSchema, gridViewCommandSchema, inferSelectorCommandSchema, inferSelectorViewSchema, inspectPageCommandSchema, inspectViewSchema, loadRecipePair, main, missingWhyTargetSchema, observedResponseSchema, openWorkspace, openWorkspaceCommandSchema, outlineBracketSchema, outlineCardSchema, outlineNodeSchema, outlineViewSchema, pageDataFindingSchema, pdfCellViewSchema, pdfDocumentViewSchema, pdfPageViewSchema, pdfRowViewSchema, pdfViewCommandSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, recordRejectedEventSchema, recordingCardEventSchema, recordingNoteEventSchema, recordingStoppedEventSchema, rejectedWhyTargetSchema, responsesSeenCommandSchema, responsesSeenViewSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveOutlineCommandSchema, saveRecipe, saveRecipeCommandSchema, sentencePartSchema, snapshotViewSchema, startPageViewSchema, startRecordingCommandSchema, startStudioServer, stopRecordingCommandSchema, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, tablePreviewBandSchema, tablePreviewCommandSchema, tablePreviewMatchSchema, tablePreviewOptionsSchema, tablePreviewTableSchema, tablePreviewViewSchema, takeSnapshotCommandSchema, traceLineEventSchema, verifySelectorCommandSchema, verifySelectorViewSchema, whyTargetSchema, whyViewSchema, workbookDocumentViewSchema, workspaceChangedEventSchema, workspaceViewSchema };
|
|
2526
4864
|
//# sourceMappingURL=index.esm.js.map
|