@opencraw/studio 0.0.4 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/opencraw-studio.mjs +0 -0
- package/dist/index.esm.js +1219 -12
- package/dist/src/index.d.ts +2 -2
- package/dist/src/page-snapshot/hidden-marks.algorithm.d.ts +23 -0
- package/dist/src/page-snapshot/index.d.ts +7 -0
- package/dist/src/page-snapshot/rewrite-document.mapper.d.ts +27 -0
- package/dist/src/page-snapshot/snapshot.store.d.ts +14 -0
- package/dist/src/page-snapshot/take-snapshot.use-case.d.ts +34 -0
- package/dist/src/sample-run/run-sample.use-case.d.ts +3 -1
- package/dist/src/scope-outline/card-sentence.mapper.d.ts +18 -0
- package/dist/src/scope-outline/index.d.ts +8 -0
- package/dist/src/scope-outline/outline-to-recipe.mapper.d.ts +17 -0
- package/dist/src/scope-outline/outline.model.d.ts +57 -0
- package/dist/src/scope-outline/recipe-to-outline.mapper.d.ts +14 -0
- package/dist/src/scope-outline/scope-at-path.algorithm.d.ts +19 -0
- package/dist/src/selector-inference/class-token.policy.d.ts +35 -0
- package/dist/src/selector-inference/dom-path.model.d.ts +73 -0
- package/dist/src/selector-inference/index.d.ts +12 -0
- package/dist/src/selector-inference/list-inference.algorithm.d.ts +43 -0
- package/dist/src/selector-inference/selector-candidates.algorithm.d.ts +22 -0
- package/dist/src/selector-inference/selector-ranking.policy.d.ts +27 -0
- package/dist/src/selector-inference/shadow-dom.error.d.ts +11 -0
- package/dist/src/selector-inference/take-kind.policy.d.ts +12 -0
- package/dist/src/studio-api/command.contract.d.ts +64 -0
- package/dist/src/studio-api/index.d.ts +10 -2
- package/dist/src/studio-api/infer-selector-view.contract.d.ts +68 -0
- package/dist/src/studio-api/outline-view.contract.d.ts +37 -0
- package/dist/src/studio-api/snapshot-view.contract.d.ts +9 -0
- package/dist/src/studio-api/verify-selector-view.contract.d.ts +18 -0
- package/dist/src/studio-api/workspace-view.contract.d.ts +4 -2
- package/dist/src/studio-server/infer-selector.handler.d.ts +18 -0
- package/dist/src/studio-server/save-outline.handler.d.ts +18 -0
- package/dist/src/studio-server/studio-http.use-case.d.ts +3 -0
- package/dist/src/studio-server/take-snapshot.handler.d.ts +16 -0
- package/dist/src/studio-server/verify-selector.handler.d.ts +18 -0
- package/dist/src/studio-server/workspace.store.d.ts +7 -2
- package/dist/ui/assets/index-BfbrsfCv.js +13 -0
- package/dist/ui/index.html +1 -1
- package/package.json +5 -3
- package/dist/ui/assets/index-CO3pSJhQ.js +0 -12
package/dist/index.esm.js
CHANGED
|
@@ -3,10 +3,39 @@ import { createServer } from 'node:http';
|
|
|
3
3
|
import { fileURLToPath } from 'node:url';
|
|
4
4
|
import { z } from 'zod';
|
|
5
5
|
import { randomBytes, createHash } from 'node:crypto';
|
|
6
|
-
import { BrowserClient, HttpClient, readRecipeSource, parseInputRecipe, parseOutputRecipe, bindRecipeSet, memorySink, createCrawler, traceLine, RecipeBindingError, RecipeValidationError } from '@opencraw/core';
|
|
6
|
+
import { BrowserClient, HttpClient, NODE_ID_ATTRIBUTE, readRecipeSource, parseInputRecipe, parseOutputRecipe, bindRecipeSet, memorySink, createCrawler, traceLine, countMatches, RecipeBindingError, RecipeValidationError } from '@opencraw/core';
|
|
7
|
+
import { load } from 'cheerio';
|
|
7
8
|
import { writeFile } from 'node:fs/promises';
|
|
8
9
|
import { existsSync, createReadStream } from 'node:fs';
|
|
9
10
|
|
|
11
|
+
const sentencePartSchema = z.object({
|
|
12
|
+
kind: z.enum(['word', 'pill', 'code']),
|
|
13
|
+
text: z.string()
|
|
14
|
+
});
|
|
15
|
+
const outlineNodeBase = {
|
|
16
|
+
path: z.string(),
|
|
17
|
+
stepType: z.string(),
|
|
18
|
+
sentence: z.array(sentencePartSchema),
|
|
19
|
+
step: z.record(z.string(), z.unknown())
|
|
20
|
+
};
|
|
21
|
+
const outlineCardSchema = z.object({
|
|
22
|
+
...outlineNodeBase,
|
|
23
|
+
kind: z.literal('card'),
|
|
24
|
+
custom: z.boolean()
|
|
25
|
+
});
|
|
26
|
+
const outlineBracketSchema = z.lazy(() => z.object({
|
|
27
|
+
...outlineNodeBase,
|
|
28
|
+
kind: z.literal('bracket'),
|
|
29
|
+
stepType: z.enum(['forEach', 'paginate', 'if']),
|
|
30
|
+
children: z.array(outlineNodeSchema),
|
|
31
|
+
elseChildren: z.array(outlineNodeSchema).optional()
|
|
32
|
+
}));
|
|
33
|
+
const outlineNodeSchema = z.union([outlineCardSchema, outlineBracketSchema]);
|
|
34
|
+
const outlineViewSchema = z.object({
|
|
35
|
+
recipe: z.record(z.string(), z.unknown()),
|
|
36
|
+
steps: z.array(outlineNodeSchema)
|
|
37
|
+
});
|
|
38
|
+
|
|
10
39
|
/** A budget for a sample run; never authored into a recipe, only asked for by the caller. */
|
|
11
40
|
const sampleBudgetSchema = z.object({
|
|
12
41
|
maxRecords: z.number().int().positive().optional(),
|
|
@@ -39,7 +68,116 @@ const fetchStartPageCommandSchema = z.object({
|
|
|
39
68
|
type: z.literal('fetch-start-page'),
|
|
40
69
|
recipeId: z.string().min(1)
|
|
41
70
|
});
|
|
42
|
-
|
|
71
|
+
/**
|
|
72
|
+
* Writes a recipe file from an edited outline: the server converts it back
|
|
73
|
+
* to the recipe's JSON (`scope-outline`'s `outlineToRecipe`) and saves it
|
|
74
|
+
* exactly as `save-recipe` would. Kept as its own command, rather than
|
|
75
|
+
* asking the Steps tab to run the outline↔recipe mappers itself, so that
|
|
76
|
+
* conversion stays server-side code (it already is: `@opencraw/studio`'s
|
|
77
|
+
* root export also carries Node-only slices, so the UI never imports
|
|
78
|
+
* runtime code from it, only types — see `apps/studio-ui`'s `steps-outline`
|
|
79
|
+
* for where this is called from).
|
|
80
|
+
*/
|
|
81
|
+
const saveOutlineCommandSchema = z.object({
|
|
82
|
+
type: z.literal('save-outline'),
|
|
83
|
+
path: z.string().min(1),
|
|
84
|
+
outline: outlineViewSchema
|
|
85
|
+
});
|
|
86
|
+
/**
|
|
87
|
+
* Captures the snapshot the content pane shows for a step (`page-snapshot`'s
|
|
88
|
+
* `take-snapshot.use-case.ts`), caching it by recipe and step path so
|
|
89
|
+
* switching the selected step does not always re-run the recipe.
|
|
90
|
+
*/
|
|
91
|
+
const takeSnapshotCommandSchema = z.object({
|
|
92
|
+
type: z.literal('take-snapshot'),
|
|
93
|
+
recipeId: z.string().min(1),
|
|
94
|
+
path: z.string().min(1)
|
|
95
|
+
});
|
|
96
|
+
/**
|
|
97
|
+
* Verifies a candidate selector against the cached snapshot and, when
|
|
98
|
+
* reachable, the live page — the studio's own selection (`@opencraw/core`'s
|
|
99
|
+
* `countMatches`), never the browser's or a guess (issue #91).
|
|
100
|
+
*/
|
|
101
|
+
const verifySelectorCommandSchema = z.object({
|
|
102
|
+
type: z.literal('verify-selector'),
|
|
103
|
+
recipeId: z.string().min(1),
|
|
104
|
+
path: z.string().min(1),
|
|
105
|
+
selector: z.string().min(1)
|
|
106
|
+
});
|
|
107
|
+
/**
|
|
108
|
+
* Turns one or two picked nodes (their `data-oc-node` ids, off the cached
|
|
109
|
+
* snapshot) into a verified selector: one id for a `Read` card's field, two
|
|
110
|
+
* for the safe item+field list shape `selector-inference`'s `inferList`
|
|
111
|
+
* builds by construction (issue #91, studio plan §3.2).
|
|
112
|
+
*/
|
|
113
|
+
const pickedNodeIdSchema = z.string().min(1);
|
|
114
|
+
const onePickedNodeSchema = z.tuple([pickedNodeIdSchema]);
|
|
115
|
+
const twoPickedNodesSchema = z.tuple([pickedNodeIdSchema, pickedNodeIdSchema]);
|
|
116
|
+
const inferSelectorCommandSchema = z.object({
|
|
117
|
+
type: z.literal('infer-selector'),
|
|
118
|
+
recipeId: z.string().min(1),
|
|
119
|
+
path: z.string().min(1),
|
|
120
|
+
nodeIds: z.union([onePickedNodeSchema, twoPickedNodesSchema])
|
|
121
|
+
});
|
|
122
|
+
const studioCommandSchema = z.discriminatedUnion('type', [openWorkspaceCommandSchema, runSampleCommandSchema, stopRunCommandSchema, saveRecipeCommandSchema, fetchStartPageCommandSchema, saveOutlineCommandSchema, takeSnapshotCommandSchema, verifySelectorCommandSchema, inferSelectorCommandSchema]);
|
|
123
|
+
|
|
124
|
+
/** What `take-snapshot` answers with: the rewritten document ready for the content pane's sandboxed iframe (studio plan §3.1, issue #91). */
|
|
125
|
+
const snapshotViewSchema = z.object({
|
|
126
|
+
html: z.string(),
|
|
127
|
+
nodeCount: z.number(),
|
|
128
|
+
baseUrl: z.string()
|
|
129
|
+
});
|
|
130
|
+
|
|
131
|
+
/**
|
|
132
|
+
* What `verify-selector` answers with: the match count against the snapshot
|
|
133
|
+
* and, when the live page could be reached, against it too (issue #91's
|
|
134
|
+
* "flag a selector that matches the snapshot but not the live page").
|
|
135
|
+
* `liveChecked` is `false` when the live page could not be fetched (offline,
|
|
136
|
+
* blocked by `allowedHosts`, a transient error) — the card shows the
|
|
137
|
+
* snapshot count alone rather than a misleading "0 on the live page".
|
|
138
|
+
*/
|
|
139
|
+
const verifySelectorViewSchema = z.object({
|
|
140
|
+
selector: z.string(),
|
|
141
|
+
snapshotMatches: z.number(),
|
|
142
|
+
liveChecked: z.boolean(),
|
|
143
|
+
liveMatches: z.number().optional(),
|
|
144
|
+
liveError: z.string().optional()
|
|
145
|
+
});
|
|
146
|
+
|
|
147
|
+
/** The candidate tiers `selector-inference`'s ranking works through, best first — see `selector-candidates.algorithm.ts`. */
|
|
148
|
+
const candidateTierSchema = z.enum(['id', 'data-attr', 'class', 'structure', 'position']);
|
|
149
|
+
/** One clicked node's field: the best selector found for it, verified against the snapshot. */
|
|
150
|
+
const fieldPickSchema = z.object({
|
|
151
|
+
selector: z.string(),
|
|
152
|
+
tier: candidateTierSchema,
|
|
153
|
+
take: z.string(),
|
|
154
|
+
matches: z.number()
|
|
155
|
+
});
|
|
156
|
+
/** A list's item shape: the best selector that matches every item, verified against the snapshot. */
|
|
157
|
+
const itemPickSchema = z.object({
|
|
158
|
+
selector: z.string(),
|
|
159
|
+
tier: candidateTierSchema,
|
|
160
|
+
matches: z.number()
|
|
161
|
+
});
|
|
162
|
+
/**
|
|
163
|
+
* What `infer-selector` answers with (issue #91, studio plan §3.2):
|
|
164
|
+
* `"field"` for one clicked node — a `Read` card's shape; `"list"` for two
|
|
165
|
+
* similar nodes — the safe `extract`(items)+`forEach`(field) shape, item and
|
|
166
|
+
* field each verified separately; `"unsupported"` when the two picks are not
|
|
167
|
+
* a coherent list (different branches, the same node twice, or shadow DOM),
|
|
168
|
+
* with a message the card can show as-is.
|
|
169
|
+
*/
|
|
170
|
+
const inferSelectorViewSchema = z.discriminatedUnion('kind', [z.object({
|
|
171
|
+
kind: z.literal('field'),
|
|
172
|
+
field: fieldPickSchema
|
|
173
|
+
}), z.object({
|
|
174
|
+
kind: z.literal('list'),
|
|
175
|
+
item: itemPickSchema,
|
|
176
|
+
field: fieldPickSchema
|
|
177
|
+
}), z.object({
|
|
178
|
+
kind: z.literal('unsupported'),
|
|
179
|
+
reason: z.string()
|
|
180
|
+
})]);
|
|
43
181
|
|
|
44
182
|
/** Why a sample run stopped before finishing on its own; mirrors `@opencraw/core`'s `RecipeReport.stoppedBy`. */
|
|
45
183
|
const stoppedBySchema = z.enum(['sample-maxRecords', 'sample-maxPages', 'sample-maxMs']);
|
|
@@ -85,7 +223,9 @@ const recipeListingSchema = z.object({
|
|
|
85
223
|
id: z.string().optional(),
|
|
86
224
|
issues: z.array(recipeIssueSchema),
|
|
87
225
|
/** The file's JSON, pretty-printed: the JSON tab's starting text (see `save-recipe` for how an edit is written back). */
|
|
88
|
-
text: z.string()
|
|
226
|
+
text: z.string(),
|
|
227
|
+
/** The Steps tab's view of this file's `steps`, from `@opencraw/studio`'s `scope-outline` slice. `input` recipes only; `undefined` for an `output` or `unknown` file, which has no steps to show. */
|
|
228
|
+
outline: outlineViewSchema.optional()
|
|
89
229
|
});
|
|
90
230
|
/** What `open-workspace` answers with: every recipe file directly inside the folder. */
|
|
91
231
|
const workspaceViewSchema = z.object({
|
|
@@ -200,19 +340,224 @@ async function fetchApi(url) {
|
|
|
200
340
|
const response = await client.send({
|
|
201
341
|
url
|
|
202
342
|
});
|
|
203
|
-
return textOf(response.body);
|
|
343
|
+
return textOf$1(response.body);
|
|
204
344
|
} finally {
|
|
205
345
|
await client.dispose();
|
|
206
346
|
}
|
|
207
347
|
}
|
|
208
348
|
/** Every body kind the studio can show as text today; others get a placeholder rather than a crash. */
|
|
209
|
-
function textOf(body) {
|
|
349
|
+
function textOf$1(body) {
|
|
210
350
|
if (body.kind === 'html') return body.html;
|
|
211
351
|
if (body.kind === 'text') return body.text;
|
|
212
352
|
if (body.kind === 'json') return JSON.stringify(body.data, null, 2);
|
|
213
353
|
return `[${body.kind} document: the content pane shows HTML and JSON responses for now]`;
|
|
214
354
|
}
|
|
215
355
|
|
|
356
|
+
/** The attribute `hiddenMarksScript` stamps on an element hidden on the live page. */
|
|
357
|
+
const HIDDEN_ATTRIBUTE = 'data-oc-hidden';
|
|
358
|
+
/**
|
|
359
|
+
* The in-page script `take-snapshot.use-case.ts` runs with `page.evaluate`
|
|
360
|
+
* *before* `page.content()`, so a `display: none`/`visibility: hidden`/
|
|
361
|
+
* zero-size element carries `data-oc-hidden="1"` into the captured markup
|
|
362
|
+
* (studio plan §3.1, issue #91) — the only way to know an element was
|
|
363
|
+
* hidden on the live page, since a static parse of the HTML afterwards has
|
|
364
|
+
* no computed style to ask.
|
|
365
|
+
*
|
|
366
|
+
* Kept as a plain string, not a TypeScript function passed to
|
|
367
|
+
* `page.evaluate`, because it references `window`/`document` as globals: `AGENTS.md`'s
|
|
368
|
+
* "things that bit before" calls out that in-page functions must not do
|
|
369
|
+
* that (the repo's `isolated-functions` lint rule), and every existing
|
|
370
|
+
* `page.evaluate` call with real DOM logic in this codebase (see
|
|
371
|
+
* `packages/core/src/captcha/resolve-captcha.use-case.ts`) already uses
|
|
372
|
+
* this same string-template shape.
|
|
373
|
+
*
|
|
374
|
+
* @param attribute - The attribute to stamp; defaults to `HIDDEN_ATTRIBUTE` (parameterised for tests and for a future caller that wants a different name).
|
|
375
|
+
* @returns A self-invoking script, ready for `page.evaluate(hiddenMarksScript())`.
|
|
376
|
+
*/
|
|
377
|
+
function hiddenMarksScript(attribute = HIDDEN_ATTRIBUTE) {
|
|
378
|
+
return `(() => {
|
|
379
|
+
var attribute = ${JSON.stringify(attribute)}
|
|
380
|
+
var elements = document.querySelectorAll('*')
|
|
381
|
+
for (var i = 0; i < elements.length; i++) {
|
|
382
|
+
var element = elements[i]
|
|
383
|
+
var style = window.getComputedStyle(element)
|
|
384
|
+
var rect = element.getBoundingClientRect()
|
|
385
|
+
var hidden = style.display === 'none' || style.visibility === 'hidden' || (rect.width === 0 && rect.height === 0)
|
|
386
|
+
if (hidden) element.setAttribute(attribute, '1')
|
|
387
|
+
}
|
|
388
|
+
})()`;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** Attributes that carry a URL and are rewritten to absolute; `srcset` is handled separately (it carries several). */
|
|
392
|
+
const URL_ATTRIBUTES = ['href', 'src', 'action', 'poster'];
|
|
393
|
+
/** Schemes left untouched: they are not a location `new URL(value, base)` should resolve, or already carry their own meaning. */
|
|
394
|
+
const SKIP_SCHEME = /^(?:javascript|mailto|tel|data|about):/i;
|
|
395
|
+
/**
|
|
396
|
+
* Rewrites captured HTML into what the content pane's sandboxed iframe shows
|
|
397
|
+
* (studio plan §3.1, issue #91): every URL made absolute against `baseUrl`,
|
|
398
|
+
* a `<base>` injected, every `<script>` removed, forms disabled (`onsubmit`
|
|
399
|
+
* stripped, fields made read-only), `<meta http-equiv="refresh">` stripped
|
|
400
|
+
* (it would otherwise navigate the sandboxed frame on its own), and every
|
|
401
|
+
* element given a stable `data-oc-node` id, in document order, for the
|
|
402
|
+
* picker overlay and `selector-inference` to address specific nodes by.
|
|
403
|
+
*
|
|
404
|
+
* Purely a markup transform: it does not know whether an element is hidden
|
|
405
|
+
* on the live page (`display: none`, `visibility: hidden`, zero size) — in
|
|
406
|
+
* web mode that is `hidden-marks.algorithm.ts`'s job, run against the live
|
|
407
|
+
* page *before* this function ever sees the markup, so a `data-oc-hidden`
|
|
408
|
+
* attribute it wrote just survives serialisation untouched here.
|
|
409
|
+
*
|
|
410
|
+
* @param html - The captured document (`page.content()` in web mode, the fetched body in api mode).
|
|
411
|
+
* @param baseUrl - The page's own URL, for resolving relative links and as the `<base>`.
|
|
412
|
+
* @returns The rewritten document and how many elements it stamped.
|
|
413
|
+
*/
|
|
414
|
+
function rewriteDocument(html, baseUrl) {
|
|
415
|
+
const $ = load(html);
|
|
416
|
+
if ($('head').length === 0) $('html').prepend('<head></head>');
|
|
417
|
+
const head = $('head').first();
|
|
418
|
+
head.find('base').remove();
|
|
419
|
+
head.prepend(`<base href="${escapeAttribute(baseUrl)}">`);
|
|
420
|
+
$('meta').each((_index, element) => {
|
|
421
|
+
if (($(element).attr('http-equiv') ?? '').toLowerCase() === 'refresh') $(element).remove();
|
|
422
|
+
});
|
|
423
|
+
$('script').remove();
|
|
424
|
+
$('form').removeAttr('onsubmit');
|
|
425
|
+
$('input, textarea').attr('readonly', 'readonly');
|
|
426
|
+
$('select, button, input[type="submit"], input[type="button"]').attr('disabled', 'disabled');
|
|
427
|
+
$('*').each((_index, element) => {
|
|
428
|
+
const node = $(element);
|
|
429
|
+
for (const attribute of URL_ATTRIBUTES) {
|
|
430
|
+
const value = node.attr(attribute);
|
|
431
|
+
if (value !== undefined) node.attr(attribute, resolveUrl(value, baseUrl) ?? value);
|
|
432
|
+
}
|
|
433
|
+
const srcset = node.attr('srcset');
|
|
434
|
+
if (srcset !== undefined) node.attr('srcset', rewriteSrcset(srcset, baseUrl));
|
|
435
|
+
});
|
|
436
|
+
let nodeCount = 0;
|
|
437
|
+
$('*').each((_index, element) => {
|
|
438
|
+
$(element).attr(NODE_ID_ATTRIBUTE, `n${nodeCount}`);
|
|
439
|
+
nodeCount += 1;
|
|
440
|
+
});
|
|
441
|
+
return {
|
|
442
|
+
html: $.html(),
|
|
443
|
+
nodeCount
|
|
444
|
+
};
|
|
445
|
+
}
|
|
446
|
+
/** Resolves one URL against `baseUrl`; a scheme that is not a location (`javascript:`, `data:`…), or a value `new URL` cannot parse, is left as it was. */
|
|
447
|
+
function resolveUrl(value, baseUrl) {
|
|
448
|
+
const trimmed = value.trim();
|
|
449
|
+
if (trimmed === '' || SKIP_SCHEME.test(trimmed)) return undefined;
|
|
450
|
+
try {
|
|
451
|
+
return new URL(trimmed, baseUrl).href;
|
|
452
|
+
} catch {
|
|
453
|
+
return undefined;
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
/** `srcset`'s own micro-syntax: comma-separated `url descriptor?` pairs. Malformed entries are passed through unresolved rather than dropped. */
|
|
457
|
+
function rewriteSrcset(value, baseUrl) {
|
|
458
|
+
return value.split(',').map(candidate => {
|
|
459
|
+
const trimmed = candidate.trim();
|
|
460
|
+
const spaceAt = trimmed.indexOf(' ');
|
|
461
|
+
const url = spaceAt === -1 ? trimmed : trimmed.slice(0, spaceAt);
|
|
462
|
+
const descriptor = spaceAt === -1 ? '' : trimmed.slice(spaceAt);
|
|
463
|
+
return `${resolveUrl(url, baseUrl) ?? url}${descriptor}`;
|
|
464
|
+
}).join(', ');
|
|
465
|
+
}
|
|
466
|
+
function escapeAttribute(value) {
|
|
467
|
+
return value.replaceAll('&', '&').replaceAll('"', '"');
|
|
468
|
+
}
|
|
469
|
+
|
|
470
|
+
/**
|
|
471
|
+
* Captures the snapshot the content pane shows (studio plan §3.1, issue
|
|
472
|
+
* #91): runs the recipe up to `stepPath`, serialises the document, marks
|
|
473
|
+
* hidden elements in web mode, and hands it to `rewrite-document.mapper.ts`
|
|
474
|
+
* for display.
|
|
475
|
+
*
|
|
476
|
+
* `stepPath` is accepted (and cached against, by `snapshot.store.ts`) for
|
|
477
|
+
* the API the UI needs — one snapshot per selected step — but v1 only
|
|
478
|
+
* actually *captures* `"start"`: the first `start` point, before any step
|
|
479
|
+
* of the recipe has run, the same reach `fetch-start-page.use-case.ts` (phase
|
|
480
|
+
* 0) already had. Snapshotting mid-recipe (after an arbitrary later step)
|
|
481
|
+
* needs a "stop the crawl after step X and hand back the live page/document"
|
|
482
|
+
* hook in `crawl-execution` that does not exist yet; building one was out of
|
|
483
|
+
* this phase's budget (see this phase's final report for the tradeoff this
|
|
484
|
+
* was weighed against). Any other `stepPath` throws, clearly, rather than
|
|
485
|
+
* silently returning the start page under a misleading label.
|
|
486
|
+
*
|
|
487
|
+
* @param input - The input recipe; only `mode` and the first `start` point are used.
|
|
488
|
+
* @param stepPath - The step to snapshot after, e.g. `"start"` or `"steps.2"` (`scope-outline`'s path format).
|
|
489
|
+
* @param browser - Browser launch settings for web mode.
|
|
490
|
+
* @returns The rewritten snapshot.
|
|
491
|
+
* @throws Error when the recipe has no start point, or `stepPath` is not `"start"`.
|
|
492
|
+
*/
|
|
493
|
+
async function takeSnapshot(input, stepPath, browser) {
|
|
494
|
+
if (stepPath !== 'start') throw new Error(`takeSnapshot: "${stepPath}" is not supported yet — only "start" (the first start point, before any step runs) is; see this function's doc comment`);
|
|
495
|
+
const point = input.start[0];
|
|
496
|
+
if (point === undefined) throw new Error(`recipe "${input.id}" has no start point`);
|
|
497
|
+
return input.mode === 'web' ? snapshotWeb(point.url, browser) : snapshotApi(point.url);
|
|
498
|
+
}
|
|
499
|
+
async function snapshotWeb(url, browser) {
|
|
500
|
+
const client = await BrowserClient.launch(browser);
|
|
501
|
+
try {
|
|
502
|
+
const session = await client.newSession();
|
|
503
|
+
try {
|
|
504
|
+
await session.page.goto(url);
|
|
505
|
+
await session.page.evaluate(hiddenMarksScript(HIDDEN_ATTRIBUTE));
|
|
506
|
+
const html = await session.page.content();
|
|
507
|
+
const baseUrl = session.page.url();
|
|
508
|
+
const rewritten = rewriteDocument(html, baseUrl);
|
|
509
|
+
return {
|
|
510
|
+
...rewritten,
|
|
511
|
+
baseUrl
|
|
512
|
+
};
|
|
513
|
+
} finally {
|
|
514
|
+
await session.close();
|
|
515
|
+
}
|
|
516
|
+
} finally {
|
|
517
|
+
await client.close();
|
|
518
|
+
}
|
|
519
|
+
}
|
|
520
|
+
async function snapshotApi(url) {
|
|
521
|
+
const client = await HttpClient.open({});
|
|
522
|
+
try {
|
|
523
|
+
const response = await client.send({
|
|
524
|
+
url
|
|
525
|
+
});
|
|
526
|
+
const rewritten = rewriteDocument(textOf(response.body), url);
|
|
527
|
+
return {
|
|
528
|
+
...rewritten,
|
|
529
|
+
baseUrl: url
|
|
530
|
+
};
|
|
531
|
+
} finally {
|
|
532
|
+
await client.dispose();
|
|
533
|
+
}
|
|
534
|
+
}
|
|
535
|
+
/** Every body kind the studio can show as text today; others get a placeholder rather than a crash (mirrors `fetch-start-page.use-case.ts`'s own `textOf`). */
|
|
536
|
+
function textOf(body) {
|
|
537
|
+
if (body.kind === 'html') return body.html;
|
|
538
|
+
if (body.kind === 'text') return body.text;
|
|
539
|
+
if (body.kind === 'json') return `<pre>${JSON.stringify(body.data, null, 2)}</pre>`;
|
|
540
|
+
return `<p>[${body.kind} document: the content pane shows HTML and JSON responses for now]</p>`;
|
|
541
|
+
}
|
|
542
|
+
|
|
543
|
+
/** @returns An empty cache, for a freshly opened workspace. */
|
|
544
|
+
function createSnapshotCache() {
|
|
545
|
+
return {
|
|
546
|
+
entries: new Map()
|
|
547
|
+
};
|
|
548
|
+
}
|
|
549
|
+
/** @returns The cached snapshot for `recipeId`/`path`, or `undefined` when none was taken yet. */
|
|
550
|
+
function cachedSnapshot(cache, recipeId, path) {
|
|
551
|
+
return cache.entries.get(keyOf(recipeId, path));
|
|
552
|
+
}
|
|
553
|
+
/** Records a freshly taken snapshot, replacing whatever was cached for the same recipe and step path. */
|
|
554
|
+
function putSnapshot(cache, recipeId, path, snapshot) {
|
|
555
|
+
cache.entries.set(keyOf(recipeId, path), snapshot);
|
|
556
|
+
}
|
|
557
|
+
function keyOf(recipeId, path) {
|
|
558
|
+
return `${recipeId}:${path}`;
|
|
559
|
+
}
|
|
560
|
+
|
|
216
561
|
/**
|
|
217
562
|
* Finds and parses one input recipe of a workspace folder, and the output
|
|
218
563
|
* recipe it names.
|
|
@@ -254,10 +599,11 @@ function idOf(content) {
|
|
|
254
599
|
* @param recipeId - The input recipe to run.
|
|
255
600
|
* @param budget - The sample budget; unset runs the recipe to completion (still capped by its own `limits`).
|
|
256
601
|
* @param callbacks - Where trace lines and records go as the run proceeds.
|
|
602
|
+
* @param browser - Browser launch settings for a web-mode recipe (an executable path override, headless…); the server-wide setting `studio-http.use-case.ts`'s `StudioServerOptions.browser` carries in, e.g. `OPENCRAW_CHROMIUM` in a sandbox with no full Playwright install.
|
|
257
603
|
* @returns A handle: `stop` closes the crawler cleanly, `result` resolves when the run ends.
|
|
258
604
|
* @throws RecipeValidationError, RecipeBindingError, Error when the recipe or its output cannot be loaded or bound.
|
|
259
605
|
*/
|
|
260
|
-
async function runSample(folder, recipeId, budget, callbacks) {
|
|
606
|
+
async function runSample(folder, recipeId, budget, callbacks, browser) {
|
|
261
607
|
const {
|
|
262
608
|
input,
|
|
263
609
|
output
|
|
@@ -268,6 +614,7 @@ async function runSample(folder, recipeId, budget, callbacks) {
|
|
|
268
614
|
sink,
|
|
269
615
|
debug: true,
|
|
270
616
|
sample: budget,
|
|
617
|
+
browser,
|
|
271
618
|
onEvent: event => {
|
|
272
619
|
const line = traceLine(event);
|
|
273
620
|
if (line !== undefined) callbacks.onTraceLine(line);
|
|
@@ -320,12 +667,759 @@ async function handleFetchStartPage(state, command) {
|
|
|
320
667
|
const {
|
|
321
668
|
input
|
|
322
669
|
} = await loadRecipePair(state.folder, command.recipeId);
|
|
323
|
-
const html = await fetchStartPage(input);
|
|
670
|
+
const html = await fetchStartPage(input, state.browser);
|
|
324
671
|
return {
|
|
325
672
|
html
|
|
326
673
|
};
|
|
327
674
|
}
|
|
328
675
|
|
|
676
|
+
const GENERATED_ID = /^(?:react-|radix-|:r|ember|css-|mui-|headlessui-)/i;
|
|
677
|
+
/**
|
|
678
|
+
* The path from the document root to `node`, one level per ancestor. Two
|
|
679
|
+
* elements that share an ancestor produce identical levels up to that
|
|
680
|
+
* ancestor and diverge exactly where their parents stop being the same node
|
|
681
|
+
* — `list-inference.algorithm.ts` leans on that to find the nearest pair of
|
|
682
|
+
* siblings two picks belong to, without ever touching cheerio again.
|
|
683
|
+
*
|
|
684
|
+
* @param $ - The parsed document.
|
|
685
|
+
* @param node - The element to describe.
|
|
686
|
+
* @returns The path, root first.
|
|
687
|
+
*/
|
|
688
|
+
function domPathOf($, node) {
|
|
689
|
+
const chain = [];
|
|
690
|
+
let current = node;
|
|
691
|
+
while (current !== undefined && current.length > 0 && isElement(current.get(0))) {
|
|
692
|
+
chain.unshift(current);
|
|
693
|
+
const parent = current.parent();
|
|
694
|
+
current = parent.length > 0 ? parent : undefined;
|
|
695
|
+
}
|
|
696
|
+
return chain.map(element => levelOf($, element));
|
|
697
|
+
}
|
|
698
|
+
function levelOf($, element) {
|
|
699
|
+
const node = element.get(0);
|
|
700
|
+
const tag = node.tagName.toLowerCase();
|
|
701
|
+
const attrs = {};
|
|
702
|
+
const entries = Object.entries(node.attribs ?? {});
|
|
703
|
+
for (const [name, value] of entries) attrs[name] = value;
|
|
704
|
+
const classes = (attrs.class ?? '').split(/\s+/).filter(token => token.length > 0);
|
|
705
|
+
const siblings = element.parent().length > 0 ? element.parent().children(tag) : $.root().children(tag);
|
|
706
|
+
const index = siblings.toArray().indexOf(node) + 1;
|
|
707
|
+
return {
|
|
708
|
+
tag,
|
|
709
|
+
id: attrs.id,
|
|
710
|
+
classes,
|
|
711
|
+
attrs,
|
|
712
|
+
index: Math.max(index, 1),
|
|
713
|
+
siblingCount: Math.max(siblings.length, 1)
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
function isElement(node) {
|
|
717
|
+
return node !== undefined && node.type === 'tag';
|
|
718
|
+
}
|
|
719
|
+
/**
|
|
720
|
+
* Whether `id` looks handed by a framework (`react-`, a CSS-module or a
|
|
721
|
+
* `radix-`/`mui-` autogenerated one) rather than by the site's own markup —
|
|
722
|
+
* `class-token.policy.ts` has the equivalent check for class tokens, kept
|
|
723
|
+
* separate because ids and classes are ranked at different tiers.
|
|
724
|
+
*/
|
|
725
|
+
function isStableId(id) {
|
|
726
|
+
return id.length > 0 && !GENERATED_ID.test(id);
|
|
727
|
+
}
|
|
728
|
+
/**
|
|
729
|
+
* Whether `path` failed to reach the real document root. `domPathOf` walks
|
|
730
|
+
* up through cheerio's own `.parent()`, which (matching real DOM semantics)
|
|
731
|
+
* never crosses from a `<template>`'s content into the light DOM outside
|
|
732
|
+
* it — a `<template shadowrootmode>` is exactly declarative shadow DOM, the
|
|
733
|
+
* only shadow-DOM shape that can show up in static, server-captured markup
|
|
734
|
+
* at all (imperative `attachShadow()` content never serialises into
|
|
735
|
+
* `page.content()`/a fetched response in the first place, so that case
|
|
736
|
+
* cannot reach this slice and needs no separate detection). A path whose
|
|
737
|
+
* first level is not `html` was cut short at exactly this kind of boundary:
|
|
738
|
+
* v1 does not pierce it, so callers report a clear "shadow DOM is not
|
|
739
|
+
* supported" error instead of building ancestor-based candidates over a
|
|
740
|
+
* chain that stops partway up.
|
|
741
|
+
*/
|
|
742
|
+
function crossesShadowRoot(path) {
|
|
743
|
+
return path.length === 0 || path[0].tag !== 'html';
|
|
744
|
+
}
|
|
745
|
+
/** Parses `html` and returns cheerio's API, for callers that need more than one path from the same document. */
|
|
746
|
+
function parseDocument$1(html) {
|
|
747
|
+
return load(html);
|
|
748
|
+
}
|
|
749
|
+
/**
|
|
750
|
+
* The path to the element the snapshot stamped with `nodeId` (its
|
|
751
|
+
* `data-oc-node` attribute — `@opencraw/core`'s `NODE_ID_ATTRIBUTE`,
|
|
752
|
+
* `packages/studio/src/page-snapshot/rewrite-document.mapper.ts`). This is
|
|
753
|
+
* how a click in the content pane's iframe — which only ever needs to read
|
|
754
|
+
* that one attribute off the element the user clicked — turns into
|
|
755
|
+
* something `selector-inference`'s other algorithms can work with, without
|
|
756
|
+
* the browser-side picker script needing to know anything about ancestry,
|
|
757
|
+
* classes or candidate ranking itself.
|
|
758
|
+
*
|
|
759
|
+
* @param html - The snapshot document (already stamped with node ids).
|
|
760
|
+
* @param nodeId - The `data-oc-node` value to find.
|
|
761
|
+
* @returns The node's path, or `undefined` when no element carries that id.
|
|
762
|
+
*/
|
|
763
|
+
function pathToNode(html, nodeId) {
|
|
764
|
+
const $ = parseDocument$1(html);
|
|
765
|
+
const found = $(`[${NODE_ID_ATTRIBUTE}="${cssAttributeValue(nodeId)}"]`);
|
|
766
|
+
return found.length === 0 ? undefined : domPathOf($, found.first());
|
|
767
|
+
}
|
|
768
|
+
function cssAttributeValue(value) {
|
|
769
|
+
return value.replaceAll('\\', String.raw`\\`).replaceAll('"', String.raw`\"`);
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
/** Attributes worth reading instead of text, in preference order, when the clicked element has one. */
|
|
773
|
+
const PREFERRED_ATTRS = {
|
|
774
|
+
a: ['href'],
|
|
775
|
+
img: ['src', 'alt']
|
|
776
|
+
};
|
|
777
|
+
/**
|
|
778
|
+
* What a `Read` card's `take` should be for the element a pick landed on
|
|
779
|
+
* (issue #91, studio plan §3.2): `attr:href` for an `<a>`, `attr:src` for an
|
|
780
|
+
* `<img>`, `attr:<name>` for the first `data-*` attribute otherwise, `json`
|
|
781
|
+
* for a `<script type="application/json">`, else the default, `text`.
|
|
782
|
+
*
|
|
783
|
+
* @param level - The clicked element's own level (the last entry of its `DomPath`).
|
|
784
|
+
* @returns The `take` value to put on the `extract` step.
|
|
785
|
+
*/
|
|
786
|
+
function takeKindFor(level) {
|
|
787
|
+
if (level.tag === 'script' && (level.attrs.type ?? '').toLowerCase() === 'application/json') return 'json';
|
|
788
|
+
const preferred = PREFERRED_ATTRS[level.tag];
|
|
789
|
+
if (preferred !== undefined) {
|
|
790
|
+
const attribute = preferred.find(name => level.attrs[name] !== undefined);
|
|
791
|
+
if (attribute !== undefined) return `attr:${attribute}`;
|
|
792
|
+
}
|
|
793
|
+
const dataAttribute = Object.keys(level.attrs).find(name => name.startsWith('data-') && name !== 'data-oc-node' && name !== 'data-oc-hidden');
|
|
794
|
+
if (dataAttribute !== undefined) return `attr:${dataAttribute}`;
|
|
795
|
+
return 'text';
|
|
796
|
+
}
|
|
797
|
+
|
|
798
|
+
/**
|
|
799
|
+
* Which class tokens are worth building a selector on. A hand-authored class
|
|
800
|
+
* (`product_pod`, `price_color`) survives a rebuild; a generated one
|
|
801
|
+
* (a content hash, a CSS-module suffix, a styled-components token) does not,
|
|
802
|
+
* so a candidate built on it looks fine on the snapshot it was picked from
|
|
803
|
+
* and breaks on the very next deploy. This is the "generated/hashed class
|
|
804
|
+
* names" tradeoff issue #91 calls out explicitly: the heuristics below are a
|
|
805
|
+
* reasoned judgement call, not a guarantee — see the doc comments on each
|
|
806
|
+
* pattern for what real-world tool it targets.
|
|
807
|
+
*/
|
|
808
|
+
const GENERATED_TOKEN_PATTERNS = [/** styled-components / emotion: `sc-bdVaJa`, `css-1x2y3z4` */
|
|
809
|
+
/^(?:sc|css)-[a-z0-9]+$/i, /** CSS Modules, most bundlers' default hash suffix: `wrapper__3fA9k`, `Button-module_root__aB3dK` */
|
|
810
|
+
/__[a-z0-9]{4,10}$/i, /** JSS / MUI: `jss123`, `MuiButton-root-145` */
|
|
811
|
+
/^(?:jss|mui[a-z]*-)[a-z0-9-]*\d{2}$/i, /** A token that is only digits: not a class a human would type. */
|
|
812
|
+
/^\d+$/, /** A bare hex hash: `a3f9c21`, `1a2b3c4d` (6+ hex chars, nothing else). */
|
|
813
|
+
/^[0-9a-f]{6,}$/i, /** Tailwind/atomic-CSS-in-JS runtime classes some bundlers emit: `_1a2b3c` */
|
|
814
|
+
/^_[0-9a-z]{5,}$/i];
|
|
815
|
+
/**
|
|
816
|
+
* Whether a class token looks hand-authored and safe to build a selector on.
|
|
817
|
+
*
|
|
818
|
+
* @param token - One class, no surrounding whitespace.
|
|
819
|
+
* @returns `false` for anything matching a known generated-class shape.
|
|
820
|
+
*/
|
|
821
|
+
function isStableClassToken(token) {
|
|
822
|
+
return token.length > 0 && GENERATED_TOKEN_PATTERNS.every(pattern => !pattern.test(token));
|
|
823
|
+
}
|
|
824
|
+
/**
|
|
825
|
+
* The subset of `tokens` worth ranking a selector candidate on, order kept.
|
|
826
|
+
*
|
|
827
|
+
* @param tokens - An element's class list.
|
|
828
|
+
* @returns Only the stable tokens.
|
|
829
|
+
*/
|
|
830
|
+
function stableClasses(tokens) {
|
|
831
|
+
return tokens.filter(token => isStableClassToken(token));
|
|
832
|
+
}
|
|
833
|
+
/**
|
|
834
|
+
* Whether two class lists describe "the same kind of element" for list
|
|
835
|
+
* inference: same stable classes, generated tokens (which often differ
|
|
836
|
+
* per-instance, e.g. a React `key`-derived class) ignored on both sides.
|
|
837
|
+
*
|
|
838
|
+
* @param a - One element's classes.
|
|
839
|
+
* @param b - Another element's classes.
|
|
840
|
+
* @returns Whether their stable class sets are equal, order ignored.
|
|
841
|
+
*/
|
|
842
|
+
function sameClassSignature(a, b) {
|
|
843
|
+
const left = new Set(stableClasses(a));
|
|
844
|
+
const right = new Set(stableClasses(b));
|
|
845
|
+
if (left.size !== right.size) return false;
|
|
846
|
+
return [...left].every(token => right.has(token));
|
|
847
|
+
}
|
|
848
|
+
|
|
849
|
+
/**
|
|
850
|
+
* Thrown when a pick or a list-inference walk crosses a declarative shadow
|
|
851
|
+
* root (`<template shadowrootmode>`). v1 does not pierce shadow DOM: rather
|
|
852
|
+
* than infer a selector that silently matches the wrong thing (or nothing,
|
|
853
|
+
* once the page hydrates), the studio surfaces this and stops.
|
|
854
|
+
*/
|
|
855
|
+
class ShadowDomUnsupportedError extends Error {
|
|
856
|
+
path;
|
|
857
|
+
constructor(path) {
|
|
858
|
+
super(`"${path}" is inside a shadow root: shadow DOM is not supported for picking (v1)`);
|
|
859
|
+
this.path = path;
|
|
860
|
+
this.name = 'ShadowDomUnsupportedError';
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
/** Attributes ranked just under `id`: named explicitly, not "any `data-*`", because a generated `data-reactid` is no more stable than a generated class. */
|
|
865
|
+
const STABLE_ATTRS = ['data-testid', 'data-id', 'itemprop'];
|
|
866
|
+
const TIER_ORDER$1 = {
|
|
867
|
+
'id': 0,
|
|
868
|
+
'data-attr': 1,
|
|
869
|
+
'class': 2,
|
|
870
|
+
'structure': 3,
|
|
871
|
+
'position': 4
|
|
872
|
+
};
|
|
873
|
+
/**
|
|
874
|
+
* Every selector candidate worth trying for the last node of `path`, ranked
|
|
875
|
+
* best-first: `id` > stable `data-*`/`itemprop` attributes > stable class
|
|
876
|
+
* tokens > ancestor structure (`parent > tag`) > position (`nth-of-type`) as
|
|
877
|
+
* a last resort (issue #91's ranking, studio plan §3.2). Structure and
|
|
878
|
+
* position candidates only look as far up as `path` itself goes — pass a
|
|
879
|
+
* sub-path (e.g. from an item's root down) to keep a field's selector
|
|
880
|
+
* relative to its item instead of absolute to the document.
|
|
881
|
+
*
|
|
882
|
+
* @param path - The node's ancestry, root (or scope root) first, itself last.
|
|
883
|
+
* @param includePosition - `false` for an item selector: a position candidate would only ever match one of the items, which is never what an item selector should do.
|
|
884
|
+
* @returns Candidates, best tier first; ties keep this function's own order.
|
|
885
|
+
*/
|
|
886
|
+
function candidatesFor(path, includePosition = true) {
|
|
887
|
+
const node = path.at(-1);
|
|
888
|
+
if (node === undefined) return [];
|
|
889
|
+
const candidates = [];
|
|
890
|
+
if (node.id !== undefined && isStableId(node.id)) candidates.push({
|
|
891
|
+
selector: `#${cssEscape(node.id)}`,
|
|
892
|
+
tier: 'id'
|
|
893
|
+
});
|
|
894
|
+
for (const name of STABLE_ATTRS) {
|
|
895
|
+
const value = node.attrs[name];
|
|
896
|
+
if (value !== undefined && value.length > 0) candidates.push({
|
|
897
|
+
selector: `${node.tag}[${name}="${cssValue(value)}"]`,
|
|
898
|
+
tier: 'data-attr'
|
|
899
|
+
});
|
|
900
|
+
}
|
|
901
|
+
const classes = stableClasses(node.classes);
|
|
902
|
+
if (classes.length > 0) candidates.push({
|
|
903
|
+
selector: `${node.tag}.${classes.map(token => cssEscape(token)).join('.')}`,
|
|
904
|
+
tier: 'class'
|
|
905
|
+
});
|
|
906
|
+
candidates.push(...structureCandidates(path));
|
|
907
|
+
if (includePosition) candidates.push(...positionCandidates(path));
|
|
908
|
+
return sortByTier(candidates);
|
|
909
|
+
}
|
|
910
|
+
/** `ancestor > ... > tag`, growing from 2 to 4 levels (or to the top of `path`), each ancestor named by its own best simple form. */
|
|
911
|
+
function structureCandidates(path) {
|
|
912
|
+
const node = path.at(-1);
|
|
913
|
+
if (node === undefined) return [];
|
|
914
|
+
const candidates = [];
|
|
915
|
+
const maxDepth = Math.min(path.length - 1, 3);
|
|
916
|
+
for (let depth = 1; depth <= maxDepth; depth++) {
|
|
917
|
+
const ancestor = path.at(-1 - depth);
|
|
918
|
+
if (ancestor === undefined) break;
|
|
919
|
+
const between = Array.from({
|
|
920
|
+
length: depth - 1
|
|
921
|
+
}, (_, index) => path.at(-depth + index)?.tag ?? '');
|
|
922
|
+
const parts = [simpleSelector(ancestor), ...between, node.tag];
|
|
923
|
+
candidates.push({
|
|
924
|
+
selector: parts.join(' > '),
|
|
925
|
+
tier: 'structure'
|
|
926
|
+
});
|
|
927
|
+
}
|
|
928
|
+
return candidates;
|
|
929
|
+
}
|
|
930
|
+
/** `parent > tag:nth-of-type(n)`, the position-only fallback; never `nth-child`, which counts every sibling tag, not just this element's own. */
|
|
931
|
+
function positionCandidates(path) {
|
|
932
|
+
const node = path.at(-1);
|
|
933
|
+
if (node === undefined) return [];
|
|
934
|
+
const own = node.siblingCount > 1 ? `${node.tag}:nth-of-type(${node.index})` : node.tag;
|
|
935
|
+
const parent = path.at(-2);
|
|
936
|
+
if (parent === undefined) return [{
|
|
937
|
+
selector: own,
|
|
938
|
+
tier: 'position'
|
|
939
|
+
}];
|
|
940
|
+
return [{
|
|
941
|
+
selector: `${simpleSelector(parent)} > ${own}`,
|
|
942
|
+
tier: 'position'
|
|
943
|
+
}];
|
|
944
|
+
}
|
|
945
|
+
/** A level's own selector fragment, no combinators: id, else stable classes, else the bare tag. */
|
|
946
|
+
function simpleSelector(level) {
|
|
947
|
+
if (level.id !== undefined && isStableId(level.id)) return `#${cssEscape(level.id)}`;
|
|
948
|
+
const classes = stableClasses(level.classes);
|
|
949
|
+
return classes.length > 0 ? `${level.tag}.${classes.map(token => cssEscape(token)).join('.')}` : level.tag;
|
|
950
|
+
}
|
|
951
|
+
function sortByTier(candidates) {
|
|
952
|
+
return [...candidates].sort((a, b) => TIER_ORDER$1[a.tier] - TIER_ORDER$1[b.tier]);
|
|
953
|
+
}
|
|
954
|
+
/** Escapes a CSS identifier segment (an id or a class token) well enough for the shapes real sites use; not a full CSS.escape polyfill. */
|
|
955
|
+
function cssEscape(token) {
|
|
956
|
+
return token.replaceAll(/([^\w-])/gi, String.raw`\$1`);
|
|
957
|
+
}
|
|
958
|
+
/** Escapes a value inside `[name="…"]`. */
|
|
959
|
+
function cssValue(value) {
|
|
960
|
+
return value.replaceAll(/(["\\])/g, String.raw`\$1`);
|
|
961
|
+
}
|
|
962
|
+
|
|
963
|
+
const TIER_ORDER = {
|
|
964
|
+
'id': 0,
|
|
965
|
+
'data-attr': 1,
|
|
966
|
+
'class': 2,
|
|
967
|
+
'structure': 3,
|
|
968
|
+
'position': 4
|
|
969
|
+
};
|
|
970
|
+
/**
|
|
971
|
+
* Verifies and orders selector candidates against real markup, by running
|
|
972
|
+
* each one through the engine's own CSS matching (`@opencraw/core`'s
|
|
973
|
+
* `countMatches`/`matchesOf`) rather than reimplementing it — issue #91's
|
|
974
|
+
* "every candidate is verified" requirement.
|
|
975
|
+
*
|
|
976
|
+
* Ordering: candidates that do not match `targetNodeId` (when given) sort
|
|
977
|
+
* last; among the rest, best tier first, and within a tier the fewest
|
|
978
|
+
* matches first (the most specific selector that still hits the target).
|
|
979
|
+
* A candidate matching nothing is dropped entirely: it is not a real option.
|
|
980
|
+
*
|
|
981
|
+
* @param html - The document to verify against (the snapshot, or the live page for the second, cross-check pass).
|
|
982
|
+
* @param candidates - Unranked candidates, e.g. from `candidatesFor`.
|
|
983
|
+
* @param targetNodeId - The `data-oc-node` id the pick landed on, when the document carries those stamps (the snapshot does; a live page fetched separately does not).
|
|
984
|
+
* @returns Every candidate that matched at least one element, ranked best first.
|
|
985
|
+
*/
|
|
986
|
+
function rankCandidates(html, candidates, targetNodeId) {
|
|
987
|
+
const ranked = candidates.map(candidate => {
|
|
988
|
+
const matches = countMatches(html, candidate.selector);
|
|
989
|
+
const hitsTarget = targetNodeId === undefined ;
|
|
990
|
+
return {
|
|
991
|
+
...candidate,
|
|
992
|
+
matches,
|
|
993
|
+
hitsTarget
|
|
994
|
+
};
|
|
995
|
+
});
|
|
996
|
+
return ranked.filter(candidate => candidate.matches > 0).sort((a, b) => Number(b.hitsTarget) - Number(a.hitsTarget) || TIER_ORDER[a.tier] - TIER_ORDER[b.tier] || a.matches - b.matches);
|
|
997
|
+
}
|
|
998
|
+
/** The single best candidate, or `undefined` when none of them matched anything. */
|
|
999
|
+
function bestCandidate(html, candidates, targetNodeId) {
|
|
1000
|
+
return rankCandidates(html, candidates, targetNodeId)[0];
|
|
1001
|
+
}
|
|
1002
|
+
|
|
1003
|
+
/**
|
|
1004
|
+
* Infers "these two picks are the same field on two items of a list" from
|
|
1005
|
+
* two node paths, by walking up from both to the nearest ancestors that are
|
|
1006
|
+
* real siblings (the same parent) with the same tag and stable-class
|
|
1007
|
+
* signature — studio plan §3.2, issue #91. That ancestor is the item; the
|
|
1008
|
+
* remainder of each path is the field, relative to it.
|
|
1009
|
+
*
|
|
1010
|
+
* Because a `DomPath` is built root-down (`dom-path.model.ts`), two paths
|
|
1011
|
+
* from the same document are byte-for-byte identical up to their real
|
|
1012
|
+
* common ancestor and diverge exactly at its differing children — so the
|
|
1013
|
+
* *first* level where they differ is always the nearest-sibling test point;
|
|
1014
|
+
* there is no need to look further down. This also makes the "wrapper trap"
|
|
1015
|
+
* (an extra single-child element around each item) resolve itself: a
|
|
1016
|
+
* wrapper's own instances are never one another's siblings, only the
|
|
1017
|
+
* genuine repeating element is, so the walk naturally lands there instead of
|
|
1018
|
+
* on the wrapper (see the wrapper-trap fixture in this slice's tests).
|
|
1019
|
+
*
|
|
1020
|
+
* @param pathA - The first click's path.
|
|
1021
|
+
* @param pathB - The second click's path.
|
|
1022
|
+
* @returns The item and field shapes (generalised) plus each pick's raw path, or `null` when the two picks are not on a coherent list (different branches entirely, the same node twice, or one path nested inside the other).
|
|
1023
|
+
* @throws ShadowDomUnsupportedError when either path crosses a declarative shadow root.
|
|
1024
|
+
*/
|
|
1025
|
+
function inferList(pathA, pathB) {
|
|
1026
|
+
if (crossesShadowRoot(pathA)) throw new ShadowDomUnsupportedError(pathToString(pathA));
|
|
1027
|
+
if (crossesShadowRoot(pathB)) throw new ShadowDomUnsupportedError(pathToString(pathB));
|
|
1028
|
+
let divergedAt = -1;
|
|
1029
|
+
const limit = Math.min(pathA.length, pathB.length);
|
|
1030
|
+
for (let index = 0; index < limit; index++) {
|
|
1031
|
+
if (!sameLevel(pathA[index], pathB[index])) {
|
|
1032
|
+
divergedAt = index;
|
|
1033
|
+
break;
|
|
1034
|
+
}
|
|
1035
|
+
}
|
|
1036
|
+
// Identical paths, or one a strict prefix of the other (the same node picked twice, or one element nested inside the other): not a list.
|
|
1037
|
+
if (divergedAt <= 0) return null;
|
|
1038
|
+
const branchA = pathA[divergedAt];
|
|
1039
|
+
const branchB = pathB[divergedAt];
|
|
1040
|
+
if (branchA.tag !== branchB.tag || !sameClassSignature(branchA.classes, branchB.classes)) return null;
|
|
1041
|
+
if (branchA.index === branchB.index) return null; // Same position: not two different items.
|
|
1042
|
+
const ancestors = pathA.slice(0, divergedAt);
|
|
1043
|
+
const fieldPathA = pathA.slice(divergedAt);
|
|
1044
|
+
const fieldPathB = pathB.slice(divergedAt);
|
|
1045
|
+
const fieldPath = mergeRelativePaths(fieldPathA, fieldPathB);
|
|
1046
|
+
return {
|
|
1047
|
+
itemPath: [...ancestors, fieldPath[0]],
|
|
1048
|
+
fieldPath,
|
|
1049
|
+
fieldPathA,
|
|
1050
|
+
fieldPathB
|
|
1051
|
+
};
|
|
1052
|
+
}
|
|
1053
|
+
function sameLevel(a, b) {
|
|
1054
|
+
return a.tag === b.tag && a.id === b.id && a.index === b.index && a.siblingCount === b.siblingCount && sameClassSignature(a.classes, b.classes);
|
|
1055
|
+
}
|
|
1056
|
+
/**
|
|
1057
|
+
* Zips two item-relative paths (item first) level by level into one that
|
|
1058
|
+
* only keeps what both share: the tag (must already match, checked by the
|
|
1059
|
+
* caller for level 0), the intersection of stable classes, and attributes
|
|
1060
|
+
* present with the same value on both sides. Shorter than the other when
|
|
1061
|
+
* the two picks' local structure differs in depth (a real possibility once
|
|
1062
|
+
* a field is missing on one item; `list-inference.algorithm.test.ts`'s
|
|
1063
|
+
* shifted-field fixture is exactly this) — the merge just stops there,
|
|
1064
|
+
* which is fine: a field selector only needs to reach as deep as the
|
|
1065
|
+
* shallower of the two picks.
|
|
1066
|
+
*/
|
|
1067
|
+
function mergeRelativePaths(a, b) {
|
|
1068
|
+
const length = Math.min(a.length, b.length);
|
|
1069
|
+
const merged = [];
|
|
1070
|
+
for (let index = 0; index < length; index++) {
|
|
1071
|
+
const left = a[index];
|
|
1072
|
+
const right = b[index];
|
|
1073
|
+
if (left.tag !== right.tag) break;
|
|
1074
|
+
merged.push(mergeLevel(left, right));
|
|
1075
|
+
}
|
|
1076
|
+
return merged.length > 0 ? merged : [a[0]];
|
|
1077
|
+
}
|
|
1078
|
+
function mergeLevel(a, b) {
|
|
1079
|
+
const classes = stableClasses(a.classes).filter(token => b.classes.includes(token));
|
|
1080
|
+
const attrs = {};
|
|
1081
|
+
for (const [name, value] of Object.entries(a.attrs)) {
|
|
1082
|
+
if (name !== 'class' && name !== 'id' && b.attrs[name] === value) attrs[name] = value;
|
|
1083
|
+
}
|
|
1084
|
+
return {
|
|
1085
|
+
tag: a.tag,
|
|
1086
|
+
id: a.id !== undefined && a.id === b.id ? a.id : undefined,
|
|
1087
|
+
classes,
|
|
1088
|
+
attrs,
|
|
1089
|
+
index: 1,
|
|
1090
|
+
siblingCount: Math.max(a.siblingCount, b.siblingCount, 1)
|
|
1091
|
+
};
|
|
1092
|
+
}
|
|
1093
|
+
function pathToString(path) {
|
|
1094
|
+
return path.map(level => level.tag + (level.id === undefined ? '' : `#${level.id}`)).join(' > ');
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
/**
|
|
1098
|
+
* Handles `infer-selector`: turns one or two picked nodes (their
|
|
1099
|
+
* `data-oc-node` ids, off the cached snapshot) into a verified selector —
|
|
1100
|
+
* one id for a `Read` card's field, two for the safe item+field list shape
|
|
1101
|
+
* `selector-inference`'s `inferList` builds by construction (issue #91,
|
|
1102
|
+
* studio plan §3.2). All the actual candidate/ranking/list-inference work
|
|
1103
|
+
* is `selector-inference`'s (pure); this handler only locates the clicked
|
|
1104
|
+
* nodes in the cached document and calls into it.
|
|
1105
|
+
*
|
|
1106
|
+
* @param state - The server state.
|
|
1107
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
1108
|
+
* @returns A field pick, a list pick, or `"unsupported"` with why.
|
|
1109
|
+
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
1110
|
+
*/
|
|
1111
|
+
function handleInferSelector(state, command) {
|
|
1112
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
1113
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
1114
|
+
const html = snapshot.html;
|
|
1115
|
+
const paths = command.nodeIds.map(nodeId => pathToNode(html, nodeId));
|
|
1116
|
+
if (paths.includes(undefined)) return {
|
|
1117
|
+
kind: 'unsupported',
|
|
1118
|
+
reason: 'one of the picked nodes is no longer in the snapshot; take a fresh snapshot and try again'
|
|
1119
|
+
};
|
|
1120
|
+
const [pathA, pathB] = paths;
|
|
1121
|
+
if (pathB === undefined) return {
|
|
1122
|
+
kind: 'field',
|
|
1123
|
+
field: fieldPick(html, pathA)
|
|
1124
|
+
};
|
|
1125
|
+
try {
|
|
1126
|
+
const result = inferList(pathA, pathB);
|
|
1127
|
+
if (result === null) return {
|
|
1128
|
+
kind: 'unsupported',
|
|
1129
|
+
reason: 'these two picks do not belong to the same list: pick the same field on two similar items'
|
|
1130
|
+
};
|
|
1131
|
+
const item = bestCandidate(html, candidatesFor(result.itemPath, false));
|
|
1132
|
+
if (item === undefined) return {
|
|
1133
|
+
kind: 'unsupported',
|
|
1134
|
+
reason: 'no selector matches every item of this list; try picking two items closer to each other in the markup'
|
|
1135
|
+
};
|
|
1136
|
+
const field = fieldPick(html, result.fieldPathA, result.fieldPath);
|
|
1137
|
+
return {
|
|
1138
|
+
kind: 'list',
|
|
1139
|
+
item: {
|
|
1140
|
+
selector: item.selector,
|
|
1141
|
+
tier: item.tier,
|
|
1142
|
+
matches: item.matches
|
|
1143
|
+
},
|
|
1144
|
+
field
|
|
1145
|
+
};
|
|
1146
|
+
} catch (error) {
|
|
1147
|
+
if (error instanceof ShadowDomUnsupportedError) return {
|
|
1148
|
+
kind: 'unsupported',
|
|
1149
|
+
reason: error.message
|
|
1150
|
+
};
|
|
1151
|
+
throw error;
|
|
1152
|
+
}
|
|
1153
|
+
}
|
|
1154
|
+
/** The best selector for one clicked node, `take` decided from the concrete clicked element; `rankSource` (a generalised path) is used for candidate generation when given, so a per-item-unique attribute of the one node picked cannot end up in the selector. */
|
|
1155
|
+
function fieldPick(html, clicked, rankSource = clicked) {
|
|
1156
|
+
const leaf = clicked.at(-1);
|
|
1157
|
+
const take = leaf === undefined ? 'text' : takeKindFor(leaf);
|
|
1158
|
+
const best = bestCandidate(html, candidatesFor(rankSource));
|
|
1159
|
+
if (best === undefined) throw new Error('no selector matches the picked node — this should not happen, since the node came from this same document');
|
|
1160
|
+
return {
|
|
1161
|
+
selector: best.selector,
|
|
1162
|
+
tier: best.tier,
|
|
1163
|
+
take,
|
|
1164
|
+
matches: best.matches
|
|
1165
|
+
};
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
/**
|
|
1169
|
+
* The outline model: an `InputRecipe`'s `steps` (and only `steps`; the rest
|
|
1170
|
+
* of the recipe passes through untouched, see `recipe-to-outline.mapper.ts`)
|
|
1171
|
+
* read as a tree of cards and brackets instead of raw JSON.
|
|
1172
|
+
*
|
|
1173
|
+
* A **card** is one step that reads as a sentence (`sentence`, built by
|
|
1174
|
+
* `card-sentence.mapper.ts`). A **bracket** is a container step (`forEach`,
|
|
1175
|
+
* `paginate`, `if`) whose body is itself a list of cards/brackets: the
|
|
1176
|
+
* indentation the plan doc describes is this nesting, not a separate layout
|
|
1177
|
+
* concept. Both kinds carry `step`, the step's own raw JSON exactly as
|
|
1178
|
+
* parsed (every key, in its original order): `outline-to-recipe.mapper.ts`
|
|
1179
|
+
* rebuilds the recipe from it, substituting only the (possibly edited)
|
|
1180
|
+
* `children`/`elseChildren` for whatever `steps`/`else` value `step` itself
|
|
1181
|
+
* holds, so an edit to one field never disturbs another's position or a
|
|
1182
|
+
* step type this outline cannot make sense of.
|
|
1183
|
+
*/
|
|
1184
|
+
/** The step types the outline renders as a bracket: their body is nested steps, not a single value. */
|
|
1185
|
+
const BRACKET_STEP_TYPES = ['forEach', 'paginate', 'if'];
|
|
1186
|
+
function isBracket(node) {
|
|
1187
|
+
return node.kind === 'bracket';
|
|
1188
|
+
}
|
|
1189
|
+
function isBracketStepType(type) {
|
|
1190
|
+
return BRACKET_STEP_TYPES.includes(type);
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
/** Step types the plan doc names as always going to a "custom" card: nothing here renders their meaning as a sentence. */
|
|
1194
|
+
const ALWAYS_CUSTOM = new Set(['evaluate', 'hook', 'captcha']);
|
|
1195
|
+
/**
|
|
1196
|
+
* Turns one step's raw JSON into its sentence parts (word/pill/code), the
|
|
1197
|
+
* way the plan doc's examples read: *Go to…*, *Read… from…*, *For each… in…
|
|
1198
|
+
* → one record*, and so on. Never throws: a step type this cannot read
|
|
1199
|
+
* becomes a short, honest fallback with `custom: true`, so the outline never
|
|
1200
|
+
* drops a step for looking unfamiliar.
|
|
1201
|
+
*
|
|
1202
|
+
* @param step - A step's raw JSON (its `type` and whatever fields it has).
|
|
1203
|
+
* @returns The sentence parts, and whether they are a real reading or a fallback.
|
|
1204
|
+
*/
|
|
1205
|
+
function cardSentence(step) {
|
|
1206
|
+
const type = typeof step.type === 'string' ? step.type : undefined;
|
|
1207
|
+
if (type === undefined || ALWAYS_CUSTOM.has(type)) return fallback(type);
|
|
1208
|
+
const build = BUILDERS[type];
|
|
1209
|
+
if (build === undefined) return fallback(type);
|
|
1210
|
+
try {
|
|
1211
|
+
return {
|
|
1212
|
+
parts: build(step),
|
|
1213
|
+
custom: false
|
|
1214
|
+
};
|
|
1215
|
+
} catch {
|
|
1216
|
+
return fallback(type);
|
|
1217
|
+
}
|
|
1218
|
+
}
|
|
1219
|
+
function fallback(type) {
|
|
1220
|
+
return {
|
|
1221
|
+
parts: [word(type ?? 'custom step')],
|
|
1222
|
+
custom: true
|
|
1223
|
+
};
|
|
1224
|
+
}
|
|
1225
|
+
function word(text) {
|
|
1226
|
+
return {
|
|
1227
|
+
kind: 'word',
|
|
1228
|
+
text
|
|
1229
|
+
};
|
|
1230
|
+
}
|
|
1231
|
+
function pill(text) {
|
|
1232
|
+
return {
|
|
1233
|
+
kind: 'pill',
|
|
1234
|
+
text
|
|
1235
|
+
};
|
|
1236
|
+
}
|
|
1237
|
+
function code(text) {
|
|
1238
|
+
return {
|
|
1239
|
+
kind: 'code',
|
|
1240
|
+
text
|
|
1241
|
+
};
|
|
1242
|
+
}
|
|
1243
|
+
/** A field that names an id: shown as a pill, `fallbackText` when it is absent. */
|
|
1244
|
+
function pillField(step, key, fallbackText) {
|
|
1245
|
+
const value = step[key];
|
|
1246
|
+
return pill(typeof value === 'string' && value.length > 0 ? value : fallbackText);
|
|
1247
|
+
}
|
|
1248
|
+
/** A field shown as a literal (a selector, a template, a value): stringified when it is not already a string. */
|
|
1249
|
+
function codeField(step, key, fallbackText = '') {
|
|
1250
|
+
const value = step[key];
|
|
1251
|
+
if (typeof value === 'string') return code(value);
|
|
1252
|
+
if (value === undefined) return code(fallbackText);
|
|
1253
|
+
return code(safeJson(value));
|
|
1254
|
+
}
|
|
1255
|
+
function safeJson(value) {
|
|
1256
|
+
try {
|
|
1257
|
+
return JSON.stringify(value) ?? String(value);
|
|
1258
|
+
} catch {
|
|
1259
|
+
return String(value);
|
|
1260
|
+
}
|
|
1261
|
+
}
|
|
1262
|
+
/** `selector` or `target`, whichever the step gives (`step.contract.ts`'s `TargetFields`: exactly one is present). */
|
|
1263
|
+
function targetField(step) {
|
|
1264
|
+
if (typeof step.target === 'string') return code(step.target);
|
|
1265
|
+
return codeField(step, 'selector');
|
|
1266
|
+
}
|
|
1267
|
+
const BUILDERS = {
|
|
1268
|
+
goto: step => [word('Go to'), codeField(step, 'url')],
|
|
1269
|
+
click: step => [word('Click'), targetField(step)],
|
|
1270
|
+
fill: step => [word('Fill'), targetField(step), word('with'), codeField(step, 'value')],
|
|
1271
|
+
press: step => [word('Press'), codeField(step, 'key'), word('in'), targetField(step)],
|
|
1272
|
+
select: step => [word('Choose'), selectChoice(step), word('in'), targetField(step)],
|
|
1273
|
+
scroll: step => [word('Scroll to'), codeField(step, 'to')],
|
|
1274
|
+
wait: step => [word('Wait for'), waitFor(step)],
|
|
1275
|
+
screenshot: step => [word('Screenshot'), codeField(step, 'path')],
|
|
1276
|
+
request: step => [word('Request'), code(`${typeof step.method === 'string' ? step.method : 'GET'} `), codeField(step, 'url')],
|
|
1277
|
+
extract: step => [word('Read'), pillField(step, 'id', '(unnamed)'), word('from'), pillField(step, 'from', 'page'), word('←'), codeField(step, 'selector')],
|
|
1278
|
+
set: step => [word('Set'), pillField(step, 'id', '(unnamed)'), word('to'), codeField(step, 'value')],
|
|
1279
|
+
collect: step => [word('Collect'), codeField(step, 'value'), word('into'), pillField(step, 'into', '(unnamed)')],
|
|
1280
|
+
forEach: step => forEachSentence(step),
|
|
1281
|
+
paginate: step => paginateSentence(step),
|
|
1282
|
+
if: step => [word('If'), codeField(step, 'test')],
|
|
1283
|
+
emit: step => emitSentence(step)
|
|
1284
|
+
};
|
|
1285
|
+
function selectChoice(step) {
|
|
1286
|
+
if (typeof step.value === 'string') return code(step.value);
|
|
1287
|
+
if (typeof step.label === 'string') return code(step.label);
|
|
1288
|
+
if (typeof step.index === 'number') return code(String(step.index));
|
|
1289
|
+
if (Array.isArray(step.values)) return code(step.values.join(', '));
|
|
1290
|
+
return code('');
|
|
1291
|
+
}
|
|
1292
|
+
function waitFor(step) {
|
|
1293
|
+
if (typeof step.selector === 'string') return code(step.selector);
|
|
1294
|
+
if (typeof step.ms === 'number') return code(`${step.ms}ms`);
|
|
1295
|
+
if (typeof step.state === 'string') return code(step.state);
|
|
1296
|
+
return code('');
|
|
1297
|
+
}
|
|
1298
|
+
function forEachSentence(step) {
|
|
1299
|
+
const parts = [word('For each'), pillField(step, 'as', '(item)'), word('in'), typeof step.over === 'string' ? pill(step.over) : targetField(step)];
|
|
1300
|
+
if (step.emit !== undefined) parts.push(word('→'), word('one record'));
|
|
1301
|
+
return parts;
|
|
1302
|
+
}
|
|
1303
|
+
function paginateSentence(step) {
|
|
1304
|
+
const parts = [word('For every page')];
|
|
1305
|
+
const next = step.next;
|
|
1306
|
+
if (next !== null && typeof next === 'object') {
|
|
1307
|
+
const record = next;
|
|
1308
|
+
if (typeof record.selector === 'string') parts.push(word('via'), code(record.selector));else if (typeof record.url === 'string') parts.push(word('via'), code(record.url));else if (typeof record.jsonpath === 'string') parts.push(word('via'), code(record.jsonpath));
|
|
1309
|
+
}
|
|
1310
|
+
if (typeof step.until === 'string') parts.push(word('until'), code(step.until));
|
|
1311
|
+
return parts;
|
|
1312
|
+
}
|
|
1313
|
+
function emitSentence(step) {
|
|
1314
|
+
if (typeof step.output === 'string') return [word('Emit'), word('to'), pill(step.output)];
|
|
1315
|
+
return [word('Emit')];
|
|
1316
|
+
}
|
|
1317
|
+
|
|
1318
|
+
/**
|
|
1319
|
+
* Reads a recipe's `steps` as an outline. Works on whatever JSON parsed
|
|
1320
|
+
* (not a validated `InputRecipe`): a recipe mid-edit in the JSON tab may be
|
|
1321
|
+
* missing fields or have the wrong shape, and the outline still has to
|
|
1322
|
+
* render something rather than crash the tab. Anything that is not a
|
|
1323
|
+
* well-formed step becomes a custom card holding what was there; anything
|
|
1324
|
+
* that is not an object at the top becomes an outline with no steps.
|
|
1325
|
+
*
|
|
1326
|
+
* @param content - The recipe's parsed JSON (the JSON tab's own `JSON.parse` of its text).
|
|
1327
|
+
* @returns The outline: the untouched top-level record plus the step tree.
|
|
1328
|
+
*/
|
|
1329
|
+
function recipeToOutline(content) {
|
|
1330
|
+
const recipe = isRecord(content) ? content : {};
|
|
1331
|
+
const steps = Array.isArray(recipe.steps) ? recipe.steps : [];
|
|
1332
|
+
return {
|
|
1333
|
+
recipe,
|
|
1334
|
+
steps: stepsToNodes(steps, 'steps')
|
|
1335
|
+
};
|
|
1336
|
+
}
|
|
1337
|
+
function stepsToNodes(steps, pathPrefix) {
|
|
1338
|
+
return steps.map((step, index) => stepToNode(step, `${pathPrefix}.${index}`));
|
|
1339
|
+
}
|
|
1340
|
+
function stepToNode(step, path) {
|
|
1341
|
+
if (!isRecord(step)) {
|
|
1342
|
+
return {
|
|
1343
|
+
kind: 'card',
|
|
1344
|
+
path,
|
|
1345
|
+
stepType: 'unknown',
|
|
1346
|
+
sentence: [{
|
|
1347
|
+
kind: 'word',
|
|
1348
|
+
text: 'custom step'
|
|
1349
|
+
}],
|
|
1350
|
+
step: {},
|
|
1351
|
+
custom: true
|
|
1352
|
+
};
|
|
1353
|
+
}
|
|
1354
|
+
const stepType = typeof step.type === 'string' ? step.type : 'unknown';
|
|
1355
|
+
const {
|
|
1356
|
+
parts,
|
|
1357
|
+
custom
|
|
1358
|
+
} = cardSentence(step);
|
|
1359
|
+
if (!isBracketStepType(stepType)) return {
|
|
1360
|
+
kind: 'card',
|
|
1361
|
+
path,
|
|
1362
|
+
stepType,
|
|
1363
|
+
sentence: parts,
|
|
1364
|
+
step,
|
|
1365
|
+
custom
|
|
1366
|
+
};
|
|
1367
|
+
const children = Array.isArray(step.steps) ? stepsToNodes(step.steps, `${path}.steps`) : [];
|
|
1368
|
+
const elseChildren = stepType === 'if' && Array.isArray(step.else) ? stepsToNodes(step.else, `${path}.else`) : undefined;
|
|
1369
|
+
return {
|
|
1370
|
+
kind: 'bracket',
|
|
1371
|
+
path,
|
|
1372
|
+
stepType,
|
|
1373
|
+
sentence: parts,
|
|
1374
|
+
step,
|
|
1375
|
+
children,
|
|
1376
|
+
...(elseChildren !== undefined && {
|
|
1377
|
+
elseChildren
|
|
1378
|
+
})
|
|
1379
|
+
};
|
|
1380
|
+
}
|
|
1381
|
+
function isRecord(value) {
|
|
1382
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
1383
|
+
}
|
|
1384
|
+
|
|
1385
|
+
/**
|
|
1386
|
+
* Rebuilds a recipe from its outline: `recipe`'s keys, in their original
|
|
1387
|
+
* order, with `steps` (when the recipe had one) replaced by the outline's
|
|
1388
|
+
* (possibly edited) step tree. A card's own raw `step` is used as-is; a
|
|
1389
|
+
* bracket's `step` supplies every field but `steps`/`else`, which come from
|
|
1390
|
+
* `children`/`elseChildren` instead, put back at the position `step` itself
|
|
1391
|
+
* held that key at (or appended, for a bracket a `+` menu just added, whose
|
|
1392
|
+
* `step` never had one) — this, together with `recipe-to-outline.mapper.ts`
|
|
1393
|
+
* always keeping the step's whole original object, is what makes
|
|
1394
|
+
* `recipe → outline → recipe` preserve every key and its order.
|
|
1395
|
+
*
|
|
1396
|
+
* @param outline - An outline, from `recipe-to-outline.mapper.ts` or edited by the UI.
|
|
1397
|
+
* @returns The recipe's plain JSON, ready for `save-recipe`.
|
|
1398
|
+
*/
|
|
1399
|
+
function outlineToRecipe(outline) {
|
|
1400
|
+
if (!('steps' in outline.recipe) && outline.steps.length === 0) return {
|
|
1401
|
+
...outline.recipe
|
|
1402
|
+
};
|
|
1403
|
+
return {
|
|
1404
|
+
...outline.recipe,
|
|
1405
|
+
steps: nodesToSteps(outline.steps)
|
|
1406
|
+
};
|
|
1407
|
+
}
|
|
1408
|
+
function nodesToSteps(nodes) {
|
|
1409
|
+
return nodes.map(node => nodeToStep(node));
|
|
1410
|
+
}
|
|
1411
|
+
function nodeToStep(node) {
|
|
1412
|
+
if (!isBracket(node)) return node.step;
|
|
1413
|
+
const keys = Object.keys(node.step);
|
|
1414
|
+
const step = {};
|
|
1415
|
+
for (const key of keys) {
|
|
1416
|
+
if (key === 'steps') step.steps = nodesToSteps(node.children);else if (key === 'else' && node.stepType === 'if') step.else = nodesToSteps(node.elseChildren ?? []);else step[key] = node.step[key];
|
|
1417
|
+
}
|
|
1418
|
+
if (!keys.includes('steps')) step.steps = nodesToSteps(node.children);
|
|
1419
|
+
if (node.stepType === 'if' && node.elseChildren !== undefined && !keys.includes('else')) step.else = nodesToSteps(node.elseChildren);
|
|
1420
|
+
return step;
|
|
1421
|
+
}
|
|
1422
|
+
|
|
329
1423
|
/**
|
|
330
1424
|
* Opens a recipe folder: every `.json`/`.jsonl` recipe directly inside it
|
|
331
1425
|
* (core's own directory reading, so relative `file:` URLs resolve the same
|
|
@@ -366,7 +1460,10 @@ async function openWorkspace(folder) {
|
|
|
366
1460
|
kind,
|
|
367
1461
|
id,
|
|
368
1462
|
issues,
|
|
369
|
-
text: `${JSON.stringify(content, null, 2)}\n
|
|
1463
|
+
text: `${JSON.stringify(content, null, 2)}\n`,
|
|
1464
|
+
// The Steps tab needs steps to render: an output/unknown file has none. recipeToOutline never
|
|
1465
|
+
// throws, so this covers a file that fails validation too (its outline just has "custom" cards).
|
|
1466
|
+
outline: kind === 'input' ? recipeToOutline(content) : undefined
|
|
370
1467
|
}))
|
|
371
1468
|
};
|
|
372
1469
|
}
|
|
@@ -449,10 +1546,11 @@ async function saveRecipe(path, recipe) {
|
|
|
449
1546
|
await writeFile(path, `${JSON.stringify(recipe, null, 2)}\n`, 'utf8');
|
|
450
1547
|
}
|
|
451
1548
|
|
|
452
|
-
/** @returns Fresh state for a new server instance: no workspace open, no run, no sockets. */
|
|
1549
|
+
/** @returns Fresh state for a new server instance: no workspace open, no run, no sockets, an empty snapshot cache, Playwright's own default browser. */
|
|
453
1550
|
function createStudioState() {
|
|
454
1551
|
return {
|
|
455
|
-
sockets: new Set()
|
|
1552
|
+
sockets: new Set(),
|
|
1553
|
+
snapshots: createSnapshotCache()
|
|
456
1554
|
};
|
|
457
1555
|
}
|
|
458
1556
|
/**
|
|
@@ -521,7 +1619,7 @@ async function handleRunSample(state, command) {
|
|
|
521
1619
|
data: record.data
|
|
522
1620
|
});
|
|
523
1621
|
}
|
|
524
|
-
});
|
|
1622
|
+
}, state.browser);
|
|
525
1623
|
state.activeRun = handle;
|
|
526
1624
|
void handle.result.then(result => {
|
|
527
1625
|
if (state.activeRun === handle) state.activeRun = undefined;
|
|
@@ -559,6 +1657,28 @@ async function handleStopRun(state, _command) {
|
|
|
559
1657
|
};
|
|
560
1658
|
}
|
|
561
1659
|
|
|
1660
|
+
/**
|
|
1661
|
+
* Handles `save-outline`: the Steps tab's edits, as the outline it holds
|
|
1662
|
+
* locally, converted back to the recipe's JSON (`scope-outline`'s
|
|
1663
|
+
* `outlineToRecipe`, so the conversion runs here rather than in the
|
|
1664
|
+
* browser) and written the same way `save-recipe` writes the JSON tab's
|
|
1665
|
+
* edits. Every connected client, including the sender, is told the
|
|
1666
|
+
* workspace changed, so both tabs re-fetch it and stay in sync.
|
|
1667
|
+
*
|
|
1668
|
+
* @param state - The server state holding the connected sockets.
|
|
1669
|
+
* @param command - The command: the file path and the edited outline.
|
|
1670
|
+
* @returns `{ saved: true }` once the write has completed.
|
|
1671
|
+
*/
|
|
1672
|
+
async function handleSaveOutline(state, command) {
|
|
1673
|
+
await saveRecipe(command.path, outlineToRecipe(command.outline));
|
|
1674
|
+
broadcast(state, {
|
|
1675
|
+
type: 'workspace-changed'
|
|
1676
|
+
});
|
|
1677
|
+
return {
|
|
1678
|
+
saved: true
|
|
1679
|
+
};
|
|
1680
|
+
}
|
|
1681
|
+
|
|
562
1682
|
/**
|
|
563
1683
|
* Handles `save-recipe`: writes the file (see `recipe-workspace/save-recipe.use-case.ts`
|
|
564
1684
|
* for the pretty-printing and its round-trip guarantee), then tells every
|
|
@@ -618,6 +1738,76 @@ function serveStatic(root, pathname, response) {
|
|
|
618
1738
|
createReadStream(file).pipe(response);
|
|
619
1739
|
}
|
|
620
1740
|
|
|
1741
|
+
/**
|
|
1742
|
+
* Handles `take-snapshot`: loads the named input recipe, captures its
|
|
1743
|
+
* snapshot at `command.path` (only `"start"` is captured for now — see
|
|
1744
|
+
* `page-snapshot`'s `take-snapshot.use-case.ts`), and caches it so a later
|
|
1745
|
+
* `verify-selector`/`infer-selector` against the same recipe and path, or
|
|
1746
|
+
* the UI reselecting the same step, does not recapture it.
|
|
1747
|
+
*
|
|
1748
|
+
* @param state - The server state: `folder` must already be set.
|
|
1749
|
+
* @param command - The command.
|
|
1750
|
+
* @returns The rewritten snapshot.
|
|
1751
|
+
* @throws Error when no workspace is open, or the recipe cannot be found or loaded.
|
|
1752
|
+
*/
|
|
1753
|
+
async function handleTakeSnapshot(state, command) {
|
|
1754
|
+
if (state.folder === undefined) throw new Error('open a workspace first');
|
|
1755
|
+
const cached = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
1756
|
+
if (cached !== undefined) return cached;
|
|
1757
|
+
const {
|
|
1758
|
+
input
|
|
1759
|
+
} = await loadRecipePair(state.folder, command.recipeId);
|
|
1760
|
+
const snapshot = await takeSnapshot(input, command.path, state.browser);
|
|
1761
|
+
putSnapshot(state.snapshots, command.recipeId, command.path, snapshot);
|
|
1762
|
+
return snapshot;
|
|
1763
|
+
}
|
|
1764
|
+
|
|
1765
|
+
/**
|
|
1766
|
+
* Handles `verify-selector`: runs a candidate through the engine's own
|
|
1767
|
+
* selection (`@opencraw/core`'s `countMatches`) against the cached
|
|
1768
|
+
* snapshot, and, best-effort, against the live page too — issue #91's "flag
|
|
1769
|
+
* a selector that matches the snapshot but not the live page". The live
|
|
1770
|
+
* page is fetched raw (`page-snapshot`'s `fetchStartPage`, the same reach
|
|
1771
|
+
* `take-snapshot` has: only `"start"` today), not through the rewritten
|
|
1772
|
+
* snapshot, so its match count is the site's own, not the studio's.
|
|
1773
|
+
*
|
|
1774
|
+
* @param state - The server state.
|
|
1775
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
1776
|
+
* @returns The match counts.
|
|
1777
|
+
* @throws Error when no workspace is open, the recipe cannot be found, or `command.path` has no cached snapshot yet.
|
|
1778
|
+
*/
|
|
1779
|
+
async function handleVerifySelector(state, command) {
|
|
1780
|
+
if (state.folder === undefined) throw new Error('open a workspace first');
|
|
1781
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
1782
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
1783
|
+
const snapshotMatches = countMatches(snapshot.html, command.selector);
|
|
1784
|
+
if (command.path !== 'start') return {
|
|
1785
|
+
selector: command.selector,
|
|
1786
|
+
snapshotMatches,
|
|
1787
|
+
liveChecked: false,
|
|
1788
|
+
liveError: 'only the start page can be checked against the live site for now'
|
|
1789
|
+
};
|
|
1790
|
+
try {
|
|
1791
|
+
const {
|
|
1792
|
+
input
|
|
1793
|
+
} = await loadRecipePair(state.folder, command.recipeId);
|
|
1794
|
+
const live = await fetchStartPage(input, state.browser);
|
|
1795
|
+
return {
|
|
1796
|
+
selector: command.selector,
|
|
1797
|
+
snapshotMatches,
|
|
1798
|
+
liveChecked: true,
|
|
1799
|
+
liveMatches: countMatches(live, command.selector)
|
|
1800
|
+
};
|
|
1801
|
+
} catch (error) {
|
|
1802
|
+
return {
|
|
1803
|
+
selector: command.selector,
|
|
1804
|
+
snapshotMatches,
|
|
1805
|
+
liveChecked: false,
|
|
1806
|
+
liveError: error instanceof Error ? error.message : String(error)
|
|
1807
|
+
};
|
|
1808
|
+
}
|
|
1809
|
+
}
|
|
1810
|
+
|
|
621
1811
|
/** RFC 6455's fixed handshake suffix. */
|
|
622
1812
|
const WS_GUID = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11';
|
|
623
1813
|
/**
|
|
@@ -734,6 +1924,7 @@ async function startStudioServer(options = {}) {
|
|
|
734
1924
|
const uiRoot = options.uiRoot ?? defaultUiRoot();
|
|
735
1925
|
const state = createStudioState();
|
|
736
1926
|
if (options.initialFolder !== undefined) state.folder = options.initialFolder;
|
|
1927
|
+
state.browser = options.browser;
|
|
737
1928
|
const server = createServer((request, response) => {
|
|
738
1929
|
void handleRequest(request, response, token, state, uiRoot);
|
|
739
1930
|
});
|
|
@@ -826,6 +2017,22 @@ function dispatch(command, state) {
|
|
|
826
2017
|
{
|
|
827
2018
|
return handleFetchStartPage(state, command);
|
|
828
2019
|
}
|
|
2020
|
+
case 'save-outline':
|
|
2021
|
+
{
|
|
2022
|
+
return handleSaveOutline(state, command);
|
|
2023
|
+
}
|
|
2024
|
+
case 'take-snapshot':
|
|
2025
|
+
{
|
|
2026
|
+
return handleTakeSnapshot(state, command);
|
|
2027
|
+
}
|
|
2028
|
+
case 'verify-selector':
|
|
2029
|
+
{
|
|
2030
|
+
return handleVerifySelector(state, command);
|
|
2031
|
+
}
|
|
2032
|
+
case 'infer-selector':
|
|
2033
|
+
{
|
|
2034
|
+
return Promise.resolve(handleInferSelector(state, command));
|
|
2035
|
+
}
|
|
829
2036
|
}
|
|
830
2037
|
}
|
|
831
2038
|
function respondJson(response, status, body) {
|
|
@@ -882,5 +2089,5 @@ async function main(argv) {
|
|
|
882
2089
|
process.on('SIGTERM', shutdown);
|
|
883
2090
|
}
|
|
884
2091
|
|
|
885
|
-
export { fetchStartPage, fetchStartPageCommandSchema, loadRecipePair, main, openWorkspace, openWorkspaceCommandSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveRecipe, saveRecipeCommandSchema, startPageViewSchema, startStudioServer, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, traceLineEventSchema, workspaceChangedEventSchema, workspaceViewSchema };
|
|
2092
|
+
export { fetchStartPage, fetchStartPageCommandSchema, fieldPickSchema, inferSelectorCommandSchema, inferSelectorViewSchema, loadRecipePair, main, openWorkspace, openWorkspaceCommandSchema, outlineBracketSchema, outlineCardSchema, outlineNodeSchema, outlineViewSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveOutlineCommandSchema, saveRecipe, saveRecipeCommandSchema, sentencePartSchema, snapshotViewSchema, startPageViewSchema, startStudioServer, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, takeSnapshotCommandSchema, traceLineEventSchema, verifySelectorCommandSchema, verifySelectorViewSchema, workspaceChangedEventSchema, workspaceViewSchema };
|
|
886
2093
|
//# sourceMappingURL=index.esm.js.map
|