@opencraw/studio 0.0.9 → 0.0.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.esm.js +1389 -1081
- package/dist/src/document-view/index.d.ts +2 -0
- package/dist/src/document-view/region-preview.use-case.d.ts +30 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/page-snapshot/index.d.ts +1 -0
- package/dist/src/page-snapshot/invalidate-stale-snapshot.use-case.d.ts +12 -0
- package/dist/src/page-snapshot/start-point.policy.d.ts +12 -0
- package/dist/src/recipe-workspace/import-document.use-case.d.ts +22 -0
- package/dist/src/recipe-workspace/index.d.ts +2 -0
- package/dist/src/studio-api/command.contract.d.ts +41 -0
- package/dist/src/studio-api/import-document-view.contract.d.ts +8 -0
- package/dist/src/studio-api/index.d.ts +6 -2
- package/dist/src/studio-api/region-preview-view.contract.d.ts +53 -0
- package/dist/src/studio-server/import-document.handler.d.ts +14 -0
- package/dist/src/studio-server/open-workspace.handler.d.ts +8 -1
- package/dist/src/studio-server/recording.handler.d.ts +4 -1
- package/dist/src/studio-server/region-preview.handler.d.ts +17 -0
- package/dist/src/studio-server/workspace.store.d.ts +2 -0
- package/dist/ui/assets/index-CCP7vHMS.js +38 -0
- package/dist/ui/assets/{pdf-De9LBx2x.js → pdf-mQRkgM7N.js} +1 -1
- package/dist/ui/index.html +1 -1
- package/package.json +4 -4
- package/dist/ui/assets/index-CiZ4i-q5.js +0 -34
package/dist/index.esm.js
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
|
-
import { join, dirname, normalize, sep,
|
|
1
|
+
import { join, dirname, basename, extname, normalize, sep, resolve } from 'node:path';
|
|
2
2
|
import { createServer } from 'node:http';
|
|
3
|
-
import { fileURLToPath } from 'node:url';
|
|
3
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
-
import { BODY_KINDS, HttpClient, BrowserClient, NODE_ID_ATTRIBUTE, rowsOfCells, parseXml, findDeckTables, findGridTables, analyzeTables, isDeckDocument, readRecipeSource, parseInputRecipe, parseOutputRecipe, bindRecipeSet, memorySink, createCrawler, traceLine, resolveMissingPolicy, isWorkbookDocument,
|
|
5
|
+
import { BODY_KINDS, HttpClient, BrowserClient, NODE_ID_ATTRIBUTE, rowsOfCells, parseXml, findDeckTables, findGridTables, analyzeTables, parseRegion, regionText, deckRegionText, isDeckDocument, readRecipeSource, parseInputRecipe, parseOutputRecipe, bindRecipeSet, memorySink, createCrawler, traceLine, UnknownHookError, resolveMissingPolicy, isWorkbookDocument, RecipeBindingError, RecipeValidationError, countMatches, matchesOf, tryParseJson, isPdfDocument, BrowserProfiles } from '@opencraw/core';
|
|
6
6
|
import { randomBytes, createHash } from 'node:crypto';
|
|
7
7
|
import { execFile } from 'node:child_process';
|
|
8
8
|
import { createRequire } from 'node:module';
|
|
9
9
|
import { promisify } from 'node:util';
|
|
10
10
|
import { load } from 'cheerio';
|
|
11
|
-
import { readFile, writeFile } from 'node:fs/promises';
|
|
11
|
+
import { readFile, writeFile, mkdir } from 'node:fs/promises';
|
|
12
12
|
import { describeJson } from '@opencraw/probe';
|
|
13
13
|
import { tmpdir } from 'node:os';
|
|
14
14
|
import { existsSync, createReadStream } from 'node:fs';
|
|
@@ -189,6 +189,19 @@ const tablePreviewCommandSchema = z.object({
|
|
|
189
189
|
path: z.string().min(1),
|
|
190
190
|
options: tablePreviewOptionsSchema
|
|
191
191
|
});
|
|
192
|
+
/**
|
|
193
|
+
* The live preview of a `region` extract's selector against a cached PDF
|
|
194
|
+
* snapshot (`document-view`'s `region-preview.use-case.ts`, issue #121: the
|
|
195
|
+
* PDF canvas's click-to-snap selection): the text the step would bind and
|
|
196
|
+
* the cells to highlight, recomputed as the box changes — call
|
|
197
|
+
* `take-snapshot` first.
|
|
198
|
+
*/
|
|
199
|
+
const regionPreviewCommandSchema = z.object({
|
|
200
|
+
type: z.literal('region-preview'),
|
|
201
|
+
recipeId: z.string().min(1),
|
|
202
|
+
path: z.string().min(1),
|
|
203
|
+
selector: z.string().min(1)
|
|
204
|
+
});
|
|
192
205
|
/**
|
|
193
206
|
* The grid canvas's sheets and cells (`document-view`'s `workbook-view.mapper.ts`,
|
|
194
207
|
* studio plan §3.4, issue #94's 5c), off the same cached snapshot
|
|
@@ -313,7 +326,23 @@ const startRecordingCommandSchema = z.object({
|
|
|
313
326
|
const stopRecordingCommandSchema = z.object({
|
|
314
327
|
type: z.literal('stop-recording')
|
|
315
328
|
});
|
|
316
|
-
|
|
329
|
+
/**
|
|
330
|
+
* Copies a document the person dropped on (or opened in) the studio into a
|
|
331
|
+
* workspace folder, so a recipe can point at it with a `file:` URL (issue
|
|
332
|
+
* #120). The bytes travel base64-encoded inside the JSON command envelope
|
|
333
|
+
* every other command uses: a local tool's own files are a few megabytes at
|
|
334
|
+
* most, not worth a second, multipart endpoint. `folder` is created when it
|
|
335
|
+
* does not exist yet, so a dropped file can also start a workspace; `name`
|
|
336
|
+
* is a plain file name (no directories), kept as the copy's name unless one
|
|
337
|
+
* is already there, in which case a numbered one is used.
|
|
338
|
+
*/
|
|
339
|
+
const importDocumentCommandSchema = z.object({
|
|
340
|
+
type: z.literal('import-document'),
|
|
341
|
+
folder: z.string().min(1),
|
|
342
|
+
name: z.string().min(1),
|
|
343
|
+
bytes: z.string()
|
|
344
|
+
});
|
|
345
|
+
const studioCommandSchema = z.discriminatedUnion('type', [openWorkspaceCommandSchema, importDocumentCommandSchema, runSampleCommandSchema, stopRunCommandSchema, saveRecipeCommandSchema, fetchStartPageCommandSchema, saveOutlineCommandSchema, takeSnapshotCommandSchema, verifySelectorCommandSchema, inferSelectorCommandSchema, explainWhyCommandSchema, inspectPageCommandSchema, responsesSeenCommandSchema, documentTreeCommandSchema, pdfViewCommandSchema, tablePreviewCommandSchema, regionPreviewCommandSchema, gridViewCommandSchema, gridPreviewCommandSchema, deckViewCommandSchema, deckPreviewCommandSchema, startRecordingCommandSchema, stopRecordingCommandSchema]);
|
|
317
346
|
|
|
318
347
|
/**
|
|
319
348
|
* What `take-snapshot` answers with: the rewritten document ready for the
|
|
@@ -480,6 +509,12 @@ const workspaceViewSchema = z.object({
|
|
|
480
509
|
recipes: z.array(recipeListingSchema)
|
|
481
510
|
});
|
|
482
511
|
|
|
512
|
+
/** `import-document`'s answer: where the copy landed, and the `file:` URL a recipe's `start.url` points at it with. */
|
|
513
|
+
const importDocumentViewSchema = z.object({
|
|
514
|
+
path: z.string().min(1),
|
|
515
|
+
url: z.string().startsWith('file:')
|
|
516
|
+
});
|
|
517
|
+
|
|
483
518
|
/** One record of a sample run's result, kept for the Records tab. */
|
|
484
519
|
const sampleRunRecordSchema = z.object({
|
|
485
520
|
key: z.string().nullable(),
|
|
@@ -669,25 +704,6 @@ const workbookDocumentViewSchema = z.object({
|
|
|
669
704
|
csv: csvFormatViewSchema.optional()
|
|
670
705
|
});
|
|
671
706
|
|
|
672
|
-
const gridPreviewRowSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
|
|
673
|
-
/** One table matched by a `table` extract's options against a workbook — mirrors `@opencraw/core`'s `GridTable`. */
|
|
674
|
-
const gridTablePreviewMatchSchema = z.object({
|
|
675
|
-
sheet: z.string(),
|
|
676
|
-
title: z.string(),
|
|
677
|
-
header: z.array(z.string()),
|
|
678
|
-
rows: z.array(gridPreviewRowSchema)
|
|
679
|
-
});
|
|
680
|
-
/**
|
|
681
|
-
* What `grid-preview` answers with (studio plan §3.4, issue #94's 5c: the
|
|
682
|
-
* live preview): every matched table, or `error` when a pattern in the
|
|
683
|
-
* options does not compile — expected while the person is still typing it,
|
|
684
|
-
* not thrown.
|
|
685
|
-
*/
|
|
686
|
-
const gridTablePreviewViewSchema = z.object({
|
|
687
|
-
matches: z.array(gridTablePreviewMatchSchema),
|
|
688
|
-
error: z.string().optional()
|
|
689
|
-
});
|
|
690
|
-
|
|
691
707
|
/** One text box on a slide, in points — mirrors `document-view/deck-view.mapper.ts`'s `DeckShapeView`, the wire shape for the UI (studio plan §3.4, issue #94's 5d). */
|
|
692
708
|
const deckShapeViewSchema = z.object({
|
|
693
709
|
x: z.number(),
|
|
@@ -727,6 +743,45 @@ const deckDocumentViewSchema = z.object({
|
|
|
727
743
|
slides: z.array(deckSlideViewSchema)
|
|
728
744
|
});
|
|
729
745
|
|
|
746
|
+
/** One page of a PDF, or one slide of a deck, a `region` extract's box reads text off — mirrors `document-view/region-preview.use-case.ts`'s `RegionPreviewMatch`: `cells` for a PDF, `shapes` for a deck, the other empty. */
|
|
747
|
+
const regionPreviewMatchSchema = z.object({
|
|
748
|
+
/** The page number of a PDF, or the slide number of a deck. */
|
|
749
|
+
page: z.number().int(),
|
|
750
|
+
text: z.string(),
|
|
751
|
+
cells: z.array(pdfCellViewSchema),
|
|
752
|
+
shapes: z.array(deckShapeViewSchema)
|
|
753
|
+
});
|
|
754
|
+
/**
|
|
755
|
+
* What `region-preview` answers with (issues #121 and #122: the PDF and
|
|
756
|
+
* deck canvases' click-to-snap selection): the engine's own matches for
|
|
757
|
+
* the selector — the text the step would bind, and the cells or text boxes
|
|
758
|
+
* to highlight — or `error` when the selector does not parse yet, or names
|
|
759
|
+
* the other document's keyword, reported rather than thrown.
|
|
760
|
+
*/
|
|
761
|
+
const regionPreviewViewSchema = z.object({
|
|
762
|
+
matches: z.array(regionPreviewMatchSchema),
|
|
763
|
+
error: z.string().optional()
|
|
764
|
+
});
|
|
765
|
+
|
|
766
|
+
const gridPreviewRowSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
|
|
767
|
+
/** One table matched by a `table` extract's options against a workbook — mirrors `@opencraw/core`'s `GridTable`. */
|
|
768
|
+
const gridTablePreviewMatchSchema = z.object({
|
|
769
|
+
sheet: z.string(),
|
|
770
|
+
title: z.string(),
|
|
771
|
+
header: z.array(z.string()),
|
|
772
|
+
rows: z.array(gridPreviewRowSchema)
|
|
773
|
+
});
|
|
774
|
+
/**
|
|
775
|
+
* What `grid-preview` answers with (studio plan §3.4, issue #94's 5c: the
|
|
776
|
+
* live preview): every matched table, or `error` when a pattern in the
|
|
777
|
+
* options does not compile — expected while the person is still typing it,
|
|
778
|
+
* not thrown.
|
|
779
|
+
*/
|
|
780
|
+
const gridTablePreviewViewSchema = z.object({
|
|
781
|
+
matches: z.array(gridTablePreviewMatchSchema),
|
|
782
|
+
error: z.string().optional()
|
|
783
|
+
});
|
|
784
|
+
|
|
730
785
|
const deckPreviewRowSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
|
|
731
786
|
/** One table matched by a `table` extract's options against a deck — mirrors `@opencraw/core`'s `DeckTable`. */
|
|
732
787
|
const deckTablePreviewMatchSchema = z.object({
|
|
@@ -1129,6 +1184,13 @@ function cachedSnapshot(cache, recipeId, path) {
|
|
|
1129
1184
|
function putSnapshot(cache, recipeId, path, snapshot) {
|
|
1130
1185
|
cache.entries.set(keyOf(recipeId, path), snapshot);
|
|
1131
1186
|
}
|
|
1187
|
+
/** Drops every cached snapshot of one recipe — called after a save, since the steps up to any path may now capture something different. */
|
|
1188
|
+
function invalidateRecipe(cache, recipeId) {
|
|
1189
|
+
const prefix = `${recipeId}:`;
|
|
1190
|
+
for (const key of cache.entries.keys()) {
|
|
1191
|
+
if (key.startsWith(prefix)) cache.entries.delete(key);
|
|
1192
|
+
}
|
|
1193
|
+
}
|
|
1132
1194
|
function keyOf(recipeId, path) {
|
|
1133
1195
|
return `${recipeId}:${path}`;
|
|
1134
1196
|
}
|
|
@@ -1184,6 +1246,50 @@ async function rereadCsv(url, override) {
|
|
|
1184
1246
|
}
|
|
1185
1247
|
}
|
|
1186
1248
|
|
|
1249
|
+
/** The keys of an input recipe that do not change what its start point shows: the steps and mapping run after it, `output` and `$schema` only name things. */
|
|
1250
|
+
const AFTER_START = new Set(['steps', 'mapping', 'output', '$schema']);
|
|
1251
|
+
/**
|
|
1252
|
+
* Whether an edit changes the page a recipe's start snapshot shows (issue
|
|
1253
|
+
* #159): its start URLs, mode, session, vars, headers… anything but the steps
|
|
1254
|
+
* and mapping that run after it. A pick only adds steps, so it keeps the
|
|
1255
|
+
* snapshot already taken instead of fetching the page again.
|
|
1256
|
+
*
|
|
1257
|
+
* @param before - The recipe as last saved (`undefined` when there was none).
|
|
1258
|
+
* @param after - The recipe being saved.
|
|
1259
|
+
* @returns `true` when the cached start snapshot may now be out of date.
|
|
1260
|
+
*/
|
|
1261
|
+
function startPointChanged(before, after) {
|
|
1262
|
+
return JSON.stringify(startPartOf(before)) !== JSON.stringify(startPartOf(after));
|
|
1263
|
+
}
|
|
1264
|
+
function startPartOf(recipe) {
|
|
1265
|
+
if (typeof recipe !== 'object' || recipe === null) return recipe;
|
|
1266
|
+
return Object.fromEntries(Object.entries(recipe).filter(([key]) => !AFTER_START.has(key)).sort(([a], [b]) => a.localeCompare(b)));
|
|
1267
|
+
}
|
|
1268
|
+
|
|
1269
|
+
/**
|
|
1270
|
+
* Drops a recipe's cached snapshots when the edit about to be saved changes
|
|
1271
|
+
* its start point (issue #159). Call it before the file is written: it reads
|
|
1272
|
+
* the recipe as it is on disk now to compare.
|
|
1273
|
+
*
|
|
1274
|
+
* @param cache - The workspace's snapshot cache.
|
|
1275
|
+
* @param file - The recipe file being saved.
|
|
1276
|
+
* @param next - The recipe being saved.
|
|
1277
|
+
*/
|
|
1278
|
+
async function invalidateStaleSnapshot(cache, file, next) {
|
|
1279
|
+
const id = next?.id;
|
|
1280
|
+
if (typeof id !== 'string') return;
|
|
1281
|
+
if (startPointChanged(await savedRecipe(file), next)) invalidateRecipe(cache, id);
|
|
1282
|
+
}
|
|
1283
|
+
/** The recipe as saved on disk now, or `undefined` when there is no such file (or it is not JSON yet). */
|
|
1284
|
+
async function savedRecipe(file) {
|
|
1285
|
+
try {
|
|
1286
|
+
const text = await readFile(file, 'utf8');
|
|
1287
|
+
return JSON.parse(text);
|
|
1288
|
+
} catch {
|
|
1289
|
+
return undefined;
|
|
1290
|
+
}
|
|
1291
|
+
}
|
|
1292
|
+
|
|
1187
1293
|
/** A cell reads as an ISO date (`2026-06-01`), an ISO moment (`2026-06-01T09:30:00`) or a bare time of day (`09:30:00`) — the three shapes `workbook-document`'s `read-xlsx.client.ts` writes for a spreadsheet date. */
|
|
1188
1294
|
const ISO_DATE_OR_TIME = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2})?$|^\d{2}:\d{2}:\d{2}$/;
|
|
1189
1295
|
/**
|
|
@@ -1719,6 +1825,56 @@ function previewDeckTable(document, options) {
|
|
|
1719
1825
|
};
|
|
1720
1826
|
}
|
|
1721
1827
|
|
|
1828
|
+
/**
|
|
1829
|
+
* Runs a `region` extract's selector against an already-read PDF or deck
|
|
1830
|
+
* (issues #121 and #122: the canvases' click-to-snap selection) — the
|
|
1831
|
+
* engine's own `regionText` / `deckRegionText`, so what the canvas
|
|
1832
|
+
* highlights and the text it shows are exactly what the step will read,
|
|
1833
|
+
* and a pure function of the document, so re-running it on every change
|
|
1834
|
+
* of the box costs nothing. A `slide=` on a PDF (or a `page=` on a deck)
|
|
1835
|
+
* is the same mismatch the step would fail with, reported as `error`.
|
|
1836
|
+
*
|
|
1837
|
+
* @param document - The read PDF or deck (`take-snapshot`'s cached body).
|
|
1838
|
+
* @param selector - The `region` selector, `page=1 x=72..252 y=640..664` or `slide=2 x=60..900 y=30..90`.
|
|
1839
|
+
* @returns The matches, or `error` when the selector does not parse or does not fit the document.
|
|
1840
|
+
*/
|
|
1841
|
+
function previewRegion(document, selector) {
|
|
1842
|
+
try {
|
|
1843
|
+
const region = parseRegion(selector);
|
|
1844
|
+
if (document.kind === 'pdf') {
|
|
1845
|
+
if (region.on !== 'page') return {
|
|
1846
|
+
matches: [],
|
|
1847
|
+
error: 'region: "slide=" reads a deck; this document is a PDF (name a "page=")'
|
|
1848
|
+
};
|
|
1849
|
+
return {
|
|
1850
|
+
matches: regionText(document, region).map(match => ({
|
|
1851
|
+
page: match.page,
|
|
1852
|
+
text: match.text,
|
|
1853
|
+
cells: match.cells,
|
|
1854
|
+
shapes: []
|
|
1855
|
+
}))
|
|
1856
|
+
};
|
|
1857
|
+
}
|
|
1858
|
+
if (region.on !== 'slide') return {
|
|
1859
|
+
matches: [],
|
|
1860
|
+
error: 'region: "page=" reads a PDF; this document is a deck (name a "slide=")'
|
|
1861
|
+
};
|
|
1862
|
+
return {
|
|
1863
|
+
matches: deckRegionText(document, region).map(match => ({
|
|
1864
|
+
page: match.slide,
|
|
1865
|
+
text: match.text,
|
|
1866
|
+
cells: [],
|
|
1867
|
+
shapes: match.shapes
|
|
1868
|
+
}))
|
|
1869
|
+
};
|
|
1870
|
+
} catch (error) {
|
|
1871
|
+
return {
|
|
1872
|
+
matches: [],
|
|
1873
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1874
|
+
};
|
|
1875
|
+
}
|
|
1876
|
+
}
|
|
1877
|
+
|
|
1722
1878
|
/**
|
|
1723
1879
|
* Handles `deck-preview`: the deck canvas's live preview of a `table`
|
|
1724
1880
|
* extract's options (`document-view`'s `table-preview.use-case.ts`'s
|
|
@@ -1863,7 +2019,7 @@ async function runSample(folder, recipeId, budget, callbacks, browser) {
|
|
|
1863
2019
|
}
|
|
1864
2020
|
}
|
|
1865
2021
|
});
|
|
1866
|
-
const result = runToResult(crawler, set, records, rejectedRecords, steps);
|
|
2022
|
+
const result = runToResult(crawler, set, recipeId, records, rejectedRecords, steps);
|
|
1867
2023
|
return {
|
|
1868
2024
|
stop: () => crawler.close(),
|
|
1869
2025
|
result
|
|
@@ -1907,7 +2063,13 @@ function trackStep(steps, event) {
|
|
|
1907
2063
|
});
|
|
1908
2064
|
}
|
|
1909
2065
|
}
|
|
1910
|
-
|
|
2066
|
+
/**
|
|
2067
|
+
* Awaits the run and shapes its result. A throw from `crawler.run` (an unknown
|
|
2068
|
+
* hook, a recipe that cannot start) becomes the run's `error` instead of a
|
|
2069
|
+
* rejection: nothing awaits `result` but a `.then`, and an unhandled rejection
|
|
2070
|
+
* would take the whole Studio server down (issue #148).
|
|
2071
|
+
*/
|
|
2072
|
+
async function runToResult(crawler, set, recipeId, records, rejectedRecords, steps) {
|
|
1911
2073
|
try {
|
|
1912
2074
|
const report = await crawler.run(set);
|
|
1913
2075
|
const recipe = report.recipes[0];
|
|
@@ -1924,10 +2086,29 @@ async function runToResult(crawler, set, records, rejectedRecords, steps) {
|
|
|
1924
2086
|
// eslint-disable-next-line unicorn/prefer-iterator-to-array -- `.toArray()` needs a `lib` newer than this repo's `es2022` (tsconfig.base.json)
|
|
1925
2087
|
steps: [...steps.values()]
|
|
1926
2088
|
};
|
|
2089
|
+
} catch (error) {
|
|
2090
|
+
return {
|
|
2091
|
+
recipeId,
|
|
2092
|
+
emitted: 0,
|
|
2093
|
+
rejected: 0,
|
|
2094
|
+
duplicates: 0,
|
|
2095
|
+
durationMs: 0,
|
|
2096
|
+
error: failureMessage(error),
|
|
2097
|
+
records,
|
|
2098
|
+
rejectedRecords,
|
|
2099
|
+
// eslint-disable-next-line unicorn/prefer-iterator-to-array -- `.toArray()` needs a `lib` newer than this repo's `es2022` (tsconfig.base.json)
|
|
2100
|
+
steps: [...steps.values()]
|
|
2101
|
+
};
|
|
1927
2102
|
} finally {
|
|
1928
2103
|
await crawler.close();
|
|
1929
2104
|
}
|
|
1930
2105
|
}
|
|
2106
|
+
/** What a run that could not start reports. A hook is the author's own code, which Studio never loads (issue #150): say so, instead of leaving "unknown hook" to read as a typo. */
|
|
2107
|
+
function failureMessage(error) {
|
|
2108
|
+
if (error instanceof UnknownHookError) return `${error.message}. Studio does not load hooks, so a recipe that uses one cannot be sample-run here; run it with the cli or the mcp server, or temporarily replace the hook step to preview the rest.`;
|
|
2109
|
+
if (error instanceof Error) return error.message;
|
|
2110
|
+
return String(error);
|
|
2111
|
+
}
|
|
1931
2112
|
|
|
1932
2113
|
/** @returns A fresh, empty cache. */
|
|
1933
2114
|
function createLastRunCache() {
|
|
@@ -2302,1204 +2483,1271 @@ async function handleGridView(state, command) {
|
|
|
2302
2483
|
return workbookDocumentView(body);
|
|
2303
2484
|
}
|
|
2304
2485
|
|
|
2305
|
-
const GENERATED_ID = /^(?:react-|radix-|:r|ember|css-|mui-|headlessui-)/i;
|
|
2306
2486
|
/**
|
|
2307
|
-
* The
|
|
2308
|
-
*
|
|
2309
|
-
*
|
|
2310
|
-
* — `list-inference.algorithm.ts` leans on that to find the nearest pair of
|
|
2311
|
-
* siblings two picks belong to, without ever touching cheerio again.
|
|
2487
|
+
* The outline model: an `InputRecipe`'s `steps` (and only `steps`; the rest
|
|
2488
|
+
* of the recipe passes through untouched, see `recipe-to-outline.mapper.ts`)
|
|
2489
|
+
* read as a tree of cards and brackets instead of raw JSON.
|
|
2312
2490
|
*
|
|
2313
|
-
*
|
|
2314
|
-
*
|
|
2315
|
-
*
|
|
2491
|
+
* A **card** is one step that reads as a sentence (`sentence`, built by
|
|
2492
|
+
* `card-sentence.mapper.ts`). A **bracket** is a container step (`forEach`,
|
|
2493
|
+
* `paginate`, `if`) whose body is itself a list of cards/brackets: the
|
|
2494
|
+
* indentation the plan doc describes is this nesting, not a separate layout
|
|
2495
|
+
* concept. Both kinds carry `step`, the step's own raw JSON exactly as
|
|
2496
|
+
* parsed (every key, in its original order): `outline-to-recipe.mapper.ts`
|
|
2497
|
+
* rebuilds the recipe from it, substituting only the (possibly edited)
|
|
2498
|
+
* `children`/`elseChildren` for whatever `steps`/`else` value `step` itself
|
|
2499
|
+
* holds, so an edit to one field never disturbs another's position or a
|
|
2500
|
+
* step type this outline cannot make sense of.
|
|
2316
2501
|
*/
|
|
2317
|
-
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
|
|
2321
|
-
chain.unshift(current);
|
|
2322
|
-
const parent = current.parent();
|
|
2323
|
-
current = parent.length > 0 ? parent : undefined;
|
|
2324
|
-
}
|
|
2325
|
-
return chain.map(element => levelOf($, element));
|
|
2326
|
-
}
|
|
2327
|
-
function levelOf($, element) {
|
|
2328
|
-
const node = element.get(0);
|
|
2329
|
-
const tag = node.tagName.toLowerCase();
|
|
2330
|
-
const attrs = {};
|
|
2331
|
-
const entries = Object.entries(node.attribs ?? {});
|
|
2332
|
-
for (const [name, value] of entries) attrs[name] = value;
|
|
2333
|
-
const classes = (attrs.class ?? '').split(/\s+/).filter(token => token.length > 0);
|
|
2334
|
-
const siblings = element.parent().length > 0 ? element.parent().children(tag) : $.root().children(tag);
|
|
2335
|
-
const index = siblings.toArray().indexOf(node) + 1;
|
|
2336
|
-
return {
|
|
2337
|
-
tag,
|
|
2338
|
-
id: attrs.id,
|
|
2339
|
-
classes,
|
|
2340
|
-
attrs,
|
|
2341
|
-
index: Math.max(index, 1),
|
|
2342
|
-
siblingCount: Math.max(siblings.length, 1)
|
|
2343
|
-
};
|
|
2502
|
+
/** The step types the outline renders as a bracket: their body is nested steps, not a single value. */
|
|
2503
|
+
const BRACKET_STEP_TYPES = ['forEach', 'paginate', 'if'];
|
|
2504
|
+
function isBracket(node) {
|
|
2505
|
+
return node.kind === 'bracket';
|
|
2344
2506
|
}
|
|
2345
|
-
function
|
|
2346
|
-
return
|
|
2507
|
+
function isBracketStepType(type) {
|
|
2508
|
+
return BRACKET_STEP_TYPES.includes(type);
|
|
2347
2509
|
}
|
|
2510
|
+
|
|
2511
|
+
/** Step types the plan doc names as always going to a "custom" card: nothing here renders their meaning as a sentence. */
|
|
2512
|
+
const ALWAYS_CUSTOM = new Set(['evaluate', 'hook', 'captcha']);
|
|
2348
2513
|
/**
|
|
2349
|
-
*
|
|
2350
|
-
*
|
|
2351
|
-
*
|
|
2352
|
-
*
|
|
2514
|
+
* Turns one step's raw JSON into its sentence parts (word/pill/code), the
|
|
2515
|
+
* way the plan doc's examples read: *Go to…*, *Read… from…*, *For each… in…
|
|
2516
|
+
* → one record*, and so on. Never throws: a step type this cannot read
|
|
2517
|
+
* becomes a short, honest fallback with `custom: true`, so the outline never
|
|
2518
|
+
* drops a step for looking unfamiliar.
|
|
2519
|
+
*
|
|
2520
|
+
* @param step - A step's raw JSON (its `type` and whatever fields it has).
|
|
2521
|
+
* @returns The sentence parts, and whether they are a real reading or a fallback.
|
|
2353
2522
|
*/
|
|
2354
|
-
function
|
|
2355
|
-
|
|
2523
|
+
function cardSentence(step) {
|
|
2524
|
+
const type = typeof step.type === 'string' ? step.type : undefined;
|
|
2525
|
+
if (type === undefined || ALWAYS_CUSTOM.has(type)) return fallback(type);
|
|
2526
|
+
const build = BUILDERS[type];
|
|
2527
|
+
if (build === undefined) return fallback(type);
|
|
2528
|
+
try {
|
|
2529
|
+
return {
|
|
2530
|
+
parts: build(step),
|
|
2531
|
+
custom: false
|
|
2532
|
+
};
|
|
2533
|
+
} catch {
|
|
2534
|
+
return fallback(type);
|
|
2535
|
+
}
|
|
2356
2536
|
}
|
|
2357
|
-
|
|
2358
|
-
|
|
2359
|
-
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
* only shadow-DOM shape that can show up in static, server-captured markup
|
|
2363
|
-
* at all (imperative `attachShadow()` content never serialises into
|
|
2364
|
-
* `page.content()`/a fetched response in the first place, so that case
|
|
2365
|
-
* cannot reach this slice and needs no separate detection). A path whose
|
|
2366
|
-
* first level is not `html` was cut short at exactly this kind of boundary:
|
|
2367
|
-
* v1 does not pierce it, so callers report a clear "shadow DOM is not
|
|
2368
|
-
* supported" error instead of building ancestor-based candidates over a
|
|
2369
|
-
* chain that stops partway up.
|
|
2370
|
-
*/
|
|
2371
|
-
function crossesShadowRoot(path) {
|
|
2372
|
-
return path.length === 0 || path[0].tag !== 'html';
|
|
2537
|
+
function fallback(type) {
|
|
2538
|
+
return {
|
|
2539
|
+
parts: [word(type ?? 'custom step')],
|
|
2540
|
+
custom: true
|
|
2541
|
+
};
|
|
2373
2542
|
}
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2543
|
+
function word(text) {
|
|
2544
|
+
return {
|
|
2545
|
+
kind: 'word',
|
|
2546
|
+
text
|
|
2547
|
+
};
|
|
2377
2548
|
}
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
* that one attribute off the element the user clicked — turns into
|
|
2384
|
-
* something `selector-inference`'s other algorithms can work with, without
|
|
2385
|
-
* the browser-side picker script needing to know anything about ancestry,
|
|
2386
|
-
* classes or candidate ranking itself.
|
|
2387
|
-
*
|
|
2388
|
-
* @param html - The snapshot document (already stamped with node ids).
|
|
2389
|
-
* @param nodeId - The `data-oc-node` value to find.
|
|
2390
|
-
* @returns The node's path, or `undefined` when no element carries that id.
|
|
2391
|
-
*/
|
|
2392
|
-
function pathToNode(html, nodeId) {
|
|
2393
|
-
const $ = parseDocument$1(html);
|
|
2394
|
-
const found = $(`[${NODE_ID_ATTRIBUTE}="${cssAttributeValue(nodeId)}"]`);
|
|
2395
|
-
return found.length === 0 ? undefined : domPathOf($, found.first());
|
|
2549
|
+
function pill(text) {
|
|
2550
|
+
return {
|
|
2551
|
+
kind: 'pill',
|
|
2552
|
+
text
|
|
2553
|
+
};
|
|
2396
2554
|
}
|
|
2397
|
-
function
|
|
2398
|
-
return
|
|
2555
|
+
function code(text) {
|
|
2556
|
+
return {
|
|
2557
|
+
kind: 'code',
|
|
2558
|
+
text
|
|
2559
|
+
};
|
|
2399
2560
|
}
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
const
|
|
2403
|
-
|
|
2404
|
-
|
|
2561
|
+
/** A field that names an id: shown as a pill, `fallbackText` when it is absent. */
|
|
2562
|
+
function pillField(step, key, fallbackText) {
|
|
2563
|
+
const value = step[key];
|
|
2564
|
+
return pill(typeof value === 'string' && value.length > 0 ? value : fallbackText);
|
|
2565
|
+
}
|
|
2566
|
+
/** A field shown as a literal (a selector, a template, a value): stringified when it is not already a string. */
|
|
2567
|
+
function codeField(step, key, fallbackText = '') {
|
|
2568
|
+
const value = step[key];
|
|
2569
|
+
if (typeof value === 'string') return code(value);
|
|
2570
|
+
if (value === undefined) return code(fallbackText);
|
|
2571
|
+
return code(safeJson(value));
|
|
2572
|
+
}
|
|
2573
|
+
function safeJson(value) {
|
|
2574
|
+
try {
|
|
2575
|
+
return JSON.stringify(value) ?? String(value);
|
|
2576
|
+
} catch {
|
|
2577
|
+
return String(value);
|
|
2578
|
+
}
|
|
2579
|
+
}
|
|
2580
|
+
/** `selector` or `target`, whichever the step gives (`step.contract.ts`'s `TargetFields`: exactly one is present). */
|
|
2581
|
+
function targetField(step) {
|
|
2582
|
+
if (typeof step.target === 'string') return code(step.target);
|
|
2583
|
+
return codeField(step, 'selector');
|
|
2584
|
+
}
|
|
2585
|
+
const BUILDERS = {
|
|
2586
|
+
goto: step => [word('Go to'), codeField(step, 'url')],
|
|
2587
|
+
click: step => [word('Click'), targetField(step)],
|
|
2588
|
+
fill: step => [word('Fill'), targetField(step), word('with'), codeField(step, 'value')],
|
|
2589
|
+
press: step => [word('Press'), codeField(step, 'key'), word('in'), targetField(step)],
|
|
2590
|
+
select: step => [word('Choose'), selectChoice(step), word('in'), targetField(step)],
|
|
2591
|
+
scroll: step => [word('Scroll to'), codeField(step, 'to')],
|
|
2592
|
+
wait: step => [word('Wait for'), waitFor(step)],
|
|
2593
|
+
screenshot: step => [word('Screenshot'), codeField(step, 'path')],
|
|
2594
|
+
request: step => [word('Request'), code(`${typeof step.method === 'string' ? step.method : 'GET'} `), codeField(step, 'url')],
|
|
2595
|
+
extract: step => [word('Read'), pillField(step, 'id', '(unnamed)'), word('from'), pillField(step, 'from', 'page'), word('←'), codeField(step, 'selector')],
|
|
2596
|
+
set: step => [word('Set'), pillField(step, 'id', '(unnamed)'), word('to'), codeField(step, 'value')],
|
|
2597
|
+
collect: step => [word('Collect'), codeField(step, 'value'), word('into'), pillField(step, 'into', '(unnamed)')],
|
|
2598
|
+
forEach: step => forEachSentence(step),
|
|
2599
|
+
paginate: step => paginateSentence(step),
|
|
2600
|
+
if: step => [word('If'), codeField(step, 'test')],
|
|
2601
|
+
emit: step => emitSentence(step)
|
|
2405
2602
|
};
|
|
2406
|
-
|
|
2407
|
-
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
if (
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
|
|
2603
|
+
function selectChoice(step) {
|
|
2604
|
+
if (typeof step.value === 'string') return code(step.value);
|
|
2605
|
+
if (typeof step.label === 'string') return code(step.label);
|
|
2606
|
+
if (typeof step.index === 'number') return code(String(step.index));
|
|
2607
|
+
if (Array.isArray(step.values)) return code(step.values.join(', '));
|
|
2608
|
+
return code('');
|
|
2609
|
+
}
|
|
2610
|
+
function waitFor(step) {
|
|
2611
|
+
if (typeof step.selector === 'string') return code(step.selector);
|
|
2612
|
+
if (typeof step.ms === 'number') return code(`${step.ms}ms`);
|
|
2613
|
+
if (typeof step.state === 'string') return code(step.state);
|
|
2614
|
+
return code('');
|
|
2615
|
+
}
|
|
2616
|
+
function forEachSentence(step) {
|
|
2617
|
+
const parts = [word('For each'), pillField(step, 'as', '(item)'), word('in'), typeof step.over === 'string' ? pill(step.over) : targetField(step)];
|
|
2618
|
+
if (step.emit !== undefined) parts.push(word('→'), word('one record'));
|
|
2619
|
+
return parts;
|
|
2620
|
+
}
|
|
2621
|
+
function paginateSentence(step) {
|
|
2622
|
+
const parts = [word('For every page')];
|
|
2623
|
+
const next = step.next;
|
|
2624
|
+
if (next !== null && typeof next === 'object') {
|
|
2625
|
+
const record = next;
|
|
2626
|
+
if (typeof record.selector === 'string') parts.push(word('via'), code(record.selector));else if (typeof record.url === 'string') parts.push(word('via'), code(record.url));else if (typeof record.jsonpath === 'string') parts.push(word('via'), code(record.jsonpath));
|
|
2421
2627
|
}
|
|
2422
|
-
|
|
2423
|
-
|
|
2424
|
-
|
|
2628
|
+
if (typeof step.until === 'string') parts.push(word('until'), code(step.until));
|
|
2629
|
+
return parts;
|
|
2630
|
+
}
|
|
2631
|
+
function emitSentence(step) {
|
|
2632
|
+
if (typeof step.output === 'string') return [word('Emit'), word('to'), pill(step.output)];
|
|
2633
|
+
return [word('Emit')];
|
|
2425
2634
|
}
|
|
2426
2635
|
|
|
2427
2636
|
/**
|
|
2428
|
-
*
|
|
2429
|
-
* (
|
|
2430
|
-
*
|
|
2431
|
-
*
|
|
2432
|
-
*
|
|
2433
|
-
*
|
|
2434
|
-
* reasoned judgement call, not a guarantee — see the doc comments on each
|
|
2435
|
-
* pattern for what real-world tool it targets.
|
|
2436
|
-
*/
|
|
2437
|
-
const GENERATED_TOKEN_PATTERNS = [/** styled-components / emotion: `sc-bdVaJa`, `css-1x2y3z4` */
|
|
2438
|
-
/^(?:sc|css)-[a-z0-9]+$/i, /** CSS Modules, most bundlers' default hash suffix: `wrapper__3fA9k`, `Button-module_root__aB3dK` */
|
|
2439
|
-
/__[a-z0-9]{4,10}$/i, /** JSS / MUI: `jss123`, `MuiButton-root-145` */
|
|
2440
|
-
/^(?:jss|mui[a-z]*-)[a-z0-9-]*\d{2}$/i, /** A token that is only digits: not a class a human would type. */
|
|
2441
|
-
/^\d+$/, /** A bare hex hash: `a3f9c21`, `1a2b3c4d` (6+ hex chars, nothing else). */
|
|
2442
|
-
/^[0-9a-f]{6,}$/i, /** Tailwind/atomic-CSS-in-JS runtime classes some bundlers emit: `_1a2b3c` */
|
|
2443
|
-
/^_[0-9a-z]{5,}$/i];
|
|
2444
|
-
/**
|
|
2445
|
-
* Whether a class token looks hand-authored and safe to build a selector on.
|
|
2637
|
+
* Reads a recipe's `steps` as an outline. Works on whatever JSON parsed
|
|
2638
|
+
* (not a validated `InputRecipe`): a recipe mid-edit in the JSON tab may be
|
|
2639
|
+
* missing fields or have the wrong shape, and the outline still has to
|
|
2640
|
+
* render something rather than crash the tab. Anything that is not a
|
|
2641
|
+
* well-formed step becomes a custom card holding what was there; anything
|
|
2642
|
+
* that is not an object at the top becomes an outline with no steps.
|
|
2446
2643
|
*
|
|
2447
|
-
* @param
|
|
2448
|
-
* @returns
|
|
2644
|
+
* @param content - The recipe's parsed JSON (the JSON tab's own `JSON.parse` of its text).
|
|
2645
|
+
* @returns The outline: the untouched top-level record plus the step tree.
|
|
2449
2646
|
*/
|
|
2450
|
-
function
|
|
2451
|
-
|
|
2647
|
+
function recipeToOutline(content) {
|
|
2648
|
+
const recipe = isRecord(content) ? content : {};
|
|
2649
|
+
const steps = Array.isArray(recipe.steps) ? recipe.steps : [];
|
|
2650
|
+
return {
|
|
2651
|
+
recipe,
|
|
2652
|
+
steps: stepsToNodes(steps, 'steps')
|
|
2653
|
+
};
|
|
2452
2654
|
}
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
*
|
|
2456
|
-
* @param tokens - An element's class list.
|
|
2457
|
-
* @returns Only the stable tokens.
|
|
2458
|
-
*/
|
|
2459
|
-
function stableClasses(tokens) {
|
|
2460
|
-
return tokens.filter(token => isStableClassToken(token));
|
|
2655
|
+
function stepsToNodes(steps, pathPrefix) {
|
|
2656
|
+
return steps.map((step, index) => stepToNode(step, `${pathPrefix}.${index}`));
|
|
2461
2657
|
}
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
|
|
2473
|
-
|
|
2474
|
-
|
|
2475
|
-
|
|
2658
|
+
function stepToNode(step, path) {
|
|
2659
|
+
if (!isRecord(step)) {
|
|
2660
|
+
return {
|
|
2661
|
+
kind: 'card',
|
|
2662
|
+
path,
|
|
2663
|
+
stepType: 'unknown',
|
|
2664
|
+
sentence: [{
|
|
2665
|
+
kind: 'word',
|
|
2666
|
+
text: 'custom step'
|
|
2667
|
+
}],
|
|
2668
|
+
step: {},
|
|
2669
|
+
custom: true
|
|
2670
|
+
};
|
|
2671
|
+
}
|
|
2672
|
+
const stepType = typeof step.type === 'string' ? step.type : 'unknown';
|
|
2673
|
+
const {
|
|
2674
|
+
parts,
|
|
2675
|
+
custom
|
|
2676
|
+
} = cardSentence(step);
|
|
2677
|
+
if (!isBracketStepType(stepType)) return {
|
|
2678
|
+
kind: 'card',
|
|
2679
|
+
path,
|
|
2680
|
+
stepType,
|
|
2681
|
+
sentence: parts,
|
|
2682
|
+
step,
|
|
2683
|
+
custom
|
|
2684
|
+
};
|
|
2685
|
+
const children = Array.isArray(step.steps) ? stepsToNodes(step.steps, `${path}.steps`) : [];
|
|
2686
|
+
const elseChildren = stepType === 'if' && Array.isArray(step.else) ? stepsToNodes(step.else, `${path}.else`) : undefined;
|
|
2687
|
+
return {
|
|
2688
|
+
kind: 'bracket',
|
|
2689
|
+
path,
|
|
2690
|
+
stepType,
|
|
2691
|
+
sentence: parts,
|
|
2692
|
+
step,
|
|
2693
|
+
children,
|
|
2694
|
+
...(elseChildren !== undefined && {
|
|
2695
|
+
elseChildren
|
|
2696
|
+
})
|
|
2697
|
+
};
|
|
2698
|
+
}
|
|
2699
|
+
function isRecord(value) {
|
|
2700
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
2476
2701
|
}
|
|
2477
2702
|
|
|
2478
2703
|
/**
|
|
2479
|
-
*
|
|
2480
|
-
*
|
|
2481
|
-
*
|
|
2482
|
-
*
|
|
2704
|
+
* Rebuilds a recipe from its outline: `recipe`'s keys, in their original
|
|
2705
|
+
* order, with `steps` (when the recipe had one) replaced by the outline's
|
|
2706
|
+
* (possibly edited) step tree. A card's own raw `step` is used as-is; a
|
|
2707
|
+
* bracket's `step` supplies every field but `steps`/`else`, which come from
|
|
2708
|
+
* `children`/`elseChildren` instead, put back at the position `step` itself
|
|
2709
|
+
* held that key at (or appended, for a bracket a `+` menu just added, whose
|
|
2710
|
+
* `step` never had one) — this, together with `recipe-to-outline.mapper.ts`
|
|
2711
|
+
* always keeping the step's whole original object, is what makes
|
|
2712
|
+
* `recipe → outline → recipe` preserve every key and its order.
|
|
2713
|
+
*
|
|
2714
|
+
* @param outline - An outline, from `recipe-to-outline.mapper.ts` or edited by the UI.
|
|
2715
|
+
* @returns The recipe's plain JSON, ready for `save-recipe`.
|
|
2483
2716
|
*/
|
|
2484
|
-
|
|
2485
|
-
|
|
2486
|
-
|
|
2487
|
-
|
|
2488
|
-
|
|
2489
|
-
|
|
2717
|
+
function outlineToRecipe(outline) {
|
|
2718
|
+
if (!('steps' in outline.recipe) && outline.steps.length === 0) return {
|
|
2719
|
+
...outline.recipe
|
|
2720
|
+
};
|
|
2721
|
+
return {
|
|
2722
|
+
...outline.recipe,
|
|
2723
|
+
steps: nodesToSteps(outline.steps)
|
|
2724
|
+
};
|
|
2725
|
+
}
|
|
2726
|
+
function nodesToSteps(nodes) {
|
|
2727
|
+
return nodes.map(node => nodeToStep(node));
|
|
2728
|
+
}
|
|
2729
|
+
function nodeToStep(node) {
|
|
2730
|
+
if (!isBracket(node)) return node.step;
|
|
2731
|
+
const keys = Object.keys(node.step);
|
|
2732
|
+
const step = {};
|
|
2733
|
+
for (const key of keys) {
|
|
2734
|
+
if (key === 'steps') step.steps = nodesToSteps(node.children);else if (key === 'else' && node.stepType === 'if') step.else = nodesToSteps(node.elseChildren ?? []);else step[key] = node.step[key];
|
|
2490
2735
|
}
|
|
2736
|
+
if (!keys.includes('steps')) step.steps = nodesToSteps(node.children);
|
|
2737
|
+
if (node.stepType === 'if' && node.elseChildren !== undefined && !keys.includes('else')) step.else = nodesToSteps(node.elseChildren);
|
|
2738
|
+
return step;
|
|
2491
2739
|
}
|
|
2492
2740
|
|
|
2493
|
-
/** Attributes ranked just under `id`: named explicitly, not "any `data-*`", because a generated `data-reactid` is no more stable than a generated class. */
|
|
2494
|
-
const STABLE_ATTRS = ['data-testid', 'data-id', 'itemprop'];
|
|
2495
|
-
const TIER_ORDER$1 = {
|
|
2496
|
-
'id': 0,
|
|
2497
|
-
'data-attr': 1,
|
|
2498
|
-
'class': 2,
|
|
2499
|
-
'structure': 3,
|
|
2500
|
-
'position': 4
|
|
2501
|
-
};
|
|
2502
2741
|
/**
|
|
2503
|
-
*
|
|
2504
|
-
*
|
|
2505
|
-
*
|
|
2506
|
-
*
|
|
2507
|
-
*
|
|
2508
|
-
*
|
|
2509
|
-
*
|
|
2510
|
-
*
|
|
2511
|
-
* @
|
|
2512
|
-
* @
|
|
2513
|
-
* @returns Candidates, best tier first; ties keep this function's own order.
|
|
2742
|
+
* Opens a recipe folder: every `.json`/`.jsonl` recipe directly inside it
|
|
2743
|
+
* (core's own directory reading, so relative `file:` URLs resolve the same
|
|
2744
|
+
* way a real run would), parsed and, for each output recipe found, bound to
|
|
2745
|
+
* the input recipes that name it. Nothing throws for a bad recipe or a
|
|
2746
|
+
* recipe that does not bind: every problem is reported on its file with the
|
|
2747
|
+
* JSON path core already computes, exactly as `opencraw validate` prints it.
|
|
2748
|
+
*
|
|
2749
|
+
* @param folder - The workspace folder.
|
|
2750
|
+
* @returns Every recipe file found, each with its validation and binding issues.
|
|
2751
|
+
* @throws Error when the folder does not exist or cannot be read.
|
|
2514
2752
|
*/
|
|
2515
|
-
function
|
|
2516
|
-
const
|
|
2517
|
-
|
|
2518
|
-
const
|
|
2519
|
-
|
|
2520
|
-
|
|
2521
|
-
|
|
2522
|
-
|
|
2523
|
-
|
|
2524
|
-
|
|
2525
|
-
|
|
2526
|
-
|
|
2527
|
-
|
|
2528
|
-
}
|
|
2753
|
+
async function openWorkspace(folder) {
|
|
2754
|
+
const documents = await readRecipeSource(folder);
|
|
2755
|
+
const entries = documents.map(document => parseDocument$1(document));
|
|
2756
|
+
const outputs = entries.filter(isParsed('output'));
|
|
2757
|
+
const inputs = entries.filter(isParsed('input'));
|
|
2758
|
+
for (const output of outputs) bindOutput(output, inputs);
|
|
2759
|
+
for (const input of inputs) {
|
|
2760
|
+
if (outputs.every(output => output.recipe.id !== input.recipe.output)) {
|
|
2761
|
+
input.issues.push({
|
|
2762
|
+
path: 'output',
|
|
2763
|
+
message: `no output recipe "${input.recipe.output}" in this workspace`,
|
|
2764
|
+
kind: 'binding'
|
|
2765
|
+
});
|
|
2766
|
+
}
|
|
2529
2767
|
}
|
|
2530
|
-
|
|
2531
|
-
|
|
2532
|
-
|
|
2533
|
-
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
|
|
2768
|
+
return {
|
|
2769
|
+
folder,
|
|
2770
|
+
recipes: entries.map(({
|
|
2771
|
+
file,
|
|
2772
|
+
kind,
|
|
2773
|
+
id,
|
|
2774
|
+
issues,
|
|
2775
|
+
content
|
|
2776
|
+
}) => ({
|
|
2777
|
+
file,
|
|
2778
|
+
kind,
|
|
2779
|
+
id,
|
|
2780
|
+
issues,
|
|
2781
|
+
text: `${JSON.stringify(content, null, 2)}\n`,
|
|
2782
|
+
// The Steps tab needs steps to render: an output/unknown file has none. recipeToOutline never
|
|
2783
|
+
// throws, so this covers a file that fails validation too (its outline just has "custom" cards).
|
|
2784
|
+
outline: kind === 'input' ? recipeToOutline(content) : undefined
|
|
2785
|
+
}))
|
|
2786
|
+
};
|
|
2538
2787
|
}
|
|
2539
|
-
/**
|
|
2540
|
-
function
|
|
2541
|
-
const
|
|
2542
|
-
|
|
2543
|
-
|
|
2544
|
-
|
|
2545
|
-
|
|
2546
|
-
const
|
|
2547
|
-
|
|
2548
|
-
|
|
2549
|
-
|
|
2550
|
-
|
|
2551
|
-
|
|
2552
|
-
|
|
2553
|
-
|
|
2554
|
-
tier: 'structure'
|
|
2555
|
-
});
|
|
2788
|
+
/** Binds one output recipe to the inputs that name it, filing every binding issue back on the recipe it names. */
|
|
2789
|
+
function bindOutput(output, inputs) {
|
|
2790
|
+
const matching = inputs.filter(input => input.recipe.output === output.recipe.id);
|
|
2791
|
+
try {
|
|
2792
|
+
bindRecipeSet(output.recipe, matching.map(input => input.recipe));
|
|
2793
|
+
} catch (error) {
|
|
2794
|
+
if (!(error instanceof RecipeBindingError)) throw error;
|
|
2795
|
+
for (const issue of error.issues) {
|
|
2796
|
+
const target = issue.recipeId === output.recipe.id ? output : matching.find(input => input.recipe.id === issue.recipeId);
|
|
2797
|
+
target?.issues.push({
|
|
2798
|
+
path: issue.path,
|
|
2799
|
+
message: issue.message,
|
|
2800
|
+
kind: 'binding'
|
|
2801
|
+
});
|
|
2802
|
+
}
|
|
2556
2803
|
}
|
|
2557
|
-
return candidates;
|
|
2558
|
-
}
|
|
2559
|
-
/** `parent > tag:nth-of-type(n)`, the position-only fallback; never `nth-child`, which counts every sibling tag, not just this element's own. */
|
|
2560
|
-
function positionCandidates(path) {
|
|
2561
|
-
const node = path.at(-1);
|
|
2562
|
-
if (node === undefined) return [];
|
|
2563
|
-
const own = node.siblingCount > 1 ? `${node.tag}:nth-of-type(${node.index})` : node.tag;
|
|
2564
|
-
const parent = path.at(-2);
|
|
2565
|
-
if (parent === undefined) return [{
|
|
2566
|
-
selector: own,
|
|
2567
|
-
tier: 'position'
|
|
2568
|
-
}];
|
|
2569
|
-
return [{
|
|
2570
|
-
selector: `${simpleSelector(parent)} > ${own}`,
|
|
2571
|
-
tier: 'position'
|
|
2572
|
-
}];
|
|
2573
|
-
}
|
|
2574
|
-
/** A level's own selector fragment, no combinators: id, else stable classes, else the bare tag. */
|
|
2575
|
-
function simpleSelector(level) {
|
|
2576
|
-
if (level.id !== undefined && isStableId(level.id)) return `#${cssEscape(level.id)}`;
|
|
2577
|
-
const classes = stableClasses(level.classes);
|
|
2578
|
-
return classes.length > 0 ? `${level.tag}.${classes.map(token => cssEscape(token)).join('.')}` : level.tag;
|
|
2579
2804
|
}
|
|
2580
|
-
function
|
|
2581
|
-
return
|
|
2805
|
+
function isParsed(kind) {
|
|
2806
|
+
return entry => entry.kind === kind && entry.recipe !== undefined;
|
|
2582
2807
|
}
|
|
2583
|
-
|
|
2584
|
-
|
|
2585
|
-
return
|
|
2808
|
+
function parseDocument$1(document) {
|
|
2809
|
+
const kind = document.content?.kind;
|
|
2810
|
+
if (kind === 'output') return parseOne(document.source, document.content, 'output', parseOutputRecipe);
|
|
2811
|
+
if (kind === 'input') return parseOne(document.source, document.content, 'input', parseInputRecipe);
|
|
2812
|
+
return {
|
|
2813
|
+
file: document.source,
|
|
2814
|
+
kind: 'unknown',
|
|
2815
|
+
content: document.content,
|
|
2816
|
+
issues: [{
|
|
2817
|
+
path: 'kind',
|
|
2818
|
+
message: '"kind" is missing or not "input"/"output"',
|
|
2819
|
+
kind: 'validation'
|
|
2820
|
+
}]
|
|
2821
|
+
};
|
|
2586
2822
|
}
|
|
2587
|
-
|
|
2588
|
-
|
|
2589
|
-
|
|
2823
|
+
function parseOne(file, content, kind, parse) {
|
|
2824
|
+
try {
|
|
2825
|
+
const recipe = parse(content, file);
|
|
2826
|
+
return {
|
|
2827
|
+
file,
|
|
2828
|
+
kind,
|
|
2829
|
+
content,
|
|
2830
|
+
id: recipe.id,
|
|
2831
|
+
issues: [],
|
|
2832
|
+
recipe
|
|
2833
|
+
};
|
|
2834
|
+
} catch (error) {
|
|
2835
|
+
if (!(error instanceof RecipeValidationError)) throw error;
|
|
2836
|
+
return {
|
|
2837
|
+
file,
|
|
2838
|
+
kind,
|
|
2839
|
+
content,
|
|
2840
|
+
issues: error.issues.map(issue => ({
|
|
2841
|
+
path: issue.path === '' ? '(root)' : issue.path,
|
|
2842
|
+
message: issue.message,
|
|
2843
|
+
kind: 'validation'
|
|
2844
|
+
}))
|
|
2845
|
+
};
|
|
2846
|
+
}
|
|
2590
2847
|
}
|
|
2591
2848
|
|
|
2592
|
-
const TIER_ORDER = {
|
|
2593
|
-
'id': 0,
|
|
2594
|
-
'data-attr': 1,
|
|
2595
|
-
'class': 2,
|
|
2596
|
-
'structure': 3,
|
|
2597
|
-
'position': 4
|
|
2598
|
-
};
|
|
2599
2849
|
/**
|
|
2600
|
-
*
|
|
2601
|
-
*
|
|
2602
|
-
*
|
|
2603
|
-
*
|
|
2850
|
+
* Writes a recipe back to its file: pretty JSON, a 2-space indent, a
|
|
2851
|
+
* trailing newline. Phase 0 takes the JSON tab's whole parsed object rather
|
|
2852
|
+
* than a patch, so this is a full replacement, not a splice: loading a file
|
|
2853
|
+
* the studio has not touched and saving it again keeps every key, value and
|
|
2854
|
+
* key order (`JSON.parse` preserves insertion order, and `JSON.stringify`
|
|
2855
|
+
* writes it back), but not necessarily the file's exact original formatting
|
|
2856
|
+
* (alignment, how arrays wrap, a trailing comment). Byte-identical
|
|
2857
|
+
* round-trip for a hand-formatted file is future work; see this file's test
|
|
2858
|
+
* for what is guaranteed today.
|
|
2604
2859
|
*
|
|
2605
|
-
*
|
|
2606
|
-
*
|
|
2607
|
-
|
|
2608
|
-
|
|
2860
|
+
* @param path - The recipe file to write.
|
|
2861
|
+
* @param recipe - The whole recipe object.
|
|
2862
|
+
*/
|
|
2863
|
+
async function saveRecipe(path, recipe) {
|
|
2864
|
+
await writeFile(path, `${JSON.stringify(recipe, null, 2)}\n`, 'utf8');
|
|
2865
|
+
}
|
|
2866
|
+
|
|
2867
|
+
/** How many `name-2.ext`, `name-3.ext`… are tried before giving up on a folder full of same-named copies. */
|
|
2868
|
+
const MAX_NUMBERED_COPIES = 100;
|
|
2869
|
+
/**
|
|
2870
|
+
* Copies a document into a workspace folder, so a recipe can read it with a
|
|
2871
|
+
* `file:` URL the same way a run would read any downloaded file (core's
|
|
2872
|
+
* `HttpClient` handles `file:` itself). The folder is created when missing,
|
|
2873
|
+
* so dropping a file is also a way to start a workspace. A name already in
|
|
2874
|
+
* the folder is never overwritten: the copy gets the next free numbered
|
|
2875
|
+
* name (`report-2.pdf`), and the returned `path`/`url` say which.
|
|
2609
2876
|
*
|
|
2610
|
-
* @param
|
|
2611
|
-
* @param
|
|
2612
|
-
* @param
|
|
2613
|
-
* @returns
|
|
2877
|
+
* @param folder - The workspace folder; created (recursively) when it does not exist.
|
|
2878
|
+
* @param name - The file's own name, no directories: the copy keeps it, extension included.
|
|
2879
|
+
* @param bytes - The file's content.
|
|
2880
|
+
* @returns Where the copy landed.
|
|
2881
|
+
* @throws Error when `name` is not a plain file name, or the folder already holds too many same-named copies.
|
|
2614
2882
|
*/
|
|
2615
|
-
function
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
return {
|
|
2620
|
-
...candidate,
|
|
2621
|
-
matches,
|
|
2622
|
-
hitsTarget
|
|
2623
|
-
};
|
|
2883
|
+
async function importDocument(folder, name, bytes) {
|
|
2884
|
+
assertPlainName(name);
|
|
2885
|
+
await mkdir(folder, {
|
|
2886
|
+
recursive: true
|
|
2624
2887
|
});
|
|
2625
|
-
|
|
2888
|
+
for (const candidate of candidateNames(name)) {
|
|
2889
|
+
const path = join(folder, candidate);
|
|
2890
|
+
try {
|
|
2891
|
+
await writeFile(path, bytes, {
|
|
2892
|
+
flag: 'wx'
|
|
2893
|
+
});
|
|
2894
|
+
return {
|
|
2895
|
+
path,
|
|
2896
|
+
url: pathToFileURL(path).href
|
|
2897
|
+
};
|
|
2898
|
+
} catch (error) {
|
|
2899
|
+
if (!isAlreadyThere(error)) throw error;
|
|
2900
|
+
}
|
|
2901
|
+
}
|
|
2902
|
+
throw new Error(`"${name}" and ${MAX_NUMBERED_COPIES} numbered copies of it are already in ${folder}`);
|
|
2626
2903
|
}
|
|
2627
|
-
|
|
2628
|
-
|
|
2629
|
-
|
|
2904
|
+
function assertPlainName(name) {
|
|
2905
|
+
if (name === '.' || name === '..' || name.includes('\0') || name !== basename(name)) {
|
|
2906
|
+
throw new Error(`"${name}" is not a plain file name`);
|
|
2907
|
+
}
|
|
2908
|
+
}
|
|
2909
|
+
function* candidateNames(name) {
|
|
2910
|
+
yield name;
|
|
2911
|
+
const extension = extname(name);
|
|
2912
|
+
const stem = name.slice(0, name.length - extension.length);
|
|
2913
|
+
for (let copy = 2; copy <= MAX_NUMBERED_COPIES + 1; copy += 1) yield `${stem}-${copy}${extension}`;
|
|
2914
|
+
}
|
|
2915
|
+
function isAlreadyThere(error) {
|
|
2916
|
+
return typeof error === 'object' && error !== null && 'code' in error && error.code === 'EEXIST';
|
|
2630
2917
|
}
|
|
2631
2918
|
|
|
2632
2919
|
/**
|
|
2633
|
-
*
|
|
2634
|
-
*
|
|
2635
|
-
*
|
|
2636
|
-
*
|
|
2637
|
-
*
|
|
2920
|
+
* Handles `import-document` (issue #120): decodes the base64 bytes and copies
|
|
2921
|
+
* the file into the folder (see `recipe-workspace/import-document.use-case.ts`
|
|
2922
|
+
* for the folder creation and the no-overwrite numbering). Nothing is
|
|
2923
|
+
* broadcast: the copy is not a recipe, the workspace listing does not change
|
|
2924
|
+
* until the UI saves the recipe pair that points at it, which `save-recipe`
|
|
2925
|
+
* announces as usual.
|
|
2638
2926
|
*
|
|
2639
|
-
*
|
|
2640
|
-
*
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
|
|
2646
|
-
|
|
2647
|
-
|
|
2927
|
+
* @param command - The folder, the file's name and its base64 content.
|
|
2928
|
+
* @returns Where the copy landed and its `file:` URL.
|
|
2929
|
+
*/
|
|
2930
|
+
async function handleImportDocument(command) {
|
|
2931
|
+
return importDocument(command.folder, command.name, new Uint8Array(Buffer.from(command.bytes, 'base64')));
|
|
2932
|
+
}
|
|
2933
|
+
|
|
2934
|
+
const GENERATED_ID = /^(?:react-|radix-|:r|ember|css-|mui-|headlessui-)/i;
|
|
2935
|
+
/**
|
|
2936
|
+
* The path from the document root to `node`, one level per ancestor. Two
|
|
2937
|
+
* elements that share an ancestor produce identical levels up to that
|
|
2938
|
+
* ancestor and diverge exactly where their parents stop being the same node
|
|
2939
|
+
* — `list-inference.algorithm.ts` leans on that to find the nearest pair of
|
|
2940
|
+
* siblings two picks belong to, without ever touching cheerio again.
|
|
2648
2941
|
*
|
|
2649
|
-
* @param
|
|
2650
|
-
* @param
|
|
2651
|
-
* @returns The
|
|
2652
|
-
* @throws ShadowDomUnsupportedError when either path crosses a declarative shadow root.
|
|
2942
|
+
* @param $ - The parsed document.
|
|
2943
|
+
* @param node - The element to describe.
|
|
2944
|
+
* @returns The path, root first.
|
|
2653
2945
|
*/
|
|
2654
|
-
function
|
|
2655
|
-
|
|
2656
|
-
|
|
2657
|
-
|
|
2658
|
-
|
|
2659
|
-
|
|
2660
|
-
|
|
2661
|
-
divergedAt = index;
|
|
2662
|
-
break;
|
|
2663
|
-
}
|
|
2946
|
+
function domPathOf($, node) {
|
|
2947
|
+
const chain = [];
|
|
2948
|
+
let current = node;
|
|
2949
|
+
while (current !== undefined && current.length > 0 && isElement(current.get(0))) {
|
|
2950
|
+
chain.unshift(current);
|
|
2951
|
+
const parent = current.parent();
|
|
2952
|
+
current = parent.length > 0 ? parent : undefined;
|
|
2664
2953
|
}
|
|
2665
|
-
|
|
2666
|
-
|
|
2667
|
-
|
|
2668
|
-
const
|
|
2669
|
-
|
|
2670
|
-
|
|
2671
|
-
const
|
|
2672
|
-
const
|
|
2673
|
-
const
|
|
2674
|
-
const
|
|
2954
|
+
return chain.map(element => levelOf($, element));
|
|
2955
|
+
}
|
|
2956
|
+
function levelOf($, element) {
|
|
2957
|
+
const node = element.get(0);
|
|
2958
|
+
const tag = node.tagName.toLowerCase();
|
|
2959
|
+
const attrs = {};
|
|
2960
|
+
const entries = Object.entries(node.attribs ?? {});
|
|
2961
|
+
for (const [name, value] of entries) attrs[name] = value;
|
|
2962
|
+
const classes = (attrs.class ?? '').split(/\s+/).filter(token => token.length > 0);
|
|
2963
|
+
const siblings = element.parent().length > 0 ? element.parent().children(tag) : $.root().children(tag);
|
|
2964
|
+
const index = siblings.toArray().indexOf(node) + 1;
|
|
2675
2965
|
return {
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
|
|
2966
|
+
tag,
|
|
2967
|
+
id: attrs.id,
|
|
2968
|
+
classes,
|
|
2969
|
+
attrs,
|
|
2970
|
+
index: Math.max(index, 1),
|
|
2971
|
+
siblingCount: Math.max(siblings.length, 1)
|
|
2680
2972
|
};
|
|
2681
2973
|
}
|
|
2682
|
-
function
|
|
2683
|
-
return
|
|
2974
|
+
function isElement(node) {
|
|
2975
|
+
return node !== undefined && node.type === 'tag';
|
|
2684
2976
|
}
|
|
2685
2977
|
/**
|
|
2686
|
-
*
|
|
2687
|
-
*
|
|
2688
|
-
*
|
|
2689
|
-
*
|
|
2690
|
-
* the two picks' local structure differs in depth (a real possibility once
|
|
2691
|
-
* a field is missing on one item; `list-inference.algorithm.test.ts`'s
|
|
2692
|
-
* shifted-field fixture is exactly this) — the merge just stops there,
|
|
2693
|
-
* which is fine: a field selector only needs to reach as deep as the
|
|
2694
|
-
* shallower of the two picks.
|
|
2978
|
+
* Whether `id` looks handed by a framework (`react-`, a CSS-module or a
|
|
2979
|
+
* `radix-`/`mui-` autogenerated one) rather than by the site's own markup —
|
|
2980
|
+
* `class-token.policy.ts` has the equivalent check for class tokens, kept
|
|
2981
|
+
* separate because ids and classes are ranked at different tiers.
|
|
2695
2982
|
*/
|
|
2696
|
-
function
|
|
2697
|
-
|
|
2698
|
-
const merged = [];
|
|
2699
|
-
for (let index = 0; index < length; index++) {
|
|
2700
|
-
const left = a[index];
|
|
2701
|
-
const right = b[index];
|
|
2702
|
-
if (left.tag !== right.tag) break;
|
|
2703
|
-
merged.push(mergeLevel(left, right));
|
|
2704
|
-
}
|
|
2705
|
-
return merged.length > 0 ? merged : [a[0]];
|
|
2983
|
+
function isStableId(id) {
|
|
2984
|
+
return id.length > 0 && !GENERATED_ID.test(id);
|
|
2706
2985
|
}
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
|
|
2716
|
-
|
|
2717
|
-
|
|
2718
|
-
|
|
2719
|
-
|
|
2720
|
-
|
|
2986
|
+
/**
|
|
2987
|
+
* Whether `path` failed to reach the real document root. `domPathOf` walks
|
|
2988
|
+
* up through cheerio's own `.parent()`, which (matching real DOM semantics)
|
|
2989
|
+
* never crosses from a `<template>`'s content into the light DOM outside
|
|
2990
|
+
* it — a `<template shadowrootmode>` is exactly declarative shadow DOM, the
|
|
2991
|
+
* only shadow-DOM shape that can show up in static, server-captured markup
|
|
2992
|
+
* at all (imperative `attachShadow()` content never serialises into
|
|
2993
|
+
* `page.content()`/a fetched response in the first place, so that case
|
|
2994
|
+
* cannot reach this slice and needs no separate detection). A path whose
|
|
2995
|
+
* first level is not `html` was cut short at exactly this kind of boundary:
|
|
2996
|
+
* v1 does not pierce it, so callers report a clear "shadow DOM is not
|
|
2997
|
+
* supported" error instead of building ancestor-based candidates over a
|
|
2998
|
+
* chain that stops partway up.
|
|
2999
|
+
*/
|
|
3000
|
+
function crossesShadowRoot(path) {
|
|
3001
|
+
return path.length === 0 || path[0].tag !== 'html';
|
|
2721
3002
|
}
|
|
2722
|
-
|
|
2723
|
-
|
|
3003
|
+
/** Parses `html` and returns cheerio's API, for callers that need more than one path from the same document. */
|
|
3004
|
+
function parseDocument(html) {
|
|
3005
|
+
return load(html);
|
|
3006
|
+
}
|
|
3007
|
+
/**
|
|
3008
|
+
* The path to the element the snapshot stamped with `nodeId` (its
|
|
3009
|
+
* `data-oc-node` attribute — `@opencraw/core`'s `NODE_ID_ATTRIBUTE`,
|
|
3010
|
+
* `packages/studio/src/page-snapshot/rewrite-document.mapper.ts`). This is
|
|
3011
|
+
* how a click in the content pane's iframe — which only ever needs to read
|
|
3012
|
+
* that one attribute off the element the user clicked — turns into
|
|
3013
|
+
* something `selector-inference`'s other algorithms can work with, without
|
|
3014
|
+
* the browser-side picker script needing to know anything about ancestry,
|
|
3015
|
+
* classes or candidate ranking itself.
|
|
3016
|
+
*
|
|
3017
|
+
* @param html - The snapshot document (already stamped with node ids).
|
|
3018
|
+
* @param nodeId - The `data-oc-node` value to find.
|
|
3019
|
+
* @returns The node's path, or `undefined` when no element carries that id.
|
|
3020
|
+
*/
|
|
3021
|
+
function pathToNode(html, nodeId) {
|
|
3022
|
+
const $ = parseDocument(html);
|
|
3023
|
+
const found = $(`[${NODE_ID_ATTRIBUTE}="${cssAttributeValue(nodeId)}"]`);
|
|
3024
|
+
return found.length === 0 ? undefined : domPathOf($, found.first());
|
|
3025
|
+
}
|
|
3026
|
+
function cssAttributeValue(value) {
|
|
3027
|
+
return value.replaceAll('\\', String.raw`\\`).replaceAll('"', String.raw`\"`);
|
|
2724
3028
|
}
|
|
2725
3029
|
|
|
3030
|
+
/** Attributes worth reading instead of text, in preference order, when the clicked element has one. */
|
|
3031
|
+
const PREFERRED_ATTRS = {
|
|
3032
|
+
a: ['href'],
|
|
3033
|
+
img: ['src', 'alt']
|
|
3034
|
+
};
|
|
2726
3035
|
/**
|
|
2727
|
-
*
|
|
2728
|
-
*
|
|
2729
|
-
*
|
|
2730
|
-
*
|
|
2731
|
-
* studio plan §3.2). All the actual candidate/ranking/list-inference work
|
|
2732
|
-
* is `selector-inference`'s (pure); this handler only locates the clicked
|
|
2733
|
-
* nodes in the cached document and calls into it.
|
|
3036
|
+
* What a `Read` card's `take` should be for the element a pick landed on
|
|
3037
|
+
* (issue #91, studio plan §3.2): `attr:href` for an `<a>`, `attr:src` for an
|
|
3038
|
+
* `<img>`, `attr:<name>` for the first `data-*` attribute otherwise, `json`
|
|
3039
|
+
* for a `<script type="application/json">`, else the default, `text`.
|
|
2734
3040
|
*
|
|
2735
|
-
* @param
|
|
2736
|
-
* @
|
|
2737
|
-
* @returns A field pick, a list pick, or `"unsupported"` with why.
|
|
2738
|
-
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
3041
|
+
* @param level - The clicked element's own level (the last entry of its `DomPath`).
|
|
3042
|
+
* @returns The `take` value to put on the `extract` step.
|
|
2739
3043
|
*/
|
|
2740
|
-
function
|
|
2741
|
-
|
|
2742
|
-
|
|
2743
|
-
|
|
2744
|
-
|
|
2745
|
-
|
|
2746
|
-
kind: 'unsupported',
|
|
2747
|
-
reason: 'one of the picked nodes is no longer in the snapshot; take a fresh snapshot and try again'
|
|
2748
|
-
};
|
|
2749
|
-
const [pathA, pathB] = paths;
|
|
2750
|
-
if (pathB === undefined) return {
|
|
2751
|
-
kind: 'field',
|
|
2752
|
-
field: fieldPick(html, pathA)
|
|
2753
|
-
};
|
|
2754
|
-
try {
|
|
2755
|
-
const result = inferList(pathA, pathB);
|
|
2756
|
-
if (result === null) return {
|
|
2757
|
-
kind: 'unsupported',
|
|
2758
|
-
reason: 'these two picks do not belong to the same list: pick the same field on two similar items'
|
|
2759
|
-
};
|
|
2760
|
-
const item = bestCandidate(html, candidatesFor(result.itemPath, false));
|
|
2761
|
-
if (item === undefined) return {
|
|
2762
|
-
kind: 'unsupported',
|
|
2763
|
-
reason: 'no selector matches every item of this list; try picking two items closer to each other in the markup'
|
|
2764
|
-
};
|
|
2765
|
-
const field = fieldPick(html, result.fieldPathA, result.fieldPath);
|
|
2766
|
-
return {
|
|
2767
|
-
kind: 'list',
|
|
2768
|
-
item: {
|
|
2769
|
-
selector: item.selector,
|
|
2770
|
-
tier: item.tier,
|
|
2771
|
-
matches: item.matches
|
|
2772
|
-
},
|
|
2773
|
-
field
|
|
2774
|
-
};
|
|
2775
|
-
} catch (error) {
|
|
2776
|
-
if (error instanceof ShadowDomUnsupportedError) return {
|
|
2777
|
-
kind: 'unsupported',
|
|
2778
|
-
reason: error.message
|
|
2779
|
-
};
|
|
2780
|
-
throw error;
|
|
3044
|
+
function takeKindFor(level) {
|
|
3045
|
+
if (level.tag === 'script' && (level.attrs.type ?? '').toLowerCase() === 'application/json') return 'json';
|
|
3046
|
+
const preferred = PREFERRED_ATTRS[level.tag];
|
|
3047
|
+
if (preferred !== undefined) {
|
|
3048
|
+
const attribute = preferred.find(name => level.attrs[name] !== undefined);
|
|
3049
|
+
if (attribute !== undefined) return `attr:${attribute}`;
|
|
2781
3050
|
}
|
|
2782
|
-
|
|
2783
|
-
|
|
2784
|
-
|
|
2785
|
-
const leaf = clicked.at(-1);
|
|
2786
|
-
const take = leaf === undefined ? 'text' : takeKindFor(leaf);
|
|
2787
|
-
const best = bestCandidate(html, candidatesFor(rankSource));
|
|
2788
|
-
if (best === undefined) throw new Error('no selector matches the picked node — this should not happen, since the node came from this same document');
|
|
2789
|
-
return {
|
|
2790
|
-
selector: best.selector,
|
|
2791
|
-
tier: best.tier,
|
|
2792
|
-
take,
|
|
2793
|
-
matches: best.matches
|
|
2794
|
-
};
|
|
3051
|
+
const dataAttribute = Object.keys(level.attrs).find(name => name.startsWith('data-') && name !== 'data-oc-node' && name !== 'data-oc-hidden');
|
|
3052
|
+
if (dataAttribute !== undefined) return `attr:${dataAttribute}`;
|
|
3053
|
+
return 'text';
|
|
2795
3054
|
}
|
|
2796
3055
|
|
|
2797
|
-
/** A leaf element's own text longer than this is cut, with an ellipsis. */
|
|
2798
|
-
const TEXT_LIMIT = 80;
|
|
2799
|
-
/** Consecutive siblings whose subtree is identical (ids aside) collapse into one row once there are at least this many — a real page can have tens of thousands of `<li>`s, and showing all of them is not "inspecting" it (issue #93). */
|
|
2800
|
-
const MIN_REPEAT_RUN = 3;
|
|
2801
3056
|
/**
|
|
2802
|
-
*
|
|
2803
|
-
*
|
|
2804
|
-
*
|
|
2805
|
-
*
|
|
2806
|
-
*
|
|
2807
|
-
*
|
|
3057
|
+
* Which class tokens are worth building a selector on. A hand-authored class
|
|
3058
|
+
* (`product_pod`, `price_color`) survives a rebuild; a generated one
|
|
3059
|
+
* (a content hash, a CSS-module suffix, a styled-components token) does not,
|
|
3060
|
+
* so a candidate built on it looks fine on the snapshot it was picked from
|
|
3061
|
+
* and breaks on the very next deploy. This is the "generated/hashed class
|
|
3062
|
+
* names" tradeoff issue #91 calls out explicitly: the heuristics below are a
|
|
3063
|
+
* reasoned judgement call, not a guarantee — see the doc comments on each
|
|
3064
|
+
* pattern for what real-world tool it targets.
|
|
3065
|
+
*/
|
|
3066
|
+
const GENERATED_TOKEN_PATTERNS = [/** styled-components / emotion: `sc-bdVaJa`, `css-1x2y3z4` */
|
|
3067
|
+
/^(?:sc|css)-[a-z0-9]+$/i, /** CSS Modules, most bundlers' default hash suffix: `wrapper__3fA9k`, `Button-module_root__aB3dK` */
|
|
3068
|
+
/__[a-z0-9]{4,10}$/i, /** JSS / MUI: `jss123`, `MuiButton-root-145` */
|
|
3069
|
+
/^(?:jss|mui[a-z]*-)[a-z0-9-]*\d{2}$/i, /** A token that is only digits: not a class a human would type. */
|
|
3070
|
+
/^\d+$/, /** A bare hex hash: `a3f9c21`, `1a2b3c4d` (6+ hex chars, nothing else). */
|
|
3071
|
+
/^[0-9a-f]{6,}$/i, /** Tailwind/atomic-CSS-in-JS runtime classes some bundlers emit: `_1a2b3c` */
|
|
3072
|
+
/^_[0-9a-z]{5,}$/i];
|
|
3073
|
+
/**
|
|
3074
|
+
* Whether a class token looks hand-authored and safe to build a selector on.
|
|
2808
3075
|
*
|
|
2809
|
-
* @param
|
|
2810
|
-
* @returns
|
|
2811
|
-
* @throws Error when the document has no `<html>` element (should not happen: `rewriteDocument` always produces one).
|
|
3076
|
+
* @param token - One class, no surrounding whitespace.
|
|
3077
|
+
* @returns `false` for anything matching a known generated-class shape.
|
|
2812
3078
|
*/
|
|
2813
|
-
function
|
|
2814
|
-
|
|
2815
|
-
const html = $('html').get(0);
|
|
2816
|
-
if (html === undefined) throw new Error('domTree: the snapshot has no <html> element');
|
|
2817
|
-
return buildNode($, html);
|
|
2818
|
-
}
|
|
2819
|
-
function buildNode($, element) {
|
|
2820
|
-
const $el = $(element);
|
|
2821
|
-
const attributes = {
|
|
2822
|
-
...element.attribs
|
|
2823
|
-
};
|
|
2824
|
-
const nodeId = attributes[NODE_ID_ATTRIBUTE] ?? '';
|
|
2825
|
-
const hidden = attributes[HIDDEN_ATTRIBUTE] === '1';
|
|
2826
|
-
delete attributes[NODE_ID_ATTRIBUTE];
|
|
2827
|
-
delete attributes[HIDDEN_ATTRIBUTE];
|
|
2828
|
-
const id = attributes.id;
|
|
2829
|
-
const classes = (attributes.class ?? '').split(/\s+/).filter(token => token !== '');
|
|
2830
|
-
const childElements = $el.children().toArray();
|
|
2831
|
-
const children = collapseRepeats(childElements.map(child => buildNode($, child)));
|
|
2832
|
-
const text = childElements.length === 0 ? abbreviate($el.text()) : undefined;
|
|
2833
|
-
return {
|
|
2834
|
-
nodeId,
|
|
2835
|
-
tag: element.tagName.toLowerCase(),
|
|
2836
|
-
...(id !== undefined && id !== '' && {
|
|
2837
|
-
id
|
|
2838
|
-
}),
|
|
2839
|
-
...(classes.length > 0 && {
|
|
2840
|
-
classes
|
|
2841
|
-
}),
|
|
2842
|
-
attributes,
|
|
2843
|
-
...(text !== undefined && text !== '' && {
|
|
2844
|
-
text
|
|
2845
|
-
}),
|
|
2846
|
-
hidden,
|
|
2847
|
-
children
|
|
2848
|
-
};
|
|
3079
|
+
function isStableClassToken(token) {
|
|
3080
|
+
return token.length > 0 && GENERATED_TOKEN_PATTERNS.every(pattern => !pattern.test(token));
|
|
2849
3081
|
}
|
|
2850
3082
|
/**
|
|
2851
|
-
*
|
|
2852
|
-
* is identical (their own `nodeId`s aside) into one representative row, the
|
|
2853
|
-
* issue's `×20` — real pages have tens of thousands of nodes, and this is
|
|
2854
|
-
* what makes the tree usable rather than merely correct.
|
|
3083
|
+
* The subset of `tokens` worth ranking a selector candidate on, order kept.
|
|
2855
3084
|
*
|
|
2856
|
-
* @param
|
|
2857
|
-
* @returns
|
|
3085
|
+
* @param tokens - An element's class list.
|
|
3086
|
+
* @returns Only the stable tokens.
|
|
2858
3087
|
*/
|
|
2859
|
-
function
|
|
2860
|
-
|
|
2861
|
-
let at = 0;
|
|
2862
|
-
while (at < nodes.length) {
|
|
2863
|
-
const signature = signatureOf(nodes[at]);
|
|
2864
|
-
let end = at + 1;
|
|
2865
|
-
while (end < nodes.length && signatureOf(nodes[end]) === signature) end += 1;
|
|
2866
|
-
const run = nodes.slice(at, end);
|
|
2867
|
-
if (run.length >= MIN_REPEAT_RUN) {
|
|
2868
|
-
result.push({
|
|
2869
|
-
...run[0],
|
|
2870
|
-
repeatCount: run.length,
|
|
2871
|
-
repeatNodeIds: run.map(node => node.nodeId)
|
|
2872
|
-
});
|
|
2873
|
-
} else {
|
|
2874
|
-
result.push(...run);
|
|
2875
|
-
}
|
|
2876
|
-
at = end;
|
|
2877
|
-
}
|
|
2878
|
-
return result;
|
|
3088
|
+
function stableClasses(tokens) {
|
|
3089
|
+
return tokens.filter(token => isStableClassToken(token));
|
|
2879
3090
|
}
|
|
2880
|
-
/**
|
|
2881
|
-
|
|
2882
|
-
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
3091
|
+
/**
|
|
3092
|
+
* Whether two class lists describe "the same kind of element" for list
|
|
3093
|
+
* inference: same stable classes, generated tokens (which often differ
|
|
3094
|
+
* per-instance, e.g. a React `key`-derived class) ignored on both sides.
|
|
3095
|
+
*
|
|
3096
|
+
* @param a - One element's classes.
|
|
3097
|
+
* @param b - Another element's classes.
|
|
3098
|
+
* @returns Whether their stable class sets are equal, order ignored.
|
|
3099
|
+
*/
|
|
3100
|
+
function sameClassSignature(a, b) {
|
|
3101
|
+
const left = new Set(stableClasses(a));
|
|
3102
|
+
const right = new Set(stableClasses(b));
|
|
3103
|
+
if (left.size !== right.size) return false;
|
|
3104
|
+
return [...left].every(token => right.has(token));
|
|
2887
3105
|
}
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
3106
|
+
|
|
3107
|
+
/**
|
|
3108
|
+
* Thrown when a pick or a list-inference walk crosses a declarative shadow
|
|
3109
|
+
* root (`<template shadowrootmode>`). v1 does not pierce shadow DOM: rather
|
|
3110
|
+
* than infer a selector that silently matches the wrong thing (or nothing,
|
|
3111
|
+
* once the page hydrates), the studio surfaces this and stops.
|
|
3112
|
+
*/
|
|
3113
|
+
class ShadowDomUnsupportedError extends Error {
|
|
3114
|
+
path;
|
|
3115
|
+
constructor(path) {
|
|
3116
|
+
super(`"${path}" is inside a shadow root: shadow DOM is not supported for picking (v1)`);
|
|
3117
|
+
this.path = path;
|
|
3118
|
+
this.name = 'ShadowDomUnsupportedError';
|
|
3119
|
+
}
|
|
2891
3120
|
}
|
|
2892
3121
|
|
|
2893
|
-
/**
|
|
2894
|
-
const
|
|
2895
|
-
const
|
|
3122
|
+
/** Attributes ranked just under `id`: named explicitly, not "any `data-*`", because a generated `data-reactid` is no more stable than a generated class. */
|
|
3123
|
+
const STABLE_ATTRS = ['data-testid', 'data-id', 'itemprop'];
|
|
3124
|
+
const TIER_ORDER$1 = {
|
|
3125
|
+
'id': 0,
|
|
3126
|
+
'data-attr': 1,
|
|
3127
|
+
'class': 2,
|
|
3128
|
+
'structure': 3,
|
|
3129
|
+
'position': 4
|
|
3130
|
+
};
|
|
2896
3131
|
/**
|
|
2897
|
-
*
|
|
2898
|
-
*
|
|
2899
|
-
*
|
|
2900
|
-
*
|
|
2901
|
-
*
|
|
2902
|
-
*
|
|
2903
|
-
*
|
|
2904
|
-
* reach it — is new).
|
|
2905
|
-
*
|
|
2906
|
-
* Takes the **raw** captured document, not the display snapshot
|
|
2907
|
-
* (`page-snapshot`'s `SnapshotResult.html`): `rewriteDocument` strips every
|
|
2908
|
-
* `<script>` for the sandboxed iframe, which would make JSON-LD, inline
|
|
2909
|
-
* state and other script-tag data invisible here. `<meta>`/`<link>` survive
|
|
2910
|
-
* rewriting untouched, so using the raw document for every kind (rather than
|
|
2911
|
-
* mixing two documents) keeps this function simple and its selectors
|
|
2912
|
-
* verified against what the engine will actually crawl.
|
|
3132
|
+
* Every selector candidate worth trying for the last node of `path`, ranked
|
|
3133
|
+
* best-first: `id` > stable `data-*`/`itemprop` attributes > stable class
|
|
3134
|
+
* tokens > ancestor structure (`parent > tag`) > position (`nth-of-type`) as
|
|
3135
|
+
* a last resort (issue #91's ranking, studio plan §3.2). Structure and
|
|
3136
|
+
* position candidates only look as far up as `path` itself goes — pass a
|
|
3137
|
+
* sub-path (e.g. from an item's root down) to keep a field's selector
|
|
3138
|
+
* relative to its item instead of absolute to the document.
|
|
2913
3139
|
*
|
|
2914
|
-
* @param
|
|
2915
|
-
* @
|
|
3140
|
+
* @param path - The node's ancestry, root (or scope root) first, itself last.
|
|
3141
|
+
* @param includePosition - `false` for an item selector: a position candidate would only ever match one of the items, which is never what an item selector should do.
|
|
3142
|
+
* @returns Candidates, best tier first; ties keep this function's own order.
|
|
2916
3143
|
*/
|
|
2917
|
-
function
|
|
2918
|
-
const
|
|
2919
|
-
|
|
2920
|
-
|
|
2921
|
-
|
|
2922
|
-
|
|
2923
|
-
|
|
2924
|
-
findings.push(...scriptFinding($, element, rawHtml, 'json-script', () => `json: ${structuralSelector($, element)}`));
|
|
2925
|
-
});
|
|
2926
|
-
$('script').each((_index, element) => {
|
|
2927
|
-
const type = $(element).attr('type');
|
|
2928
|
-
if (type !== undefined && type !== 'text/javascript' && type !== 'module') return;
|
|
2929
|
-
const text = $(element).text();
|
|
2930
|
-
const match = INLINE_STATE_PATTERN.exec(text);
|
|
2931
|
-
if (match === null) return;
|
|
2932
|
-
const value = tryParseJson(text);
|
|
2933
|
-
const selector = structuralSelector($, element);
|
|
2934
|
-
findings.push({
|
|
2935
|
-
kind: 'inline-state',
|
|
2936
|
-
label: match[1],
|
|
2937
|
-
selector,
|
|
2938
|
-
selectorKind: 'css',
|
|
2939
|
-
matches: countMatches(rawHtml, selector),
|
|
2940
|
-
keys: keysOf(value)
|
|
2941
|
-
});
|
|
3144
|
+
function candidatesFor(path, includePosition = true) {
|
|
3145
|
+
const node = path.at(-1);
|
|
3146
|
+
if (node === undefined) return [];
|
|
3147
|
+
const candidates = [];
|
|
3148
|
+
if (node.id !== undefined && isStableId(node.id)) candidates.push({
|
|
3149
|
+
selector: `#${cssEscape(node.id)}`,
|
|
3150
|
+
tier: 'id'
|
|
2942
3151
|
});
|
|
2943
|
-
|
|
2944
|
-
const
|
|
2945
|
-
|
|
2946
|
-
|
|
2947
|
-
|
|
2948
|
-
label: `meta: ${name}`,
|
|
2949
|
-
selector,
|
|
2950
|
-
selectorKind: 'css',
|
|
2951
|
-
matches: countMatches(rawHtml, selector),
|
|
2952
|
-
attribute: 'content'
|
|
3152
|
+
for (const name of STABLE_ATTRS) {
|
|
3153
|
+
const value = node.attrs[name];
|
|
3154
|
+
if (value !== undefined && value.length > 0) candidates.push({
|
|
3155
|
+
selector: `${node.tag}[${name}="${cssValue(value)}"]`,
|
|
3156
|
+
tier: 'data-attr'
|
|
2953
3157
|
});
|
|
3158
|
+
}
|
|
3159
|
+
const classes = stableClasses(node.classes);
|
|
3160
|
+
if (classes.length > 0) candidates.push({
|
|
3161
|
+
selector: `${node.tag}.${classes.map(token => cssEscape(token)).join('.')}`,
|
|
3162
|
+
tier: 'class'
|
|
2954
3163
|
});
|
|
2955
|
-
|
|
2956
|
-
|
|
2957
|
-
|
|
2958
|
-
|
|
2959
|
-
|
|
2960
|
-
|
|
2961
|
-
|
|
2962
|
-
|
|
2963
|
-
|
|
2964
|
-
|
|
3164
|
+
candidates.push(...structureCandidates(path));
|
|
3165
|
+
if (includePosition) candidates.push(...positionCandidates(path));
|
|
3166
|
+
return sortByTier(candidates);
|
|
3167
|
+
}
|
|
3168
|
+
/** `ancestor > ... > tag`, growing from 2 to 4 levels (or to the top of `path`), each ancestor named by its own best simple form. */
|
|
3169
|
+
function structureCandidates(path) {
|
|
3170
|
+
const node = path.at(-1);
|
|
3171
|
+
if (node === undefined) return [];
|
|
3172
|
+
const candidates = [];
|
|
3173
|
+
const maxDepth = Math.min(path.length - 1, 3);
|
|
3174
|
+
for (let depth = 1; depth <= maxDepth; depth++) {
|
|
3175
|
+
const ancestor = path.at(-1 - depth);
|
|
3176
|
+
if (ancestor === undefined) break;
|
|
3177
|
+
const between = Array.from({
|
|
3178
|
+
length: depth - 1
|
|
3179
|
+
}, (_, index) => path.at(-depth + index)?.tag ?? '');
|
|
3180
|
+
const parts = [simpleSelector(ancestor), ...between, node.tag];
|
|
3181
|
+
candidates.push({
|
|
3182
|
+
selector: parts.join(' > '),
|
|
3183
|
+
tier: 'structure'
|
|
2965
3184
|
});
|
|
2966
|
-
}
|
|
2967
|
-
return
|
|
3185
|
+
}
|
|
3186
|
+
return candidates;
|
|
2968
3187
|
}
|
|
2969
|
-
|
|
2970
|
-
|
|
2971
|
-
|
|
2972
|
-
|
|
3188
|
+
/** `parent > tag:nth-of-type(n)`, the position-only fallback; never `nth-child`, which counts every sibling tag, not just this element's own. */
|
|
3189
|
+
function positionCandidates(path) {
|
|
3190
|
+
const node = path.at(-1);
|
|
3191
|
+
if (node === undefined) return [];
|
|
3192
|
+
const own = node.siblingCount > 1 ? `${node.tag}:nth-of-type(${node.index})` : node.tag;
|
|
3193
|
+
const parent = path.at(-2);
|
|
3194
|
+
if (parent === undefined) return [{
|
|
3195
|
+
selector: own,
|
|
3196
|
+
tier: 'position'
|
|
3197
|
+
}];
|
|
2973
3198
|
return [{
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
selector,
|
|
2977
|
-
selectorKind: 'css',
|
|
2978
|
-
matches: countMatches(rawHtml, selector),
|
|
2979
|
-
keys: keysOf(value)
|
|
3199
|
+
selector: `${simpleSelector(parent)} > ${own}`,
|
|
3200
|
+
tier: 'position'
|
|
2980
3201
|
}];
|
|
2981
3202
|
}
|
|
2982
|
-
|
|
2983
|
-
|
|
2984
|
-
if (
|
|
2985
|
-
|
|
2986
|
-
|
|
2987
|
-
|
|
2988
|
-
|
|
2989
|
-
|
|
2990
|
-
|
|
2991
|
-
|
|
3203
|
+
/** A level's own selector fragment, no combinators: id, else stable classes, else the bare tag. */
|
|
3204
|
+
function simpleSelector(level) {
|
|
3205
|
+
if (level.id !== undefined && isStableId(level.id)) return `#${cssEscape(level.id)}`;
|
|
3206
|
+
const classes = stableClasses(level.classes);
|
|
3207
|
+
return classes.length > 0 ? `${level.tag}.${classes.map(token => cssEscape(token)).join('.')}` : level.tag;
|
|
3208
|
+
}
|
|
3209
|
+
function sortByTier(candidates) {
|
|
3210
|
+
return [...candidates].sort((a, b) => TIER_ORDER$1[a.tier] - TIER_ORDER$1[b.tier]);
|
|
3211
|
+
}
|
|
3212
|
+
/** Escapes a CSS identifier segment (an id or a class token) well enough for the shapes real sites use; not a full CSS.escape polyfill. */
|
|
3213
|
+
function cssEscape(token) {
|
|
3214
|
+
return token.replaceAll(/([^\w-])/gi, String.raw`\$1`);
|
|
3215
|
+
}
|
|
3216
|
+
/** Escapes a value inside `[name="…"]`. */
|
|
3217
|
+
function cssValue(value) {
|
|
3218
|
+
return value.replaceAll(/(["\\])/g, String.raw`\$1`);
|
|
2992
3219
|
}
|
|
3220
|
+
|
|
3221
|
+
const TIER_ORDER = {
|
|
3222
|
+
'id': 0,
|
|
3223
|
+
'data-attr': 1,
|
|
3224
|
+
'class': 2,
|
|
3225
|
+
'structure': 3,
|
|
3226
|
+
'position': 4
|
|
3227
|
+
};
|
|
2993
3228
|
/**
|
|
2994
|
-
*
|
|
2995
|
-
*
|
|
2996
|
-
*
|
|
2997
|
-
*
|
|
2998
|
-
* element, on the raw document this ran against and on the real page (a
|
|
2999
|
-
* purely structural selector, unaffected by the snapshot's own rewriting).
|
|
3229
|
+
* Verifies and orders selector candidates against real markup, by running
|
|
3230
|
+
* each one through the engine's own CSS matching (`@opencraw/core`'s
|
|
3231
|
+
* `countMatches`/`matchesOf`) rather than reimplementing it — issue #91's
|
|
3232
|
+
* "every candidate is verified" requirement.
|
|
3000
3233
|
*
|
|
3001
|
-
*
|
|
3002
|
-
*
|
|
3003
|
-
*
|
|
3234
|
+
* Ordering: candidates that do not match `targetNodeId` (when given) sort
|
|
3235
|
+
* last; among the rest, best tier first, and within a tier the fewest
|
|
3236
|
+
* matches first (the most specific selector that still hits the target).
|
|
3237
|
+
* A candidate matching nothing is dropped entirely: it is not a real option.
|
|
3238
|
+
*
|
|
3239
|
+
* @param html - The document to verify against (the snapshot, or the live page for the second, cross-check pass).
|
|
3240
|
+
* @param candidates - Unranked candidates, e.g. from `candidatesFor`.
|
|
3241
|
+
* @param targetNodeId - The `data-oc-node` id the pick landed on, when the document carries those stamps (the snapshot does; a live page fetched separately does not).
|
|
3242
|
+
* @returns Every candidate that matched at least one element, ranked best first.
|
|
3004
3243
|
*/
|
|
3005
|
-
function
|
|
3006
|
-
const
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3013
|
-
|
|
3014
|
-
|
|
3015
|
-
|
|
3016
|
-
|
|
3017
|
-
|
|
3018
|
-
|
|
3244
|
+
function rankCandidates(html, candidates, targetNodeId) {
|
|
3245
|
+
const ranked = candidates.map(candidate => {
|
|
3246
|
+
const matches = countMatches(html, candidate.selector);
|
|
3247
|
+
const hitsTarget = targetNodeId === undefined || matches === 0 ? targetNodeId === undefined : matchesOf(html, candidate.selector).includes(targetNodeId);
|
|
3248
|
+
return {
|
|
3249
|
+
...candidate,
|
|
3250
|
+
matches,
|
|
3251
|
+
hitsTarget
|
|
3252
|
+
};
|
|
3253
|
+
});
|
|
3254
|
+
return ranked.filter(candidate => candidate.matches > 0).sort((a, b) => Number(b.hitsTarget) - Number(a.hitsTarget) || TIER_ORDER[a.tier] - TIER_ORDER[b.tier] || a.matches - b.matches);
|
|
3255
|
+
}
|
|
3256
|
+
/** The single best candidate, or `undefined` when none of them matched anything. */
|
|
3257
|
+
function bestCandidate(html, candidates, targetNodeId) {
|
|
3258
|
+
return rankCandidates(html, candidates, targetNodeId)[0];
|
|
3019
3259
|
}
|
|
3020
3260
|
|
|
3021
|
-
/** How long the browser is given to settle after `goto`, watching for JSON responses (mirrors `packages/cli`'s `probe --browser`, `observeBrowserJson`). */
|
|
3022
|
-
const OBSERVE_MS = 4000;
|
|
3023
|
-
/** Responses kept, largest/most-recent-first ordering left to the caller; just a sane cap on what the panel lists. */
|
|
3024
|
-
const RESPONSE_LIMIT = 15;
|
|
3025
3261
|
/**
|
|
3026
|
-
*
|
|
3027
|
-
*
|
|
3028
|
-
*
|
|
3029
|
-
*
|
|
3030
|
-
*
|
|
3031
|
-
* `take-snapshot.use-case.ts`: that capture reads `page.content()` right
|
|
3032
|
-
* after `goto` resolves, with no observation window, and changing its shape
|
|
3033
|
-
* to also collect responses was a larger, riskier change to already-shipped,
|
|
3034
|
-
* tested code than this phase's budget allowed (see this phase's final
|
|
3035
|
-
* report) — so this opens its own short-lived browser session instead of
|
|
3036
|
-
* piggy-backing on the cached snapshot.
|
|
3262
|
+
* Infers "these two picks are the same field on two items of a list" from
|
|
3263
|
+
* two node paths, by walking up from both to the nearest ancestors that are
|
|
3264
|
+
* real siblings (the same parent) with the same tag and stable-class
|
|
3265
|
+
* signature — studio plan §3.2, issue #91. That ancestor is the item; the
|
|
3266
|
+
* remainder of each path is the field, relative to it.
|
|
3037
3267
|
*
|
|
3038
|
-
*
|
|
3039
|
-
*
|
|
3040
|
-
*
|
|
3041
|
-
*
|
|
3268
|
+
* Because a `DomPath` is built root-down (`dom-path.model.ts`), two paths
|
|
3269
|
+
* from the same document are byte-for-byte identical up to their real
|
|
3270
|
+
* common ancestor and diverge exactly at its differing children — so the
|
|
3271
|
+
* *first* level where they differ is always the nearest-sibling test point;
|
|
3272
|
+
* there is no need to look further down. This also makes the "wrapper trap"
|
|
3273
|
+
* (an extra single-child element around each item) resolve itself: a
|
|
3274
|
+
* wrapper's own instances are never one another's siblings, only the
|
|
3275
|
+
* genuine repeating element is, so the walk naturally lands there instead of
|
|
3276
|
+
* on the wrapper (see the wrapper-trap fixture in this slice's tests).
|
|
3042
3277
|
*
|
|
3043
|
-
* @param
|
|
3044
|
-
* @param
|
|
3045
|
-
* @
|
|
3046
|
-
* @
|
|
3047
|
-
* @throws Error when the recipe has no start point, or `stepPath` is not `"start"`.
|
|
3278
|
+
* @param pathA - The first click's path.
|
|
3279
|
+
* @param pathB - The second click's path.
|
|
3280
|
+
* @returns The item and field shapes (generalised) plus each pick's raw path, or `null` when the two picks are not on a coherent list (different branches entirely, the same node twice, or one path nested inside the other).
|
|
3281
|
+
* @throws ShadowDomUnsupportedError when either path crosses a declarative shadow root.
|
|
3048
3282
|
*/
|
|
3049
|
-
|
|
3050
|
-
if (
|
|
3051
|
-
if (
|
|
3052
|
-
|
|
3053
|
-
|
|
3054
|
-
|
|
3055
|
-
|
|
3056
|
-
|
|
3057
|
-
|
|
3058
|
-
const responses = [];
|
|
3059
|
-
const seen = new Set();
|
|
3060
|
-
const pending = [];
|
|
3061
|
-
session.page.on('response', response => {
|
|
3062
|
-
if (!(response.headers()['content-type'] ?? '').includes('json')) return;
|
|
3063
|
-
const key = `${response.status()} ${response.url()}`;
|
|
3064
|
-
if (seen.has(key)) return;
|
|
3065
|
-
seen.add(key);
|
|
3066
|
-
pending.push(recordResponse(response, responses));
|
|
3067
|
-
});
|
|
3068
|
-
try {
|
|
3069
|
-
await session.page.goto(point.url, {
|
|
3070
|
-
waitUntil: 'networkidle'
|
|
3071
|
-
});
|
|
3072
|
-
} catch {
|
|
3073
|
-
// a page that never idles (long-poll, websocket) still yields whatever was observed by the timeout below
|
|
3074
|
-
}
|
|
3075
|
-
await session.page.waitForTimeout(OBSERVE_MS);
|
|
3076
|
-
await Promise.all(pending);
|
|
3077
|
-
return responses.slice(0, RESPONSE_LIMIT);
|
|
3078
|
-
} finally {
|
|
3079
|
-
await session.close();
|
|
3283
|
+
function inferList(pathA, pathB) {
|
|
3284
|
+
if (crossesShadowRoot(pathA)) throw new ShadowDomUnsupportedError(pathToString(pathA));
|
|
3285
|
+
if (crossesShadowRoot(pathB)) throw new ShadowDomUnsupportedError(pathToString(pathB));
|
|
3286
|
+
let divergedAt = -1;
|
|
3287
|
+
const limit = Math.min(pathA.length, pathB.length);
|
|
3288
|
+
for (let index = 0; index < limit; index++) {
|
|
3289
|
+
if (!sameLevel(pathA[index], pathB[index])) {
|
|
3290
|
+
divergedAt = index;
|
|
3291
|
+
break;
|
|
3080
3292
|
}
|
|
3081
|
-
} finally {
|
|
3082
|
-
await client.close();
|
|
3083
3293
|
}
|
|
3294
|
+
// Identical paths, or one a strict prefix of the other (the same node picked twice, or one element nested inside the other): not a list.
|
|
3295
|
+
if (divergedAt <= 0) return null;
|
|
3296
|
+
const branchA = pathA[divergedAt];
|
|
3297
|
+
const branchB = pathB[divergedAt];
|
|
3298
|
+
if (branchA.tag !== branchB.tag || !sameClassSignature(branchA.classes, branchB.classes)) return null;
|
|
3299
|
+
if (branchA.index === branchB.index) return null; // Same position: not two different items.
|
|
3300
|
+
const ancestors = pathA.slice(0, divergedAt);
|
|
3301
|
+
const fieldPathA = pathA.slice(divergedAt);
|
|
3302
|
+
const fieldPathB = pathB.slice(divergedAt);
|
|
3303
|
+
const fieldPath = mergeRelativePaths(fieldPathA, fieldPathB);
|
|
3304
|
+
return {
|
|
3305
|
+
itemPath: [...ancestors, fieldPath[0]],
|
|
3306
|
+
fieldPath,
|
|
3307
|
+
fieldPathA,
|
|
3308
|
+
fieldPathB
|
|
3309
|
+
};
|
|
3084
3310
|
}
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
|
|
3089
|
-
|
|
3090
|
-
|
|
3091
|
-
|
|
3092
|
-
|
|
3093
|
-
|
|
3094
|
-
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3311
|
+
function sameLevel(a, b) {
|
|
3312
|
+
return a.tag === b.tag && a.id === b.id && a.index === b.index && a.siblingCount === b.siblingCount && sameClassSignature(a.classes, b.classes);
|
|
3313
|
+
}
|
|
3314
|
+
/**
|
|
3315
|
+
* Zips two item-relative paths (item first) level by level into one that
|
|
3316
|
+
* only keeps what both share: the tag (must already match, checked by the
|
|
3317
|
+
* caller for level 0), the intersection of stable classes, and attributes
|
|
3318
|
+
* present with the same value on both sides. Shorter than the other when
|
|
3319
|
+
* the two picks' local structure differs in depth (a real possibility once
|
|
3320
|
+
* a field is missing on one item; `list-inference.algorithm.test.ts`'s
|
|
3321
|
+
* shifted-field fixture is exactly this) — the merge just stops there,
|
|
3322
|
+
* which is fine: a field selector only needs to reach as deep as the
|
|
3323
|
+
* shallower of the two picks.
|
|
3324
|
+
*/
|
|
3325
|
+
function mergeRelativePaths(a, b) {
|
|
3326
|
+
const length = Math.min(a.length, b.length);
|
|
3327
|
+
const merged = [];
|
|
3328
|
+
for (let index = 0; index < length; index++) {
|
|
3329
|
+
const left = a[index];
|
|
3330
|
+
const right = b[index];
|
|
3331
|
+
if (left.tag !== right.tag) break;
|
|
3332
|
+
merged.push(mergeLevel(left, right));
|
|
3333
|
+
}
|
|
3334
|
+
return merged.length > 0 ? merged : [a[0]];
|
|
3335
|
+
}
|
|
3336
|
+
function mergeLevel(a, b) {
|
|
3337
|
+
const classes = stableClasses(a.classes).filter(token => b.classes.includes(token));
|
|
3338
|
+
const attrs = {};
|
|
3339
|
+
for (const [name, value] of Object.entries(a.attrs)) {
|
|
3340
|
+
if (name !== 'class' && name !== 'id' && b.attrs[name] === value) attrs[name] = value;
|
|
3098
3341
|
}
|
|
3342
|
+
return {
|
|
3343
|
+
tag: a.tag,
|
|
3344
|
+
id: a.id !== undefined && a.id === b.id ? a.id : undefined,
|
|
3345
|
+
classes,
|
|
3346
|
+
attrs,
|
|
3347
|
+
index: 1,
|
|
3348
|
+
siblingCount: Math.max(a.siblingCount, b.siblingCount, 1)
|
|
3349
|
+
};
|
|
3350
|
+
}
|
|
3351
|
+
function pathToString(path) {
|
|
3352
|
+
return path.map(level => level.tag + (level.id === undefined ? '' : `#${level.id}`)).join(' > ');
|
|
3099
3353
|
}
|
|
3100
3354
|
|
|
3101
3355
|
/**
|
|
3102
|
-
* Handles `
|
|
3103
|
-
*
|
|
3104
|
-
*
|
|
3105
|
-
*
|
|
3106
|
-
*
|
|
3107
|
-
*
|
|
3108
|
-
*
|
|
3356
|
+
* Handles `infer-selector`: turns one or two picked nodes (their
|
|
3357
|
+
* `data-oc-node` ids, off the cached snapshot) into a verified selector —
|
|
3358
|
+
* one id for a `Read` card's field, two for the safe item+field list shape
|
|
3359
|
+
* `selector-inference`'s `inferList` builds by construction (issue #91,
|
|
3360
|
+
* studio plan §3.2). All the actual candidate/ranking/list-inference work
|
|
3361
|
+
* is `selector-inference`'s (pure); this handler only locates the clicked
|
|
3362
|
+
* nodes in the cached document and calls into it.
|
|
3109
3363
|
*
|
|
3110
3364
|
* @param state - The server state.
|
|
3111
3365
|
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
3112
|
-
* @returns
|
|
3366
|
+
* @returns A field pick, a list pick, or `"unsupported"` with why.
|
|
3113
3367
|
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
3114
3368
|
*/
|
|
3115
|
-
function
|
|
3369
|
+
function handleInferSelector(state, command) {
|
|
3116
3370
|
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
3117
3371
|
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
3118
|
-
|
|
3119
|
-
|
|
3120
|
-
|
|
3121
|
-
|
|
3122
|
-
|
|
3123
|
-
|
|
3124
|
-
|
|
3125
|
-
|
|
3126
|
-
|
|
3127
|
-
|
|
3128
|
-
|
|
3129
|
-
* A **card** is one step that reads as a sentence (`sentence`, built by
|
|
3130
|
-
* `card-sentence.mapper.ts`). A **bracket** is a container step (`forEach`,
|
|
3131
|
-
* `paginate`, `if`) whose body is itself a list of cards/brackets: the
|
|
3132
|
-
* indentation the plan doc describes is this nesting, not a separate layout
|
|
3133
|
-
* concept. Both kinds carry `step`, the step's own raw JSON exactly as
|
|
3134
|
-
* parsed (every key, in its original order): `outline-to-recipe.mapper.ts`
|
|
3135
|
-
* rebuilds the recipe from it, substituting only the (possibly edited)
|
|
3136
|
-
* `children`/`elseChildren` for whatever `steps`/`else` value `step` itself
|
|
3137
|
-
* holds, so an edit to one field never disturbs another's position or a
|
|
3138
|
-
* step type this outline cannot make sense of.
|
|
3139
|
-
*/
|
|
3140
|
-
/** The step types the outline renders as a bracket: their body is nested steps, not a single value. */
|
|
3141
|
-
const BRACKET_STEP_TYPES = ['forEach', 'paginate', 'if'];
|
|
3142
|
-
function isBracket(node) {
|
|
3143
|
-
return node.kind === 'bracket';
|
|
3144
|
-
}
|
|
3145
|
-
function isBracketStepType(type) {
|
|
3146
|
-
return BRACKET_STEP_TYPES.includes(type);
|
|
3147
|
-
}
|
|
3148
|
-
|
|
3149
|
-
/** Step types the plan doc names as always going to a "custom" card: nothing here renders their meaning as a sentence. */
|
|
3150
|
-
const ALWAYS_CUSTOM = new Set(['evaluate', 'hook', 'captcha']);
|
|
3151
|
-
/**
|
|
3152
|
-
* Turns one step's raw JSON into its sentence parts (word/pill/code), the
|
|
3153
|
-
* way the plan doc's examples read: *Go to…*, *Read… from…*, *For each… in…
|
|
3154
|
-
* → one record*, and so on. Never throws: a step type this cannot read
|
|
3155
|
-
* becomes a short, honest fallback with `custom: true`, so the outline never
|
|
3156
|
-
* drops a step for looking unfamiliar.
|
|
3157
|
-
*
|
|
3158
|
-
* @param step - A step's raw JSON (its `type` and whatever fields it has).
|
|
3159
|
-
* @returns The sentence parts, and whether they are a real reading or a fallback.
|
|
3160
|
-
*/
|
|
3161
|
-
function cardSentence(step) {
|
|
3162
|
-
const type = typeof step.type === 'string' ? step.type : undefined;
|
|
3163
|
-
if (type === undefined || ALWAYS_CUSTOM.has(type)) return fallback(type);
|
|
3164
|
-
const build = BUILDERS[type];
|
|
3165
|
-
if (build === undefined) return fallback(type);
|
|
3372
|
+
const html = snapshot.html;
|
|
3373
|
+
const paths = command.nodeIds.map(nodeId => pathToNode(html, nodeId));
|
|
3374
|
+
if (paths.includes(undefined)) return {
|
|
3375
|
+
kind: 'unsupported',
|
|
3376
|
+
reason: 'one of the picked nodes is no longer in the snapshot; take a fresh snapshot and try again'
|
|
3377
|
+
};
|
|
3378
|
+
const [pathA, pathB] = paths;
|
|
3379
|
+
if (pathB === undefined) return {
|
|
3380
|
+
kind: 'field',
|
|
3381
|
+
field: fieldPick(html, pathA)
|
|
3382
|
+
};
|
|
3166
3383
|
try {
|
|
3384
|
+
const result = inferList(pathA, pathB);
|
|
3385
|
+
if (result === null) return {
|
|
3386
|
+
kind: 'unsupported',
|
|
3387
|
+
reason: 'these two picks do not belong to the same list: pick the same field on two similar items'
|
|
3388
|
+
};
|
|
3389
|
+
const item = bestCandidate(html, candidatesFor(result.itemPath, false));
|
|
3390
|
+
if (item === undefined) return {
|
|
3391
|
+
kind: 'unsupported',
|
|
3392
|
+
reason: 'no selector matches every item of this list; try picking two items closer to each other in the markup'
|
|
3393
|
+
};
|
|
3394
|
+
const field = fieldPick(html, result.fieldPathA, result.fieldPath);
|
|
3167
3395
|
return {
|
|
3168
|
-
|
|
3169
|
-
|
|
3396
|
+
kind: 'list',
|
|
3397
|
+
item: {
|
|
3398
|
+
selector: item.selector,
|
|
3399
|
+
tier: item.tier,
|
|
3400
|
+
matches: item.matches
|
|
3401
|
+
},
|
|
3402
|
+
field
|
|
3170
3403
|
};
|
|
3171
|
-
} catch {
|
|
3172
|
-
|
|
3404
|
+
} catch (error) {
|
|
3405
|
+
if (error instanceof ShadowDomUnsupportedError) return {
|
|
3406
|
+
kind: 'unsupported',
|
|
3407
|
+
reason: error.message
|
|
3408
|
+
};
|
|
3409
|
+
throw error;
|
|
3173
3410
|
}
|
|
3174
3411
|
}
|
|
3175
|
-
|
|
3412
|
+
/** The best selector for one clicked node, `take` decided from the concrete clicked element; `rankSource` (a generalised path) is used for candidate generation when given, so a per-item-unique attribute of the one node picked cannot end up in the selector. */
|
|
3413
|
+
function fieldPick(html, clicked, rankSource = clicked) {
|
|
3414
|
+
const leaf = clicked.at(-1);
|
|
3415
|
+
const take = leaf === undefined ? 'text' : takeKindFor(leaf);
|
|
3416
|
+
const best = bestCandidate(html, candidatesFor(rankSource));
|
|
3417
|
+
if (best === undefined) throw new Error('no selector matches the picked node — this should not happen, since the node came from this same document');
|
|
3176
3418
|
return {
|
|
3177
|
-
|
|
3178
|
-
|
|
3419
|
+
selector: best.selector,
|
|
3420
|
+
tier: best.tier,
|
|
3421
|
+
take,
|
|
3422
|
+
matches: best.matches
|
|
3179
3423
|
};
|
|
3180
3424
|
}
|
|
3181
|
-
|
|
3182
|
-
|
|
3183
|
-
|
|
3184
|
-
|
|
3185
|
-
|
|
3425
|
+
|
|
3426
|
+
/** A leaf element's own text longer than this is cut, with an ellipsis. */
|
|
3427
|
+
const TEXT_LIMIT = 80;
|
|
3428
|
+
/** Consecutive siblings whose subtree is identical (ids aside) collapse into one row once there are at least this many — a real page can have tens of thousands of `<li>`s, and showing all of them is not "inspecting" it (issue #93). */
|
|
3429
|
+
const MIN_REPEAT_RUN = 3;
|
|
3430
|
+
/**
|
|
3431
|
+
* Builds the Inspect panel's DOM tree from a captured snapshot (studio plan
|
|
3432
|
+
* §3.3, issue #93): the same rewritten, marked-up document the content pane's
|
|
3433
|
+
* iframe shows (`page-snapshot`'s `take-snapshot.use-case.ts`), so a tree
|
|
3434
|
+
* row's `nodeId` addresses exactly the element the iframe would highlight —
|
|
3435
|
+
* no separate capture, per issue #93's scope note to reuse phase 2's
|
|
3436
|
+
* snapshot rather than recapture.
|
|
3437
|
+
*
|
|
3438
|
+
* @param snapshotHtml - The cached, rewritten snapshot (`SnapshotResult.html`).
|
|
3439
|
+
* @returns The tree's root (the document's `<html>` element).
|
|
3440
|
+
* @throws Error when the document has no `<html>` element (should not happen: `rewriteDocument` always produces one).
|
|
3441
|
+
*/
|
|
3442
|
+
function domTree(snapshotHtml) {
|
|
3443
|
+
const $ = load(snapshotHtml);
|
|
3444
|
+
const html = $('html').get(0);
|
|
3445
|
+
if (html === undefined) throw new Error('domTree: the snapshot has no <html> element');
|
|
3446
|
+
return buildNode($, html);
|
|
3186
3447
|
}
|
|
3187
|
-
function
|
|
3188
|
-
|
|
3189
|
-
|
|
3190
|
-
|
|
3448
|
+
function buildNode($, element) {
|
|
3449
|
+
const $el = $(element);
|
|
3450
|
+
const attributes = {
|
|
3451
|
+
...element.attribs
|
|
3191
3452
|
};
|
|
3192
|
-
|
|
3193
|
-
|
|
3453
|
+
const nodeId = attributes[NODE_ID_ATTRIBUTE] ?? '';
|
|
3454
|
+
const hidden = attributes[HIDDEN_ATTRIBUTE] === '1';
|
|
3455
|
+
delete attributes[NODE_ID_ATTRIBUTE];
|
|
3456
|
+
delete attributes[HIDDEN_ATTRIBUTE];
|
|
3457
|
+
const id = attributes.id;
|
|
3458
|
+
const classes = (attributes.class ?? '').split(/\s+/).filter(token => token !== '');
|
|
3459
|
+
const childElements = $el.children().toArray();
|
|
3460
|
+
const children = collapseRepeats(childElements.map(child => buildNode($, child)));
|
|
3461
|
+
const text = childElements.length === 0 ? abbreviate($el.text()) : undefined;
|
|
3194
3462
|
return {
|
|
3195
|
-
|
|
3196
|
-
|
|
3463
|
+
nodeId,
|
|
3464
|
+
tag: element.tagName.toLowerCase(),
|
|
3465
|
+
...(id !== undefined && id !== '' && {
|
|
3466
|
+
id
|
|
3467
|
+
}),
|
|
3468
|
+
...(classes.length > 0 && {
|
|
3469
|
+
classes
|
|
3470
|
+
}),
|
|
3471
|
+
attributes,
|
|
3472
|
+
...(text !== undefined && text !== '' && {
|
|
3473
|
+
text
|
|
3474
|
+
}),
|
|
3475
|
+
hidden,
|
|
3476
|
+
children
|
|
3197
3477
|
};
|
|
3198
3478
|
}
|
|
3199
|
-
/** A field that names an id: shown as a pill, `fallbackText` when it is absent. */
|
|
3200
|
-
function pillField(step, key, fallbackText) {
|
|
3201
|
-
const value = step[key];
|
|
3202
|
-
return pill(typeof value === 'string' && value.length > 0 ? value : fallbackText);
|
|
3203
|
-
}
|
|
3204
|
-
/** A field shown as a literal (a selector, a template, a value): stringified when it is not already a string. */
|
|
3205
|
-
function codeField(step, key, fallbackText = '') {
|
|
3206
|
-
const value = step[key];
|
|
3207
|
-
if (typeof value === 'string') return code(value);
|
|
3208
|
-
if (value === undefined) return code(fallbackText);
|
|
3209
|
-
return code(safeJson(value));
|
|
3210
|
-
}
|
|
3211
|
-
function safeJson(value) {
|
|
3212
|
-
try {
|
|
3213
|
-
return JSON.stringify(value) ?? String(value);
|
|
3214
|
-
} catch {
|
|
3215
|
-
return String(value);
|
|
3216
|
-
}
|
|
3217
|
-
}
|
|
3218
|
-
/** `selector` or `target`, whichever the step gives (`step.contract.ts`'s `TargetFields`: exactly one is present). */
|
|
3219
|
-
function targetField(step) {
|
|
3220
|
-
if (typeof step.target === 'string') return code(step.target);
|
|
3221
|
-
return codeField(step, 'selector');
|
|
3222
|
-
}
|
|
3223
|
-
const BUILDERS = {
|
|
3224
|
-
goto: step => [word('Go to'), codeField(step, 'url')],
|
|
3225
|
-
click: step => [word('Click'), targetField(step)],
|
|
3226
|
-
fill: step => [word('Fill'), targetField(step), word('with'), codeField(step, 'value')],
|
|
3227
|
-
press: step => [word('Press'), codeField(step, 'key'), word('in'), targetField(step)],
|
|
3228
|
-
select: step => [word('Choose'), selectChoice(step), word('in'), targetField(step)],
|
|
3229
|
-
scroll: step => [word('Scroll to'), codeField(step, 'to')],
|
|
3230
|
-
wait: step => [word('Wait for'), waitFor(step)],
|
|
3231
|
-
screenshot: step => [word('Screenshot'), codeField(step, 'path')],
|
|
3232
|
-
request: step => [word('Request'), code(`${typeof step.method === 'string' ? step.method : 'GET'} `), codeField(step, 'url')],
|
|
3233
|
-
extract: step => [word('Read'), pillField(step, 'id', '(unnamed)'), word('from'), pillField(step, 'from', 'page'), word('←'), codeField(step, 'selector')],
|
|
3234
|
-
set: step => [word('Set'), pillField(step, 'id', '(unnamed)'), word('to'), codeField(step, 'value')],
|
|
3235
|
-
collect: step => [word('Collect'), codeField(step, 'value'), word('into'), pillField(step, 'into', '(unnamed)')],
|
|
3236
|
-
forEach: step => forEachSentence(step),
|
|
3237
|
-
paginate: step => paginateSentence(step),
|
|
3238
|
-
if: step => [word('If'), codeField(step, 'test')],
|
|
3239
|
-
emit: step => emitSentence(step)
|
|
3240
|
-
};
|
|
3241
|
-
function selectChoice(step) {
|
|
3242
|
-
if (typeof step.value === 'string') return code(step.value);
|
|
3243
|
-
if (typeof step.label === 'string') return code(step.label);
|
|
3244
|
-
if (typeof step.index === 'number') return code(String(step.index));
|
|
3245
|
-
if (Array.isArray(step.values)) return code(step.values.join(', '));
|
|
3246
|
-
return code('');
|
|
3247
|
-
}
|
|
3248
|
-
function waitFor(step) {
|
|
3249
|
-
if (typeof step.selector === 'string') return code(step.selector);
|
|
3250
|
-
if (typeof step.ms === 'number') return code(`${step.ms}ms`);
|
|
3251
|
-
if (typeof step.state === 'string') return code(step.state);
|
|
3252
|
-
return code('');
|
|
3253
|
-
}
|
|
3254
|
-
function forEachSentence(step) {
|
|
3255
|
-
const parts = [word('For each'), pillField(step, 'as', '(item)'), word('in'), typeof step.over === 'string' ? pill(step.over) : targetField(step)];
|
|
3256
|
-
if (step.emit !== undefined) parts.push(word('→'), word('one record'));
|
|
3257
|
-
return parts;
|
|
3258
|
-
}
|
|
3259
|
-
function paginateSentence(step) {
|
|
3260
|
-
const parts = [word('For every page')];
|
|
3261
|
-
const next = step.next;
|
|
3262
|
-
if (next !== null && typeof next === 'object') {
|
|
3263
|
-
const record = next;
|
|
3264
|
-
if (typeof record.selector === 'string') parts.push(word('via'), code(record.selector));else if (typeof record.url === 'string') parts.push(word('via'), code(record.url));else if (typeof record.jsonpath === 'string') parts.push(word('via'), code(record.jsonpath));
|
|
3265
|
-
}
|
|
3266
|
-
if (typeof step.until === 'string') parts.push(word('until'), code(step.until));
|
|
3267
|
-
return parts;
|
|
3268
|
-
}
|
|
3269
|
-
function emitSentence(step) {
|
|
3270
|
-
if (typeof step.output === 'string') return [word('Emit'), word('to'), pill(step.output)];
|
|
3271
|
-
return [word('Emit')];
|
|
3272
|
-
}
|
|
3273
|
-
|
|
3274
3479
|
/**
|
|
3275
|
-
*
|
|
3276
|
-
* (
|
|
3277
|
-
*
|
|
3278
|
-
*
|
|
3279
|
-
* well-formed step becomes a custom card holding what was there; anything
|
|
3280
|
-
* that is not an object at the top becomes an outline with no steps.
|
|
3480
|
+
* Merges runs of `MIN_REPEAT_RUN` or more consecutive siblings whose subtree
|
|
3481
|
+
* is identical (their own `nodeId`s aside) into one representative row, the
|
|
3482
|
+
* issue's `×20` — real pages have tens of thousands of nodes, and this is
|
|
3483
|
+
* what makes the tree usable rather than merely correct.
|
|
3281
3484
|
*
|
|
3282
|
-
* @param
|
|
3283
|
-
* @returns The
|
|
3485
|
+
* @param nodes - Already-built sibling rows, in document order.
|
|
3486
|
+
* @returns The same rows, with each long-enough identical run replaced by one.
|
|
3284
3487
|
*/
|
|
3285
|
-
function
|
|
3286
|
-
const
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
|
|
3291
|
-
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3295
|
-
|
|
3296
|
-
|
|
3297
|
-
|
|
3298
|
-
|
|
3299
|
-
|
|
3300
|
-
|
|
3301
|
-
|
|
3302
|
-
|
|
3303
|
-
kind: 'word',
|
|
3304
|
-
text: 'custom step'
|
|
3305
|
-
}],
|
|
3306
|
-
step: {},
|
|
3307
|
-
custom: true
|
|
3308
|
-
};
|
|
3488
|
+
function collapseRepeats(nodes) {
|
|
3489
|
+
const result = [];
|
|
3490
|
+
let at = 0;
|
|
3491
|
+
while (at < nodes.length) {
|
|
3492
|
+
const signature = signatureOf(nodes[at]);
|
|
3493
|
+
let end = at + 1;
|
|
3494
|
+
while (end < nodes.length && signatureOf(nodes[end]) === signature) end += 1;
|
|
3495
|
+
const run = nodes.slice(at, end);
|
|
3496
|
+
if (run.length >= MIN_REPEAT_RUN) {
|
|
3497
|
+
result.push({
|
|
3498
|
+
...run[0],
|
|
3499
|
+
repeatCount: run.length,
|
|
3500
|
+
repeatNodeIds: run.map(node => node.nodeId)
|
|
3501
|
+
});
|
|
3502
|
+
} else {
|
|
3503
|
+
result.push(...run);
|
|
3504
|
+
}
|
|
3505
|
+
at = end;
|
|
3309
3506
|
}
|
|
3310
|
-
|
|
3311
|
-
const {
|
|
3312
|
-
parts,
|
|
3313
|
-
custom
|
|
3314
|
-
} = cardSentence(step);
|
|
3315
|
-
if (!isBracketStepType(stepType)) return {
|
|
3316
|
-
kind: 'card',
|
|
3317
|
-
path,
|
|
3318
|
-
stepType,
|
|
3319
|
-
sentence: parts,
|
|
3320
|
-
step,
|
|
3321
|
-
custom
|
|
3322
|
-
};
|
|
3323
|
-
const children = Array.isArray(step.steps) ? stepsToNodes(step.steps, `${path}.steps`) : [];
|
|
3324
|
-
const elseChildren = stepType === 'if' && Array.isArray(step.else) ? stepsToNodes(step.else, `${path}.else`) : undefined;
|
|
3325
|
-
return {
|
|
3326
|
-
kind: 'bracket',
|
|
3327
|
-
path,
|
|
3328
|
-
stepType,
|
|
3329
|
-
sentence: parts,
|
|
3330
|
-
step,
|
|
3331
|
-
children,
|
|
3332
|
-
...(elseChildren !== undefined && {
|
|
3333
|
-
elseChildren
|
|
3334
|
-
})
|
|
3335
|
-
};
|
|
3507
|
+
return result;
|
|
3336
3508
|
}
|
|
3337
|
-
|
|
3338
|
-
|
|
3509
|
+
/** A run member's identity for collapsing: everything but its own `nodeId` (every instance has a different one by construction). */
|
|
3510
|
+
function signatureOf(node) {
|
|
3511
|
+
return JSON.stringify({
|
|
3512
|
+
...node,
|
|
3513
|
+
nodeId: undefined,
|
|
3514
|
+
children: node.children.map(child => signatureOf(child))
|
|
3515
|
+
});
|
|
3516
|
+
}
|
|
3517
|
+
function abbreviate(text) {
|
|
3518
|
+
const trimmed = text.trim().replaceAll(/\s+/g, ' ');
|
|
3519
|
+
return trimmed.length > TEXT_LIMIT ? `${trimmed.slice(0, TEXT_LIMIT)}…` : trimmed;
|
|
3339
3520
|
}
|
|
3340
3521
|
|
|
3522
|
+
/** The global names sites hydrate state into, that `describeHtml`'s own JSON-LD/table scan does not cover (issue #93's "inline state assignments"). */
|
|
3523
|
+
const INLINE_STATE_NAMES = ['__NEXT_DATA__', '__NUXT__', '__INITIAL_STATE__', '__APOLLO_STATE__', '__PRELOADED_STATE__', '__STATE__'];
|
|
3524
|
+
const INLINE_STATE_PATTERN = new RegExp(String.raw`(?:window\.)?(${INLINE_STATE_NAMES.join('|')})\s*=`);
|
|
3341
3525
|
/**
|
|
3342
|
-
*
|
|
3343
|
-
*
|
|
3344
|
-
*
|
|
3345
|
-
*
|
|
3346
|
-
*
|
|
3347
|
-
*
|
|
3348
|
-
*
|
|
3349
|
-
*
|
|
3350
|
-
* `recipe → outline → recipe` preserve every key and its order.
|
|
3526
|
+
* Finds the data-in-the-page findings the Inspect panel lists apart from the
|
|
3527
|
+
* DOM tree (studio plan §3.3, issue #93): JSON-LD and `application/json`
|
|
3528
|
+
* script blocks, inline state assignments, `<meta>` name/property values and
|
|
3529
|
+
* `<link rel>`s — reusing `@opencraw/probe`'s `describeJson` for a found
|
|
3530
|
+
* value's shape (its own `find-data.algorithm.ts`/`html-findings.mapper.ts`
|
|
3531
|
+
* scan a page's raw text for *reporting*, with no element to point a pick at;
|
|
3532
|
+
* this function's own job — locating each finding's element so a pick can
|
|
3533
|
+
* reach it — is new).
|
|
3351
3534
|
*
|
|
3352
|
-
*
|
|
3353
|
-
*
|
|
3535
|
+
* Takes the **raw** captured document, not the display snapshot
|
|
3536
|
+
* (`page-snapshot`'s `SnapshotResult.html`): `rewriteDocument` strips every
|
|
3537
|
+
* `<script>` for the sandboxed iframe, which would make JSON-LD, inline
|
|
3538
|
+
* state and other script-tag data invisible here. `<meta>`/`<link>` survive
|
|
3539
|
+
* rewriting untouched, so using the raw document for every kind (rather than
|
|
3540
|
+
* mixing two documents) keeps this function simple and its selectors
|
|
3541
|
+
* verified against what the engine will actually crawl.
|
|
3542
|
+
*
|
|
3543
|
+
* @param rawHtml - The document as fetched/rendered, before `rewriteDocument` strips scripts.
|
|
3544
|
+
* @returns The findings, in document order.
|
|
3354
3545
|
*/
|
|
3355
|
-
function
|
|
3356
|
-
|
|
3357
|
-
|
|
3358
|
-
|
|
3359
|
-
|
|
3360
|
-
|
|
3361
|
-
|
|
3362
|
-
|
|
3546
|
+
function pageData(rawHtml) {
|
|
3547
|
+
const $ = load(rawHtml);
|
|
3548
|
+
const findings = [];
|
|
3549
|
+
$('script[type="application/ld+json"]').each((_index, element) => {
|
|
3550
|
+
findings.push(...scriptFinding($, element, rawHtml, 'ld-json', ldJsonLabel));
|
|
3551
|
+
});
|
|
3552
|
+
$('script[type="application/json"]').each((_index, element) => {
|
|
3553
|
+
findings.push(...scriptFinding($, element, rawHtml, 'json-script', () => `json: ${structuralSelector($, element)}`));
|
|
3554
|
+
});
|
|
3555
|
+
$('script').each((_index, element) => {
|
|
3556
|
+
const type = $(element).attr('type');
|
|
3557
|
+
if (type !== undefined && type !== 'text/javascript' && type !== 'module') return;
|
|
3558
|
+
const text = $(element).text();
|
|
3559
|
+
const match = INLINE_STATE_PATTERN.exec(text);
|
|
3560
|
+
if (match === null) return;
|
|
3561
|
+
const value = tryParseJson(text);
|
|
3562
|
+
const selector = structuralSelector($, element);
|
|
3563
|
+
findings.push({
|
|
3564
|
+
kind: 'inline-state',
|
|
3565
|
+
label: match[1],
|
|
3566
|
+
selector,
|
|
3567
|
+
selectorKind: 'css',
|
|
3568
|
+
matches: countMatches(rawHtml, selector),
|
|
3569
|
+
keys: keysOf(value)
|
|
3570
|
+
});
|
|
3571
|
+
});
|
|
3572
|
+
$('meta[name], meta[property]').each((_index, element) => {
|
|
3573
|
+
const name = $(element).attr('name') ?? $(element).attr('property') ?? '';
|
|
3574
|
+
const selector = structuralSelector($, element);
|
|
3575
|
+
findings.push({
|
|
3576
|
+
kind: 'meta',
|
|
3577
|
+
label: `meta: ${name}`,
|
|
3578
|
+
selector,
|
|
3579
|
+
selectorKind: 'css',
|
|
3580
|
+
matches: countMatches(rawHtml, selector),
|
|
3581
|
+
attribute: 'content'
|
|
3582
|
+
});
|
|
3583
|
+
});
|
|
3584
|
+
$('link[rel]').each((_index, element) => {
|
|
3585
|
+
const rel = $(element).attr('rel') ?? '';
|
|
3586
|
+
const selector = structuralSelector($, element);
|
|
3587
|
+
findings.push({
|
|
3588
|
+
kind: 'link',
|
|
3589
|
+
label: `link: ${rel}`,
|
|
3590
|
+
selector,
|
|
3591
|
+
selectorKind: 'css',
|
|
3592
|
+
matches: countMatches(rawHtml, selector),
|
|
3593
|
+
attribute: 'href'
|
|
3594
|
+
});
|
|
3595
|
+
});
|
|
3596
|
+
return findings;
|
|
3363
3597
|
}
|
|
3364
|
-
function
|
|
3365
|
-
|
|
3598
|
+
function scriptFinding($, element, rawHtml, kind, labelOf) {
|
|
3599
|
+
const value = tryParseJson($(element).text());
|
|
3600
|
+
if (value === undefined) return [];
|
|
3601
|
+
const selector = structuralSelector($, element);
|
|
3602
|
+
return [{
|
|
3603
|
+
kind,
|
|
3604
|
+
label: labelOf(value),
|
|
3605
|
+
selector,
|
|
3606
|
+
selectorKind: 'css',
|
|
3607
|
+
matches: countMatches(rawHtml, selector),
|
|
3608
|
+
keys: keysOf(value)
|
|
3609
|
+
}];
|
|
3366
3610
|
}
|
|
3367
|
-
function
|
|
3368
|
-
if (
|
|
3369
|
-
|
|
3370
|
-
|
|
3371
|
-
|
|
3372
|
-
if (key === 'steps') step.steps = nodesToSteps(node.children);else if (key === 'else' && node.stepType === 'if') step.else = nodesToSteps(node.elseChildren ?? []);else step[key] = node.step[key];
|
|
3611
|
+
function ldJsonLabel(value) {
|
|
3612
|
+
if (Array.isArray(value)) return `ld+json: array (${value.length})`;
|
|
3613
|
+
if (typeof value === 'object' && value !== null) {
|
|
3614
|
+
const type = value['@type'];
|
|
3615
|
+
if (typeof type === 'string') return `ld+json: ${type}`;
|
|
3373
3616
|
}
|
|
3374
|
-
|
|
3375
|
-
|
|
3376
|
-
|
|
3617
|
+
return `ld+json: ${describeJson(value, 'json').type}`;
|
|
3618
|
+
}
|
|
3619
|
+
function keysOf(value) {
|
|
3620
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value) ? Object.keys(value) : undefined;
|
|
3377
3621
|
}
|
|
3378
|
-
|
|
3379
3622
|
/**
|
|
3380
|
-
*
|
|
3381
|
-
* (
|
|
3382
|
-
*
|
|
3383
|
-
* the
|
|
3384
|
-
*
|
|
3385
|
-
*
|
|
3623
|
+
* A unique CSS path for an element with no stable attribute of its own
|
|
3624
|
+
* (a `<script>` tag has no id or class to speak of): `<ancestor> > tag`
|
|
3625
|
+
* from `<html>` down, `:nth-of-type` added only where a tag has siblings of
|
|
3626
|
+
* the same name under the same parent. Always resolves to exactly one
|
|
3627
|
+
* element, on the raw document this ran against and on the real page (a
|
|
3628
|
+
* purely structural selector, unaffected by the snapshot's own rewriting).
|
|
3386
3629
|
*
|
|
3387
|
-
* @param
|
|
3388
|
-
* @
|
|
3389
|
-
* @
|
|
3630
|
+
* @param $ - The document.
|
|
3631
|
+
* @param element - The element to address.
|
|
3632
|
+
* @returns The selector.
|
|
3390
3633
|
*/
|
|
3391
|
-
|
|
3392
|
-
const
|
|
3393
|
-
|
|
3394
|
-
|
|
3395
|
-
|
|
3396
|
-
|
|
3397
|
-
|
|
3398
|
-
|
|
3399
|
-
|
|
3400
|
-
|
|
3401
|
-
|
|
3402
|
-
|
|
3403
|
-
});
|
|
3404
|
-
}
|
|
3634
|
+
function structuralSelector($, element) {
|
|
3635
|
+
const segments = [];
|
|
3636
|
+
let $current = $(element);
|
|
3637
|
+
while ($current.length > 0) {
|
|
3638
|
+
const tag = $current.prop('tagName')?.toLowerCase();
|
|
3639
|
+
if (tag === undefined) break;
|
|
3640
|
+
const $parent = $current.parent();
|
|
3641
|
+
const siblingsOfType = $parent.length > 0 ? $parent.children(tag) : $current;
|
|
3642
|
+
const index = siblingsOfType.toArray().indexOf($current.get(0)) + 1;
|
|
3643
|
+
segments.unshift(siblingsOfType.length > 1 ? `${tag}:nth-of-type(${index})` : tag);
|
|
3644
|
+
if (tag === 'html' || $parent.length === 0) break;
|
|
3645
|
+
$current = $parent;
|
|
3405
3646
|
}
|
|
3406
|
-
return
|
|
3407
|
-
folder,
|
|
3408
|
-
recipes: entries.map(({
|
|
3409
|
-
file,
|
|
3410
|
-
kind,
|
|
3411
|
-
id,
|
|
3412
|
-
issues,
|
|
3413
|
-
content
|
|
3414
|
-
}) => ({
|
|
3415
|
-
file,
|
|
3416
|
-
kind,
|
|
3417
|
-
id,
|
|
3418
|
-
issues,
|
|
3419
|
-
text: `${JSON.stringify(content, null, 2)}\n`,
|
|
3420
|
-
// The Steps tab needs steps to render: an output/unknown file has none. recipeToOutline never
|
|
3421
|
-
// throws, so this covers a file that fails validation too (its outline just has "custom" cards).
|
|
3422
|
-
outline: kind === 'input' ? recipeToOutline(content) : undefined
|
|
3423
|
-
}))
|
|
3424
|
-
};
|
|
3647
|
+
return segments.join(' > ');
|
|
3425
3648
|
}
|
|
3426
|
-
|
|
3427
|
-
|
|
3428
|
-
|
|
3649
|
+
|
|
3650
|
+
/** How long the browser is given to settle after `goto`, watching for JSON responses (mirrors `packages/cli`'s `probe --browser`, `observeBrowserJson`). */
|
|
3651
|
+
const OBSERVE_MS = 4000;
|
|
3652
|
+
/** Responses kept, largest/most-recent-first ordering left to the caller; just a sane cap on what the panel lists. */
|
|
3653
|
+
const RESPONSE_LIMIT = 15;
|
|
3654
|
+
/**
|
|
3655
|
+
* Records the JSON responses seen while an input recipe's start page
|
|
3656
|
+
* rendered (studio plan §3.3, issue #93): the same technique `packages/cli`'s
|
|
3657
|
+
* `probe --browser` uses (`observeBrowserJson`) — render the page, listen for
|
|
3658
|
+
* `response` events whose content type is JSON, give it a few seconds to
|
|
3659
|
+
* settle. Kept as its own capture, independent of `page-snapshot`'s
|
|
3660
|
+
* `take-snapshot.use-case.ts`: that capture reads `page.content()` right
|
|
3661
|
+
* after `goto` resolves, with no observation window, and changing its shape
|
|
3662
|
+
* to also collect responses was a larger, riskier change to already-shipped,
|
|
3663
|
+
* tested code than this phase's budget allowed (see this phase's final
|
|
3664
|
+
* report) — so this opens its own short-lived browser session instead of
|
|
3665
|
+
* piggy-backing on the cached snapshot.
|
|
3666
|
+
*
|
|
3667
|
+
* Only `"start"` is supported, mirroring `take-snapshot.use-case.ts`'s own
|
|
3668
|
+
* reach; api-mode recipes have no page load to observe, so they always
|
|
3669
|
+
* answer with an empty list rather than an error (there is nothing wrong
|
|
3670
|
+
* with asking, the answer is just "none").
|
|
3671
|
+
*
|
|
3672
|
+
* @param input - The input recipe; only `mode` and the first `start` point are used.
|
|
3673
|
+
* @param stepPath - The step to observe after, e.g. `"start"`.
|
|
3674
|
+
* @param browser - Browser launch settings, shared with `take-snapshot`/`verify-selector`/`run-sample`.
|
|
3675
|
+
* @returns The distinct JSON responses observed, at most `RESPONSE_LIMIT`.
|
|
3676
|
+
* @throws Error when the recipe has no start point, or `stepPath` is not `"start"`.
|
|
3677
|
+
*/
|
|
3678
|
+
async function responsesSeen(input, stepPath, browser) {
|
|
3679
|
+
if (stepPath !== 'start') throw new Error(`responsesSeen: "${stepPath}" is not supported yet — only "start" (the first start point, before any step ran) is`);
|
|
3680
|
+
if (input.mode !== 'web') return [];
|
|
3681
|
+
const point = input.start[0];
|
|
3682
|
+
if (point === undefined) throw new Error(`recipe "${input.id}" has no start point`);
|
|
3683
|
+
const client = await ensureBrowserLaunch(browser, () => BrowserClient.launch(browser));
|
|
3429
3684
|
try {
|
|
3430
|
-
|
|
3431
|
-
|
|
3432
|
-
|
|
3433
|
-
|
|
3434
|
-
const
|
|
3435
|
-
|
|
3436
|
-
|
|
3437
|
-
|
|
3438
|
-
|
|
3685
|
+
const session = await client.newSession();
|
|
3686
|
+
try {
|
|
3687
|
+
const responses = [];
|
|
3688
|
+
const seen = new Set();
|
|
3689
|
+
const pending = [];
|
|
3690
|
+
session.page.on('response', response => {
|
|
3691
|
+
if (!(response.headers()['content-type'] ?? '').includes('json')) return;
|
|
3692
|
+
const key = `${response.status()} ${response.url()}`;
|
|
3693
|
+
if (seen.has(key)) return;
|
|
3694
|
+
seen.add(key);
|
|
3695
|
+
pending.push(recordResponse(response, responses));
|
|
3439
3696
|
});
|
|
3697
|
+
try {
|
|
3698
|
+
await session.page.goto(point.url, {
|
|
3699
|
+
waitUntil: 'networkidle'
|
|
3700
|
+
});
|
|
3701
|
+
} catch {
|
|
3702
|
+
// a page that never idles (long-poll, websocket) still yields whatever was observed by the timeout below
|
|
3703
|
+
}
|
|
3704
|
+
await session.page.waitForTimeout(OBSERVE_MS);
|
|
3705
|
+
await Promise.all(pending);
|
|
3706
|
+
return responses.slice(0, RESPONSE_LIMIT);
|
|
3707
|
+
} finally {
|
|
3708
|
+
await session.close();
|
|
3440
3709
|
}
|
|
3710
|
+
} finally {
|
|
3711
|
+
await client.close();
|
|
3441
3712
|
}
|
|
3442
3713
|
}
|
|
3443
|
-
|
|
3444
|
-
|
|
3445
|
-
}
|
|
3446
|
-
function parseDocument(document) {
|
|
3447
|
-
const kind = document.content?.kind;
|
|
3448
|
-
if (kind === 'output') return parseOne(document.source, document.content, 'output', parseOutputRecipe);
|
|
3449
|
-
if (kind === 'input') return parseOne(document.source, document.content, 'input', parseInputRecipe);
|
|
3450
|
-
return {
|
|
3451
|
-
file: document.source,
|
|
3452
|
-
kind: 'unknown',
|
|
3453
|
-
content: document.content,
|
|
3454
|
-
issues: [{
|
|
3455
|
-
path: 'kind',
|
|
3456
|
-
message: '"kind" is missing or not "input"/"output"',
|
|
3457
|
-
kind: 'validation'
|
|
3458
|
-
}]
|
|
3459
|
-
};
|
|
3460
|
-
}
|
|
3461
|
-
function parseOne(file, content, kind, parse) {
|
|
3714
|
+
/** Reads one response's body, when it still can be (not aborted, not redirected away); a response that cannot be read is silently left out rather than failing the whole capture. */
|
|
3715
|
+
async function recordResponse(response, into) {
|
|
3462
3716
|
try {
|
|
3463
|
-
const
|
|
3464
|
-
|
|
3465
|
-
|
|
3466
|
-
|
|
3467
|
-
|
|
3468
|
-
|
|
3469
|
-
|
|
3470
|
-
|
|
3471
|
-
|
|
3472
|
-
|
|
3473
|
-
if (!(error instanceof RecipeValidationError)) throw error;
|
|
3474
|
-
return {
|
|
3475
|
-
file,
|
|
3476
|
-
kind,
|
|
3477
|
-
content,
|
|
3478
|
-
issues: error.issues.map(issue => ({
|
|
3479
|
-
path: issue.path === '' ? '(root)' : issue.path,
|
|
3480
|
-
message: issue.message,
|
|
3481
|
-
kind: 'validation'
|
|
3482
|
-
}))
|
|
3483
|
-
};
|
|
3717
|
+
const body = await response.text();
|
|
3718
|
+
const value = tryParseJson(body);
|
|
3719
|
+
into.push({
|
|
3720
|
+
url: response.url(),
|
|
3721
|
+
status: response.status(),
|
|
3722
|
+
size: body.length,
|
|
3723
|
+
shape: value === undefined ? 'text' : describeJson(value, 'json').type
|
|
3724
|
+
});
|
|
3725
|
+
} catch {
|
|
3726
|
+
// response body unavailable (aborted, redirected, the page navigated away) — skip it
|
|
3484
3727
|
}
|
|
3485
3728
|
}
|
|
3486
3729
|
|
|
3487
3730
|
/**
|
|
3488
|
-
*
|
|
3489
|
-
*
|
|
3490
|
-
*
|
|
3491
|
-
*
|
|
3492
|
-
*
|
|
3493
|
-
*
|
|
3494
|
-
*
|
|
3495
|
-
* round-trip for a hand-formatted file is future work; see this file's test
|
|
3496
|
-
* for what is guaranteed today.
|
|
3731
|
+
* Handles `inspect-page`: the Inspect panel's DOM tree and data-in-the-page
|
|
3732
|
+
* findings, off the same cached snapshot `take-snapshot`/`infer-selector`
|
|
3733
|
+
* already use (`page-inspector`, issue #93). `domTree` reads the cached,
|
|
3734
|
+
* rewritten snapshot (the same markup the content pane's iframe shows, so a
|
|
3735
|
+
* tree row's `nodeId` lines up with it exactly); `pageData` reads the raw
|
|
3736
|
+
* capture (`SnapshotResult.rawHtml`), since the display snapshot has had its
|
|
3737
|
+
* `<script>` tags stripped for the sandboxed iframe.
|
|
3497
3738
|
*
|
|
3498
|
-
* @param
|
|
3499
|
-
* @param
|
|
3739
|
+
* @param state - The server state.
|
|
3740
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
3741
|
+
* @returns The tree and the data-in-the-page findings.
|
|
3742
|
+
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
3500
3743
|
*/
|
|
3501
|
-
|
|
3502
|
-
|
|
3744
|
+
function handleInspectPage(state, command) {
|
|
3745
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
3746
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
3747
|
+
return {
|
|
3748
|
+
tree: domTree(snapshot.html),
|
|
3749
|
+
pageData: pageData(snapshot.rawHtml)
|
|
3750
|
+
};
|
|
3503
3751
|
}
|
|
3504
3752
|
|
|
3505
3753
|
/** Where the studio's own recorder profile lives when `StudioServerOptions.recorderProfileDir` is not given: never a crawl's own `session.browserProfile` directory, never the person's default browser profile. */
|
|
@@ -3537,7 +3785,14 @@ function broadcast(state, event) {
|
|
|
3537
3785
|
/**
|
|
3538
3786
|
* Handles `open-workspace`: reads the folder, remembers it as the server's
|
|
3539
3787
|
* current workspace (every later command that names a recipe resolves it
|
|
3540
|
-
* against this folder), and
|
|
3788
|
+
* against this folder), and, when that is a different folder than before,
|
|
3789
|
+
* tells every connected client the workspace changed.
|
|
3790
|
+
*
|
|
3791
|
+
* Only on an actual switch: the UI answers `workspace-changed` with another
|
|
3792
|
+
* `open-workspace` of the same folder (its workspace query re-fetches), so a
|
|
3793
|
+
* broadcast on every open would answer that re-fetch with another broadcast,
|
|
3794
|
+
* for ever — and each round cancels the fetch in flight, so a recipe saved
|
|
3795
|
+
* meanwhile never showed up in the listing (found by issue #120's drop flow).
|
|
3541
3796
|
*
|
|
3542
3797
|
* @param state - The server state to update.
|
|
3543
3798
|
* @param command - The command.
|
|
@@ -3545,8 +3800,9 @@ function broadcast(state, event) {
|
|
|
3545
3800
|
*/
|
|
3546
3801
|
async function handleOpenWorkspace(state, command) {
|
|
3547
3802
|
const view = await openWorkspace(command.folder);
|
|
3803
|
+
const switched = state.folder !== command.folder;
|
|
3548
3804
|
state.folder = command.folder;
|
|
3549
|
-
broadcast(state, {
|
|
3805
|
+
if (switched) broadcast(state, {
|
|
3550
3806
|
type: 'workspace-changed'
|
|
3551
3807
|
});
|
|
3552
3808
|
return view;
|
|
@@ -4235,13 +4491,13 @@ async function detectNavigation(page, urlBefore) {
|
|
|
4235
4491
|
*/
|
|
4236
4492
|
async function handleStartRecording(state, command) {
|
|
4237
4493
|
if (state.folder === undefined) throw new Error('open a workspace first');
|
|
4238
|
-
await state
|
|
4494
|
+
await settleRecording(state);
|
|
4239
4495
|
const {
|
|
4240
4496
|
input
|
|
4241
4497
|
} = await loadRecipePair(state.folder, command.recipeId);
|
|
4242
4498
|
const point = input.start[0];
|
|
4243
4499
|
if (point === undefined) throw new Error(`recipe "${command.recipeId}" has no start point`);
|
|
4244
|
-
const
|
|
4500
|
+
const opening = openRecorderSession({
|
|
4245
4501
|
startUrl: point.url,
|
|
4246
4502
|
profileDir: state.recorderProfileDir,
|
|
4247
4503
|
browser: state.browser
|
|
@@ -4261,7 +4517,12 @@ async function handleStartRecording(state, command) {
|
|
|
4261
4517
|
});
|
|
4262
4518
|
}
|
|
4263
4519
|
});
|
|
4264
|
-
state.
|
|
4520
|
+
state.openingRecording = opening;
|
|
4521
|
+
try {
|
|
4522
|
+
state.activeRecording = await opening;
|
|
4523
|
+
} finally {
|
|
4524
|
+
state.openingRecording = undefined;
|
|
4525
|
+
}
|
|
4265
4526
|
return {
|
|
4266
4527
|
started: true
|
|
4267
4528
|
};
|
|
@@ -4272,21 +4533,22 @@ async function handleStartRecording(state, command) {
|
|
|
4272
4533
|
* build "make this the login" (`session.bootstrap`) or "keep as steps"
|
|
4273
4534
|
* without asking the server again. Also broadcasts `recording-stopped`, so
|
|
4274
4535
|
* every connected client (not only the one that asked) knows the window
|
|
4275
|
-
* closed.
|
|
4536
|
+
* closed, even when nothing was open (a UI waiting on it would stay stuck).
|
|
4537
|
+
* A stop that arrives while the window is still opening waits for it.
|
|
4276
4538
|
*
|
|
4277
4539
|
* @param state - The server state holding the active recording, if any.
|
|
4278
4540
|
* @param _command - The command (carries nothing beyond its type).
|
|
4279
4541
|
* @returns Every step recorded, in order; empty when nothing was recording.
|
|
4280
4542
|
*/
|
|
4281
4543
|
async function handleStopRecording(state, _command) {
|
|
4544
|
+
await waitForOpening(state);
|
|
4282
4545
|
const recording = state.activeRecording;
|
|
4283
4546
|
state.activeRecording = undefined;
|
|
4284
|
-
if (recording === undefined) return {
|
|
4285
|
-
steps: []
|
|
4286
|
-
};
|
|
4287
4547
|
const {
|
|
4288
4548
|
steps
|
|
4289
|
-
} =
|
|
4549
|
+
} = recording === undefined ? {
|
|
4550
|
+
steps: []
|
|
4551
|
+
} : await recording.stop();
|
|
4290
4552
|
broadcast(state, {
|
|
4291
4553
|
type: 'recording-stopped',
|
|
4292
4554
|
steps
|
|
@@ -4295,6 +4557,20 @@ async function handleStopRecording(state, _command) {
|
|
|
4295
4557
|
steps
|
|
4296
4558
|
};
|
|
4297
4559
|
}
|
|
4560
|
+
/** Closes the recording in progress, waiting first for one still opening (issue #174). */
|
|
4561
|
+
async function settleRecording(state) {
|
|
4562
|
+
await waitForOpening(state);
|
|
4563
|
+
await state.activeRecording?.stop();
|
|
4564
|
+
state.activeRecording = undefined;
|
|
4565
|
+
}
|
|
4566
|
+
/** Waits for a window still opening; a failed open is `start-recording`'s own error to report, not the stopper's. */
|
|
4567
|
+
async function waitForOpening(state) {
|
|
4568
|
+
try {
|
|
4569
|
+
await state.openingRecording;
|
|
4570
|
+
} catch {
|
|
4571
|
+
// reported to the caller of start-recording
|
|
4572
|
+
}
|
|
4573
|
+
}
|
|
4298
4574
|
|
|
4299
4575
|
/**
|
|
4300
4576
|
* Handles `responses-seen`: the JSON responses observed while a recipe's
|
|
@@ -4319,6 +4595,26 @@ async function handleResponsesSeen(state, command) {
|
|
|
4319
4595
|
};
|
|
4320
4596
|
}
|
|
4321
4597
|
|
|
4598
|
+
/**
|
|
4599
|
+
* Handles `region-preview`: the PDF and deck canvases' live preview of a
|
|
4600
|
+
* `region` extract's selector (`document-view`'s
|
|
4601
|
+
* `region-preview.use-case.ts`, issues #121 and #122), off the same cached
|
|
4602
|
+
* snapshot `take-snapshot` already captured — a pure computation over it,
|
|
4603
|
+
* so this answers instantly on every change of the box, no re-fetch or
|
|
4604
|
+
* re-parse (mirrors `table-preview.handler.ts`).
|
|
4605
|
+
*
|
|
4606
|
+
* @param state - The server state.
|
|
4607
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
4608
|
+
* @returns The matches, or `error` when the selector does not parse or does not fit the document.
|
|
4609
|
+
* @throws Error when `command.path` has no cached snapshot yet, or the snapshot is neither a PDF nor a deck.
|
|
4610
|
+
*/
|
|
4611
|
+
function handleRegionPreview(state, command) {
|
|
4612
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
4613
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
4614
|
+
if (!isPdfDocument(snapshot.body) && !isDeckDocument(snapshot.body)) throw new Error(`"${command.recipeId}" at "${command.path}" is neither a PDF nor a deck snapshot: region-preview only reads a read PDF or PowerPoint document`);
|
|
4615
|
+
return previewRegion(snapshot.body, command.selector);
|
|
4616
|
+
}
|
|
4617
|
+
|
|
4322
4618
|
/**
|
|
4323
4619
|
* Handles `run-sample`: stops whatever sample was still running (one run at
|
|
4324
4620
|
* a time), starts the new one, and returns once it has started, not once it
|
|
@@ -4411,7 +4707,9 @@ async function handleStopRun(state, _command) {
|
|
|
4411
4707
|
* @returns `{ saved: true }` once the write has completed.
|
|
4412
4708
|
*/
|
|
4413
4709
|
async function handleSaveOutline(state, command) {
|
|
4414
|
-
|
|
4710
|
+
const recipe = outlineToRecipe(command.outline);
|
|
4711
|
+
await invalidateStaleSnapshot(state.snapshots, command.path, recipe);
|
|
4712
|
+
await saveRecipe(command.path, recipe);
|
|
4415
4713
|
broadcast(state, {
|
|
4416
4714
|
type: 'workspace-changed'
|
|
4417
4715
|
});
|
|
@@ -4430,7 +4728,9 @@ async function handleSaveOutline(state, command) {
|
|
|
4430
4728
|
* @returns `{ saved: true }` once the write has completed.
|
|
4431
4729
|
*/
|
|
4432
4730
|
async function handleSaveRecipe(state, command) {
|
|
4433
|
-
|
|
4731
|
+
const recipe = command.recipe;
|
|
4732
|
+
await invalidateStaleSnapshot(state.snapshots, command.path, recipe);
|
|
4733
|
+
await saveRecipe(command.path, recipe);
|
|
4434
4734
|
broadcast(state, {
|
|
4435
4735
|
type: 'workspace-changed'
|
|
4436
4736
|
});
|
|
@@ -4718,7 +5018,7 @@ async function startStudioServer(options = {}) {
|
|
|
4718
5018
|
}
|
|
4719
5019
|
async function stop(server, state) {
|
|
4720
5020
|
await state.activeRun?.stop();
|
|
4721
|
-
await state
|
|
5021
|
+
await settleRecording(state);
|
|
4722
5022
|
server.closeAllConnections();
|
|
4723
5023
|
await new Promise((resolve, reject) => {
|
|
4724
5024
|
server.close(error => error === undefined ? resolve() : reject(error));
|
|
@@ -4806,6 +5106,10 @@ function dispatch(command, state) {
|
|
|
4806
5106
|
{
|
|
4807
5107
|
return handleOpenWorkspace(state, command);
|
|
4808
5108
|
}
|
|
5109
|
+
case 'import-document':
|
|
5110
|
+
{
|
|
5111
|
+
return handleImportDocument(command);
|
|
5112
|
+
}
|
|
4809
5113
|
case 'run-sample':
|
|
4810
5114
|
{
|
|
4811
5115
|
return handleRunSample(state, command);
|
|
@@ -4862,6 +5166,10 @@ function dispatch(command, state) {
|
|
|
4862
5166
|
{
|
|
4863
5167
|
return Promise.resolve(handleTablePreview(state, command));
|
|
4864
5168
|
}
|
|
5169
|
+
case 'region-preview':
|
|
5170
|
+
{
|
|
5171
|
+
return Promise.resolve(handleRegionPreview(state, command));
|
|
5172
|
+
}
|
|
4865
5173
|
case 'grid-view':
|
|
4866
5174
|
{
|
|
4867
5175
|
return handleGridView(state, command);
|
|
@@ -4942,5 +5250,5 @@ async function main(argv) {
|
|
|
4942
5250
|
process.on('SIGTERM', shutdown);
|
|
4943
5251
|
}
|
|
4944
5252
|
|
|
4945
|
-
export { deckChartSeriesViewSchema, deckChartViewSchema, deckDocumentViewSchema, deckPreviewCommandSchema, deckPreviewOptionsSchema, deckShapeViewSchema, deckSlideViewSchema, deckTablePreviewMatchSchema, deckTablePreviewViewSchema, deckViewCommandSchema, documentTreeCommandSchema, documentTreeNodeViewSchema, documentTreeViewSchema, domTreeNodeViewSchema, explainWhyCommandSchema, fetchStartPage, fetchStartPageCommandSchema, fieldPickSchema, fieldTraceSchema, gridCellViewSchema, gridMergeViewSchema, gridPreviewCommandSchema, gridPreviewOptionsSchema, gridSheetViewSchema, gridTablePreviewMatchSchema, gridTablePreviewViewSchema, gridViewCommandSchema, inferSelectorCommandSchema, inferSelectorViewSchema, inspectPageCommandSchema, inspectViewSchema, loadRecipePair, main, missingWhyTargetSchema, observedResponseSchema, openWorkspace, openWorkspaceCommandSchema, outlineBracketSchema, outlineCardSchema, outlineNodeSchema, outlineViewSchema, pageDataFindingSchema, pdfCellViewSchema, pdfDocumentViewSchema, pdfPageViewSchema, pdfRowViewSchema, pdfViewCommandSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, recordRejectedEventSchema, recordingCardEventSchema, recordingNoteEventSchema, recordingStoppedEventSchema, rejectedWhyTargetSchema, responsesSeenCommandSchema, responsesSeenViewSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveOutlineCommandSchema, saveRecipe, saveRecipeCommandSchema, sentencePartSchema, snapshotViewSchema, startPageViewSchema, startRecordingCommandSchema, startStudioServer, stopRecordingCommandSchema, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, tablePreviewBandSchema, tablePreviewCommandSchema, tablePreviewMatchSchema, tablePreviewOptionsSchema, tablePreviewTableSchema, tablePreviewViewSchema, takeSnapshotCommandSchema, traceLineEventSchema, verifySelectorCommandSchema, verifySelectorViewSchema, whyTargetSchema, whyViewSchema, workbookDocumentViewSchema, workspaceChangedEventSchema, workspaceViewSchema };
|
|
5253
|
+
export { deckChartSeriesViewSchema, deckChartViewSchema, deckDocumentViewSchema, deckPreviewCommandSchema, deckPreviewOptionsSchema, deckShapeViewSchema, deckSlideViewSchema, deckTablePreviewMatchSchema, deckTablePreviewViewSchema, deckViewCommandSchema, documentTreeCommandSchema, documentTreeNodeViewSchema, documentTreeViewSchema, domTreeNodeViewSchema, explainWhyCommandSchema, fetchStartPage, fetchStartPageCommandSchema, fieldPickSchema, fieldTraceSchema, gridCellViewSchema, gridMergeViewSchema, gridPreviewCommandSchema, gridPreviewOptionsSchema, gridSheetViewSchema, gridTablePreviewMatchSchema, gridTablePreviewViewSchema, gridViewCommandSchema, importDocument, importDocumentCommandSchema, importDocumentViewSchema, inferSelectorCommandSchema, inferSelectorViewSchema, inspectPageCommandSchema, inspectViewSchema, loadRecipePair, main, missingWhyTargetSchema, observedResponseSchema, openWorkspace, openWorkspaceCommandSchema, outlineBracketSchema, outlineCardSchema, outlineNodeSchema, outlineViewSchema, pageDataFindingSchema, pdfCellViewSchema, pdfDocumentViewSchema, pdfPageViewSchema, pdfRowViewSchema, pdfViewCommandSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, recordRejectedEventSchema, recordingCardEventSchema, recordingNoteEventSchema, recordingStoppedEventSchema, regionPreviewCommandSchema, regionPreviewMatchSchema, regionPreviewViewSchema, rejectedWhyTargetSchema, responsesSeenCommandSchema, responsesSeenViewSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveOutlineCommandSchema, saveRecipe, saveRecipeCommandSchema, sentencePartSchema, snapshotViewSchema, startPageViewSchema, startRecordingCommandSchema, startStudioServer, stopRecordingCommandSchema, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, tablePreviewBandSchema, tablePreviewCommandSchema, tablePreviewMatchSchema, tablePreviewOptionsSchema, tablePreviewTableSchema, tablePreviewViewSchema, takeSnapshotCommandSchema, traceLineEventSchema, verifySelectorCommandSchema, verifySelectorViewSchema, whyTargetSchema, whyViewSchema, workbookDocumentViewSchema, workspaceChangedEventSchema, workspaceViewSchema };
|
|
4946
5254
|
//# sourceMappingURL=index.esm.js.map
|