@opencraw/studio 0.0.8 → 0.0.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +31 -5
- package/dist/index.esm.js +1362 -1072
- package/dist/src/browser-provisioning/ensure-browser-launch.use-case.d.ts +12 -0
- package/dist/src/browser-provisioning/index.d.ts +2 -0
- package/dist/src/browser-provisioning/install-chromium.use-case.d.ts +2 -0
- package/dist/src/browser-provisioning/missing-browser.policy.d.ts +3 -0
- package/dist/src/document-view/index.d.ts +2 -0
- package/dist/src/document-view/region-preview.use-case.d.ts +30 -0
- package/dist/src/index.d.ts +4 -3
- package/dist/src/recipe-workspace/import-document.use-case.d.ts +22 -0
- package/dist/src/recipe-workspace/index.d.ts +2 -0
- package/dist/src/studio-api/command.contract.d.ts +41 -0
- package/dist/src/studio-api/import-document-view.contract.d.ts +8 -0
- package/dist/src/studio-api/index.d.ts +6 -2
- package/dist/src/studio-api/region-preview-view.contract.d.ts +53 -0
- package/dist/src/studio-server/import-document.handler.d.ts +14 -0
- package/dist/src/studio-server/open-workspace.handler.d.ts +8 -1
- package/dist/src/studio-server/region-preview.handler.d.ts +17 -0
- package/dist/ui/assets/index-CxiFCU_G.js +37 -0
- package/dist/ui/assets/{pdf-lqXfRfkR.js → pdf-4CYQHobd.js} +1 -1
- package/dist/ui/index.html +1 -1
- package/package.json +3 -3
- package/dist/ui/assets/index-DdiGdaTN.js +0 -13
package/dist/index.esm.js
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
|
-
import { join,
|
|
1
|
+
import { join, dirname, basename, extname, normalize, sep, resolve } from 'node:path';
|
|
2
2
|
import { createServer } from 'node:http';
|
|
3
|
-
import { fileURLToPath } from 'node:url';
|
|
3
|
+
import { fileURLToPath, pathToFileURL } from 'node:url';
|
|
4
4
|
import { z } from 'zod';
|
|
5
|
-
import { BODY_KINDS,
|
|
5
|
+
import { BODY_KINDS, HttpClient, BrowserClient, NODE_ID_ATTRIBUTE, rowsOfCells, parseXml, findDeckTables, findGridTables, analyzeTables, parseRegion, regionText, deckRegionText, isDeckDocument, readRecipeSource, parseInputRecipe, parseOutputRecipe, bindRecipeSet, memorySink, createCrawler, traceLine, resolveMissingPolicy, isWorkbookDocument, RecipeBindingError, RecipeValidationError, countMatches, matchesOf, tryParseJson, isPdfDocument, BrowserProfiles } from '@opencraw/core';
|
|
6
6
|
import { randomBytes, createHash } from 'node:crypto';
|
|
7
|
+
import { execFile } from 'node:child_process';
|
|
8
|
+
import { createRequire } from 'node:module';
|
|
9
|
+
import { promisify } from 'node:util';
|
|
7
10
|
import { load } from 'cheerio';
|
|
8
|
-
import { readFile, writeFile } from 'node:fs/promises';
|
|
11
|
+
import { readFile, writeFile, mkdir } from 'node:fs/promises';
|
|
9
12
|
import { describeJson } from '@opencraw/probe';
|
|
10
13
|
import { tmpdir } from 'node:os';
|
|
11
14
|
import { existsSync, createReadStream } from 'node:fs';
|
|
@@ -186,6 +189,19 @@ const tablePreviewCommandSchema = z.object({
|
|
|
186
189
|
path: z.string().min(1),
|
|
187
190
|
options: tablePreviewOptionsSchema
|
|
188
191
|
});
|
|
192
|
+
/**
|
|
193
|
+
* The live preview of a `region` extract's selector against a cached PDF
|
|
194
|
+
* snapshot (`document-view`'s `region-preview.use-case.ts`, issue #121: the
|
|
195
|
+
* PDF canvas's click-to-snap selection): the text the step would bind and
|
|
196
|
+
* the cells to highlight, recomputed as the box changes — call
|
|
197
|
+
* `take-snapshot` first.
|
|
198
|
+
*/
|
|
199
|
+
const regionPreviewCommandSchema = z.object({
|
|
200
|
+
type: z.literal('region-preview'),
|
|
201
|
+
recipeId: z.string().min(1),
|
|
202
|
+
path: z.string().min(1),
|
|
203
|
+
selector: z.string().min(1)
|
|
204
|
+
});
|
|
189
205
|
/**
|
|
190
206
|
* The grid canvas's sheets and cells (`document-view`'s `workbook-view.mapper.ts`,
|
|
191
207
|
* studio plan §3.4, issue #94's 5c), off the same cached snapshot
|
|
@@ -310,7 +326,23 @@ const startRecordingCommandSchema = z.object({
|
|
|
310
326
|
const stopRecordingCommandSchema = z.object({
|
|
311
327
|
type: z.literal('stop-recording')
|
|
312
328
|
});
|
|
313
|
-
|
|
329
|
+
/**
|
|
330
|
+
* Copies a document the person dropped on (or opened in) the studio into a
|
|
331
|
+
* workspace folder, so a recipe can point at it with a `file:` URL (issue
|
|
332
|
+
* #120). The bytes travel base64-encoded inside the JSON command envelope
|
|
333
|
+
* every other command uses: a local tool's own files are a few megabytes at
|
|
334
|
+
* most, not worth a second, multipart endpoint. `folder` is created when it
|
|
335
|
+
* does not exist yet, so a dropped file can also start a workspace; `name`
|
|
336
|
+
* is a plain file name (no directories), kept as the copy's name unless one
|
|
337
|
+
* is already there, in which case a numbered one is used.
|
|
338
|
+
*/
|
|
339
|
+
const importDocumentCommandSchema = z.object({
|
|
340
|
+
type: z.literal('import-document'),
|
|
341
|
+
folder: z.string().min(1),
|
|
342
|
+
name: z.string().min(1),
|
|
343
|
+
bytes: z.string()
|
|
344
|
+
});
|
|
345
|
+
const studioCommandSchema = z.discriminatedUnion('type', [openWorkspaceCommandSchema, importDocumentCommandSchema, runSampleCommandSchema, stopRunCommandSchema, saveRecipeCommandSchema, fetchStartPageCommandSchema, saveOutlineCommandSchema, takeSnapshotCommandSchema, verifySelectorCommandSchema, inferSelectorCommandSchema, explainWhyCommandSchema, inspectPageCommandSchema, responsesSeenCommandSchema, documentTreeCommandSchema, pdfViewCommandSchema, tablePreviewCommandSchema, regionPreviewCommandSchema, gridViewCommandSchema, gridPreviewCommandSchema, deckViewCommandSchema, deckPreviewCommandSchema, startRecordingCommandSchema, stopRecordingCommandSchema]);
|
|
314
346
|
|
|
315
347
|
/**
|
|
316
348
|
* What `take-snapshot` answers with: the rewritten document ready for the
|
|
@@ -477,6 +509,12 @@ const workspaceViewSchema = z.object({
|
|
|
477
509
|
recipes: z.array(recipeListingSchema)
|
|
478
510
|
});
|
|
479
511
|
|
|
512
|
+
/** `import-document`'s answer: where the copy landed, and the `file:` URL a recipe's `start.url` points at it with. */
|
|
513
|
+
const importDocumentViewSchema = z.object({
|
|
514
|
+
path: z.string().min(1),
|
|
515
|
+
url: z.string().startsWith('file:')
|
|
516
|
+
});
|
|
517
|
+
|
|
480
518
|
/** One record of a sample run's result, kept for the Records tab. */
|
|
481
519
|
const sampleRunRecordSchema = z.object({
|
|
482
520
|
key: z.string().nullable(),
|
|
@@ -666,25 +704,6 @@ const workbookDocumentViewSchema = z.object({
|
|
|
666
704
|
csv: csvFormatViewSchema.optional()
|
|
667
705
|
});
|
|
668
706
|
|
|
669
|
-
const gridPreviewRowSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
|
|
670
|
-
/** One table matched by a `table` extract's options against a workbook — mirrors `@opencraw/core`'s `GridTable`. */
|
|
671
|
-
const gridTablePreviewMatchSchema = z.object({
|
|
672
|
-
sheet: z.string(),
|
|
673
|
-
title: z.string(),
|
|
674
|
-
header: z.array(z.string()),
|
|
675
|
-
rows: z.array(gridPreviewRowSchema)
|
|
676
|
-
});
|
|
677
|
-
/**
|
|
678
|
-
* What `grid-preview` answers with (studio plan §3.4, issue #94's 5c: the
|
|
679
|
-
* live preview): every matched table, or `error` when a pattern in the
|
|
680
|
-
* options does not compile — expected while the person is still typing it,
|
|
681
|
-
* not thrown.
|
|
682
|
-
*/
|
|
683
|
-
const gridTablePreviewViewSchema = z.object({
|
|
684
|
-
matches: z.array(gridTablePreviewMatchSchema),
|
|
685
|
-
error: z.string().optional()
|
|
686
|
-
});
|
|
687
|
-
|
|
688
707
|
/** One text box on a slide, in points — mirrors `document-view/deck-view.mapper.ts`'s `DeckShapeView`, the wire shape for the UI (studio plan §3.4, issue #94's 5d). */
|
|
689
708
|
const deckShapeViewSchema = z.object({
|
|
690
709
|
x: z.number(),
|
|
@@ -724,6 +743,45 @@ const deckDocumentViewSchema = z.object({
|
|
|
724
743
|
slides: z.array(deckSlideViewSchema)
|
|
725
744
|
});
|
|
726
745
|
|
|
746
|
+
/** One page of a PDF, or one slide of a deck, a `region` extract's box reads text off — mirrors `document-view/region-preview.use-case.ts`'s `RegionPreviewMatch`: `cells` for a PDF, `shapes` for a deck, the other empty. */
|
|
747
|
+
const regionPreviewMatchSchema = z.object({
|
|
748
|
+
/** The page number of a PDF, or the slide number of a deck. */
|
|
749
|
+
page: z.number().int(),
|
|
750
|
+
text: z.string(),
|
|
751
|
+
cells: z.array(pdfCellViewSchema),
|
|
752
|
+
shapes: z.array(deckShapeViewSchema)
|
|
753
|
+
});
|
|
754
|
+
/**
|
|
755
|
+
* What `region-preview` answers with (issues #121 and #122: the PDF and
|
|
756
|
+
* deck canvases' click-to-snap selection): the engine's own matches for
|
|
757
|
+
* the selector — the text the step would bind, and the cells or text boxes
|
|
758
|
+
* to highlight — or `error` when the selector does not parse yet, or names
|
|
759
|
+
* the other document's keyword, reported rather than thrown.
|
|
760
|
+
*/
|
|
761
|
+
const regionPreviewViewSchema = z.object({
|
|
762
|
+
matches: z.array(regionPreviewMatchSchema),
|
|
763
|
+
error: z.string().optional()
|
|
764
|
+
});
|
|
765
|
+
|
|
766
|
+
const gridPreviewRowSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
|
|
767
|
+
/** One table matched by a `table` extract's options against a workbook — mirrors `@opencraw/core`'s `GridTable`. */
|
|
768
|
+
const gridTablePreviewMatchSchema = z.object({
|
|
769
|
+
sheet: z.string(),
|
|
770
|
+
title: z.string(),
|
|
771
|
+
header: z.array(z.string()),
|
|
772
|
+
rows: z.array(gridPreviewRowSchema)
|
|
773
|
+
});
|
|
774
|
+
/**
|
|
775
|
+
* What `grid-preview` answers with (studio plan §3.4, issue #94's 5c: the
|
|
776
|
+
* live preview): every matched table, or `error` when a pattern in the
|
|
777
|
+
* options does not compile — expected while the person is still typing it,
|
|
778
|
+
* not thrown.
|
|
779
|
+
*/
|
|
780
|
+
const gridTablePreviewViewSchema = z.object({
|
|
781
|
+
matches: z.array(gridTablePreviewMatchSchema),
|
|
782
|
+
error: z.string().optional()
|
|
783
|
+
});
|
|
784
|
+
|
|
727
785
|
const deckPreviewRowSchema = z.record(z.string(), z.union([z.string(), z.number(), z.boolean()]));
|
|
728
786
|
/** One table matched by a `table` extract's options against a deck — mirrors `@opencraw/core`'s `DeckTable`. */
|
|
729
787
|
const deckTablePreviewMatchSchema = z.object({
|
|
@@ -786,6 +844,85 @@ function cookieToken(header) {
|
|
|
786
844
|
return undefined;
|
|
787
845
|
}
|
|
788
846
|
|
|
847
|
+
const execFileAsync = promisify(execFile);
|
|
848
|
+
/** `playwright`'s own `exports` map doesn't whitelist a `./cli` subpath, so its CLI is resolved off `playwright/package.json`'s real location — the same technique `@opencraw/captcha-tesseract`'s `bundledLangPath()` uses for a bundled data path. */
|
|
849
|
+
function playwrightCliPath() {
|
|
850
|
+
const require = createRequire(import.meta.url);
|
|
851
|
+
return join(dirname(require.resolve('playwright/package.json')), 'cli.js');
|
|
852
|
+
}
|
|
853
|
+
async function runPlaywrightInstall() {
|
|
854
|
+
await execFileAsync(process.execPath, [playwrightCliPath(), 'install', 'chromium']);
|
|
855
|
+
}
|
|
856
|
+
let installing;
|
|
857
|
+
/**
|
|
858
|
+
* Runs `playwright install chromium` at most once per process. A failed install clears the memo,
|
|
859
|
+
* so the next browser launch attempt retries instead of remembering failure forever.
|
|
860
|
+
*/
|
|
861
|
+
async function runOnce(run) {
|
|
862
|
+
try {
|
|
863
|
+
await run();
|
|
864
|
+
console.log('@opencraw/studio: Chromium installed.');
|
|
865
|
+
} catch (error) {
|
|
866
|
+
installing = undefined;
|
|
867
|
+
console.log('@opencraw/studio: "playwright install chromium" failed.');
|
|
868
|
+
throw error;
|
|
869
|
+
}
|
|
870
|
+
}
|
|
871
|
+
function installChromiumOnce(run = runPlaywrightInstall) {
|
|
872
|
+
if (installing === undefined) {
|
|
873
|
+
console.log('@opencraw/studio: Playwright\'s Chromium is not downloaded yet — running "playwright install chromium" once...');
|
|
874
|
+
installing = runOnce(run);
|
|
875
|
+
}
|
|
876
|
+
return installing;
|
|
877
|
+
}
|
|
878
|
+
|
|
879
|
+
/** Recognises Playwright's own "no browser downloaded" failure so a launch can be retried after installing one, instead of surfacing Playwright's raw ASCII-art error. */
|
|
880
|
+
function isMissingBrowserError(error) {
|
|
881
|
+
return error instanceof Error && /Executable doesn't exist at/.test(error.message);
|
|
882
|
+
}
|
|
883
|
+
|
|
884
|
+
/**
|
|
885
|
+
* Runs a browser-launching function, and — only when it fails with Playwright's own "no browser
|
|
886
|
+
* downloaded" error, only for a default chromium launch with no explicit binary configured — installs
|
|
887
|
+
* Chromium once and retries, instead of letting Playwright's raw error reach the studio's caller.
|
|
888
|
+
*
|
|
889
|
+
* An explicit `executablePath` or `OPENCRAW_CHROMIUM` means the person (or this repo's own sandbox)
|
|
890
|
+
* already chose a specific binary; a missing-browser failure there is their setup to fix, not
|
|
891
|
+
* something to silently install over.
|
|
892
|
+
*/
|
|
893
|
+
async function ensureBrowserLaunch(browser, launch, installChromium = installChromiumOnce) {
|
|
894
|
+
try {
|
|
895
|
+
return await launch();
|
|
896
|
+
} catch (error) {
|
|
897
|
+
if (!isMissingBrowserError(error)) throw error;
|
|
898
|
+
if (browser?.executablePath !== undefined || process.env.OPENCRAW_CHROMIUM !== undefined) {
|
|
899
|
+
throw new Error('@opencraw/studio: no browser found at the configured executable path. Check it, or unset it and let the studio install Chromium itself.', {
|
|
900
|
+
cause: error
|
|
901
|
+
});
|
|
902
|
+
}
|
|
903
|
+
if ((browser?.browserType ?? 'chromium') !== 'chromium') {
|
|
904
|
+
throw new Error(`@opencraw/studio: no ${browser?.browserType} browser found. Run "npx playwright install ${browser?.browserType}" and try again.`, {
|
|
905
|
+
cause: error
|
|
906
|
+
});
|
|
907
|
+
}
|
|
908
|
+
try {
|
|
909
|
+
await installChromium();
|
|
910
|
+
} catch (installError) {
|
|
911
|
+
throw new Error('@opencraw/studio: could not install Chromium automatically. Run "npx playwright install chromium" yourself and try again.', {
|
|
912
|
+
cause: installError
|
|
913
|
+
});
|
|
914
|
+
}
|
|
915
|
+
try {
|
|
916
|
+
return await launch();
|
|
917
|
+
} catch (retryError) {
|
|
918
|
+
if (!isMissingBrowserError(retryError)) throw retryError;
|
|
919
|
+
throw new Error('@opencraw/studio: Chromium was installed but is still not found. Run "npx playwright install chromium" yourself and try again.', {
|
|
920
|
+
cause: retryError
|
|
921
|
+
});
|
|
922
|
+
}
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
|
|
789
926
|
/**
|
|
790
927
|
* Fetches an input recipe's start page as the engine would see it: a fresh
|
|
791
928
|
* Playwright page in web mode, core's HTTP client in api mode. Returns plain
|
|
@@ -809,7 +946,7 @@ async function fetchStartPage(input, browser) {
|
|
|
809
946
|
return input.mode === 'web' ? fetchWeb(point.url, browser) : fetchApi(point.url);
|
|
810
947
|
}
|
|
811
948
|
async function fetchWeb(url, browser) {
|
|
812
|
-
const client = await BrowserClient.launch(browser);
|
|
949
|
+
const client = await ensureBrowserLaunch(browser, () => BrowserClient.launch(browser));
|
|
813
950
|
try {
|
|
814
951
|
const session = await client.newSession();
|
|
815
952
|
try {
|
|
@@ -985,7 +1122,7 @@ async function takeSnapshot(input, stepPath, browser) {
|
|
|
985
1122
|
return input.mode === 'web' ? snapshotWeb(point.url, browser) : snapshotApi(point.url);
|
|
986
1123
|
}
|
|
987
1124
|
async function snapshotWeb(url, browser) {
|
|
988
|
-
const client = await BrowserClient.launch(browser);
|
|
1125
|
+
const client = await ensureBrowserLaunch(browser, () => BrowserClient.launch(browser));
|
|
989
1126
|
try {
|
|
990
1127
|
const session = await client.newSession();
|
|
991
1128
|
try {
|
|
@@ -1637,6 +1774,56 @@ function previewDeckTable(document, options) {
|
|
|
1637
1774
|
};
|
|
1638
1775
|
}
|
|
1639
1776
|
|
|
1777
|
+
/**
|
|
1778
|
+
* Runs a `region` extract's selector against an already-read PDF or deck
|
|
1779
|
+
* (issues #121 and #122: the canvases' click-to-snap selection) — the
|
|
1780
|
+
* engine's own `regionText` / `deckRegionText`, so what the canvas
|
|
1781
|
+
* highlights and the text it shows are exactly what the step will read,
|
|
1782
|
+
* and a pure function of the document, so re-running it on every change
|
|
1783
|
+
* of the box costs nothing. A `slide=` on a PDF (or a `page=` on a deck)
|
|
1784
|
+
* is the same mismatch the step would fail with, reported as `error`.
|
|
1785
|
+
*
|
|
1786
|
+
* @param document - The read PDF or deck (`take-snapshot`'s cached body).
|
|
1787
|
+
* @param selector - The `region` selector, `page=1 x=72..252 y=640..664` or `slide=2 x=60..900 y=30..90`.
|
|
1788
|
+
* @returns The matches, or `error` when the selector does not parse or does not fit the document.
|
|
1789
|
+
*/
|
|
1790
|
+
function previewRegion(document, selector) {
|
|
1791
|
+
try {
|
|
1792
|
+
const region = parseRegion(selector);
|
|
1793
|
+
if (document.kind === 'pdf') {
|
|
1794
|
+
if (region.on !== 'page') return {
|
|
1795
|
+
matches: [],
|
|
1796
|
+
error: 'region: "slide=" reads a deck; this document is a PDF (name a "page=")'
|
|
1797
|
+
};
|
|
1798
|
+
return {
|
|
1799
|
+
matches: regionText(document, region).map(match => ({
|
|
1800
|
+
page: match.page,
|
|
1801
|
+
text: match.text,
|
|
1802
|
+
cells: match.cells,
|
|
1803
|
+
shapes: []
|
|
1804
|
+
}))
|
|
1805
|
+
};
|
|
1806
|
+
}
|
|
1807
|
+
if (region.on !== 'slide') return {
|
|
1808
|
+
matches: [],
|
|
1809
|
+
error: 'region: "page=" reads a PDF; this document is a deck (name a "slide=")'
|
|
1810
|
+
};
|
|
1811
|
+
return {
|
|
1812
|
+
matches: deckRegionText(document, region).map(match => ({
|
|
1813
|
+
page: match.slide,
|
|
1814
|
+
text: match.text,
|
|
1815
|
+
cells: [],
|
|
1816
|
+
shapes: match.shapes
|
|
1817
|
+
}))
|
|
1818
|
+
};
|
|
1819
|
+
} catch (error) {
|
|
1820
|
+
return {
|
|
1821
|
+
matches: [],
|
|
1822
|
+
error: error instanceof Error ? error.message : String(error)
|
|
1823
|
+
};
|
|
1824
|
+
}
|
|
1825
|
+
}
|
|
1826
|
+
|
|
1640
1827
|
/**
|
|
1641
1828
|
* Handles `deck-preview`: the deck canvas's live preview of a `table`
|
|
1642
1829
|
* extract's options (`document-view`'s `table-preview.use-case.ts`'s
|
|
@@ -2220,1204 +2407,1271 @@ async function handleGridView(state, command) {
|
|
|
2220
2407
|
return workbookDocumentView(body);
|
|
2221
2408
|
}
|
|
2222
2409
|
|
|
2223
|
-
const GENERATED_ID = /^(?:react-|radix-|:r|ember|css-|mui-|headlessui-)/i;
|
|
2224
2410
|
/**
|
|
2225
|
-
* The
|
|
2226
|
-
*
|
|
2227
|
-
*
|
|
2228
|
-
* — `list-inference.algorithm.ts` leans on that to find the nearest pair of
|
|
2229
|
-
* siblings two picks belong to, without ever touching cheerio again.
|
|
2411
|
+
* The outline model: an `InputRecipe`'s `steps` (and only `steps`; the rest
|
|
2412
|
+
* of the recipe passes through untouched, see `recipe-to-outline.mapper.ts`)
|
|
2413
|
+
* read as a tree of cards and brackets instead of raw JSON.
|
|
2230
2414
|
*
|
|
2231
|
-
*
|
|
2232
|
-
*
|
|
2233
|
-
*
|
|
2415
|
+
* A **card** is one step that reads as a sentence (`sentence`, built by
|
|
2416
|
+
* `card-sentence.mapper.ts`). A **bracket** is a container step (`forEach`,
|
|
2417
|
+
* `paginate`, `if`) whose body is itself a list of cards/brackets: the
|
|
2418
|
+
* indentation the plan doc describes is this nesting, not a separate layout
|
|
2419
|
+
* concept. Both kinds carry `step`, the step's own raw JSON exactly as
|
|
2420
|
+
* parsed (every key, in its original order): `outline-to-recipe.mapper.ts`
|
|
2421
|
+
* rebuilds the recipe from it, substituting only the (possibly edited)
|
|
2422
|
+
* `children`/`elseChildren` for whatever `steps`/`else` value `step` itself
|
|
2423
|
+
* holds, so an edit to one field never disturbs another's position or a
|
|
2424
|
+
* step type this outline cannot make sense of.
|
|
2234
2425
|
*/
|
|
2235
|
-
|
|
2236
|
-
|
|
2237
|
-
|
|
2238
|
-
|
|
2239
|
-
chain.unshift(current);
|
|
2240
|
-
const parent = current.parent();
|
|
2241
|
-
current = parent.length > 0 ? parent : undefined;
|
|
2242
|
-
}
|
|
2243
|
-
return chain.map(element => levelOf($, element));
|
|
2244
|
-
}
|
|
2245
|
-
function levelOf($, element) {
|
|
2246
|
-
const node = element.get(0);
|
|
2247
|
-
const tag = node.tagName.toLowerCase();
|
|
2248
|
-
const attrs = {};
|
|
2249
|
-
const entries = Object.entries(node.attribs ?? {});
|
|
2250
|
-
for (const [name, value] of entries) attrs[name] = value;
|
|
2251
|
-
const classes = (attrs.class ?? '').split(/\s+/).filter(token => token.length > 0);
|
|
2252
|
-
const siblings = element.parent().length > 0 ? element.parent().children(tag) : $.root().children(tag);
|
|
2253
|
-
const index = siblings.toArray().indexOf(node) + 1;
|
|
2254
|
-
return {
|
|
2255
|
-
tag,
|
|
2256
|
-
id: attrs.id,
|
|
2257
|
-
classes,
|
|
2258
|
-
attrs,
|
|
2259
|
-
index: Math.max(index, 1),
|
|
2260
|
-
siblingCount: Math.max(siblings.length, 1)
|
|
2261
|
-
};
|
|
2426
|
+
/** The step types the outline renders as a bracket: their body is nested steps, not a single value. */
|
|
2427
|
+
const BRACKET_STEP_TYPES = ['forEach', 'paginate', 'if'];
|
|
2428
|
+
function isBracket(node) {
|
|
2429
|
+
return node.kind === 'bracket';
|
|
2262
2430
|
}
|
|
2263
|
-
function
|
|
2264
|
-
return
|
|
2431
|
+
function isBracketStepType(type) {
|
|
2432
|
+
return BRACKET_STEP_TYPES.includes(type);
|
|
2265
2433
|
}
|
|
2434
|
+
|
|
2435
|
+
/** Step types the plan doc names as always going to a "custom" card: nothing here renders their meaning as a sentence. */
|
|
2436
|
+
const ALWAYS_CUSTOM = new Set(['evaluate', 'hook', 'captcha']);
|
|
2266
2437
|
/**
|
|
2267
|
-
*
|
|
2268
|
-
*
|
|
2269
|
-
*
|
|
2270
|
-
*
|
|
2438
|
+
* Turns one step's raw JSON into its sentence parts (word/pill/code), the
|
|
2439
|
+
* way the plan doc's examples read: *Go to…*, *Read… from…*, *For each… in…
|
|
2440
|
+
* → one record*, and so on. Never throws: a step type this cannot read
|
|
2441
|
+
* becomes a short, honest fallback with `custom: true`, so the outline never
|
|
2442
|
+
* drops a step for looking unfamiliar.
|
|
2443
|
+
*
|
|
2444
|
+
* @param step - A step's raw JSON (its `type` and whatever fields it has).
|
|
2445
|
+
* @returns The sentence parts, and whether they are a real reading or a fallback.
|
|
2271
2446
|
*/
|
|
2272
|
-
function
|
|
2273
|
-
|
|
2447
|
+
function cardSentence(step) {
|
|
2448
|
+
const type = typeof step.type === 'string' ? step.type : undefined;
|
|
2449
|
+
if (type === undefined || ALWAYS_CUSTOM.has(type)) return fallback(type);
|
|
2450
|
+
const build = BUILDERS[type];
|
|
2451
|
+
if (build === undefined) return fallback(type);
|
|
2452
|
+
try {
|
|
2453
|
+
return {
|
|
2454
|
+
parts: build(step),
|
|
2455
|
+
custom: false
|
|
2456
|
+
};
|
|
2457
|
+
} catch {
|
|
2458
|
+
return fallback(type);
|
|
2459
|
+
}
|
|
2274
2460
|
}
|
|
2275
|
-
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
* only shadow-DOM shape that can show up in static, server-captured markup
|
|
2281
|
-
* at all (imperative `attachShadow()` content never serialises into
|
|
2282
|
-
* `page.content()`/a fetched response in the first place, so that case
|
|
2283
|
-
* cannot reach this slice and needs no separate detection). A path whose
|
|
2284
|
-
* first level is not `html` was cut short at exactly this kind of boundary:
|
|
2285
|
-
* v1 does not pierce it, so callers report a clear "shadow DOM is not
|
|
2286
|
-
* supported" error instead of building ancestor-based candidates over a
|
|
2287
|
-
* chain that stops partway up.
|
|
2288
|
-
*/
|
|
2289
|
-
function crossesShadowRoot(path) {
|
|
2290
|
-
return path.length === 0 || path[0].tag !== 'html';
|
|
2461
|
+
function fallback(type) {
|
|
2462
|
+
return {
|
|
2463
|
+
parts: [word(type ?? 'custom step')],
|
|
2464
|
+
custom: true
|
|
2465
|
+
};
|
|
2291
2466
|
}
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2467
|
+
function word(text) {
|
|
2468
|
+
return {
|
|
2469
|
+
kind: 'word',
|
|
2470
|
+
text
|
|
2471
|
+
};
|
|
2295
2472
|
}
|
|
2296
|
-
|
|
2297
|
-
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
|
|
2301
|
-
* that one attribute off the element the user clicked — turns into
|
|
2302
|
-
* something `selector-inference`'s other algorithms can work with, without
|
|
2303
|
-
* the browser-side picker script needing to know anything about ancestry,
|
|
2304
|
-
* classes or candidate ranking itself.
|
|
2305
|
-
*
|
|
2306
|
-
* @param html - The snapshot document (already stamped with node ids).
|
|
2307
|
-
* @param nodeId - The `data-oc-node` value to find.
|
|
2308
|
-
* @returns The node's path, or `undefined` when no element carries that id.
|
|
2309
|
-
*/
|
|
2310
|
-
function pathToNode(html, nodeId) {
|
|
2311
|
-
const $ = parseDocument$1(html);
|
|
2312
|
-
const found = $(`[${NODE_ID_ATTRIBUTE}="${cssAttributeValue(nodeId)}"]`);
|
|
2313
|
-
return found.length === 0 ? undefined : domPathOf($, found.first());
|
|
2473
|
+
function pill(text) {
|
|
2474
|
+
return {
|
|
2475
|
+
kind: 'pill',
|
|
2476
|
+
text
|
|
2477
|
+
};
|
|
2314
2478
|
}
|
|
2315
|
-
function
|
|
2316
|
-
return
|
|
2479
|
+
function code(text) {
|
|
2480
|
+
return {
|
|
2481
|
+
kind: 'code',
|
|
2482
|
+
text
|
|
2483
|
+
};
|
|
2317
2484
|
}
|
|
2318
|
-
|
|
2319
|
-
|
|
2320
|
-
const
|
|
2321
|
-
|
|
2322
|
-
|
|
2485
|
+
/** A field that names an id: shown as a pill, `fallbackText` when it is absent. */
|
|
2486
|
+
function pillField(step, key, fallbackText) {
|
|
2487
|
+
const value = step[key];
|
|
2488
|
+
return pill(typeof value === 'string' && value.length > 0 ? value : fallbackText);
|
|
2489
|
+
}
|
|
2490
|
+
/** A field shown as a literal (a selector, a template, a value): stringified when it is not already a string. */
|
|
2491
|
+
function codeField(step, key, fallbackText = '') {
|
|
2492
|
+
const value = step[key];
|
|
2493
|
+
if (typeof value === 'string') return code(value);
|
|
2494
|
+
if (value === undefined) return code(fallbackText);
|
|
2495
|
+
return code(safeJson(value));
|
|
2496
|
+
}
|
|
2497
|
+
function safeJson(value) {
|
|
2498
|
+
try {
|
|
2499
|
+
return JSON.stringify(value) ?? String(value);
|
|
2500
|
+
} catch {
|
|
2501
|
+
return String(value);
|
|
2502
|
+
}
|
|
2503
|
+
}
|
|
2504
|
+
/** `selector` or `target`, whichever the step gives (`step.contract.ts`'s `TargetFields`: exactly one is present). */
|
|
2505
|
+
function targetField(step) {
|
|
2506
|
+
if (typeof step.target === 'string') return code(step.target);
|
|
2507
|
+
return codeField(step, 'selector');
|
|
2508
|
+
}
|
|
2509
|
+
const BUILDERS = {
|
|
2510
|
+
goto: step => [word('Go to'), codeField(step, 'url')],
|
|
2511
|
+
click: step => [word('Click'), targetField(step)],
|
|
2512
|
+
fill: step => [word('Fill'), targetField(step), word('with'), codeField(step, 'value')],
|
|
2513
|
+
press: step => [word('Press'), codeField(step, 'key'), word('in'), targetField(step)],
|
|
2514
|
+
select: step => [word('Choose'), selectChoice(step), word('in'), targetField(step)],
|
|
2515
|
+
scroll: step => [word('Scroll to'), codeField(step, 'to')],
|
|
2516
|
+
wait: step => [word('Wait for'), waitFor(step)],
|
|
2517
|
+
screenshot: step => [word('Screenshot'), codeField(step, 'path')],
|
|
2518
|
+
request: step => [word('Request'), code(`${typeof step.method === 'string' ? step.method : 'GET'} `), codeField(step, 'url')],
|
|
2519
|
+
extract: step => [word('Read'), pillField(step, 'id', '(unnamed)'), word('from'), pillField(step, 'from', 'page'), word('←'), codeField(step, 'selector')],
|
|
2520
|
+
set: step => [word('Set'), pillField(step, 'id', '(unnamed)'), word('to'), codeField(step, 'value')],
|
|
2521
|
+
collect: step => [word('Collect'), codeField(step, 'value'), word('into'), pillField(step, 'into', '(unnamed)')],
|
|
2522
|
+
forEach: step => forEachSentence(step),
|
|
2523
|
+
paginate: step => paginateSentence(step),
|
|
2524
|
+
if: step => [word('If'), codeField(step, 'test')],
|
|
2525
|
+
emit: step => emitSentence(step)
|
|
2323
2526
|
};
|
|
2324
|
-
|
|
2325
|
-
|
|
2326
|
-
|
|
2327
|
-
|
|
2328
|
-
|
|
2329
|
-
|
|
2330
|
-
|
|
2331
|
-
|
|
2332
|
-
|
|
2333
|
-
|
|
2334
|
-
if (
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2527
|
+
function selectChoice(step) {
|
|
2528
|
+
if (typeof step.value === 'string') return code(step.value);
|
|
2529
|
+
if (typeof step.label === 'string') return code(step.label);
|
|
2530
|
+
if (typeof step.index === 'number') return code(String(step.index));
|
|
2531
|
+
if (Array.isArray(step.values)) return code(step.values.join(', '));
|
|
2532
|
+
return code('');
|
|
2533
|
+
}
|
|
2534
|
+
function waitFor(step) {
|
|
2535
|
+
if (typeof step.selector === 'string') return code(step.selector);
|
|
2536
|
+
if (typeof step.ms === 'number') return code(`${step.ms}ms`);
|
|
2537
|
+
if (typeof step.state === 'string') return code(step.state);
|
|
2538
|
+
return code('');
|
|
2539
|
+
}
|
|
2540
|
+
function forEachSentence(step) {
|
|
2541
|
+
const parts = [word('For each'), pillField(step, 'as', '(item)'), word('in'), typeof step.over === 'string' ? pill(step.over) : targetField(step)];
|
|
2542
|
+
if (step.emit !== undefined) parts.push(word('→'), word('one record'));
|
|
2543
|
+
return parts;
|
|
2544
|
+
}
|
|
2545
|
+
function paginateSentence(step) {
|
|
2546
|
+
const parts = [word('For every page')];
|
|
2547
|
+
const next = step.next;
|
|
2548
|
+
if (next !== null && typeof next === 'object') {
|
|
2549
|
+
const record = next;
|
|
2550
|
+
if (typeof record.selector === 'string') parts.push(word('via'), code(record.selector));else if (typeof record.url === 'string') parts.push(word('via'), code(record.url));else if (typeof record.jsonpath === 'string') parts.push(word('via'), code(record.jsonpath));
|
|
2339
2551
|
}
|
|
2340
|
-
|
|
2341
|
-
|
|
2342
|
-
|
|
2552
|
+
if (typeof step.until === 'string') parts.push(word('until'), code(step.until));
|
|
2553
|
+
return parts;
|
|
2554
|
+
}
|
|
2555
|
+
function emitSentence(step) {
|
|
2556
|
+
if (typeof step.output === 'string') return [word('Emit'), word('to'), pill(step.output)];
|
|
2557
|
+
return [word('Emit')];
|
|
2343
2558
|
}
|
|
2344
2559
|
|
|
2345
2560
|
/**
|
|
2346
|
-
*
|
|
2347
|
-
* (
|
|
2348
|
-
*
|
|
2349
|
-
*
|
|
2350
|
-
*
|
|
2351
|
-
*
|
|
2352
|
-
* reasoned judgement call, not a guarantee — see the doc comments on each
|
|
2353
|
-
* pattern for what real-world tool it targets.
|
|
2354
|
-
*/
|
|
2355
|
-
const GENERATED_TOKEN_PATTERNS = [/** styled-components / emotion: `sc-bdVaJa`, `css-1x2y3z4` */
|
|
2356
|
-
/^(?:sc|css)-[a-z0-9]+$/i, /** CSS Modules, most bundlers' default hash suffix: `wrapper__3fA9k`, `Button-module_root__aB3dK` */
|
|
2357
|
-
/__[a-z0-9]{4,10}$/i, /** JSS / MUI: `jss123`, `MuiButton-root-145` */
|
|
2358
|
-
/^(?:jss|mui[a-z]*-)[a-z0-9-]*\d{2}$/i, /** A token that is only digits: not a class a human would type. */
|
|
2359
|
-
/^\d+$/, /** A bare hex hash: `a3f9c21`, `1a2b3c4d` (6+ hex chars, nothing else). */
|
|
2360
|
-
/^[0-9a-f]{6,}$/i, /** Tailwind/atomic-CSS-in-JS runtime classes some bundlers emit: `_1a2b3c` */
|
|
2361
|
-
/^_[0-9a-z]{5,}$/i];
|
|
2362
|
-
/**
|
|
2363
|
-
* Whether a class token looks hand-authored and safe to build a selector on.
|
|
2561
|
+
* Reads a recipe's `steps` as an outline. Works on whatever JSON parsed
|
|
2562
|
+
* (not a validated `InputRecipe`): a recipe mid-edit in the JSON tab may be
|
|
2563
|
+
* missing fields or have the wrong shape, and the outline still has to
|
|
2564
|
+
* render something rather than crash the tab. Anything that is not a
|
|
2565
|
+
* well-formed step becomes a custom card holding what was there; anything
|
|
2566
|
+
* that is not an object at the top becomes an outline with no steps.
|
|
2364
2567
|
*
|
|
2365
|
-
* @param
|
|
2366
|
-
* @returns
|
|
2568
|
+
* @param content - The recipe's parsed JSON (the JSON tab's own `JSON.parse` of its text).
|
|
2569
|
+
* @returns The outline: the untouched top-level record plus the step tree.
|
|
2367
2570
|
*/
|
|
2368
|
-
function
|
|
2369
|
-
|
|
2571
|
+
function recipeToOutline(content) {
|
|
2572
|
+
const recipe = isRecord(content) ? content : {};
|
|
2573
|
+
const steps = Array.isArray(recipe.steps) ? recipe.steps : [];
|
|
2574
|
+
return {
|
|
2575
|
+
recipe,
|
|
2576
|
+
steps: stepsToNodes(steps, 'steps')
|
|
2577
|
+
};
|
|
2370
2578
|
}
|
|
2371
|
-
|
|
2372
|
-
|
|
2373
|
-
*
|
|
2374
|
-
* @param tokens - An element's class list.
|
|
2375
|
-
* @returns Only the stable tokens.
|
|
2376
|
-
*/
|
|
2377
|
-
function stableClasses(tokens) {
|
|
2378
|
-
return tokens.filter(token => isStableClassToken(token));
|
|
2579
|
+
function stepsToNodes(steps, pathPrefix) {
|
|
2580
|
+
return steps.map((step, index) => stepToNode(step, `${pathPrefix}.${index}`));
|
|
2379
2581
|
}
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
|
|
2383
|
-
|
|
2384
|
-
|
|
2385
|
-
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2582
|
+
function stepToNode(step, path) {
|
|
2583
|
+
if (!isRecord(step)) {
|
|
2584
|
+
return {
|
|
2585
|
+
kind: 'card',
|
|
2586
|
+
path,
|
|
2587
|
+
stepType: 'unknown',
|
|
2588
|
+
sentence: [{
|
|
2589
|
+
kind: 'word',
|
|
2590
|
+
text: 'custom step'
|
|
2591
|
+
}],
|
|
2592
|
+
step: {},
|
|
2593
|
+
custom: true
|
|
2594
|
+
};
|
|
2595
|
+
}
|
|
2596
|
+
const stepType = typeof step.type === 'string' ? step.type : 'unknown';
|
|
2597
|
+
const {
|
|
2598
|
+
parts,
|
|
2599
|
+
custom
|
|
2600
|
+
} = cardSentence(step);
|
|
2601
|
+
if (!isBracketStepType(stepType)) return {
|
|
2602
|
+
kind: 'card',
|
|
2603
|
+
path,
|
|
2604
|
+
stepType,
|
|
2605
|
+
sentence: parts,
|
|
2606
|
+
step,
|
|
2607
|
+
custom
|
|
2608
|
+
};
|
|
2609
|
+
const children = Array.isArray(step.steps) ? stepsToNodes(step.steps, `${path}.steps`) : [];
|
|
2610
|
+
const elseChildren = stepType === 'if' && Array.isArray(step.else) ? stepsToNodes(step.else, `${path}.else`) : undefined;
|
|
2611
|
+
return {
|
|
2612
|
+
kind: 'bracket',
|
|
2613
|
+
path,
|
|
2614
|
+
stepType,
|
|
2615
|
+
sentence: parts,
|
|
2616
|
+
step,
|
|
2617
|
+
children,
|
|
2618
|
+
...(elseChildren !== undefined && {
|
|
2619
|
+
elseChildren
|
|
2620
|
+
})
|
|
2621
|
+
};
|
|
2622
|
+
}
|
|
2623
|
+
function isRecord(value) {
|
|
2624
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
2394
2625
|
}
|
|
2395
2626
|
|
|
2396
2627
|
/**
|
|
2397
|
-
*
|
|
2398
|
-
*
|
|
2399
|
-
*
|
|
2400
|
-
*
|
|
2628
|
+
* Rebuilds a recipe from its outline: `recipe`'s keys, in their original
|
|
2629
|
+
* order, with `steps` (when the recipe had one) replaced by the outline's
|
|
2630
|
+
* (possibly edited) step tree. A card's own raw `step` is used as-is; a
|
|
2631
|
+
* bracket's `step` supplies every field but `steps`/`else`, which come from
|
|
2632
|
+
* `children`/`elseChildren` instead, put back at the position `step` itself
|
|
2633
|
+
* held that key at (or appended, for a bracket a `+` menu just added, whose
|
|
2634
|
+
* `step` never had one) — this, together with `recipe-to-outline.mapper.ts`
|
|
2635
|
+
* always keeping the step's whole original object, is what makes
|
|
2636
|
+
* `recipe → outline → recipe` preserve every key and its order.
|
|
2637
|
+
*
|
|
2638
|
+
* @param outline - An outline, from `recipe-to-outline.mapper.ts` or edited by the UI.
|
|
2639
|
+
* @returns The recipe's plain JSON, ready for `save-recipe`.
|
|
2401
2640
|
*/
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
|
|
2406
|
-
|
|
2407
|
-
|
|
2641
|
+
function outlineToRecipe(outline) {
|
|
2642
|
+
if (!('steps' in outline.recipe) && outline.steps.length === 0) return {
|
|
2643
|
+
...outline.recipe
|
|
2644
|
+
};
|
|
2645
|
+
return {
|
|
2646
|
+
...outline.recipe,
|
|
2647
|
+
steps: nodesToSteps(outline.steps)
|
|
2648
|
+
};
|
|
2649
|
+
}
|
|
2650
|
+
function nodesToSteps(nodes) {
|
|
2651
|
+
return nodes.map(node => nodeToStep(node));
|
|
2652
|
+
}
|
|
2653
|
+
function nodeToStep(node) {
|
|
2654
|
+
if (!isBracket(node)) return node.step;
|
|
2655
|
+
const keys = Object.keys(node.step);
|
|
2656
|
+
const step = {};
|
|
2657
|
+
for (const key of keys) {
|
|
2658
|
+
if (key === 'steps') step.steps = nodesToSteps(node.children);else if (key === 'else' && node.stepType === 'if') step.else = nodesToSteps(node.elseChildren ?? []);else step[key] = node.step[key];
|
|
2408
2659
|
}
|
|
2660
|
+
if (!keys.includes('steps')) step.steps = nodesToSteps(node.children);
|
|
2661
|
+
if (node.stepType === 'if' && node.elseChildren !== undefined && !keys.includes('else')) step.else = nodesToSteps(node.elseChildren);
|
|
2662
|
+
return step;
|
|
2409
2663
|
}
|
|
2410
2664
|
|
|
2411
|
-
/** Attributes ranked just under `id`: named explicitly, not "any `data-*`", because a generated `data-reactid` is no more stable than a generated class. */
|
|
2412
|
-
const STABLE_ATTRS = ['data-testid', 'data-id', 'itemprop'];
|
|
2413
|
-
const TIER_ORDER$1 = {
|
|
2414
|
-
'id': 0,
|
|
2415
|
-
'data-attr': 1,
|
|
2416
|
-
'class': 2,
|
|
2417
|
-
'structure': 3,
|
|
2418
|
-
'position': 4
|
|
2419
|
-
};
|
|
2420
2665
|
/**
|
|
2421
|
-
*
|
|
2422
|
-
*
|
|
2423
|
-
*
|
|
2424
|
-
*
|
|
2425
|
-
*
|
|
2426
|
-
*
|
|
2427
|
-
* relative to its item instead of absolute to the document.
|
|
2666
|
+
* Opens a recipe folder: every `.json`/`.jsonl` recipe directly inside it
|
|
2667
|
+
* (core's own directory reading, so relative `file:` URLs resolve the same
|
|
2668
|
+
* way a real run would), parsed and, for each output recipe found, bound to
|
|
2669
|
+
* the input recipes that name it. Nothing throws for a bad recipe or a
|
|
2670
|
+
* recipe that does not bind: every problem is reported on its file with the
|
|
2671
|
+
* JSON path core already computes, exactly as `opencraw validate` prints it.
|
|
2428
2672
|
*
|
|
2429
|
-
* @param
|
|
2430
|
-
* @
|
|
2431
|
-
* @
|
|
2673
|
+
* @param folder - The workspace folder.
|
|
2674
|
+
* @returns Every recipe file found, each with its validation and binding issues.
|
|
2675
|
+
* @throws Error when the folder does not exist or cannot be read.
|
|
2432
2676
|
*/
|
|
2433
|
-
function
|
|
2434
|
-
const
|
|
2435
|
-
|
|
2436
|
-
const
|
|
2437
|
-
|
|
2438
|
-
|
|
2439
|
-
|
|
2440
|
-
|
|
2441
|
-
|
|
2442
|
-
|
|
2443
|
-
|
|
2444
|
-
|
|
2445
|
-
|
|
2446
|
-
}
|
|
2677
|
+
async function openWorkspace(folder) {
|
|
2678
|
+
const documents = await readRecipeSource(folder);
|
|
2679
|
+
const entries = documents.map(document => parseDocument$1(document));
|
|
2680
|
+
const outputs = entries.filter(isParsed('output'));
|
|
2681
|
+
const inputs = entries.filter(isParsed('input'));
|
|
2682
|
+
for (const output of outputs) bindOutput(output, inputs);
|
|
2683
|
+
for (const input of inputs) {
|
|
2684
|
+
if (outputs.every(output => output.recipe.id !== input.recipe.output)) {
|
|
2685
|
+
input.issues.push({
|
|
2686
|
+
path: 'output',
|
|
2687
|
+
message: `no output recipe "${input.recipe.output}" in this workspace`,
|
|
2688
|
+
kind: 'binding'
|
|
2689
|
+
});
|
|
2690
|
+
}
|
|
2447
2691
|
}
|
|
2448
|
-
|
|
2449
|
-
|
|
2450
|
-
|
|
2451
|
-
|
|
2452
|
-
|
|
2453
|
-
|
|
2454
|
-
|
|
2455
|
-
|
|
2692
|
+
return {
|
|
2693
|
+
folder,
|
|
2694
|
+
recipes: entries.map(({
|
|
2695
|
+
file,
|
|
2696
|
+
kind,
|
|
2697
|
+
id,
|
|
2698
|
+
issues,
|
|
2699
|
+
content
|
|
2700
|
+
}) => ({
|
|
2701
|
+
file,
|
|
2702
|
+
kind,
|
|
2703
|
+
id,
|
|
2704
|
+
issues,
|
|
2705
|
+
text: `${JSON.stringify(content, null, 2)}\n`,
|
|
2706
|
+
// The Steps tab needs steps to render: an output/unknown file has none. recipeToOutline never
|
|
2707
|
+
// throws, so this covers a file that fails validation too (its outline just has "custom" cards).
|
|
2708
|
+
outline: kind === 'input' ? recipeToOutline(content) : undefined
|
|
2709
|
+
}))
|
|
2710
|
+
};
|
|
2456
2711
|
}
|
|
2457
|
-
/**
|
|
2458
|
-
function
|
|
2459
|
-
const
|
|
2460
|
-
|
|
2461
|
-
|
|
2462
|
-
|
|
2463
|
-
|
|
2464
|
-
const
|
|
2465
|
-
|
|
2466
|
-
|
|
2467
|
-
|
|
2468
|
-
|
|
2469
|
-
|
|
2470
|
-
|
|
2471
|
-
|
|
2472
|
-
tier: 'structure'
|
|
2473
|
-
});
|
|
2712
|
+
/** Binds one output recipe to the inputs that name it, filing every binding issue back on the recipe it names. */
|
|
2713
|
+
function bindOutput(output, inputs) {
|
|
2714
|
+
const matching = inputs.filter(input => input.recipe.output === output.recipe.id);
|
|
2715
|
+
try {
|
|
2716
|
+
bindRecipeSet(output.recipe, matching.map(input => input.recipe));
|
|
2717
|
+
} catch (error) {
|
|
2718
|
+
if (!(error instanceof RecipeBindingError)) throw error;
|
|
2719
|
+
for (const issue of error.issues) {
|
|
2720
|
+
const target = issue.recipeId === output.recipe.id ? output : matching.find(input => input.recipe.id === issue.recipeId);
|
|
2721
|
+
target?.issues.push({
|
|
2722
|
+
path: issue.path,
|
|
2723
|
+
message: issue.message,
|
|
2724
|
+
kind: 'binding'
|
|
2725
|
+
});
|
|
2726
|
+
}
|
|
2474
2727
|
}
|
|
2475
|
-
return candidates;
|
|
2476
|
-
}
|
|
2477
|
-
/** `parent > tag:nth-of-type(n)`, the position-only fallback; never `nth-child`, which counts every sibling tag, not just this element's own. */
|
|
2478
|
-
function positionCandidates(path) {
|
|
2479
|
-
const node = path.at(-1);
|
|
2480
|
-
if (node === undefined) return [];
|
|
2481
|
-
const own = node.siblingCount > 1 ? `${node.tag}:nth-of-type(${node.index})` : node.tag;
|
|
2482
|
-
const parent = path.at(-2);
|
|
2483
|
-
if (parent === undefined) return [{
|
|
2484
|
-
selector: own,
|
|
2485
|
-
tier: 'position'
|
|
2486
|
-
}];
|
|
2487
|
-
return [{
|
|
2488
|
-
selector: `${simpleSelector(parent)} > ${own}`,
|
|
2489
|
-
tier: 'position'
|
|
2490
|
-
}];
|
|
2491
|
-
}
|
|
2492
|
-
/** A level's own selector fragment, no combinators: id, else stable classes, else the bare tag. */
|
|
2493
|
-
function simpleSelector(level) {
|
|
2494
|
-
if (level.id !== undefined && isStableId(level.id)) return `#${cssEscape(level.id)}`;
|
|
2495
|
-
const classes = stableClasses(level.classes);
|
|
2496
|
-
return classes.length > 0 ? `${level.tag}.${classes.map(token => cssEscape(token)).join('.')}` : level.tag;
|
|
2497
2728
|
}
|
|
2498
|
-
function
|
|
2499
|
-
return
|
|
2729
|
+
function isParsed(kind) {
|
|
2730
|
+
return entry => entry.kind === kind && entry.recipe !== undefined;
|
|
2500
2731
|
}
|
|
2501
|
-
|
|
2502
|
-
|
|
2503
|
-
return
|
|
2732
|
+
function parseDocument$1(document) {
|
|
2733
|
+
const kind = document.content?.kind;
|
|
2734
|
+
if (kind === 'output') return parseOne(document.source, document.content, 'output', parseOutputRecipe);
|
|
2735
|
+
if (kind === 'input') return parseOne(document.source, document.content, 'input', parseInputRecipe);
|
|
2736
|
+
return {
|
|
2737
|
+
file: document.source,
|
|
2738
|
+
kind: 'unknown',
|
|
2739
|
+
content: document.content,
|
|
2740
|
+
issues: [{
|
|
2741
|
+
path: 'kind',
|
|
2742
|
+
message: '"kind" is missing or not "input"/"output"',
|
|
2743
|
+
kind: 'validation'
|
|
2744
|
+
}]
|
|
2745
|
+
};
|
|
2504
2746
|
}
|
|
2505
|
-
|
|
2506
|
-
|
|
2507
|
-
|
|
2747
|
+
function parseOne(file, content, kind, parse) {
|
|
2748
|
+
try {
|
|
2749
|
+
const recipe = parse(content, file);
|
|
2750
|
+
return {
|
|
2751
|
+
file,
|
|
2752
|
+
kind,
|
|
2753
|
+
content,
|
|
2754
|
+
id: recipe.id,
|
|
2755
|
+
issues: [],
|
|
2756
|
+
recipe
|
|
2757
|
+
};
|
|
2758
|
+
} catch (error) {
|
|
2759
|
+
if (!(error instanceof RecipeValidationError)) throw error;
|
|
2760
|
+
return {
|
|
2761
|
+
file,
|
|
2762
|
+
kind,
|
|
2763
|
+
content,
|
|
2764
|
+
issues: error.issues.map(issue => ({
|
|
2765
|
+
path: issue.path === '' ? '(root)' : issue.path,
|
|
2766
|
+
message: issue.message,
|
|
2767
|
+
kind: 'validation'
|
|
2768
|
+
}))
|
|
2769
|
+
};
|
|
2770
|
+
}
|
|
2508
2771
|
}
|
|
2509
2772
|
|
|
2510
|
-
const TIER_ORDER = {
|
|
2511
|
-
'id': 0,
|
|
2512
|
-
'data-attr': 1,
|
|
2513
|
-
'class': 2,
|
|
2514
|
-
'structure': 3,
|
|
2515
|
-
'position': 4
|
|
2516
|
-
};
|
|
2517
2773
|
/**
|
|
2518
|
-
*
|
|
2519
|
-
*
|
|
2520
|
-
*
|
|
2521
|
-
*
|
|
2774
|
+
* Writes a recipe back to its file: pretty JSON, a 2-space indent, a
|
|
2775
|
+
* trailing newline. Phase 0 takes the JSON tab's whole parsed object rather
|
|
2776
|
+
* than a patch, so this is a full replacement, not a splice: loading a file
|
|
2777
|
+
* the studio has not touched and saving it again keeps every key, value and
|
|
2778
|
+
* key order (`JSON.parse` preserves insertion order, and `JSON.stringify`
|
|
2779
|
+
* writes it back), but not necessarily the file's exact original formatting
|
|
2780
|
+
* (alignment, how arrays wrap, a trailing comment). Byte-identical
|
|
2781
|
+
* round-trip for a hand-formatted file is future work; see this file's test
|
|
2782
|
+
* for what is guaranteed today.
|
|
2522
2783
|
*
|
|
2523
|
-
*
|
|
2524
|
-
*
|
|
2525
|
-
|
|
2526
|
-
|
|
2784
|
+
* @param path - The recipe file to write.
|
|
2785
|
+
* @param recipe - The whole recipe object.
|
|
2786
|
+
*/
|
|
2787
|
+
async function saveRecipe(path, recipe) {
|
|
2788
|
+
await writeFile(path, `${JSON.stringify(recipe, null, 2)}\n`, 'utf8');
|
|
2789
|
+
}
|
|
2790
|
+
|
|
2791
|
+
/** How many `name-2.ext`, `name-3.ext`… are tried before giving up on a folder full of same-named copies. */
|
|
2792
|
+
const MAX_NUMBERED_COPIES = 100;
|
|
2793
|
+
/**
|
|
2794
|
+
* Copies a document into a workspace folder, so a recipe can read it with a
|
|
2795
|
+
* `file:` URL the same way a run would read any downloaded file (core's
|
|
2796
|
+
* `HttpClient` handles `file:` itself). The folder is created when missing,
|
|
2797
|
+
* so dropping a file is also a way to start a workspace. A name already in
|
|
2798
|
+
* the folder is never overwritten: the copy gets the next free numbered
|
|
2799
|
+
* name (`report-2.pdf`), and the returned `path`/`url` say which.
|
|
2527
2800
|
*
|
|
2528
|
-
* @param
|
|
2529
|
-
* @param
|
|
2530
|
-
* @param
|
|
2531
|
-
* @returns
|
|
2801
|
+
* @param folder - The workspace folder; created (recursively) when it does not exist.
|
|
2802
|
+
* @param name - The file's own name, no directories: the copy keeps it, extension included.
|
|
2803
|
+
* @param bytes - The file's content.
|
|
2804
|
+
* @returns Where the copy landed.
|
|
2805
|
+
* @throws Error when `name` is not a plain file name, or the folder already holds too many same-named copies.
|
|
2532
2806
|
*/
|
|
2533
|
-
function
|
|
2534
|
-
|
|
2535
|
-
|
|
2536
|
-
|
|
2537
|
-
return {
|
|
2538
|
-
...candidate,
|
|
2539
|
-
matches,
|
|
2540
|
-
hitsTarget
|
|
2541
|
-
};
|
|
2807
|
+
async function importDocument(folder, name, bytes) {
|
|
2808
|
+
assertPlainName(name);
|
|
2809
|
+
await mkdir(folder, {
|
|
2810
|
+
recursive: true
|
|
2542
2811
|
});
|
|
2543
|
-
|
|
2812
|
+
for (const candidate of candidateNames(name)) {
|
|
2813
|
+
const path = join(folder, candidate);
|
|
2814
|
+
try {
|
|
2815
|
+
await writeFile(path, bytes, {
|
|
2816
|
+
flag: 'wx'
|
|
2817
|
+
});
|
|
2818
|
+
return {
|
|
2819
|
+
path,
|
|
2820
|
+
url: pathToFileURL(path).href
|
|
2821
|
+
};
|
|
2822
|
+
} catch (error) {
|
|
2823
|
+
if (!isAlreadyThere(error)) throw error;
|
|
2824
|
+
}
|
|
2825
|
+
}
|
|
2826
|
+
throw new Error(`"${name}" and ${MAX_NUMBERED_COPIES} numbered copies of it are already in ${folder}`);
|
|
2544
2827
|
}
|
|
2545
|
-
|
|
2546
|
-
|
|
2547
|
-
|
|
2828
|
+
function assertPlainName(name) {
|
|
2829
|
+
if (name === '.' || name === '..' || name.includes('\0') || name !== basename(name)) {
|
|
2830
|
+
throw new Error(`"${name}" is not a plain file name`);
|
|
2831
|
+
}
|
|
2832
|
+
}
|
|
2833
|
+
function* candidateNames(name) {
|
|
2834
|
+
yield name;
|
|
2835
|
+
const extension = extname(name);
|
|
2836
|
+
const stem = name.slice(0, name.length - extension.length);
|
|
2837
|
+
for (let copy = 2; copy <= MAX_NUMBERED_COPIES + 1; copy += 1) yield `${stem}-${copy}${extension}`;
|
|
2838
|
+
}
|
|
2839
|
+
function isAlreadyThere(error) {
|
|
2840
|
+
return typeof error === 'object' && error !== null && 'code' in error && error.code === 'EEXIST';
|
|
2548
2841
|
}
|
|
2549
2842
|
|
|
2550
2843
|
/**
|
|
2551
|
-
*
|
|
2552
|
-
*
|
|
2553
|
-
*
|
|
2554
|
-
*
|
|
2555
|
-
*
|
|
2556
|
-
*
|
|
2557
|
-
* Because a `DomPath` is built root-down (`dom-path.model.ts`), two paths
|
|
2558
|
-
* from the same document are byte-for-byte identical up to their real
|
|
2559
|
-
* common ancestor and diverge exactly at its differing children — so the
|
|
2560
|
-
* *first* level where they differ is always the nearest-sibling test point;
|
|
2561
|
-
* there is no need to look further down. This also makes the "wrapper trap"
|
|
2562
|
-
* (an extra single-child element around each item) resolve itself: a
|
|
2563
|
-
* wrapper's own instances are never one another's siblings, only the
|
|
2564
|
-
* genuine repeating element is, so the walk naturally lands there instead of
|
|
2565
|
-
* on the wrapper (see the wrapper-trap fixture in this slice's tests).
|
|
2844
|
+
* Handles `import-document` (issue #120): decodes the base64 bytes and copies
|
|
2845
|
+
* the file into the folder (see `recipe-workspace/import-document.use-case.ts`
|
|
2846
|
+
* for the folder creation and the no-overwrite numbering). Nothing is
|
|
2847
|
+
* broadcast: the copy is not a recipe, the workspace listing does not change
|
|
2848
|
+
* until the UI saves the recipe pair that points at it, which `save-recipe`
|
|
2849
|
+
* announces as usual.
|
|
2566
2850
|
*
|
|
2567
|
-
* @param
|
|
2568
|
-
* @
|
|
2569
|
-
* @returns The item and field shapes (generalised) plus each pick's raw path, or `null` when the two picks are not on a coherent list (different branches entirely, the same node twice, or one path nested inside the other).
|
|
2570
|
-
* @throws ShadowDomUnsupportedError when either path crosses a declarative shadow root.
|
|
2851
|
+
* @param command - The folder, the file's name and its base64 content.
|
|
2852
|
+
* @returns Where the copy landed and its `file:` URL.
|
|
2571
2853
|
*/
|
|
2572
|
-
function
|
|
2573
|
-
|
|
2574
|
-
if (crossesShadowRoot(pathB)) throw new ShadowDomUnsupportedError(pathToString(pathB));
|
|
2575
|
-
let divergedAt = -1;
|
|
2576
|
-
const limit = Math.min(pathA.length, pathB.length);
|
|
2577
|
-
for (let index = 0; index < limit; index++) {
|
|
2578
|
-
if (!sameLevel(pathA[index], pathB[index])) {
|
|
2579
|
-
divergedAt = index;
|
|
2580
|
-
break;
|
|
2581
|
-
}
|
|
2582
|
-
}
|
|
2583
|
-
// Identical paths, or one a strict prefix of the other (the same node picked twice, or one element nested inside the other): not a list.
|
|
2584
|
-
if (divergedAt <= 0) return null;
|
|
2585
|
-
const branchA = pathA[divergedAt];
|
|
2586
|
-
const branchB = pathB[divergedAt];
|
|
2587
|
-
if (branchA.tag !== branchB.tag || !sameClassSignature(branchA.classes, branchB.classes)) return null;
|
|
2588
|
-
if (branchA.index === branchB.index) return null; // Same position: not two different items.
|
|
2589
|
-
const ancestors = pathA.slice(0, divergedAt);
|
|
2590
|
-
const fieldPathA = pathA.slice(divergedAt);
|
|
2591
|
-
const fieldPathB = pathB.slice(divergedAt);
|
|
2592
|
-
const fieldPath = mergeRelativePaths(fieldPathA, fieldPathB);
|
|
2593
|
-
return {
|
|
2594
|
-
itemPath: [...ancestors, fieldPath[0]],
|
|
2595
|
-
fieldPath,
|
|
2596
|
-
fieldPathA,
|
|
2597
|
-
fieldPathB
|
|
2598
|
-
};
|
|
2599
|
-
}
|
|
2600
|
-
function sameLevel(a, b) {
|
|
2601
|
-
return a.tag === b.tag && a.id === b.id && a.index === b.index && a.siblingCount === b.siblingCount && sameClassSignature(a.classes, b.classes);
|
|
2854
|
+
async function handleImportDocument(command) {
|
|
2855
|
+
return importDocument(command.folder, command.name, new Uint8Array(Buffer.from(command.bytes, 'base64')));
|
|
2602
2856
|
}
|
|
2857
|
+
|
|
2858
|
+
const GENERATED_ID = /^(?:react-|radix-|:r|ember|css-|mui-|headlessui-)/i;
|
|
2603
2859
|
/**
|
|
2604
|
-
*
|
|
2605
|
-
*
|
|
2606
|
-
*
|
|
2607
|
-
*
|
|
2608
|
-
*
|
|
2609
|
-
*
|
|
2610
|
-
*
|
|
2611
|
-
*
|
|
2612
|
-
*
|
|
2860
|
+
* The path from the document root to `node`, one level per ancestor. Two
|
|
2861
|
+
* elements that share an ancestor produce identical levels up to that
|
|
2862
|
+
* ancestor and diverge exactly where their parents stop being the same node
|
|
2863
|
+
* — `list-inference.algorithm.ts` leans on that to find the nearest pair of
|
|
2864
|
+
* siblings two picks belong to, without ever touching cheerio again.
|
|
2865
|
+
*
|
|
2866
|
+
* @param $ - The parsed document.
|
|
2867
|
+
* @param node - The element to describe.
|
|
2868
|
+
* @returns The path, root first.
|
|
2613
2869
|
*/
|
|
2614
|
-
function
|
|
2615
|
-
const
|
|
2616
|
-
|
|
2617
|
-
|
|
2618
|
-
|
|
2619
|
-
const
|
|
2620
|
-
|
|
2621
|
-
merged.push(mergeLevel(left, right));
|
|
2870
|
+
function domPathOf($, node) {
|
|
2871
|
+
const chain = [];
|
|
2872
|
+
let current = node;
|
|
2873
|
+
while (current !== undefined && current.length > 0 && isElement(current.get(0))) {
|
|
2874
|
+
chain.unshift(current);
|
|
2875
|
+
const parent = current.parent();
|
|
2876
|
+
current = parent.length > 0 ? parent : undefined;
|
|
2622
2877
|
}
|
|
2623
|
-
return
|
|
2878
|
+
return chain.map(element => levelOf($, element));
|
|
2624
2879
|
}
|
|
2625
|
-
function
|
|
2626
|
-
const
|
|
2880
|
+
function levelOf($, element) {
|
|
2881
|
+
const node = element.get(0);
|
|
2882
|
+
const tag = node.tagName.toLowerCase();
|
|
2627
2883
|
const attrs = {};
|
|
2628
|
-
|
|
2629
|
-
|
|
2630
|
-
|
|
2884
|
+
const entries = Object.entries(node.attribs ?? {});
|
|
2885
|
+
for (const [name, value] of entries) attrs[name] = value;
|
|
2886
|
+
const classes = (attrs.class ?? '').split(/\s+/).filter(token => token.length > 0);
|
|
2887
|
+
const siblings = element.parent().length > 0 ? element.parent().children(tag) : $.root().children(tag);
|
|
2888
|
+
const index = siblings.toArray().indexOf(node) + 1;
|
|
2631
2889
|
return {
|
|
2632
|
-
tag
|
|
2633
|
-
id:
|
|
2890
|
+
tag,
|
|
2891
|
+
id: attrs.id,
|
|
2634
2892
|
classes,
|
|
2635
2893
|
attrs,
|
|
2636
|
-
index: 1,
|
|
2637
|
-
siblingCount: Math.max(
|
|
2894
|
+
index: Math.max(index, 1),
|
|
2895
|
+
siblingCount: Math.max(siblings.length, 1)
|
|
2638
2896
|
};
|
|
2639
2897
|
}
|
|
2640
|
-
function
|
|
2641
|
-
return
|
|
2898
|
+
function isElement(node) {
|
|
2899
|
+
return node !== undefined && node.type === 'tag';
|
|
2642
2900
|
}
|
|
2643
|
-
|
|
2644
2901
|
/**
|
|
2645
|
-
*
|
|
2646
|
-
* `
|
|
2647
|
-
*
|
|
2648
|
-
*
|
|
2649
|
-
* studio plan §3.2). All the actual candidate/ranking/list-inference work
|
|
2650
|
-
* is `selector-inference`'s (pure); this handler only locates the clicked
|
|
2651
|
-
* nodes in the cached document and calls into it.
|
|
2652
|
-
*
|
|
2653
|
-
* @param state - The server state.
|
|
2654
|
-
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
2655
|
-
* @returns A field pick, a list pick, or `"unsupported"` with why.
|
|
2656
|
-
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
2902
|
+
* Whether `id` looks handed by a framework (`react-`, a CSS-module or a
|
|
2903
|
+
* `radix-`/`mui-` autogenerated one) rather than by the site's own markup —
|
|
2904
|
+
* `class-token.policy.ts` has the equivalent check for class tokens, kept
|
|
2905
|
+
* separate because ids and classes are ranked at different tiers.
|
|
2657
2906
|
*/
|
|
2658
|
-
function
|
|
2659
|
-
|
|
2660
|
-
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
2661
|
-
const html = snapshot.html;
|
|
2662
|
-
const paths = command.nodeIds.map(nodeId => pathToNode(html, nodeId));
|
|
2663
|
-
if (paths.includes(undefined)) return {
|
|
2664
|
-
kind: 'unsupported',
|
|
2665
|
-
reason: 'one of the picked nodes is no longer in the snapshot; take a fresh snapshot and try again'
|
|
2666
|
-
};
|
|
2667
|
-
const [pathA, pathB] = paths;
|
|
2668
|
-
if (pathB === undefined) return {
|
|
2669
|
-
kind: 'field',
|
|
2670
|
-
field: fieldPick(html, pathA)
|
|
2671
|
-
};
|
|
2672
|
-
try {
|
|
2673
|
-
const result = inferList(pathA, pathB);
|
|
2674
|
-
if (result === null) return {
|
|
2675
|
-
kind: 'unsupported',
|
|
2676
|
-
reason: 'these two picks do not belong to the same list: pick the same field on two similar items'
|
|
2677
|
-
};
|
|
2678
|
-
const item = bestCandidate(html, candidatesFor(result.itemPath, false));
|
|
2679
|
-
if (item === undefined) return {
|
|
2680
|
-
kind: 'unsupported',
|
|
2681
|
-
reason: 'no selector matches every item of this list; try picking two items closer to each other in the markup'
|
|
2682
|
-
};
|
|
2683
|
-
const field = fieldPick(html, result.fieldPathA, result.fieldPath);
|
|
2684
|
-
return {
|
|
2685
|
-
kind: 'list',
|
|
2686
|
-
item: {
|
|
2687
|
-
selector: item.selector,
|
|
2688
|
-
tier: item.tier,
|
|
2689
|
-
matches: item.matches
|
|
2690
|
-
},
|
|
2691
|
-
field
|
|
2692
|
-
};
|
|
2693
|
-
} catch (error) {
|
|
2694
|
-
if (error instanceof ShadowDomUnsupportedError) return {
|
|
2695
|
-
kind: 'unsupported',
|
|
2696
|
-
reason: error.message
|
|
2697
|
-
};
|
|
2698
|
-
throw error;
|
|
2699
|
-
}
|
|
2907
|
+
function isStableId(id) {
|
|
2908
|
+
return id.length > 0 && !GENERATED_ID.test(id);
|
|
2700
2909
|
}
|
|
2701
|
-
/**
|
|
2702
|
-
|
|
2703
|
-
|
|
2704
|
-
|
|
2705
|
-
|
|
2706
|
-
|
|
2707
|
-
|
|
2708
|
-
|
|
2709
|
-
|
|
2710
|
-
|
|
2711
|
-
|
|
2712
|
-
|
|
2910
|
+
/**
|
|
2911
|
+
* Whether `path` failed to reach the real document root. `domPathOf` walks
|
|
2912
|
+
* up through cheerio's own `.parent()`, which (matching real DOM semantics)
|
|
2913
|
+
* never crosses from a `<template>`'s content into the light DOM outside
|
|
2914
|
+
* it — a `<template shadowrootmode>` is exactly declarative shadow DOM, the
|
|
2915
|
+
* only shadow-DOM shape that can show up in static, server-captured markup
|
|
2916
|
+
* at all (imperative `attachShadow()` content never serialises into
|
|
2917
|
+
* `page.content()`/a fetched response in the first place, so that case
|
|
2918
|
+
* cannot reach this slice and needs no separate detection). A path whose
|
|
2919
|
+
* first level is not `html` was cut short at exactly this kind of boundary:
|
|
2920
|
+
* v1 does not pierce it, so callers report a clear "shadow DOM is not
|
|
2921
|
+
* supported" error instead of building ancestor-based candidates over a
|
|
2922
|
+
* chain that stops partway up.
|
|
2923
|
+
*/
|
|
2924
|
+
function crossesShadowRoot(path) {
|
|
2925
|
+
return path.length === 0 || path[0].tag !== 'html';
|
|
2926
|
+
}
|
|
2927
|
+
/** Parses `html` and returns cheerio's API, for callers that need more than one path from the same document. */
|
|
2928
|
+
function parseDocument(html) {
|
|
2929
|
+
return load(html);
|
|
2713
2930
|
}
|
|
2714
|
-
|
|
2715
|
-
/** A leaf element's own text longer than this is cut, with an ellipsis. */
|
|
2716
|
-
const TEXT_LIMIT = 80;
|
|
2717
|
-
/** Consecutive siblings whose subtree is identical (ids aside) collapse into one row once there are at least this many — a real page can have tens of thousands of `<li>`s, and showing all of them is not "inspecting" it (issue #93). */
|
|
2718
|
-
const MIN_REPEAT_RUN = 3;
|
|
2719
2931
|
/**
|
|
2720
|
-
*
|
|
2721
|
-
*
|
|
2722
|
-
*
|
|
2723
|
-
*
|
|
2724
|
-
*
|
|
2725
|
-
*
|
|
2932
|
+
* The path to the element the snapshot stamped with `nodeId` (its
|
|
2933
|
+
* `data-oc-node` attribute — `@opencraw/core`'s `NODE_ID_ATTRIBUTE`,
|
|
2934
|
+
* `packages/studio/src/page-snapshot/rewrite-document.mapper.ts`). This is
|
|
2935
|
+
* how a click in the content pane's iframe — which only ever needs to read
|
|
2936
|
+
* that one attribute off the element the user clicked — turns into
|
|
2937
|
+
* something `selector-inference`'s other algorithms can work with, without
|
|
2938
|
+
* the browser-side picker script needing to know anything about ancestry,
|
|
2939
|
+
* classes or candidate ranking itself.
|
|
2726
2940
|
*
|
|
2727
|
-
* @param
|
|
2728
|
-
* @
|
|
2729
|
-
* @
|
|
2941
|
+
* @param html - The snapshot document (already stamped with node ids).
|
|
2942
|
+
* @param nodeId - The `data-oc-node` value to find.
|
|
2943
|
+
* @returns The node's path, or `undefined` when no element carries that id.
|
|
2730
2944
|
*/
|
|
2731
|
-
function
|
|
2732
|
-
const $ =
|
|
2733
|
-
const
|
|
2734
|
-
|
|
2735
|
-
return buildNode($, html);
|
|
2945
|
+
function pathToNode(html, nodeId) {
|
|
2946
|
+
const $ = parseDocument(html);
|
|
2947
|
+
const found = $(`[${NODE_ID_ATTRIBUTE}="${cssAttributeValue(nodeId)}"]`);
|
|
2948
|
+
return found.length === 0 ? undefined : domPathOf($, found.first());
|
|
2736
2949
|
}
|
|
2737
|
-
function
|
|
2738
|
-
|
|
2739
|
-
const attributes = {
|
|
2740
|
-
...element.attribs
|
|
2741
|
-
};
|
|
2742
|
-
const nodeId = attributes[NODE_ID_ATTRIBUTE] ?? '';
|
|
2743
|
-
const hidden = attributes[HIDDEN_ATTRIBUTE] === '1';
|
|
2744
|
-
delete attributes[NODE_ID_ATTRIBUTE];
|
|
2745
|
-
delete attributes[HIDDEN_ATTRIBUTE];
|
|
2746
|
-
const id = attributes.id;
|
|
2747
|
-
const classes = (attributes.class ?? '').split(/\s+/).filter(token => token !== '');
|
|
2748
|
-
const childElements = $el.children().toArray();
|
|
2749
|
-
const children = collapseRepeats(childElements.map(child => buildNode($, child)));
|
|
2750
|
-
const text = childElements.length === 0 ? abbreviate($el.text()) : undefined;
|
|
2751
|
-
return {
|
|
2752
|
-
nodeId,
|
|
2753
|
-
tag: element.tagName.toLowerCase(),
|
|
2754
|
-
...(id !== undefined && id !== '' && {
|
|
2755
|
-
id
|
|
2756
|
-
}),
|
|
2757
|
-
...(classes.length > 0 && {
|
|
2758
|
-
classes
|
|
2759
|
-
}),
|
|
2760
|
-
attributes,
|
|
2761
|
-
...(text !== undefined && text !== '' && {
|
|
2762
|
-
text
|
|
2763
|
-
}),
|
|
2764
|
-
hidden,
|
|
2765
|
-
children
|
|
2766
|
-
};
|
|
2950
|
+
function cssAttributeValue(value) {
|
|
2951
|
+
return value.replaceAll('\\', String.raw`\\`).replaceAll('"', String.raw`\"`);
|
|
2767
2952
|
}
|
|
2953
|
+
|
|
2954
|
+
/** Attributes worth reading instead of text, in preference order, when the clicked element has one. */
|
|
2955
|
+
const PREFERRED_ATTRS = {
|
|
2956
|
+
a: ['href'],
|
|
2957
|
+
img: ['src', 'alt']
|
|
2958
|
+
};
|
|
2768
2959
|
/**
|
|
2769
|
-
*
|
|
2770
|
-
*
|
|
2771
|
-
*
|
|
2772
|
-
*
|
|
2960
|
+
* What a `Read` card's `take` should be for the element a pick landed on
|
|
2961
|
+
* (issue #91, studio plan §3.2): `attr:href` for an `<a>`, `attr:src` for an
|
|
2962
|
+
* `<img>`, `attr:<name>` for the first `data-*` attribute otherwise, `json`
|
|
2963
|
+
* for a `<script type="application/json">`, else the default, `text`.
|
|
2773
2964
|
*
|
|
2774
|
-
* @param
|
|
2775
|
-
* @returns The
|
|
2965
|
+
* @param level - The clicked element's own level (the last entry of its `DomPath`).
|
|
2966
|
+
* @returns The `take` value to put on the `extract` step.
|
|
2776
2967
|
*/
|
|
2777
|
-
function
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
|
|
2781
|
-
const
|
|
2782
|
-
|
|
2783
|
-
while (end < nodes.length && signatureOf(nodes[end]) === signature) end += 1;
|
|
2784
|
-
const run = nodes.slice(at, end);
|
|
2785
|
-
if (run.length >= MIN_REPEAT_RUN) {
|
|
2786
|
-
result.push({
|
|
2787
|
-
...run[0],
|
|
2788
|
-
repeatCount: run.length,
|
|
2789
|
-
repeatNodeIds: run.map(node => node.nodeId)
|
|
2790
|
-
});
|
|
2791
|
-
} else {
|
|
2792
|
-
result.push(...run);
|
|
2793
|
-
}
|
|
2794
|
-
at = end;
|
|
2968
|
+
function takeKindFor(level) {
|
|
2969
|
+
if (level.tag === 'script' && (level.attrs.type ?? '').toLowerCase() === 'application/json') return 'json';
|
|
2970
|
+
const preferred = PREFERRED_ATTRS[level.tag];
|
|
2971
|
+
if (preferred !== undefined) {
|
|
2972
|
+
const attribute = preferred.find(name => level.attrs[name] !== undefined);
|
|
2973
|
+
if (attribute !== undefined) return `attr:${attribute}`;
|
|
2795
2974
|
}
|
|
2796
|
-
|
|
2797
|
-
}
|
|
2798
|
-
|
|
2799
|
-
function signatureOf(node) {
|
|
2800
|
-
return JSON.stringify({
|
|
2801
|
-
...node,
|
|
2802
|
-
nodeId: undefined,
|
|
2803
|
-
children: node.children.map(child => signatureOf(child))
|
|
2804
|
-
});
|
|
2805
|
-
}
|
|
2806
|
-
function abbreviate(text) {
|
|
2807
|
-
const trimmed = text.trim().replaceAll(/\s+/g, ' ');
|
|
2808
|
-
return trimmed.length > TEXT_LIMIT ? `${trimmed.slice(0, TEXT_LIMIT)}…` : trimmed;
|
|
2975
|
+
const dataAttribute = Object.keys(level.attrs).find(name => name.startsWith('data-') && name !== 'data-oc-node' && name !== 'data-oc-hidden');
|
|
2976
|
+
if (dataAttribute !== undefined) return `attr:${dataAttribute}`;
|
|
2977
|
+
return 'text';
|
|
2809
2978
|
}
|
|
2810
2979
|
|
|
2811
|
-
/** The global names sites hydrate state into, that `describeHtml`'s own JSON-LD/table scan does not cover (issue #93's "inline state assignments"). */
|
|
2812
|
-
const INLINE_STATE_NAMES = ['__NEXT_DATA__', '__NUXT__', '__INITIAL_STATE__', '__APOLLO_STATE__', '__PRELOADED_STATE__', '__STATE__'];
|
|
2813
|
-
const INLINE_STATE_PATTERN = new RegExp(String.raw`(?:window\.)?(${INLINE_STATE_NAMES.join('|')})\s*=`);
|
|
2814
2980
|
/**
|
|
2815
|
-
*
|
|
2816
|
-
*
|
|
2817
|
-
*
|
|
2818
|
-
*
|
|
2819
|
-
*
|
|
2820
|
-
*
|
|
2821
|
-
*
|
|
2822
|
-
*
|
|
2823
|
-
|
|
2824
|
-
|
|
2825
|
-
|
|
2826
|
-
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2981
|
+
* Which class tokens are worth building a selector on. A hand-authored class
|
|
2982
|
+
* (`product_pod`, `price_color`) survives a rebuild; a generated one
|
|
2983
|
+
* (a content hash, a CSS-module suffix, a styled-components token) does not,
|
|
2984
|
+
* so a candidate built on it looks fine on the snapshot it was picked from
|
|
2985
|
+
* and breaks on the very next deploy. This is the "generated/hashed class
|
|
2986
|
+
* names" tradeoff issue #91 calls out explicitly: the heuristics below are a
|
|
2987
|
+
* reasoned judgement call, not a guarantee — see the doc comments on each
|
|
2988
|
+
* pattern for what real-world tool it targets.
|
|
2989
|
+
*/
|
|
2990
|
+
const GENERATED_TOKEN_PATTERNS = [/** styled-components / emotion: `sc-bdVaJa`, `css-1x2y3z4` */
|
|
2991
|
+
/^(?:sc|css)-[a-z0-9]+$/i, /** CSS Modules, most bundlers' default hash suffix: `wrapper__3fA9k`, `Button-module_root__aB3dK` */
|
|
2992
|
+
/__[a-z0-9]{4,10}$/i, /** JSS / MUI: `jss123`, `MuiButton-root-145` */
|
|
2993
|
+
/^(?:jss|mui[a-z]*-)[a-z0-9-]*\d{2}$/i, /** A token that is only digits: not a class a human would type. */
|
|
2994
|
+
/^\d+$/, /** A bare hex hash: `a3f9c21`, `1a2b3c4d` (6+ hex chars, nothing else). */
|
|
2995
|
+
/^[0-9a-f]{6,}$/i, /** Tailwind/atomic-CSS-in-JS runtime classes some bundlers emit: `_1a2b3c` */
|
|
2996
|
+
/^_[0-9a-z]{5,}$/i];
|
|
2997
|
+
/**
|
|
2998
|
+
* Whether a class token looks hand-authored and safe to build a selector on.
|
|
2831
2999
|
*
|
|
2832
|
-
* @param
|
|
2833
|
-
* @returns
|
|
3000
|
+
* @param token - One class, no surrounding whitespace.
|
|
3001
|
+
* @returns `false` for anything matching a known generated-class shape.
|
|
2834
3002
|
*/
|
|
2835
|
-
function
|
|
2836
|
-
|
|
2837
|
-
const findings = [];
|
|
2838
|
-
$('script[type="application/ld+json"]').each((_index, element) => {
|
|
2839
|
-
findings.push(...scriptFinding($, element, rawHtml, 'ld-json', ldJsonLabel));
|
|
2840
|
-
});
|
|
2841
|
-
$('script[type="application/json"]').each((_index, element) => {
|
|
2842
|
-
findings.push(...scriptFinding($, element, rawHtml, 'json-script', () => `json: ${structuralSelector($, element)}`));
|
|
2843
|
-
});
|
|
2844
|
-
$('script').each((_index, element) => {
|
|
2845
|
-
const type = $(element).attr('type');
|
|
2846
|
-
if (type !== undefined && type !== 'text/javascript' && type !== 'module') return;
|
|
2847
|
-
const text = $(element).text();
|
|
2848
|
-
const match = INLINE_STATE_PATTERN.exec(text);
|
|
2849
|
-
if (match === null) return;
|
|
2850
|
-
const value = tryParseJson(text);
|
|
2851
|
-
const selector = structuralSelector($, element);
|
|
2852
|
-
findings.push({
|
|
2853
|
-
kind: 'inline-state',
|
|
2854
|
-
label: match[1],
|
|
2855
|
-
selector,
|
|
2856
|
-
selectorKind: 'css',
|
|
2857
|
-
matches: countMatches(rawHtml, selector),
|
|
2858
|
-
keys: keysOf(value)
|
|
2859
|
-
});
|
|
2860
|
-
});
|
|
2861
|
-
$('meta[name], meta[property]').each((_index, element) => {
|
|
2862
|
-
const name = $(element).attr('name') ?? $(element).attr('property') ?? '';
|
|
2863
|
-
const selector = structuralSelector($, element);
|
|
2864
|
-
findings.push({
|
|
2865
|
-
kind: 'meta',
|
|
2866
|
-
label: `meta: ${name}`,
|
|
2867
|
-
selector,
|
|
2868
|
-
selectorKind: 'css',
|
|
2869
|
-
matches: countMatches(rawHtml, selector),
|
|
2870
|
-
attribute: 'content'
|
|
2871
|
-
});
|
|
2872
|
-
});
|
|
2873
|
-
$('link[rel]').each((_index, element) => {
|
|
2874
|
-
const rel = $(element).attr('rel') ?? '';
|
|
2875
|
-
const selector = structuralSelector($, element);
|
|
2876
|
-
findings.push({
|
|
2877
|
-
kind: 'link',
|
|
2878
|
-
label: `link: ${rel}`,
|
|
2879
|
-
selector,
|
|
2880
|
-
selectorKind: 'css',
|
|
2881
|
-
matches: countMatches(rawHtml, selector),
|
|
2882
|
-
attribute: 'href'
|
|
2883
|
-
});
|
|
2884
|
-
});
|
|
2885
|
-
return findings;
|
|
2886
|
-
}
|
|
2887
|
-
function scriptFinding($, element, rawHtml, kind, labelOf) {
|
|
2888
|
-
const value = tryParseJson($(element).text());
|
|
2889
|
-
if (value === undefined) return [];
|
|
2890
|
-
const selector = structuralSelector($, element);
|
|
2891
|
-
return [{
|
|
2892
|
-
kind,
|
|
2893
|
-
label: labelOf(value),
|
|
2894
|
-
selector,
|
|
2895
|
-
selectorKind: 'css',
|
|
2896
|
-
matches: countMatches(rawHtml, selector),
|
|
2897
|
-
keys: keysOf(value)
|
|
2898
|
-
}];
|
|
2899
|
-
}
|
|
2900
|
-
function ldJsonLabel(value) {
|
|
2901
|
-
if (Array.isArray(value)) return `ld+json: array (${value.length})`;
|
|
2902
|
-
if (typeof value === 'object' && value !== null) {
|
|
2903
|
-
const type = value['@type'];
|
|
2904
|
-
if (typeof type === 'string') return `ld+json: ${type}`;
|
|
2905
|
-
}
|
|
2906
|
-
return `ld+json: ${describeJson(value, 'json').type}`;
|
|
3003
|
+
function isStableClassToken(token) {
|
|
3004
|
+
return token.length > 0 && GENERATED_TOKEN_PATTERNS.every(pattern => !pattern.test(token));
|
|
2907
3005
|
}
|
|
2908
|
-
|
|
2909
|
-
|
|
3006
|
+
/**
|
|
3007
|
+
* The subset of `tokens` worth ranking a selector candidate on, order kept.
|
|
3008
|
+
*
|
|
3009
|
+
* @param tokens - An element's class list.
|
|
3010
|
+
* @returns Only the stable tokens.
|
|
3011
|
+
*/
|
|
3012
|
+
function stableClasses(tokens) {
|
|
3013
|
+
return tokens.filter(token => isStableClassToken(token));
|
|
2910
3014
|
}
|
|
2911
3015
|
/**
|
|
2912
|
-
*
|
|
2913
|
-
*
|
|
2914
|
-
*
|
|
2915
|
-
* the same name under the same parent. Always resolves to exactly one
|
|
2916
|
-
* element, on the raw document this ran against and on the real page (a
|
|
2917
|
-
* purely structural selector, unaffected by the snapshot's own rewriting).
|
|
3016
|
+
* Whether two class lists describe "the same kind of element" for list
|
|
3017
|
+
* inference: same stable classes, generated tokens (which often differ
|
|
3018
|
+
* per-instance, e.g. a React `key`-derived class) ignored on both sides.
|
|
2918
3019
|
*
|
|
2919
|
-
* @param
|
|
2920
|
-
* @param
|
|
2921
|
-
* @returns
|
|
3020
|
+
* @param a - One element's classes.
|
|
3021
|
+
* @param b - Another element's classes.
|
|
3022
|
+
* @returns Whether their stable class sets are equal, order ignored.
|
|
2922
3023
|
*/
|
|
2923
|
-
function
|
|
2924
|
-
const
|
|
2925
|
-
|
|
2926
|
-
|
|
2927
|
-
|
|
2928
|
-
|
|
2929
|
-
|
|
2930
|
-
|
|
2931
|
-
|
|
2932
|
-
|
|
2933
|
-
|
|
2934
|
-
|
|
3024
|
+
function sameClassSignature(a, b) {
|
|
3025
|
+
const left = new Set(stableClasses(a));
|
|
3026
|
+
const right = new Set(stableClasses(b));
|
|
3027
|
+
if (left.size !== right.size) return false;
|
|
3028
|
+
return [...left].every(token => right.has(token));
|
|
3029
|
+
}
|
|
3030
|
+
|
|
3031
|
+
/**
|
|
3032
|
+
* Thrown when a pick or a list-inference walk crosses a declarative shadow
|
|
3033
|
+
* root (`<template shadowrootmode>`). v1 does not pierce shadow DOM: rather
|
|
3034
|
+
* than infer a selector that silently matches the wrong thing (or nothing,
|
|
3035
|
+
* once the page hydrates), the studio surfaces this and stops.
|
|
3036
|
+
*/
|
|
3037
|
+
class ShadowDomUnsupportedError extends Error {
|
|
3038
|
+
path;
|
|
3039
|
+
constructor(path) {
|
|
3040
|
+
super(`"${path}" is inside a shadow root: shadow DOM is not supported for picking (v1)`);
|
|
3041
|
+
this.path = path;
|
|
3042
|
+
this.name = 'ShadowDomUnsupportedError';
|
|
2935
3043
|
}
|
|
2936
|
-
return segments.join(' > ');
|
|
2937
3044
|
}
|
|
2938
3045
|
|
|
2939
|
-
/**
|
|
2940
|
-
const
|
|
2941
|
-
|
|
2942
|
-
|
|
3046
|
+
/** Attributes ranked just under `id`: named explicitly, not "any `data-*`", because a generated `data-reactid` is no more stable than a generated class. */
|
|
3047
|
+
const STABLE_ATTRS = ['data-testid', 'data-id', 'itemprop'];
|
|
3048
|
+
const TIER_ORDER$1 = {
|
|
3049
|
+
'id': 0,
|
|
3050
|
+
'data-attr': 1,
|
|
3051
|
+
'class': 2,
|
|
3052
|
+
'structure': 3,
|
|
3053
|
+
'position': 4
|
|
3054
|
+
};
|
|
2943
3055
|
/**
|
|
2944
|
-
*
|
|
2945
|
-
*
|
|
2946
|
-
*
|
|
2947
|
-
*
|
|
2948
|
-
*
|
|
2949
|
-
*
|
|
2950
|
-
*
|
|
2951
|
-
* to also collect responses was a larger, riskier change to already-shipped,
|
|
2952
|
-
* tested code than this phase's budget allowed (see this phase's final
|
|
2953
|
-
* report) — so this opens its own short-lived browser session instead of
|
|
2954
|
-
* piggy-backing on the cached snapshot.
|
|
2955
|
-
*
|
|
2956
|
-
* Only `"start"` is supported, mirroring `take-snapshot.use-case.ts`'s own
|
|
2957
|
-
* reach; api-mode recipes have no page load to observe, so they always
|
|
2958
|
-
* answer with an empty list rather than an error (there is nothing wrong
|
|
2959
|
-
* with asking, the answer is just "none").
|
|
3056
|
+
* Every selector candidate worth trying for the last node of `path`, ranked
|
|
3057
|
+
* best-first: `id` > stable `data-*`/`itemprop` attributes > stable class
|
|
3058
|
+
* tokens > ancestor structure (`parent > tag`) > position (`nth-of-type`) as
|
|
3059
|
+
* a last resort (issue #91's ranking, studio plan §3.2). Structure and
|
|
3060
|
+
* position candidates only look as far up as `path` itself goes — pass a
|
|
3061
|
+
* sub-path (e.g. from an item's root down) to keep a field's selector
|
|
3062
|
+
* relative to its item instead of absolute to the document.
|
|
2960
3063
|
*
|
|
2961
|
-
* @param
|
|
2962
|
-
* @param
|
|
2963
|
-
* @
|
|
2964
|
-
* @returns The distinct JSON responses observed, at most `RESPONSE_LIMIT`.
|
|
2965
|
-
* @throws Error when the recipe has no start point, or `stepPath` is not `"start"`.
|
|
3064
|
+
* @param path - The node's ancestry, root (or scope root) first, itself last.
|
|
3065
|
+
* @param includePosition - `false` for an item selector: a position candidate would only ever match one of the items, which is never what an item selector should do.
|
|
3066
|
+
* @returns Candidates, best tier first; ties keep this function's own order.
|
|
2966
3067
|
*/
|
|
2967
|
-
|
|
2968
|
-
|
|
2969
|
-
if (
|
|
2970
|
-
const
|
|
2971
|
-
if (
|
|
2972
|
-
|
|
2973
|
-
|
|
2974
|
-
|
|
2975
|
-
|
|
2976
|
-
|
|
2977
|
-
|
|
2978
|
-
|
|
2979
|
-
|
|
2980
|
-
|
|
2981
|
-
const key = `${response.status()} ${response.url()}`;
|
|
2982
|
-
if (seen.has(key)) return;
|
|
2983
|
-
seen.add(key);
|
|
2984
|
-
pending.push(recordResponse(response, responses));
|
|
2985
|
-
});
|
|
2986
|
-
try {
|
|
2987
|
-
await session.page.goto(point.url, {
|
|
2988
|
-
waitUntil: 'networkidle'
|
|
2989
|
-
});
|
|
2990
|
-
} catch {
|
|
2991
|
-
// a page that never idles (long-poll, websocket) still yields whatever was observed by the timeout below
|
|
2992
|
-
}
|
|
2993
|
-
await session.page.waitForTimeout(OBSERVE_MS);
|
|
2994
|
-
await Promise.all(pending);
|
|
2995
|
-
return responses.slice(0, RESPONSE_LIMIT);
|
|
2996
|
-
} finally {
|
|
2997
|
-
await session.close();
|
|
2998
|
-
}
|
|
2999
|
-
} finally {
|
|
3000
|
-
await client.close();
|
|
3068
|
+
function candidatesFor(path, includePosition = true) {
|
|
3069
|
+
const node = path.at(-1);
|
|
3070
|
+
if (node === undefined) return [];
|
|
3071
|
+
const candidates = [];
|
|
3072
|
+
if (node.id !== undefined && isStableId(node.id)) candidates.push({
|
|
3073
|
+
selector: `#${cssEscape(node.id)}`,
|
|
3074
|
+
tier: 'id'
|
|
3075
|
+
});
|
|
3076
|
+
for (const name of STABLE_ATTRS) {
|
|
3077
|
+
const value = node.attrs[name];
|
|
3078
|
+
if (value !== undefined && value.length > 0) candidates.push({
|
|
3079
|
+
selector: `${node.tag}[${name}="${cssValue(value)}"]`,
|
|
3080
|
+
tier: 'data-attr'
|
|
3081
|
+
});
|
|
3001
3082
|
}
|
|
3083
|
+
const classes = stableClasses(node.classes);
|
|
3084
|
+
if (classes.length > 0) candidates.push({
|
|
3085
|
+
selector: `${node.tag}.${classes.map(token => cssEscape(token)).join('.')}`,
|
|
3086
|
+
tier: 'class'
|
|
3087
|
+
});
|
|
3088
|
+
candidates.push(...structureCandidates(path));
|
|
3089
|
+
if (includePosition) candidates.push(...positionCandidates(path));
|
|
3090
|
+
return sortByTier(candidates);
|
|
3002
3091
|
}
|
|
3003
|
-
/**
|
|
3004
|
-
|
|
3005
|
-
|
|
3006
|
-
|
|
3007
|
-
|
|
3008
|
-
|
|
3009
|
-
|
|
3010
|
-
|
|
3011
|
-
|
|
3012
|
-
|
|
3092
|
+
/** `ancestor > ... > tag`, growing from 2 to 4 levels (or to the top of `path`), each ancestor named by its own best simple form. */
|
|
3093
|
+
function structureCandidates(path) {
|
|
3094
|
+
const node = path.at(-1);
|
|
3095
|
+
if (node === undefined) return [];
|
|
3096
|
+
const candidates = [];
|
|
3097
|
+
const maxDepth = Math.min(path.length - 1, 3);
|
|
3098
|
+
for (let depth = 1; depth <= maxDepth; depth++) {
|
|
3099
|
+
const ancestor = path.at(-1 - depth);
|
|
3100
|
+
if (ancestor === undefined) break;
|
|
3101
|
+
const between = Array.from({
|
|
3102
|
+
length: depth - 1
|
|
3103
|
+
}, (_, index) => path.at(-depth + index)?.tag ?? '');
|
|
3104
|
+
const parts = [simpleSelector(ancestor), ...between, node.tag];
|
|
3105
|
+
candidates.push({
|
|
3106
|
+
selector: parts.join(' > '),
|
|
3107
|
+
tier: 'structure'
|
|
3013
3108
|
});
|
|
3014
|
-
} catch {
|
|
3015
|
-
// response body unavailable (aborted, redirected, the page navigated away) — skip it
|
|
3016
3109
|
}
|
|
3110
|
+
return candidates;
|
|
3017
3111
|
}
|
|
3018
|
-
|
|
3019
|
-
|
|
3020
|
-
|
|
3021
|
-
|
|
3022
|
-
|
|
3023
|
-
|
|
3024
|
-
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
|
|
3029
|
-
|
|
3030
|
-
|
|
3031
|
-
|
|
3032
|
-
|
|
3033
|
-
|
|
3034
|
-
|
|
3035
|
-
if (
|
|
3036
|
-
|
|
3037
|
-
|
|
3038
|
-
|
|
3039
|
-
|
|
3112
|
+
/** `parent > tag:nth-of-type(n)`, the position-only fallback; never `nth-child`, which counts every sibling tag, not just this element's own. */
|
|
3113
|
+
function positionCandidates(path) {
|
|
3114
|
+
const node = path.at(-1);
|
|
3115
|
+
if (node === undefined) return [];
|
|
3116
|
+
const own = node.siblingCount > 1 ? `${node.tag}:nth-of-type(${node.index})` : node.tag;
|
|
3117
|
+
const parent = path.at(-2);
|
|
3118
|
+
if (parent === undefined) return [{
|
|
3119
|
+
selector: own,
|
|
3120
|
+
tier: 'position'
|
|
3121
|
+
}];
|
|
3122
|
+
return [{
|
|
3123
|
+
selector: `${simpleSelector(parent)} > ${own}`,
|
|
3124
|
+
tier: 'position'
|
|
3125
|
+
}];
|
|
3126
|
+
}
|
|
3127
|
+
/** A level's own selector fragment, no combinators: id, else stable classes, else the bare tag. */
|
|
3128
|
+
function simpleSelector(level) {
|
|
3129
|
+
if (level.id !== undefined && isStableId(level.id)) return `#${cssEscape(level.id)}`;
|
|
3130
|
+
const classes = stableClasses(level.classes);
|
|
3131
|
+
return classes.length > 0 ? `${level.tag}.${classes.map(token => cssEscape(token)).join('.')}` : level.tag;
|
|
3132
|
+
}
|
|
3133
|
+
function sortByTier(candidates) {
|
|
3134
|
+
return [...candidates].sort((a, b) => TIER_ORDER$1[a.tier] - TIER_ORDER$1[b.tier]);
|
|
3135
|
+
}
|
|
3136
|
+
/** Escapes a CSS identifier segment (an id or a class token) well enough for the shapes real sites use; not a full CSS.escape polyfill. */
|
|
3137
|
+
function cssEscape(token) {
|
|
3138
|
+
return token.replaceAll(/([^\w-])/gi, String.raw`\$1`);
|
|
3139
|
+
}
|
|
3140
|
+
/** Escapes a value inside `[name="…"]`. */
|
|
3141
|
+
function cssValue(value) {
|
|
3142
|
+
return value.replaceAll(/(["\\])/g, String.raw`\$1`);
|
|
3040
3143
|
}
|
|
3041
3144
|
|
|
3145
|
+
const TIER_ORDER = {
|
|
3146
|
+
'id': 0,
|
|
3147
|
+
'data-attr': 1,
|
|
3148
|
+
'class': 2,
|
|
3149
|
+
'structure': 3,
|
|
3150
|
+
'position': 4
|
|
3151
|
+
};
|
|
3042
3152
|
/**
|
|
3043
|
-
*
|
|
3044
|
-
*
|
|
3045
|
-
*
|
|
3153
|
+
* Verifies and orders selector candidates against real markup, by running
|
|
3154
|
+
* each one through the engine's own CSS matching (`@opencraw/core`'s
|
|
3155
|
+
* `countMatches`/`matchesOf`) rather than reimplementing it — issue #91's
|
|
3156
|
+
* "every candidate is verified" requirement.
|
|
3046
3157
|
*
|
|
3047
|
-
*
|
|
3048
|
-
*
|
|
3049
|
-
*
|
|
3050
|
-
*
|
|
3051
|
-
*
|
|
3052
|
-
*
|
|
3053
|
-
*
|
|
3054
|
-
*
|
|
3055
|
-
*
|
|
3056
|
-
* step type this outline cannot make sense of.
|
|
3158
|
+
* Ordering: candidates that do not match `targetNodeId` (when given) sort
|
|
3159
|
+
* last; among the rest, best tier first, and within a tier the fewest
|
|
3160
|
+
* matches first (the most specific selector that still hits the target).
|
|
3161
|
+
* A candidate matching nothing is dropped entirely: it is not a real option.
|
|
3162
|
+
*
|
|
3163
|
+
* @param html - The document to verify against (the snapshot, or the live page for the second, cross-check pass).
|
|
3164
|
+
* @param candidates - Unranked candidates, e.g. from `candidatesFor`.
|
|
3165
|
+
* @param targetNodeId - The `data-oc-node` id the pick landed on, when the document carries those stamps (the snapshot does; a live page fetched separately does not).
|
|
3166
|
+
* @returns Every candidate that matched at least one element, ranked best first.
|
|
3057
3167
|
*/
|
|
3058
|
-
|
|
3059
|
-
const
|
|
3060
|
-
|
|
3061
|
-
|
|
3168
|
+
function rankCandidates(html, candidates, targetNodeId) {
|
|
3169
|
+
const ranked = candidates.map(candidate => {
|
|
3170
|
+
const matches = countMatches(html, candidate.selector);
|
|
3171
|
+
const hitsTarget = targetNodeId === undefined || matches === 0 ? targetNodeId === undefined : matchesOf(html, candidate.selector).includes(targetNodeId);
|
|
3172
|
+
return {
|
|
3173
|
+
...candidate,
|
|
3174
|
+
matches,
|
|
3175
|
+
hitsTarget
|
|
3176
|
+
};
|
|
3177
|
+
});
|
|
3178
|
+
return ranked.filter(candidate => candidate.matches > 0).sort((a, b) => Number(b.hitsTarget) - Number(a.hitsTarget) || TIER_ORDER[a.tier] - TIER_ORDER[b.tier] || a.matches - b.matches);
|
|
3062
3179
|
}
|
|
3063
|
-
|
|
3064
|
-
|
|
3180
|
+
/** The single best candidate, or `undefined` when none of them matched anything. */
|
|
3181
|
+
function bestCandidate(html, candidates, targetNodeId) {
|
|
3182
|
+
return rankCandidates(html, candidates, targetNodeId)[0];
|
|
3065
3183
|
}
|
|
3066
3184
|
|
|
3067
|
-
/** Step types the plan doc names as always going to a "custom" card: nothing here renders their meaning as a sentence. */
|
|
3068
|
-
const ALWAYS_CUSTOM = new Set(['evaluate', 'hook', 'captcha']);
|
|
3069
3185
|
/**
|
|
3070
|
-
*
|
|
3071
|
-
*
|
|
3072
|
-
*
|
|
3073
|
-
*
|
|
3074
|
-
*
|
|
3186
|
+
* Infers "these two picks are the same field on two items of a list" from
|
|
3187
|
+
* two node paths, by walking up from both to the nearest ancestors that are
|
|
3188
|
+
* real siblings (the same parent) with the same tag and stable-class
|
|
3189
|
+
* signature — studio plan §3.2, issue #91. That ancestor is the item; the
|
|
3190
|
+
* remainder of each path is the field, relative to it.
|
|
3075
3191
|
*
|
|
3076
|
-
*
|
|
3077
|
-
*
|
|
3192
|
+
* Because a `DomPath` is built root-down (`dom-path.model.ts`), two paths
|
|
3193
|
+
* from the same document are byte-for-byte identical up to their real
|
|
3194
|
+
* common ancestor and diverge exactly at its differing children — so the
|
|
3195
|
+
* *first* level where they differ is always the nearest-sibling test point;
|
|
3196
|
+
* there is no need to look further down. This also makes the "wrapper trap"
|
|
3197
|
+
* (an extra single-child element around each item) resolve itself: a
|
|
3198
|
+
* wrapper's own instances are never one another's siblings, only the
|
|
3199
|
+
* genuine repeating element is, so the walk naturally lands there instead of
|
|
3200
|
+
* on the wrapper (see the wrapper-trap fixture in this slice's tests).
|
|
3201
|
+
*
|
|
3202
|
+
* @param pathA - The first click's path.
|
|
3203
|
+
* @param pathB - The second click's path.
|
|
3204
|
+
* @returns The item and field shapes (generalised) plus each pick's raw path, or `null` when the two picks are not on a coherent list (different branches entirely, the same node twice, or one path nested inside the other).
|
|
3205
|
+
* @throws ShadowDomUnsupportedError when either path crosses a declarative shadow root.
|
|
3078
3206
|
*/
|
|
3079
|
-
function
|
|
3080
|
-
|
|
3081
|
-
if (
|
|
3082
|
-
|
|
3083
|
-
|
|
3084
|
-
|
|
3085
|
-
|
|
3086
|
-
|
|
3087
|
-
|
|
3088
|
-
}
|
|
3089
|
-
} catch {
|
|
3090
|
-
return fallback(type);
|
|
3207
|
+
function inferList(pathA, pathB) {
|
|
3208
|
+
if (crossesShadowRoot(pathA)) throw new ShadowDomUnsupportedError(pathToString(pathA));
|
|
3209
|
+
if (crossesShadowRoot(pathB)) throw new ShadowDomUnsupportedError(pathToString(pathB));
|
|
3210
|
+
let divergedAt = -1;
|
|
3211
|
+
const limit = Math.min(pathA.length, pathB.length);
|
|
3212
|
+
for (let index = 0; index < limit; index++) {
|
|
3213
|
+
if (!sameLevel(pathA[index], pathB[index])) {
|
|
3214
|
+
divergedAt = index;
|
|
3215
|
+
break;
|
|
3216
|
+
}
|
|
3091
3217
|
}
|
|
3092
|
-
|
|
3093
|
-
|
|
3218
|
+
// Identical paths, or one a strict prefix of the other (the same node picked twice, or one element nested inside the other): not a list.
|
|
3219
|
+
if (divergedAt <= 0) return null;
|
|
3220
|
+
const branchA = pathA[divergedAt];
|
|
3221
|
+
const branchB = pathB[divergedAt];
|
|
3222
|
+
if (branchA.tag !== branchB.tag || !sameClassSignature(branchA.classes, branchB.classes)) return null;
|
|
3223
|
+
if (branchA.index === branchB.index) return null; // Same position: not two different items.
|
|
3224
|
+
const ancestors = pathA.slice(0, divergedAt);
|
|
3225
|
+
const fieldPathA = pathA.slice(divergedAt);
|
|
3226
|
+
const fieldPathB = pathB.slice(divergedAt);
|
|
3227
|
+
const fieldPath = mergeRelativePaths(fieldPathA, fieldPathB);
|
|
3094
3228
|
return {
|
|
3095
|
-
|
|
3096
|
-
|
|
3097
|
-
|
|
3098
|
-
|
|
3099
|
-
function word(text) {
|
|
3100
|
-
return {
|
|
3101
|
-
kind: 'word',
|
|
3102
|
-
text
|
|
3103
|
-
};
|
|
3104
|
-
}
|
|
3105
|
-
function pill(text) {
|
|
3106
|
-
return {
|
|
3107
|
-
kind: 'pill',
|
|
3108
|
-
text
|
|
3109
|
-
};
|
|
3110
|
-
}
|
|
3111
|
-
function code(text) {
|
|
3112
|
-
return {
|
|
3113
|
-
kind: 'code',
|
|
3114
|
-
text
|
|
3229
|
+
itemPath: [...ancestors, fieldPath[0]],
|
|
3230
|
+
fieldPath,
|
|
3231
|
+
fieldPathA,
|
|
3232
|
+
fieldPathB
|
|
3115
3233
|
};
|
|
3116
3234
|
}
|
|
3117
|
-
|
|
3118
|
-
|
|
3119
|
-
const value = step[key];
|
|
3120
|
-
return pill(typeof value === 'string' && value.length > 0 ? value : fallbackText);
|
|
3121
|
-
}
|
|
3122
|
-
/** A field shown as a literal (a selector, a template, a value): stringified when it is not already a string. */
|
|
3123
|
-
function codeField(step, key, fallbackText = '') {
|
|
3124
|
-
const value = step[key];
|
|
3125
|
-
if (typeof value === 'string') return code(value);
|
|
3126
|
-
if (value === undefined) return code(fallbackText);
|
|
3127
|
-
return code(safeJson(value));
|
|
3235
|
+
function sameLevel(a, b) {
|
|
3236
|
+
return a.tag === b.tag && a.id === b.id && a.index === b.index && a.siblingCount === b.siblingCount && sameClassSignature(a.classes, b.classes);
|
|
3128
3237
|
}
|
|
3129
|
-
|
|
3130
|
-
|
|
3131
|
-
|
|
3132
|
-
|
|
3133
|
-
|
|
3238
|
+
/**
|
|
3239
|
+
* Zips two item-relative paths (item first) level by level into one that
|
|
3240
|
+
* only keeps what both share: the tag (must already match, checked by the
|
|
3241
|
+
* caller for level 0), the intersection of stable classes, and attributes
|
|
3242
|
+
* present with the same value on both sides. Shorter than the other when
|
|
3243
|
+
* the two picks' local structure differs in depth (a real possibility once
|
|
3244
|
+
* a field is missing on one item; `list-inference.algorithm.test.ts`'s
|
|
3245
|
+
* shifted-field fixture is exactly this) — the merge just stops there,
|
|
3246
|
+
* which is fine: a field selector only needs to reach as deep as the
|
|
3247
|
+
* shallower of the two picks.
|
|
3248
|
+
*/
|
|
3249
|
+
function mergeRelativePaths(a, b) {
|
|
3250
|
+
const length = Math.min(a.length, b.length);
|
|
3251
|
+
const merged = [];
|
|
3252
|
+
for (let index = 0; index < length; index++) {
|
|
3253
|
+
const left = a[index];
|
|
3254
|
+
const right = b[index];
|
|
3255
|
+
if (left.tag !== right.tag) break;
|
|
3256
|
+
merged.push(mergeLevel(left, right));
|
|
3134
3257
|
}
|
|
3258
|
+
return merged.length > 0 ? merged : [a[0]];
|
|
3135
3259
|
}
|
|
3136
|
-
|
|
3137
|
-
|
|
3138
|
-
|
|
3139
|
-
|
|
3140
|
-
|
|
3141
|
-
const BUILDERS = {
|
|
3142
|
-
goto: step => [word('Go to'), codeField(step, 'url')],
|
|
3143
|
-
click: step => [word('Click'), targetField(step)],
|
|
3144
|
-
fill: step => [word('Fill'), targetField(step), word('with'), codeField(step, 'value')],
|
|
3145
|
-
press: step => [word('Press'), codeField(step, 'key'), word('in'), targetField(step)],
|
|
3146
|
-
select: step => [word('Choose'), selectChoice(step), word('in'), targetField(step)],
|
|
3147
|
-
scroll: step => [word('Scroll to'), codeField(step, 'to')],
|
|
3148
|
-
wait: step => [word('Wait for'), waitFor(step)],
|
|
3149
|
-
screenshot: step => [word('Screenshot'), codeField(step, 'path')],
|
|
3150
|
-
request: step => [word('Request'), code(`${typeof step.method === 'string' ? step.method : 'GET'} `), codeField(step, 'url')],
|
|
3151
|
-
extract: step => [word('Read'), pillField(step, 'id', '(unnamed)'), word('from'), pillField(step, 'from', 'page'), word('←'), codeField(step, 'selector')],
|
|
3152
|
-
set: step => [word('Set'), pillField(step, 'id', '(unnamed)'), word('to'), codeField(step, 'value')],
|
|
3153
|
-
collect: step => [word('Collect'), codeField(step, 'value'), word('into'), pillField(step, 'into', '(unnamed)')],
|
|
3154
|
-
forEach: step => forEachSentence(step),
|
|
3155
|
-
paginate: step => paginateSentence(step),
|
|
3156
|
-
if: step => [word('If'), codeField(step, 'test')],
|
|
3157
|
-
emit: step => emitSentence(step)
|
|
3158
|
-
};
|
|
3159
|
-
function selectChoice(step) {
|
|
3160
|
-
if (typeof step.value === 'string') return code(step.value);
|
|
3161
|
-
if (typeof step.label === 'string') return code(step.label);
|
|
3162
|
-
if (typeof step.index === 'number') return code(String(step.index));
|
|
3163
|
-
if (Array.isArray(step.values)) return code(step.values.join(', '));
|
|
3164
|
-
return code('');
|
|
3165
|
-
}
|
|
3166
|
-
function waitFor(step) {
|
|
3167
|
-
if (typeof step.selector === 'string') return code(step.selector);
|
|
3168
|
-
if (typeof step.ms === 'number') return code(`${step.ms}ms`);
|
|
3169
|
-
if (typeof step.state === 'string') return code(step.state);
|
|
3170
|
-
return code('');
|
|
3171
|
-
}
|
|
3172
|
-
function forEachSentence(step) {
|
|
3173
|
-
const parts = [word('For each'), pillField(step, 'as', '(item)'), word('in'), typeof step.over === 'string' ? pill(step.over) : targetField(step)];
|
|
3174
|
-
if (step.emit !== undefined) parts.push(word('→'), word('one record'));
|
|
3175
|
-
return parts;
|
|
3176
|
-
}
|
|
3177
|
-
function paginateSentence(step) {
|
|
3178
|
-
const parts = [word('For every page')];
|
|
3179
|
-
const next = step.next;
|
|
3180
|
-
if (next !== null && typeof next === 'object') {
|
|
3181
|
-
const record = next;
|
|
3182
|
-
if (typeof record.selector === 'string') parts.push(word('via'), code(record.selector));else if (typeof record.url === 'string') parts.push(word('via'), code(record.url));else if (typeof record.jsonpath === 'string') parts.push(word('via'), code(record.jsonpath));
|
|
3260
|
+
function mergeLevel(a, b) {
|
|
3261
|
+
const classes = stableClasses(a.classes).filter(token => b.classes.includes(token));
|
|
3262
|
+
const attrs = {};
|
|
3263
|
+
for (const [name, value] of Object.entries(a.attrs)) {
|
|
3264
|
+
if (name !== 'class' && name !== 'id' && b.attrs[name] === value) attrs[name] = value;
|
|
3183
3265
|
}
|
|
3184
|
-
|
|
3185
|
-
|
|
3266
|
+
return {
|
|
3267
|
+
tag: a.tag,
|
|
3268
|
+
id: a.id !== undefined && a.id === b.id ? a.id : undefined,
|
|
3269
|
+
classes,
|
|
3270
|
+
attrs,
|
|
3271
|
+
index: 1,
|
|
3272
|
+
siblingCount: Math.max(a.siblingCount, b.siblingCount, 1)
|
|
3273
|
+
};
|
|
3186
3274
|
}
|
|
3187
|
-
function
|
|
3188
|
-
|
|
3189
|
-
return [word('Emit')];
|
|
3275
|
+
function pathToString(path) {
|
|
3276
|
+
return path.map(level => level.tag + (level.id === undefined ? '' : `#${level.id}`)).join(' > ');
|
|
3190
3277
|
}
|
|
3191
3278
|
|
|
3192
3279
|
/**
|
|
3193
|
-
*
|
|
3194
|
-
*
|
|
3195
|
-
*
|
|
3196
|
-
*
|
|
3197
|
-
*
|
|
3198
|
-
*
|
|
3280
|
+
* Handles `infer-selector`: turns one or two picked nodes (their
|
|
3281
|
+
* `data-oc-node` ids, off the cached snapshot) into a verified selector —
|
|
3282
|
+
* one id for a `Read` card's field, two for the safe item+field list shape
|
|
3283
|
+
* `selector-inference`'s `inferList` builds by construction (issue #91,
|
|
3284
|
+
* studio plan §3.2). All the actual candidate/ranking/list-inference work
|
|
3285
|
+
* is `selector-inference`'s (pure); this handler only locates the clicked
|
|
3286
|
+
* nodes in the cached document and calls into it.
|
|
3199
3287
|
*
|
|
3200
|
-
* @param
|
|
3201
|
-
* @
|
|
3288
|
+
* @param state - The server state.
|
|
3289
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
3290
|
+
* @returns A field pick, a list pick, or `"unsupported"` with why.
|
|
3291
|
+
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
3202
3292
|
*/
|
|
3203
|
-
function
|
|
3204
|
-
const
|
|
3205
|
-
|
|
3206
|
-
|
|
3207
|
-
|
|
3208
|
-
|
|
3293
|
+
function handleInferSelector(state, command) {
|
|
3294
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
3295
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
3296
|
+
const html = snapshot.html;
|
|
3297
|
+
const paths = command.nodeIds.map(nodeId => pathToNode(html, nodeId));
|
|
3298
|
+
if (paths.includes(undefined)) return {
|
|
3299
|
+
kind: 'unsupported',
|
|
3300
|
+
reason: 'one of the picked nodes is no longer in the snapshot; take a fresh snapshot and try again'
|
|
3209
3301
|
};
|
|
3210
|
-
|
|
3211
|
-
|
|
3212
|
-
|
|
3213
|
-
|
|
3214
|
-
|
|
3215
|
-
|
|
3302
|
+
const [pathA, pathB] = paths;
|
|
3303
|
+
if (pathB === undefined) return {
|
|
3304
|
+
kind: 'field',
|
|
3305
|
+
field: fieldPick(html, pathA)
|
|
3306
|
+
};
|
|
3307
|
+
try {
|
|
3308
|
+
const result = inferList(pathA, pathB);
|
|
3309
|
+
if (result === null) return {
|
|
3310
|
+
kind: 'unsupported',
|
|
3311
|
+
reason: 'these two picks do not belong to the same list: pick the same field on two similar items'
|
|
3312
|
+
};
|
|
3313
|
+
const item = bestCandidate(html, candidatesFor(result.itemPath, false));
|
|
3314
|
+
if (item === undefined) return {
|
|
3315
|
+
kind: 'unsupported',
|
|
3316
|
+
reason: 'no selector matches every item of this list; try picking two items closer to each other in the markup'
|
|
3317
|
+
};
|
|
3318
|
+
const field = fieldPick(html, result.fieldPathA, result.fieldPath);
|
|
3216
3319
|
return {
|
|
3217
|
-
kind: '
|
|
3218
|
-
|
|
3219
|
-
|
|
3220
|
-
|
|
3221
|
-
|
|
3222
|
-
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3320
|
+
kind: 'list',
|
|
3321
|
+
item: {
|
|
3322
|
+
selector: item.selector,
|
|
3323
|
+
tier: item.tier,
|
|
3324
|
+
matches: item.matches
|
|
3325
|
+
},
|
|
3326
|
+
field
|
|
3327
|
+
};
|
|
3328
|
+
} catch (error) {
|
|
3329
|
+
if (error instanceof ShadowDomUnsupportedError) return {
|
|
3330
|
+
kind: 'unsupported',
|
|
3331
|
+
reason: error.message
|
|
3226
3332
|
};
|
|
3333
|
+
throw error;
|
|
3227
3334
|
}
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
3233
|
-
|
|
3234
|
-
|
|
3235
|
-
path,
|
|
3236
|
-
stepType,
|
|
3237
|
-
sentence: parts,
|
|
3238
|
-
step,
|
|
3239
|
-
custom
|
|
3240
|
-
};
|
|
3241
|
-
const children = Array.isArray(step.steps) ? stepsToNodes(step.steps, `${path}.steps`) : [];
|
|
3242
|
-
const elseChildren = stepType === 'if' && Array.isArray(step.else) ? stepsToNodes(step.else, `${path}.else`) : undefined;
|
|
3335
|
+
}
|
|
3336
|
+
/** The best selector for one clicked node, `take` decided from the concrete clicked element; `rankSource` (a generalised path) is used for candidate generation when given, so a per-item-unique attribute of the one node picked cannot end up in the selector. */
|
|
3337
|
+
function fieldPick(html, clicked, rankSource = clicked) {
|
|
3338
|
+
const leaf = clicked.at(-1);
|
|
3339
|
+
const take = leaf === undefined ? 'text' : takeKindFor(leaf);
|
|
3340
|
+
const best = bestCandidate(html, candidatesFor(rankSource));
|
|
3341
|
+
if (best === undefined) throw new Error('no selector matches the picked node — this should not happen, since the node came from this same document');
|
|
3243
3342
|
return {
|
|
3244
|
-
|
|
3245
|
-
|
|
3246
|
-
|
|
3247
|
-
|
|
3248
|
-
step,
|
|
3249
|
-
children,
|
|
3250
|
-
...(elseChildren !== undefined && {
|
|
3251
|
-
elseChildren
|
|
3252
|
-
})
|
|
3343
|
+
selector: best.selector,
|
|
3344
|
+
tier: best.tier,
|
|
3345
|
+
take,
|
|
3346
|
+
matches: best.matches
|
|
3253
3347
|
};
|
|
3254
3348
|
}
|
|
3255
|
-
function isRecord(value) {
|
|
3256
|
-
return typeof value === 'object' && value !== null && !Array.isArray(value);
|
|
3257
|
-
}
|
|
3258
3349
|
|
|
3350
|
+
/** A leaf element's own text longer than this is cut, with an ellipsis. */
|
|
3351
|
+
const TEXT_LIMIT = 80;
|
|
3352
|
+
/** Consecutive siblings whose subtree is identical (ids aside) collapse into one row once there are at least this many — a real page can have tens of thousands of `<li>`s, and showing all of them is not "inspecting" it (issue #93). */
|
|
3353
|
+
const MIN_REPEAT_RUN = 3;
|
|
3259
3354
|
/**
|
|
3260
|
-
*
|
|
3261
|
-
*
|
|
3262
|
-
*
|
|
3263
|
-
*
|
|
3264
|
-
*
|
|
3265
|
-
*
|
|
3266
|
-
* `step` never had one) — this, together with `recipe-to-outline.mapper.ts`
|
|
3267
|
-
* always keeping the step's whole original object, is what makes
|
|
3268
|
-
* `recipe → outline → recipe` preserve every key and its order.
|
|
3355
|
+
* Builds the Inspect panel's DOM tree from a captured snapshot (studio plan
|
|
3356
|
+
* §3.3, issue #93): the same rewritten, marked-up document the content pane's
|
|
3357
|
+
* iframe shows (`page-snapshot`'s `take-snapshot.use-case.ts`), so a tree
|
|
3358
|
+
* row's `nodeId` addresses exactly the element the iframe would highlight —
|
|
3359
|
+
* no separate capture, per issue #93's scope note to reuse phase 2's
|
|
3360
|
+
* snapshot rather than recapture.
|
|
3269
3361
|
*
|
|
3270
|
-
* @param
|
|
3271
|
-
* @returns The
|
|
3362
|
+
* @param snapshotHtml - The cached, rewritten snapshot (`SnapshotResult.html`).
|
|
3363
|
+
* @returns The tree's root (the document's `<html>` element).
|
|
3364
|
+
* @throws Error when the document has no `<html>` element (should not happen: `rewriteDocument` always produces one).
|
|
3272
3365
|
*/
|
|
3273
|
-
function
|
|
3274
|
-
|
|
3275
|
-
|
|
3366
|
+
function domTree(snapshotHtml) {
|
|
3367
|
+
const $ = load(snapshotHtml);
|
|
3368
|
+
const html = $('html').get(0);
|
|
3369
|
+
if (html === undefined) throw new Error('domTree: the snapshot has no <html> element');
|
|
3370
|
+
return buildNode($, html);
|
|
3371
|
+
}
|
|
3372
|
+
function buildNode($, element) {
|
|
3373
|
+
const $el = $(element);
|
|
3374
|
+
const attributes = {
|
|
3375
|
+
...element.attribs
|
|
3276
3376
|
};
|
|
3377
|
+
const nodeId = attributes[NODE_ID_ATTRIBUTE] ?? '';
|
|
3378
|
+
const hidden = attributes[HIDDEN_ATTRIBUTE] === '1';
|
|
3379
|
+
delete attributes[NODE_ID_ATTRIBUTE];
|
|
3380
|
+
delete attributes[HIDDEN_ATTRIBUTE];
|
|
3381
|
+
const id = attributes.id;
|
|
3382
|
+
const classes = (attributes.class ?? '').split(/\s+/).filter(token => token !== '');
|
|
3383
|
+
const childElements = $el.children().toArray();
|
|
3384
|
+
const children = collapseRepeats(childElements.map(child => buildNode($, child)));
|
|
3385
|
+
const text = childElements.length === 0 ? abbreviate($el.text()) : undefined;
|
|
3277
3386
|
return {
|
|
3278
|
-
|
|
3279
|
-
|
|
3387
|
+
nodeId,
|
|
3388
|
+
tag: element.tagName.toLowerCase(),
|
|
3389
|
+
...(id !== undefined && id !== '' && {
|
|
3390
|
+
id
|
|
3391
|
+
}),
|
|
3392
|
+
...(classes.length > 0 && {
|
|
3393
|
+
classes
|
|
3394
|
+
}),
|
|
3395
|
+
attributes,
|
|
3396
|
+
...(text !== undefined && text !== '' && {
|
|
3397
|
+
text
|
|
3398
|
+
}),
|
|
3399
|
+
hidden,
|
|
3400
|
+
children
|
|
3280
3401
|
};
|
|
3281
3402
|
}
|
|
3282
|
-
|
|
3283
|
-
|
|
3403
|
+
/**
|
|
3404
|
+
* Merges runs of `MIN_REPEAT_RUN` or more consecutive siblings whose subtree
|
|
3405
|
+
* is identical (their own `nodeId`s aside) into one representative row, the
|
|
3406
|
+
* issue's `×20` — real pages have tens of thousands of nodes, and this is
|
|
3407
|
+
* what makes the tree usable rather than merely correct.
|
|
3408
|
+
*
|
|
3409
|
+
* @param nodes - Already-built sibling rows, in document order.
|
|
3410
|
+
* @returns The same rows, with each long-enough identical run replaced by one.
|
|
3411
|
+
*/
|
|
3412
|
+
function collapseRepeats(nodes) {
|
|
3413
|
+
const result = [];
|
|
3414
|
+
let at = 0;
|
|
3415
|
+
while (at < nodes.length) {
|
|
3416
|
+
const signature = signatureOf(nodes[at]);
|
|
3417
|
+
let end = at + 1;
|
|
3418
|
+
while (end < nodes.length && signatureOf(nodes[end]) === signature) end += 1;
|
|
3419
|
+
const run = nodes.slice(at, end);
|
|
3420
|
+
if (run.length >= MIN_REPEAT_RUN) {
|
|
3421
|
+
result.push({
|
|
3422
|
+
...run[0],
|
|
3423
|
+
repeatCount: run.length,
|
|
3424
|
+
repeatNodeIds: run.map(node => node.nodeId)
|
|
3425
|
+
});
|
|
3426
|
+
} else {
|
|
3427
|
+
result.push(...run);
|
|
3428
|
+
}
|
|
3429
|
+
at = end;
|
|
3430
|
+
}
|
|
3431
|
+
return result;
|
|
3432
|
+
}
|
|
3433
|
+
/** A run member's identity for collapsing: everything but its own `nodeId` (every instance has a different one by construction). */
|
|
3434
|
+
function signatureOf(node) {
|
|
3435
|
+
return JSON.stringify({
|
|
3436
|
+
...node,
|
|
3437
|
+
nodeId: undefined,
|
|
3438
|
+
children: node.children.map(child => signatureOf(child))
|
|
3439
|
+
});
|
|
3440
|
+
}
|
|
3441
|
+
function abbreviate(text) {
|
|
3442
|
+
const trimmed = text.trim().replaceAll(/\s+/g, ' ');
|
|
3443
|
+
return trimmed.length > TEXT_LIMIT ? `${trimmed.slice(0, TEXT_LIMIT)}…` : trimmed;
|
|
3444
|
+
}
|
|
3445
|
+
|
|
3446
|
+
/** The global names sites hydrate state into, that `describeHtml`'s own JSON-LD/table scan does not cover (issue #93's "inline state assignments"). */
|
|
3447
|
+
const INLINE_STATE_NAMES = ['__NEXT_DATA__', '__NUXT__', '__INITIAL_STATE__', '__APOLLO_STATE__', '__PRELOADED_STATE__', '__STATE__'];
|
|
3448
|
+
const INLINE_STATE_PATTERN = new RegExp(String.raw`(?:window\.)?(${INLINE_STATE_NAMES.join('|')})\s*=`);
|
|
3449
|
+
/**
|
|
3450
|
+
* Finds the data-in-the-page findings the Inspect panel lists apart from the
|
|
3451
|
+
* DOM tree (studio plan §3.3, issue #93): JSON-LD and `application/json`
|
|
3452
|
+
* script blocks, inline state assignments, `<meta>` name/property values and
|
|
3453
|
+
* `<link rel>`s — reusing `@opencraw/probe`'s `describeJson` for a found
|
|
3454
|
+
* value's shape (its own `find-data.algorithm.ts`/`html-findings.mapper.ts`
|
|
3455
|
+
* scan a page's raw text for *reporting*, with no element to point a pick at;
|
|
3456
|
+
* this function's own job — locating each finding's element so a pick can
|
|
3457
|
+
* reach it — is new).
|
|
3458
|
+
*
|
|
3459
|
+
* Takes the **raw** captured document, not the display snapshot
|
|
3460
|
+
* (`page-snapshot`'s `SnapshotResult.html`): `rewriteDocument` strips every
|
|
3461
|
+
* `<script>` for the sandboxed iframe, which would make JSON-LD, inline
|
|
3462
|
+
* state and other script-tag data invisible here. `<meta>`/`<link>` survive
|
|
3463
|
+
* rewriting untouched, so using the raw document for every kind (rather than
|
|
3464
|
+
* mixing two documents) keeps this function simple and its selectors
|
|
3465
|
+
* verified against what the engine will actually crawl.
|
|
3466
|
+
*
|
|
3467
|
+
* @param rawHtml - The document as fetched/rendered, before `rewriteDocument` strips scripts.
|
|
3468
|
+
* @returns The findings, in document order.
|
|
3469
|
+
*/
|
|
3470
|
+
function pageData(rawHtml) {
|
|
3471
|
+
const $ = load(rawHtml);
|
|
3472
|
+
const findings = [];
|
|
3473
|
+
$('script[type="application/ld+json"]').each((_index, element) => {
|
|
3474
|
+
findings.push(...scriptFinding($, element, rawHtml, 'ld-json', ldJsonLabel));
|
|
3475
|
+
});
|
|
3476
|
+
$('script[type="application/json"]').each((_index, element) => {
|
|
3477
|
+
findings.push(...scriptFinding($, element, rawHtml, 'json-script', () => `json: ${structuralSelector($, element)}`));
|
|
3478
|
+
});
|
|
3479
|
+
$('script').each((_index, element) => {
|
|
3480
|
+
const type = $(element).attr('type');
|
|
3481
|
+
if (type !== undefined && type !== 'text/javascript' && type !== 'module') return;
|
|
3482
|
+
const text = $(element).text();
|
|
3483
|
+
const match = INLINE_STATE_PATTERN.exec(text);
|
|
3484
|
+
if (match === null) return;
|
|
3485
|
+
const value = tryParseJson(text);
|
|
3486
|
+
const selector = structuralSelector($, element);
|
|
3487
|
+
findings.push({
|
|
3488
|
+
kind: 'inline-state',
|
|
3489
|
+
label: match[1],
|
|
3490
|
+
selector,
|
|
3491
|
+
selectorKind: 'css',
|
|
3492
|
+
matches: countMatches(rawHtml, selector),
|
|
3493
|
+
keys: keysOf(value)
|
|
3494
|
+
});
|
|
3495
|
+
});
|
|
3496
|
+
$('meta[name], meta[property]').each((_index, element) => {
|
|
3497
|
+
const name = $(element).attr('name') ?? $(element).attr('property') ?? '';
|
|
3498
|
+
const selector = structuralSelector($, element);
|
|
3499
|
+
findings.push({
|
|
3500
|
+
kind: 'meta',
|
|
3501
|
+
label: `meta: ${name}`,
|
|
3502
|
+
selector,
|
|
3503
|
+
selectorKind: 'css',
|
|
3504
|
+
matches: countMatches(rawHtml, selector),
|
|
3505
|
+
attribute: 'content'
|
|
3506
|
+
});
|
|
3507
|
+
});
|
|
3508
|
+
$('link[rel]').each((_index, element) => {
|
|
3509
|
+
const rel = $(element).attr('rel') ?? '';
|
|
3510
|
+
const selector = structuralSelector($, element);
|
|
3511
|
+
findings.push({
|
|
3512
|
+
kind: 'link',
|
|
3513
|
+
label: `link: ${rel}`,
|
|
3514
|
+
selector,
|
|
3515
|
+
selectorKind: 'css',
|
|
3516
|
+
matches: countMatches(rawHtml, selector),
|
|
3517
|
+
attribute: 'href'
|
|
3518
|
+
});
|
|
3519
|
+
});
|
|
3520
|
+
return findings;
|
|
3521
|
+
}
|
|
3522
|
+
function scriptFinding($, element, rawHtml, kind, labelOf) {
|
|
3523
|
+
const value = tryParseJson($(element).text());
|
|
3524
|
+
if (value === undefined) return [];
|
|
3525
|
+
const selector = structuralSelector($, element);
|
|
3526
|
+
return [{
|
|
3527
|
+
kind,
|
|
3528
|
+
label: labelOf(value),
|
|
3529
|
+
selector,
|
|
3530
|
+
selectorKind: 'css',
|
|
3531
|
+
matches: countMatches(rawHtml, selector),
|
|
3532
|
+
keys: keysOf(value)
|
|
3533
|
+
}];
|
|
3284
3534
|
}
|
|
3285
|
-
function
|
|
3286
|
-
if (
|
|
3287
|
-
|
|
3288
|
-
|
|
3289
|
-
|
|
3290
|
-
if (key === 'steps') step.steps = nodesToSteps(node.children);else if (key === 'else' && node.stepType === 'if') step.else = nodesToSteps(node.elseChildren ?? []);else step[key] = node.step[key];
|
|
3535
|
+
function ldJsonLabel(value) {
|
|
3536
|
+
if (Array.isArray(value)) return `ld+json: array (${value.length})`;
|
|
3537
|
+
if (typeof value === 'object' && value !== null) {
|
|
3538
|
+
const type = value['@type'];
|
|
3539
|
+
if (typeof type === 'string') return `ld+json: ${type}`;
|
|
3291
3540
|
}
|
|
3292
|
-
|
|
3293
|
-
|
|
3294
|
-
|
|
3541
|
+
return `ld+json: ${describeJson(value, 'json').type}`;
|
|
3542
|
+
}
|
|
3543
|
+
function keysOf(value) {
|
|
3544
|
+
return typeof value === 'object' && value !== null && !Array.isArray(value) ? Object.keys(value) : undefined;
|
|
3295
3545
|
}
|
|
3296
|
-
|
|
3297
3546
|
/**
|
|
3298
|
-
*
|
|
3299
|
-
* (
|
|
3300
|
-
*
|
|
3301
|
-
* the
|
|
3302
|
-
*
|
|
3303
|
-
*
|
|
3547
|
+
* A unique CSS path for an element with no stable attribute of its own
|
|
3548
|
+
* (a `<script>` tag has no id or class to speak of): `<ancestor> > tag`
|
|
3549
|
+
* from `<html>` down, `:nth-of-type` added only where a tag has siblings of
|
|
3550
|
+
* the same name under the same parent. Always resolves to exactly one
|
|
3551
|
+
* element, on the raw document this ran against and on the real page (a
|
|
3552
|
+
* purely structural selector, unaffected by the snapshot's own rewriting).
|
|
3304
3553
|
*
|
|
3305
|
-
* @param
|
|
3306
|
-
* @
|
|
3307
|
-
* @
|
|
3554
|
+
* @param $ - The document.
|
|
3555
|
+
* @param element - The element to address.
|
|
3556
|
+
* @returns The selector.
|
|
3308
3557
|
*/
|
|
3309
|
-
|
|
3310
|
-
const
|
|
3311
|
-
|
|
3312
|
-
|
|
3313
|
-
|
|
3314
|
-
|
|
3315
|
-
|
|
3316
|
-
|
|
3317
|
-
|
|
3318
|
-
|
|
3319
|
-
|
|
3320
|
-
|
|
3321
|
-
});
|
|
3322
|
-
}
|
|
3558
|
+
function structuralSelector($, element) {
|
|
3559
|
+
const segments = [];
|
|
3560
|
+
let $current = $(element);
|
|
3561
|
+
while ($current.length > 0) {
|
|
3562
|
+
const tag = $current.prop('tagName')?.toLowerCase();
|
|
3563
|
+
if (tag === undefined) break;
|
|
3564
|
+
const $parent = $current.parent();
|
|
3565
|
+
const siblingsOfType = $parent.length > 0 ? $parent.children(tag) : $current;
|
|
3566
|
+
const index = siblingsOfType.toArray().indexOf($current.get(0)) + 1;
|
|
3567
|
+
segments.unshift(siblingsOfType.length > 1 ? `${tag}:nth-of-type(${index})` : tag);
|
|
3568
|
+
if (tag === 'html' || $parent.length === 0) break;
|
|
3569
|
+
$current = $parent;
|
|
3323
3570
|
}
|
|
3324
|
-
return
|
|
3325
|
-
folder,
|
|
3326
|
-
recipes: entries.map(({
|
|
3327
|
-
file,
|
|
3328
|
-
kind,
|
|
3329
|
-
id,
|
|
3330
|
-
issues,
|
|
3331
|
-
content
|
|
3332
|
-
}) => ({
|
|
3333
|
-
file,
|
|
3334
|
-
kind,
|
|
3335
|
-
id,
|
|
3336
|
-
issues,
|
|
3337
|
-
text: `${JSON.stringify(content, null, 2)}\n`,
|
|
3338
|
-
// The Steps tab needs steps to render: an output/unknown file has none. recipeToOutline never
|
|
3339
|
-
// throws, so this covers a file that fails validation too (its outline just has "custom" cards).
|
|
3340
|
-
outline: kind === 'input' ? recipeToOutline(content) : undefined
|
|
3341
|
-
}))
|
|
3342
|
-
};
|
|
3571
|
+
return segments.join(' > ');
|
|
3343
3572
|
}
|
|
3344
|
-
|
|
3345
|
-
|
|
3346
|
-
|
|
3573
|
+
|
|
3574
|
+
/** How long the browser is given to settle after `goto`, watching for JSON responses (mirrors `packages/cli`'s `probe --browser`, `observeBrowserJson`). */
|
|
3575
|
+
const OBSERVE_MS = 4000;
|
|
3576
|
+
/** Responses kept, largest/most-recent-first ordering left to the caller; just a sane cap on what the panel lists. */
|
|
3577
|
+
const RESPONSE_LIMIT = 15;
|
|
3578
|
+
/**
|
|
3579
|
+
* Records the JSON responses seen while an input recipe's start page
|
|
3580
|
+
* rendered (studio plan §3.3, issue #93): the same technique `packages/cli`'s
|
|
3581
|
+
* `probe --browser` uses (`observeBrowserJson`) — render the page, listen for
|
|
3582
|
+
* `response` events whose content type is JSON, give it a few seconds to
|
|
3583
|
+
* settle. Kept as its own capture, independent of `page-snapshot`'s
|
|
3584
|
+
* `take-snapshot.use-case.ts`: that capture reads `page.content()` right
|
|
3585
|
+
* after `goto` resolves, with no observation window, and changing its shape
|
|
3586
|
+
* to also collect responses was a larger, riskier change to already-shipped,
|
|
3587
|
+
* tested code than this phase's budget allowed (see this phase's final
|
|
3588
|
+
* report) — so this opens its own short-lived browser session instead of
|
|
3589
|
+
* piggy-backing on the cached snapshot.
|
|
3590
|
+
*
|
|
3591
|
+
* Only `"start"` is supported, mirroring `take-snapshot.use-case.ts`'s own
|
|
3592
|
+
* reach; api-mode recipes have no page load to observe, so they always
|
|
3593
|
+
* answer with an empty list rather than an error (there is nothing wrong
|
|
3594
|
+
* with asking, the answer is just "none").
|
|
3595
|
+
*
|
|
3596
|
+
* @param input - The input recipe; only `mode` and the first `start` point are used.
|
|
3597
|
+
* @param stepPath - The step to observe after, e.g. `"start"`.
|
|
3598
|
+
* @param browser - Browser launch settings, shared with `take-snapshot`/`verify-selector`/`run-sample`.
|
|
3599
|
+
* @returns The distinct JSON responses observed, at most `RESPONSE_LIMIT`.
|
|
3600
|
+
* @throws Error when the recipe has no start point, or `stepPath` is not `"start"`.
|
|
3601
|
+
*/
|
|
3602
|
+
async function responsesSeen(input, stepPath, browser) {
|
|
3603
|
+
if (stepPath !== 'start') throw new Error(`responsesSeen: "${stepPath}" is not supported yet — only "start" (the first start point, before any step ran) is`);
|
|
3604
|
+
if (input.mode !== 'web') return [];
|
|
3605
|
+
const point = input.start[0];
|
|
3606
|
+
if (point === undefined) throw new Error(`recipe "${input.id}" has no start point`);
|
|
3607
|
+
const client = await ensureBrowserLaunch(browser, () => BrowserClient.launch(browser));
|
|
3347
3608
|
try {
|
|
3348
|
-
|
|
3349
|
-
|
|
3350
|
-
|
|
3351
|
-
|
|
3352
|
-
const
|
|
3353
|
-
|
|
3354
|
-
|
|
3355
|
-
|
|
3356
|
-
|
|
3609
|
+
const session = await client.newSession();
|
|
3610
|
+
try {
|
|
3611
|
+
const responses = [];
|
|
3612
|
+
const seen = new Set();
|
|
3613
|
+
const pending = [];
|
|
3614
|
+
session.page.on('response', response => {
|
|
3615
|
+
if (!(response.headers()['content-type'] ?? '').includes('json')) return;
|
|
3616
|
+
const key = `${response.status()} ${response.url()}`;
|
|
3617
|
+
if (seen.has(key)) return;
|
|
3618
|
+
seen.add(key);
|
|
3619
|
+
pending.push(recordResponse(response, responses));
|
|
3357
3620
|
});
|
|
3621
|
+
try {
|
|
3622
|
+
await session.page.goto(point.url, {
|
|
3623
|
+
waitUntil: 'networkidle'
|
|
3624
|
+
});
|
|
3625
|
+
} catch {
|
|
3626
|
+
// a page that never idles (long-poll, websocket) still yields whatever was observed by the timeout below
|
|
3627
|
+
}
|
|
3628
|
+
await session.page.waitForTimeout(OBSERVE_MS);
|
|
3629
|
+
await Promise.all(pending);
|
|
3630
|
+
return responses.slice(0, RESPONSE_LIMIT);
|
|
3631
|
+
} finally {
|
|
3632
|
+
await session.close();
|
|
3358
3633
|
}
|
|
3634
|
+
} finally {
|
|
3635
|
+
await client.close();
|
|
3359
3636
|
}
|
|
3360
3637
|
}
|
|
3361
|
-
|
|
3362
|
-
|
|
3363
|
-
}
|
|
3364
|
-
function parseDocument(document) {
|
|
3365
|
-
const kind = document.content?.kind;
|
|
3366
|
-
if (kind === 'output') return parseOne(document.source, document.content, 'output', parseOutputRecipe);
|
|
3367
|
-
if (kind === 'input') return parseOne(document.source, document.content, 'input', parseInputRecipe);
|
|
3368
|
-
return {
|
|
3369
|
-
file: document.source,
|
|
3370
|
-
kind: 'unknown',
|
|
3371
|
-
content: document.content,
|
|
3372
|
-
issues: [{
|
|
3373
|
-
path: 'kind',
|
|
3374
|
-
message: '"kind" is missing or not "input"/"output"',
|
|
3375
|
-
kind: 'validation'
|
|
3376
|
-
}]
|
|
3377
|
-
};
|
|
3378
|
-
}
|
|
3379
|
-
function parseOne(file, content, kind, parse) {
|
|
3638
|
+
/** Reads one response's body, when it still can be (not aborted, not redirected away); a response that cannot be read is silently left out rather than failing the whole capture. */
|
|
3639
|
+
async function recordResponse(response, into) {
|
|
3380
3640
|
try {
|
|
3381
|
-
const
|
|
3382
|
-
|
|
3383
|
-
|
|
3384
|
-
|
|
3385
|
-
|
|
3386
|
-
|
|
3387
|
-
|
|
3388
|
-
|
|
3389
|
-
|
|
3390
|
-
|
|
3391
|
-
if (!(error instanceof RecipeValidationError)) throw error;
|
|
3392
|
-
return {
|
|
3393
|
-
file,
|
|
3394
|
-
kind,
|
|
3395
|
-
content,
|
|
3396
|
-
issues: error.issues.map(issue => ({
|
|
3397
|
-
path: issue.path === '' ? '(root)' : issue.path,
|
|
3398
|
-
message: issue.message,
|
|
3399
|
-
kind: 'validation'
|
|
3400
|
-
}))
|
|
3401
|
-
};
|
|
3641
|
+
const body = await response.text();
|
|
3642
|
+
const value = tryParseJson(body);
|
|
3643
|
+
into.push({
|
|
3644
|
+
url: response.url(),
|
|
3645
|
+
status: response.status(),
|
|
3646
|
+
size: body.length,
|
|
3647
|
+
shape: value === undefined ? 'text' : describeJson(value, 'json').type
|
|
3648
|
+
});
|
|
3649
|
+
} catch {
|
|
3650
|
+
// response body unavailable (aborted, redirected, the page navigated away) — skip it
|
|
3402
3651
|
}
|
|
3403
3652
|
}
|
|
3404
3653
|
|
|
3405
3654
|
/**
|
|
3406
|
-
*
|
|
3407
|
-
*
|
|
3408
|
-
*
|
|
3409
|
-
*
|
|
3410
|
-
*
|
|
3411
|
-
*
|
|
3412
|
-
*
|
|
3413
|
-
* round-trip for a hand-formatted file is future work; see this file's test
|
|
3414
|
-
* for what is guaranteed today.
|
|
3655
|
+
* Handles `inspect-page`: the Inspect panel's DOM tree and data-in-the-page
|
|
3656
|
+
* findings, off the same cached snapshot `take-snapshot`/`infer-selector`
|
|
3657
|
+
* already use (`page-inspector`, issue #93). `domTree` reads the cached,
|
|
3658
|
+
* rewritten snapshot (the same markup the content pane's iframe shows, so a
|
|
3659
|
+
* tree row's `nodeId` lines up with it exactly); `pageData` reads the raw
|
|
3660
|
+
* capture (`SnapshotResult.rawHtml`), since the display snapshot has had its
|
|
3661
|
+
* `<script>` tags stripped for the sandboxed iframe.
|
|
3415
3662
|
*
|
|
3416
|
-
* @param
|
|
3417
|
-
* @param
|
|
3663
|
+
* @param state - The server state.
|
|
3664
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
3665
|
+
* @returns The tree and the data-in-the-page findings.
|
|
3666
|
+
* @throws Error when no workspace is open, or `command.path` has no cached snapshot yet.
|
|
3418
3667
|
*/
|
|
3419
|
-
|
|
3420
|
-
|
|
3668
|
+
function handleInspectPage(state, command) {
|
|
3669
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
3670
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
3671
|
+
return {
|
|
3672
|
+
tree: domTree(snapshot.html),
|
|
3673
|
+
pageData: pageData(snapshot.rawHtml)
|
|
3674
|
+
};
|
|
3421
3675
|
}
|
|
3422
3676
|
|
|
3423
3677
|
/** Where the studio's own recorder profile lives when `StudioServerOptions.recorderProfileDir` is not given: never a crawl's own `session.browserProfile` directory, never the person's default browser profile. */
|
|
@@ -3455,7 +3709,14 @@ function broadcast(state, event) {
|
|
|
3455
3709
|
/**
|
|
3456
3710
|
* Handles `open-workspace`: reads the folder, remembers it as the server's
|
|
3457
3711
|
* current workspace (every later command that names a recipe resolves it
|
|
3458
|
-
* against this folder), and
|
|
3712
|
+
* against this folder), and, when that is a different folder than before,
|
|
3713
|
+
* tells every connected client the workspace changed.
|
|
3714
|
+
*
|
|
3715
|
+
* Only on an actual switch: the UI answers `workspace-changed` with another
|
|
3716
|
+
* `open-workspace` of the same folder (its workspace query re-fetches), so a
|
|
3717
|
+
* broadcast on every open would answer that re-fetch with another broadcast,
|
|
3718
|
+
* for ever — and each round cancels the fetch in flight, so a recipe saved
|
|
3719
|
+
* meanwhile never showed up in the listing (found by issue #120's drop flow).
|
|
3459
3720
|
*
|
|
3460
3721
|
* @param state - The server state to update.
|
|
3461
3722
|
* @param command - The command.
|
|
@@ -3463,8 +3724,9 @@ function broadcast(state, event) {
|
|
|
3463
3724
|
*/
|
|
3464
3725
|
async function handleOpenWorkspace(state, command) {
|
|
3465
3726
|
const view = await openWorkspace(command.folder);
|
|
3727
|
+
const switched = state.folder !== command.folder;
|
|
3466
3728
|
state.folder = command.folder;
|
|
3467
|
-
broadcast(state, {
|
|
3729
|
+
if (switched) broadcast(state, {
|
|
3468
3730
|
type: 'workspace-changed'
|
|
3469
3731
|
});
|
|
3470
3732
|
return view;
|
|
@@ -3957,7 +4219,7 @@ async function openRecorderSession(options, callbacks) {
|
|
|
3957
4219
|
...options.browser,
|
|
3958
4220
|
headless: false
|
|
3959
4221
|
});
|
|
3960
|
-
const session = await profiles.open(RECORDER_PROFILE_NAME, {}, {});
|
|
4222
|
+
const session = await ensureBrowserLaunch(options.browser, () => profiles.open(RECORDER_PROFILE_NAME, {}, {}));
|
|
3961
4223
|
const context = session.context;
|
|
3962
4224
|
const steps = [];
|
|
3963
4225
|
let closed = false;
|
|
@@ -4237,6 +4499,26 @@ async function handleResponsesSeen(state, command) {
|
|
|
4237
4499
|
};
|
|
4238
4500
|
}
|
|
4239
4501
|
|
|
4502
|
+
/**
|
|
4503
|
+
* Handles `region-preview`: the PDF and deck canvases' live preview of a
|
|
4504
|
+
* `region` extract's selector (`document-view`'s
|
|
4505
|
+
* `region-preview.use-case.ts`, issues #121 and #122), off the same cached
|
|
4506
|
+
* snapshot `take-snapshot` already captured — a pure computation over it,
|
|
4507
|
+
* so this answers instantly on every change of the box, no re-fetch or
|
|
4508
|
+
* re-parse (mirrors `table-preview.handler.ts`).
|
|
4509
|
+
*
|
|
4510
|
+
* @param state - The server state.
|
|
4511
|
+
* @param command - The command; `command.path` must already have a cached snapshot (call `take-snapshot` first).
|
|
4512
|
+
* @returns The matches, or `error` when the selector does not parse or does not fit the document.
|
|
4513
|
+
* @throws Error when `command.path` has no cached snapshot yet, or the snapshot is neither a PDF nor a deck.
|
|
4514
|
+
*/
|
|
4515
|
+
function handleRegionPreview(state, command) {
|
|
4516
|
+
const snapshot = cachedSnapshot(state.snapshots, command.recipeId, command.path);
|
|
4517
|
+
if (snapshot === undefined) throw new Error(`no cached snapshot for "${command.recipeId}" at "${command.path}": call take-snapshot first`);
|
|
4518
|
+
if (!isPdfDocument(snapshot.body) && !isDeckDocument(snapshot.body)) throw new Error(`"${command.recipeId}" at "${command.path}" is neither a PDF nor a deck snapshot: region-preview only reads a read PDF or PowerPoint document`);
|
|
4519
|
+
return previewRegion(snapshot.body, command.selector);
|
|
4520
|
+
}
|
|
4521
|
+
|
|
4240
4522
|
/**
|
|
4241
4523
|
* Handles `run-sample`: stops whatever sample was still running (one run at
|
|
4242
4524
|
* a time), starts the new one, and returns once it has started, not once it
|
|
@@ -4724,6 +5006,10 @@ function dispatch(command, state) {
|
|
|
4724
5006
|
{
|
|
4725
5007
|
return handleOpenWorkspace(state, command);
|
|
4726
5008
|
}
|
|
5009
|
+
case 'import-document':
|
|
5010
|
+
{
|
|
5011
|
+
return handleImportDocument(command);
|
|
5012
|
+
}
|
|
4727
5013
|
case 'run-sample':
|
|
4728
5014
|
{
|
|
4729
5015
|
return handleRunSample(state, command);
|
|
@@ -4780,6 +5066,10 @@ function dispatch(command, state) {
|
|
|
4780
5066
|
{
|
|
4781
5067
|
return Promise.resolve(handleTablePreview(state, command));
|
|
4782
5068
|
}
|
|
5069
|
+
case 'region-preview':
|
|
5070
|
+
{
|
|
5071
|
+
return Promise.resolve(handleRegionPreview(state, command));
|
|
5072
|
+
}
|
|
4783
5073
|
case 'grid-view':
|
|
4784
5074
|
{
|
|
4785
5075
|
return handleGridView(state, command);
|
|
@@ -4860,5 +5150,5 @@ async function main(argv) {
|
|
|
4860
5150
|
process.on('SIGTERM', shutdown);
|
|
4861
5151
|
}
|
|
4862
5152
|
|
|
4863
|
-
export { deckChartSeriesViewSchema, deckChartViewSchema, deckDocumentViewSchema, deckPreviewCommandSchema, deckPreviewOptionsSchema, deckShapeViewSchema, deckSlideViewSchema, deckTablePreviewMatchSchema, deckTablePreviewViewSchema, deckViewCommandSchema, documentTreeCommandSchema, documentTreeNodeViewSchema, documentTreeViewSchema, domTreeNodeViewSchema, explainWhyCommandSchema, fetchStartPage, fetchStartPageCommandSchema, fieldPickSchema, fieldTraceSchema, gridCellViewSchema, gridMergeViewSchema, gridPreviewCommandSchema, gridPreviewOptionsSchema, gridSheetViewSchema, gridTablePreviewMatchSchema, gridTablePreviewViewSchema, gridViewCommandSchema, inferSelectorCommandSchema, inferSelectorViewSchema, inspectPageCommandSchema, inspectViewSchema, loadRecipePair, main, missingWhyTargetSchema, observedResponseSchema, openWorkspace, openWorkspaceCommandSchema, outlineBracketSchema, outlineCardSchema, outlineNodeSchema, outlineViewSchema, pageDataFindingSchema, pdfCellViewSchema, pdfDocumentViewSchema, pdfPageViewSchema, pdfRowViewSchema, pdfViewCommandSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, recordRejectedEventSchema, recordingCardEventSchema, recordingNoteEventSchema, recordingStoppedEventSchema, rejectedWhyTargetSchema, responsesSeenCommandSchema, responsesSeenViewSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveOutlineCommandSchema, saveRecipe, saveRecipeCommandSchema, sentencePartSchema, snapshotViewSchema, startPageViewSchema, startRecordingCommandSchema, startStudioServer, stopRecordingCommandSchema, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, tablePreviewBandSchema, tablePreviewCommandSchema, tablePreviewMatchSchema, tablePreviewOptionsSchema, tablePreviewTableSchema, tablePreviewViewSchema, takeSnapshotCommandSchema, traceLineEventSchema, verifySelectorCommandSchema, verifySelectorViewSchema, whyTargetSchema, whyViewSchema, workbookDocumentViewSchema, workspaceChangedEventSchema, workspaceViewSchema };
|
|
5153
|
+
export { deckChartSeriesViewSchema, deckChartViewSchema, deckDocumentViewSchema, deckPreviewCommandSchema, deckPreviewOptionsSchema, deckShapeViewSchema, deckSlideViewSchema, deckTablePreviewMatchSchema, deckTablePreviewViewSchema, deckViewCommandSchema, documentTreeCommandSchema, documentTreeNodeViewSchema, documentTreeViewSchema, domTreeNodeViewSchema, explainWhyCommandSchema, fetchStartPage, fetchStartPageCommandSchema, fieldPickSchema, fieldTraceSchema, gridCellViewSchema, gridMergeViewSchema, gridPreviewCommandSchema, gridPreviewOptionsSchema, gridSheetViewSchema, gridTablePreviewMatchSchema, gridTablePreviewViewSchema, gridViewCommandSchema, importDocument, importDocumentCommandSchema, importDocumentViewSchema, inferSelectorCommandSchema, inferSelectorViewSchema, inspectPageCommandSchema, inspectViewSchema, loadRecipePair, main, missingWhyTargetSchema, observedResponseSchema, openWorkspace, openWorkspaceCommandSchema, outlineBracketSchema, outlineCardSchema, outlineNodeSchema, outlineViewSchema, pageDataFindingSchema, pdfCellViewSchema, pdfDocumentViewSchema, pdfPageViewSchema, pdfRowViewSchema, pdfViewCommandSchema, recipeIssueSchema, recipeListingSchema, recordEventSchema, recordRejectedEventSchema, recordingCardEventSchema, recordingNoteEventSchema, recordingStoppedEventSchema, regionPreviewCommandSchema, regionPreviewMatchSchema, regionPreviewViewSchema, rejectedWhyTargetSchema, responsesSeenCommandSchema, responsesSeenViewSchema, runFinishedEventSchema, runSample, runSampleCommandSchema, sampleBudgetSchema, sampleRunRecordSchema, sampleRunViewSchema, saveOutlineCommandSchema, saveRecipe, saveRecipeCommandSchema, sentencePartSchema, snapshotViewSchema, startPageViewSchema, startRecordingCommandSchema, startStudioServer, stopRecordingCommandSchema, stopRunCommandSchema, stoppedBySchema, studioCommandSchema, studioEventSchema, tablePreviewBandSchema, tablePreviewCommandSchema, tablePreviewMatchSchema, tablePreviewOptionsSchema, tablePreviewTableSchema, tablePreviewViewSchema, takeSnapshotCommandSchema, traceLineEventSchema, verifySelectorCommandSchema, verifySelectorViewSchema, whyTargetSchema, whyViewSchema, workbookDocumentViewSchema, workspaceChangedEventSchema, workspaceViewSchema };
|
|
4864
5154
|
//# sourceMappingURL=index.esm.js.map
|