@showrun/core 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/__tests__/dsl-validation.test.js +185 -0
- package/dist/__tests__/httpReplay.test.js +62 -0
- package/dist/__tests__/proxy.test.d.ts +2 -0
- package/dist/__tests__/proxy.test.d.ts.map +1 -0
- package/dist/__tests__/proxy.test.js +117 -0
- package/dist/__tests__/registry-client.test.d.ts +2 -0
- package/dist/__tests__/registry-client.test.d.ts.map +1 -0
- package/dist/__tests__/registry-client.test.js +228 -0
- package/dist/browserLauncher.d.ts +15 -0
- package/dist/browserLauncher.d.ts.map +1 -1
- package/dist/browserLauncher.js +62 -12
- package/dist/config.d.ts +1 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +4 -3
- package/dist/dsl/builders.d.ts +22 -1
- package/dist/dsl/builders.d.ts.map +1 -1
- package/dist/dsl/builders.js +23 -0
- package/dist/dsl/interpreter.d.ts +5 -0
- package/dist/dsl/interpreter.d.ts.map +1 -1
- package/dist/dsl/interpreter.js +1 -0
- package/dist/dsl/stepHandlers.d.ts +3 -0
- package/dist/dsl/stepHandlers.d.ts.map +1 -1
- package/dist/dsl/stepHandlers.js +62 -2
- package/dist/dsl/types.d.ts +53 -1
- package/dist/dsl/types.d.ts.map +1 -1
- package/dist/dsl/validation.d.ts.map +1 -1
- package/dist/dsl/validation.js +90 -1
- package/dist/httpReplay.d.ts +5 -0
- package/dist/httpReplay.d.ts.map +1 -1
- package/dist/httpReplay.js +46 -2
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/jsonPackValidator.js +1 -1
- package/dist/proxy/index.d.ts +4 -0
- package/dist/proxy/index.d.ts.map +1 -0
- package/dist/proxy/index.js +3 -0
- package/dist/proxy/oxylabs.d.ts +15 -0
- package/dist/proxy/oxylabs.d.ts.map +1 -0
- package/dist/proxy/oxylabs.js +34 -0
- package/dist/proxy/proxyService.d.ts +34 -0
- package/dist/proxy/proxyService.d.ts.map +1 -0
- package/dist/proxy/proxyService.js +69 -0
- package/dist/proxy/types.d.ts +59 -0
- package/dist/proxy/types.d.ts.map +1 -0
- package/dist/proxy/types.js +4 -0
- package/dist/registry/client.d.ts +33 -0
- package/dist/registry/client.d.ts.map +1 -0
- package/dist/registry/client.js +261 -0
- package/dist/registry/index.d.ts +4 -0
- package/dist/registry/index.d.ts.map +1 -0
- package/dist/registry/index.js +3 -0
- package/dist/registry/tokenStore.d.ts +13 -0
- package/dist/registry/tokenStore.d.ts.map +1 -0
- package/dist/registry/tokenStore.js +54 -0
- package/dist/registry/types.d.ts +110 -0
- package/dist/registry/types.d.ts.map +1 -0
- package/dist/registry/types.js +4 -0
- package/dist/runner.d.ts +4 -0
- package/dist/runner.d.ts.map +1 -1
- package/dist/runner.js +18 -8
- package/dist/types.d.ts +6 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +1 -1
package/dist/dsl/validation.js
CHANGED
|
@@ -235,6 +235,7 @@ const ALLOWED_PARAMS = {
|
|
|
235
235
|
frame: ['frame', 'action'],
|
|
236
236
|
new_tab: ['url', 'saveTabIndexAs'],
|
|
237
237
|
switch_tab: ['tab', 'closeCurrentTab'],
|
|
238
|
+
dom_scrape: ['selector', 'target', 'collect', 'skip_empty', 'out', 'hint', 'scope', 'near'],
|
|
238
239
|
};
|
|
239
240
|
/**
|
|
240
241
|
* Validates a single step, pushing errors to the array.
|
|
@@ -922,8 +923,96 @@ function validateStep(step, stepIndex, errors) {
|
|
|
922
923
|
errors.push(`${prefix}: SwitchTab step "closeCurrentTab" must be a boolean`);
|
|
923
924
|
}
|
|
924
925
|
break;
|
|
926
|
+
case 'dom_scrape': {
|
|
927
|
+
// Must have either selector (legacy) or target (new)
|
|
928
|
+
if (!params.selector && !params.target) {
|
|
929
|
+
errors.push(`${prefix}: DomScrape step must have either "selector" or "target" in params`);
|
|
930
|
+
}
|
|
931
|
+
if (params.selector !== undefined && typeof params.selector !== 'string') {
|
|
932
|
+
errors.push(`${prefix}: DomScrape step "selector" must be a string`);
|
|
933
|
+
}
|
|
934
|
+
if (params.target !== undefined) {
|
|
935
|
+
validateTargetOrAnyOf(params.target, errors, prefix);
|
|
936
|
+
}
|
|
937
|
+
if (typeof params.out !== 'string' || !params.out) {
|
|
938
|
+
errors.push(`${prefix}: DomScrape step must have a non-empty string "out" in params`);
|
|
939
|
+
}
|
|
940
|
+
if (!Array.isArray(params.collect) || params.collect.length === 0) {
|
|
941
|
+
errors.push(`${prefix}: DomScrape step must have a non-empty "collect" array in params`);
|
|
942
|
+
}
|
|
943
|
+
else {
|
|
944
|
+
const seenKeys = new Set();
|
|
945
|
+
const ALLOWED_COLLECT_FIELDS = new Set(['key', 'target', 'extract', 'attribute']);
|
|
946
|
+
for (let ci = 0; ci < params.collect.length; ci++) {
|
|
947
|
+
const field = params.collect[ci];
|
|
948
|
+
if (!field || typeof field !== 'object') {
|
|
949
|
+
errors.push(`${prefix}: DomScrape collect[${ci}] must be an object`);
|
|
950
|
+
continue;
|
|
951
|
+
}
|
|
952
|
+
const f = field;
|
|
953
|
+
// Check for unknown fields
|
|
954
|
+
const unknownFields = Object.keys(f).filter(k => !ALLOWED_COLLECT_FIELDS.has(k));
|
|
955
|
+
if (unknownFields.length > 0) {
|
|
956
|
+
errors.push(`${prefix}: DomScrape collect[${ci}] has unknown field(s): ${unknownFields.map(k => `"${k}"`).join(', ')}. Allowed: key, target, extract, attribute`);
|
|
957
|
+
}
|
|
958
|
+
// key
|
|
959
|
+
if (typeof f.key !== 'string' || !f.key) {
|
|
960
|
+
errors.push(`${prefix}: DomScrape collect[${ci}] must have a non-empty string "key"`);
|
|
961
|
+
}
|
|
962
|
+
else {
|
|
963
|
+
if (seenKeys.has(f.key)) {
|
|
964
|
+
errors.push(`${prefix}: DomScrape collect has duplicate key "${f.key}"`);
|
|
965
|
+
}
|
|
966
|
+
seenKeys.add(f.key);
|
|
967
|
+
}
|
|
968
|
+
// target
|
|
969
|
+
if (!f.target) {
|
|
970
|
+
errors.push(`${prefix}: DomScrape collect[${ci}] must have a "target"`);
|
|
971
|
+
}
|
|
972
|
+
else {
|
|
973
|
+
validateTarget(f.target, errors, `${prefix} collect[${ci}]`);
|
|
974
|
+
}
|
|
975
|
+
// extract
|
|
976
|
+
if (f.extract !== undefined) {
|
|
977
|
+
if (f.extract !== 'text' && f.extract !== 'attribute' && f.extract !== 'html') {
|
|
978
|
+
errors.push(`${prefix}: DomScrape collect[${ci}] "extract" must be "text", "attribute", or "html"`);
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
// attribute required when extract === 'attribute'
|
|
982
|
+
if (f.extract === 'attribute') {
|
|
983
|
+
if (typeof f.attribute !== 'string' || !f.attribute) {
|
|
984
|
+
errors.push(`${prefix}: DomScrape collect[${ci}] requires "attribute" when extract is "attribute"`);
|
|
985
|
+
}
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
}
|
|
989
|
+
if (params.skip_empty !== undefined && typeof params.skip_empty !== 'boolean') {
|
|
990
|
+
errors.push(`${prefix}: DomScrape step "skip_empty" must be a boolean`);
|
|
991
|
+
}
|
|
992
|
+
if (params.hint !== undefined && typeof params.hint !== 'string') {
|
|
993
|
+
errors.push(`${prefix}: DomScrape step "hint" must be a string`);
|
|
994
|
+
}
|
|
995
|
+
if (params.scope !== undefined) {
|
|
996
|
+
validateTarget(params.scope, errors, prefix);
|
|
997
|
+
}
|
|
998
|
+
if (params.near !== undefined && params.near !== null) {
|
|
999
|
+
const near = params.near;
|
|
1000
|
+
if (typeof near !== 'object' || near.kind !== 'text') {
|
|
1001
|
+
errors.push(`${prefix}: DomScrape step "near" must be an object with kind: "text"`);
|
|
1002
|
+
}
|
|
1003
|
+
else {
|
|
1004
|
+
if (typeof near.text !== 'string') {
|
|
1005
|
+
errors.push(`${prefix}: DomScrape step "near.text" must be a string`);
|
|
1006
|
+
}
|
|
1007
|
+
if (near.exact !== undefined && typeof near.exact !== 'boolean') {
|
|
1008
|
+
errors.push(`${prefix}: DomScrape step "near.exact" must be a boolean`);
|
|
1009
|
+
}
|
|
1010
|
+
}
|
|
1011
|
+
}
|
|
1012
|
+
break;
|
|
1013
|
+
}
|
|
925
1014
|
default:
|
|
926
|
-
errors.push(`${prefix}: Unknown step type: ${s.type}. Supported types: navigate, extract_title, extract_text, extract_attribute, sleep, wait_for, click, fill, assert, set_var, network_find, network_replay, network_extract, select_option, press_key, upload_file, frame, new_tab, switch_tab`);
|
|
1015
|
+
errors.push(`${prefix}: Unknown step type: ${s.type}. Supported types: navigate, extract_title, extract_text, extract_attribute, sleep, wait_for, click, fill, assert, set_var, network_find, network_replay, network_extract, select_option, press_key, upload_file, frame, new_tab, switch_tab, dom_scrape`);
|
|
927
1016
|
}
|
|
928
1017
|
// Check for unknown params
|
|
929
1018
|
const allowed = ALLOWED_PARAMS[s.type];
|
package/dist/httpReplay.d.ts
CHANGED
|
@@ -7,6 +7,7 @@
|
|
|
7
7
|
*/
|
|
8
8
|
import type { DslStep } from './dsl/types.js';
|
|
9
9
|
import { type RequestSnapshot, type SnapshotFile, type ValidationResult } from './requestSnapshot.js';
|
|
10
|
+
import type { ResolvedProxy } from './proxy/types.js';
|
|
10
11
|
export interface HttpReplayResult {
|
|
11
12
|
status: number;
|
|
12
13
|
contentType?: string;
|
|
@@ -19,6 +20,8 @@ export interface HttpReplayResult {
|
|
|
19
20
|
* Requirements:
|
|
20
21
|
* 1. Every `network_replay` step has a corresponding, non-stale snapshot.
|
|
21
22
|
* 2. No DOM extraction steps exist in the flow.
|
|
23
|
+
* 3. No skipped steps contain dynamic templates — templates in skipped steps
|
|
24
|
+
* would never be evaluated, so the snapshot replays stale data.
|
|
22
25
|
*/
|
|
23
26
|
export declare function isFlowHttpCompatible(steps: DslStep[], snapshots: SnapshotFile | null): boolean;
|
|
24
27
|
/**
|
|
@@ -28,6 +31,7 @@ export declare function isFlowHttpCompatible(steps: DslStep[], snapshots: Snapsh
|
|
|
28
31
|
export declare function replayFromSnapshot(snapshot: RequestSnapshot, inputs: Record<string, unknown>, vars: Record<string, unknown>, options?: {
|
|
29
32
|
secrets?: Record<string, string>;
|
|
30
33
|
timeoutMs?: number;
|
|
34
|
+
proxy?: ResolvedProxy;
|
|
31
35
|
}): Promise<HttpReplayResult>;
|
|
32
36
|
/**
|
|
33
37
|
* Replay a snapshot and validate the response.
|
|
@@ -36,6 +40,7 @@ export declare function replayFromSnapshot(snapshot: RequestSnapshot, inputs: Re
|
|
|
36
40
|
export declare function replayAndValidate(snapshot: RequestSnapshot, inputs: Record<string, unknown>, vars: Record<string, unknown>, options?: {
|
|
37
41
|
secrets?: Record<string, string>;
|
|
38
42
|
timeoutMs?: number;
|
|
43
|
+
proxy?: ResolvedProxy;
|
|
39
44
|
}): Promise<{
|
|
40
45
|
result: HttpReplayResult;
|
|
41
46
|
validation: ValidationResult;
|
package/dist/httpReplay.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"httpReplay.d.ts","sourceRoot":"","sources":["../src/httpReplay.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,YAAY,EAIjB,KAAK,gBAAgB,EACtB,MAAM,sBAAsB,CAAC;
|
|
1
|
+
{"version":3,"file":"httpReplay.d.ts","sourceRoot":"","sources":["../src/httpReplay.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,gBAAgB,CAAC;AAC9C,OAAO,EACL,KAAK,eAAe,EACpB,KAAK,YAAY,EAIjB,KAAK,gBAAgB,EACtB,MAAM,sBAAsB,CAAC;AAC9B,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAatD,MAAM,WAAW,gBAAgB;IAC/B,MAAM,EAAE,MAAM,CAAC;IACf,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CAClB;AA6BD;;;;;;;;GAQG;AACH,wBAAgB,oBAAoB,CAClC,KAAK,EAAE,OAAO,EAAE,EAChB,SAAS,EAAE,YAAY,GAAG,IAAI,GAC7B,OAAO,CA4BT;AAMD;;;GAGG;AACH,wBAAsB,kBAAkB,CACtC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,OAAO,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,aAAa,CAAA;CAAE,GACxF,OAAO,CAAC,gBAAgB,CAAC,CA6D3B;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,QAAQ,EAAE,eAAe,EACzB,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC/B,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAC7B,OAAO,CAAC,EAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAAC,SAAS,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,CAAC,EAAE,aAAa,CAAA;CAAE,GACxF,OAAO,CAAC;IAAE,MAAM,EAAE,gBAAgB,CAAC;IAAC,UAAU,EAAE,gBAAgB,CAAA;CAAE,CAAC,CAIrE"}
|
package/dist/httpReplay.js
CHANGED
|
@@ -15,22 +15,50 @@ const DEFAULT_TIMEOUT_MS = 30_000;
|
|
|
15
15
|
// HTTP-only compatibility check
|
|
16
16
|
// ---------------------------------------------------------------------------
|
|
17
17
|
/** Step types that require DOM access for data extraction (force browser mode). */
|
|
18
|
-
const DOM_EXTRACTION_STEPS = new Set(['extract_text', 'extract_title', 'extract_attribute']);
|
|
18
|
+
const DOM_EXTRACTION_STEPS = new Set(['extract_text', 'extract_title', 'extract_attribute', 'dom_scrape']);
|
|
19
|
+
/**
|
|
20
|
+
* Step types that are silently skipped in HTTP mode.
|
|
21
|
+
* Must match HTTP_MODE_SKIP_STEPS in stepHandlers.ts.
|
|
22
|
+
*/
|
|
23
|
+
const HTTP_SKIPPED_STEPS = new Set([
|
|
24
|
+
'navigate', 'click', 'fill', 'select_option', 'press_key',
|
|
25
|
+
'upload_file', 'wait_for', 'assert', 'frame', 'new_tab',
|
|
26
|
+
'switch_tab', 'network_find', 'dom_scrape',
|
|
27
|
+
]);
|
|
28
|
+
/** Check if a value contains Nunjucks template expressions. */
|
|
29
|
+
function containsTemplate(value) {
|
|
30
|
+
if (typeof value === 'string')
|
|
31
|
+
return value.includes('{{');
|
|
32
|
+
if (Array.isArray(value))
|
|
33
|
+
return value.some(containsTemplate);
|
|
34
|
+
if (value && typeof value === 'object') {
|
|
35
|
+
return Object.values(value).some(containsTemplate);
|
|
36
|
+
}
|
|
37
|
+
return false;
|
|
38
|
+
}
|
|
19
39
|
/**
|
|
20
40
|
* Check whether a flow can run in HTTP-only mode.
|
|
21
41
|
*
|
|
22
42
|
* Requirements:
|
|
23
43
|
* 1. Every `network_replay` step has a corresponding, non-stale snapshot.
|
|
24
44
|
* 2. No DOM extraction steps exist in the flow.
|
|
45
|
+
* 3. No skipped steps contain dynamic templates — templates in skipped steps
|
|
46
|
+
* would never be evaluated, so the snapshot replays stale data.
|
|
25
47
|
*/
|
|
26
48
|
export function isFlowHttpCompatible(steps, snapshots) {
|
|
27
49
|
if (!snapshots)
|
|
28
50
|
return false;
|
|
29
|
-
// Check for DOM extraction steps
|
|
30
51
|
for (const step of steps) {
|
|
52
|
+
// DOM extraction steps force browser mode
|
|
31
53
|
if (DOM_EXTRACTION_STEPS.has(step.type)) {
|
|
32
54
|
return false;
|
|
33
55
|
}
|
|
56
|
+
// Steps skipped in HTTP mode must not contain templates — those templates
|
|
57
|
+
// affect what data the API returns but would never be evaluated, causing
|
|
58
|
+
// the snapshot to replay stale/wrong data regardless of input values.
|
|
59
|
+
if (HTTP_SKIPPED_STEPS.has(step.type) && containsTemplate(step.params)) {
|
|
60
|
+
return false;
|
|
61
|
+
}
|
|
34
62
|
}
|
|
35
63
|
// Check that every network_replay step has a valid snapshot
|
|
36
64
|
const replaySteps = steps.filter((s) => s.type === 'network_replay');
|
|
@@ -70,6 +98,22 @@ export async function replayFromSnapshot(snapshot, inputs, vars, options) {
|
|
|
70
98
|
if (body && method !== 'GET' && method !== 'HEAD') {
|
|
71
99
|
fetchOptions.body = body;
|
|
72
100
|
}
|
|
101
|
+
// When proxy is provided, create a ProxyAgent dispatcher for undici-backed fetch.
|
|
102
|
+
// undici is bundled with Node but may not have separate type declarations.
|
|
103
|
+
if (options?.proxy) {
|
|
104
|
+
try {
|
|
105
|
+
// @ts-expect-error undici types may not be installed; runtime import is fine
|
|
106
|
+
const undiciModule = await import('undici');
|
|
107
|
+
const ProxyAgentClass = undiciModule.ProxyAgent;
|
|
108
|
+
if (ProxyAgentClass) {
|
|
109
|
+
const proxyUrl = options.proxy.server.replace('://', `://${encodeURIComponent(options.proxy.username)}:${encodeURIComponent(options.proxy.password)}@`);
|
|
110
|
+
fetchOptions.dispatcher = new ProxyAgentClass(proxyUrl);
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
catch {
|
|
114
|
+
console.warn('[httpReplay] Failed to load undici ProxyAgent, making direct request');
|
|
115
|
+
}
|
|
116
|
+
}
|
|
73
117
|
try {
|
|
74
118
|
const response = await fetch(url, fetchOptions);
|
|
75
119
|
const responseBody = await response.text();
|
package/dist/index.d.ts
CHANGED
|
@@ -13,6 +13,7 @@ export * from './packVersioning.js';
|
|
|
13
13
|
export * from './config.js';
|
|
14
14
|
export * from './requestSnapshot.js';
|
|
15
15
|
export * from './httpReplay.js';
|
|
16
|
+
export * from './proxy/index.js';
|
|
16
17
|
export * from './storage/index.js';
|
|
17
18
|
export * from './dsl/types.js';
|
|
18
19
|
export * from './dsl/builders.js';
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,yBAAyB,CAAC;AACxC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,qBAAqB,CAAC;AACpC,cAAc,aAAa,CAAC;AAC5B,cAAc,sBAAsB,CAAC;AACrC,cAAc,iBAAiB,CAAC;AAGhC,cAAc,oBAAoB,CAAC;AAGnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC"}
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,cAAc,qBAAqB,CAAC;AACpC,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,aAAa,CAAC;AAC5B,cAAc,wBAAwB,CAAC;AACvC,cAAc,qBAAqB,CAAC;AACpC,cAAc,sBAAsB,CAAC;AACrC,cAAc,yBAAyB,CAAC;AACxC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,qBAAqB,CAAC;AACpC,cAAc,aAAa,CAAC;AAC5B,cAAc,sBAAsB,CAAC;AACrC,cAAc,iBAAiB,CAAC;AAGhC,cAAc,kBAAkB,CAAC;AAGjC,cAAc,oBAAoB,CAAC;AAGnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,qBAAqB,CAAC;AACpC,cAAc,qBAAqB,CAAC;AACpC,cAAc,iBAAiB,CAAC;AAChC,cAAc,qBAAqB,CAAC"}
|
package/dist/index.js
CHANGED
|
@@ -13,6 +13,8 @@ export * from './packVersioning.js';
|
|
|
13
13
|
export * from './config.js';
|
|
14
14
|
export * from './requestSnapshot.js';
|
|
15
15
|
export * from './httpReplay.js';
|
|
16
|
+
// Proxy exports
|
|
17
|
+
export * from './proxy/index.js';
|
|
16
18
|
// Storage exports
|
|
17
19
|
export * from './storage/index.js';
|
|
18
20
|
// DSL exports
|
|
@@ -2,7 +2,7 @@ import { validateFlow, ValidationError } from './dsl/validation.js';
|
|
|
2
2
|
/** Step types that produce collectible output via an "out" parameter */
|
|
3
3
|
const STEPS_WITH_OUT = new Set([
|
|
4
4
|
'extract_title', 'extract_text', 'extract_attribute',
|
|
5
|
-
'network_replay', 'network_extract',
|
|
5
|
+
'network_replay', 'network_extract', 'dom_scrape',
|
|
6
6
|
]);
|
|
7
7
|
/**
|
|
8
8
|
* Validates that collectibles referenced in flow steps exist
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/proxy/index.ts"],"names":[],"mappings":"AAAA,cAAc,YAAY,CAAC;AAC3B,cAAc,cAAc,CAAC;AAC7B,cAAc,mBAAmB,CAAC"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OxyLabs residential proxy provider.
|
|
3
|
+
*
|
|
4
|
+
* Endpoint: pr.oxylabs.io:7777
|
|
5
|
+
* Username format:
|
|
6
|
+
* Random: customer-{USERNAME}[-cc-{COUNTRY}]
|
|
7
|
+
* Session: customer-{USERNAME}[-cc-{COUNTRY}]-sessid-{UUID}-sesstime-{MIN}
|
|
8
|
+
*/
|
|
9
|
+
import type { ProxyConfig, ProxyCredentials, ProxyProvider, ResolvedProxy } from './types.js';
|
|
10
|
+
export declare class OxylabsProvider implements ProxyProvider {
|
|
11
|
+
readonly name = "oxylabs";
|
|
12
|
+
resolve(config: ProxyConfig, credentials: ProxyCredentials): ResolvedProxy;
|
|
13
|
+
requiredCredentialKeys(): string[];
|
|
14
|
+
}
|
|
15
|
+
//# sourceMappingURL=oxylabs.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"oxylabs.d.ts","sourceRoot":"","sources":["../../src/proxy/oxylabs.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,WAAW,EAAE,gBAAgB,EAAE,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAK9F,qBAAa,eAAgB,YAAW,aAAa;IACnD,QAAQ,CAAC,IAAI,aAAa;IAE1B,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,GAAG,aAAa;IAqB1E,sBAAsB,IAAI,MAAM,EAAE;CAGnC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OxyLabs residential proxy provider.
|
|
3
|
+
*
|
|
4
|
+
* Endpoint: pr.oxylabs.io:7777
|
|
5
|
+
* Username format:
|
|
6
|
+
* Random: customer-{USERNAME}[-cc-{COUNTRY}]
|
|
7
|
+
* Session: customer-{USERNAME}[-cc-{COUNTRY}]-sessid-{UUID}-sesstime-{MIN}
|
|
8
|
+
*/
|
|
9
|
+
import { randomUUID } from 'crypto';
|
|
10
|
+
const OXYLABS_ENDPOINT = 'http://pr.oxylabs.io:7777';
|
|
11
|
+
const DEFAULT_SESSION_DURATION_MINUTES = 10;
|
|
12
|
+
export class OxylabsProvider {
|
|
13
|
+
name = 'oxylabs';
|
|
14
|
+
resolve(config, credentials) {
|
|
15
|
+
const mode = config.mode ?? 'session';
|
|
16
|
+
let username = `customer-${credentials.username}`;
|
|
17
|
+
if (config.country) {
|
|
18
|
+
username += `-cc-${config.country.toUpperCase()}`;
|
|
19
|
+
}
|
|
20
|
+
if (mode === 'session') {
|
|
21
|
+
const sessionId = randomUUID().replace(/-/g, '');
|
|
22
|
+
const duration = config.sessionDurationMinutes ?? DEFAULT_SESSION_DURATION_MINUTES;
|
|
23
|
+
username += `-sessid-${sessionId}-sesstime-${duration}`;
|
|
24
|
+
}
|
|
25
|
+
return {
|
|
26
|
+
server: OXYLABS_ENDPOINT,
|
|
27
|
+
username,
|
|
28
|
+
password: credentials.password,
|
|
29
|
+
};
|
|
30
|
+
}
|
|
31
|
+
requiredCredentialKeys() {
|
|
32
|
+
return ['USERNAME', 'PASSWORD'];
|
|
33
|
+
}
|
|
34
|
+
}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Proxy service: provider registry and resolution.
|
|
3
|
+
*
|
|
4
|
+
* Reads credentials from environment variables:
|
|
5
|
+
* SHOWRUN_PROXY_USERNAME, SHOWRUN_PROXY_PASSWORD
|
|
6
|
+
* Provider from config or SHOWRUN_PROXY_PROVIDER env var (default: 'oxylabs').
|
|
7
|
+
*
|
|
8
|
+
* Graceful fallback: if env vars are missing, returns null (no proxy) instead of throwing.
|
|
9
|
+
*/
|
|
10
|
+
import type { ProxyConfig, ProxyProvider, ResolvedProxy } from './types.js';
|
|
11
|
+
/**
|
|
12
|
+
* Register a custom proxy provider.
|
|
13
|
+
*/
|
|
14
|
+
export declare function registerProxyProvider(provider: ProxyProvider): void;
|
|
15
|
+
/**
|
|
16
|
+
* Get a registered proxy provider by name.
|
|
17
|
+
*/
|
|
18
|
+
export declare function getProxyProvider(name: string): ProxyProvider | undefined;
|
|
19
|
+
/**
|
|
20
|
+
* List all registered proxy provider names.
|
|
21
|
+
*/
|
|
22
|
+
export declare function listProxyProviders(): string[];
|
|
23
|
+
/**
|
|
24
|
+
* Resolve a proxy configuration into concrete connection details.
|
|
25
|
+
*
|
|
26
|
+
* Returns `null` when:
|
|
27
|
+
* - config is undefined or disabled
|
|
28
|
+
* - required env vars are missing (graceful fallback, logs warning)
|
|
29
|
+
*
|
|
30
|
+
* Throws when:
|
|
31
|
+
* - provider name is unknown
|
|
32
|
+
*/
|
|
33
|
+
export declare function resolveProxy(config: ProxyConfig | undefined): ResolvedProxy | null;
|
|
34
|
+
//# sourceMappingURL=proxyService.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"proxyService.d.ts","sourceRoot":"","sources":["../../src/proxy/proxyService.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,KAAK,EAAE,WAAW,EAAoB,aAAa,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAW9F;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,QAAQ,EAAE,aAAa,GAAG,IAAI,CAEnE;AAED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,IAAI,EAAE,MAAM,GAAG,aAAa,GAAG,SAAS,CAExE;AAED;;GAEG;AACH,wBAAgB,kBAAkB,IAAI,MAAM,EAAE,CAE7C;AAaD;;;;;;;;;GASG;AACH,wBAAgB,YAAY,CAAC,MAAM,EAAE,WAAW,GAAG,SAAS,GAAG,aAAa,GAAG,IAAI,CAsBlF"}
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Proxy service: provider registry and resolution.
|
|
3
|
+
*
|
|
4
|
+
* Reads credentials from environment variables:
|
|
5
|
+
* SHOWRUN_PROXY_USERNAME, SHOWRUN_PROXY_PASSWORD
|
|
6
|
+
* Provider from config or SHOWRUN_PROXY_PROVIDER env var (default: 'oxylabs').
|
|
7
|
+
*
|
|
8
|
+
* Graceful fallback: if env vars are missing, returns null (no proxy) instead of throwing.
|
|
9
|
+
*/
|
|
10
|
+
import { OxylabsProvider } from './oxylabs.js';
|
|
11
|
+
// ── Provider registry ──────────────────────────────────────────────────
|
|
12
|
+
const providers = new Map();
|
|
13
|
+
// Register OxyLabs by default
|
|
14
|
+
const defaultProvider = new OxylabsProvider();
|
|
15
|
+
providers.set(defaultProvider.name, defaultProvider);
|
|
16
|
+
/**
|
|
17
|
+
* Register a custom proxy provider.
|
|
18
|
+
*/
|
|
19
|
+
export function registerProxyProvider(provider) {
|
|
20
|
+
providers.set(provider.name, provider);
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Get a registered proxy provider by name.
|
|
24
|
+
*/
|
|
25
|
+
export function getProxyProvider(name) {
|
|
26
|
+
return providers.get(name);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* List all registered proxy provider names.
|
|
30
|
+
*/
|
|
31
|
+
export function listProxyProviders() {
|
|
32
|
+
return [...providers.keys()];
|
|
33
|
+
}
|
|
34
|
+
// ── Credential helpers ─────────────────────────────────────────────────
|
|
35
|
+
function readCredentialsFromEnv() {
|
|
36
|
+
const username = process.env.SHOWRUN_PROXY_USERNAME;
|
|
37
|
+
const password = process.env.SHOWRUN_PROXY_PASSWORD;
|
|
38
|
+
if (!username || !password)
|
|
39
|
+
return null;
|
|
40
|
+
return { username, password };
|
|
41
|
+
}
|
|
42
|
+
// ── Resolution ─────────────────────────────────────────────────────────
|
|
43
|
+
/**
|
|
44
|
+
* Resolve a proxy configuration into concrete connection details.
|
|
45
|
+
*
|
|
46
|
+
* Returns `null` when:
|
|
47
|
+
* - config is undefined or disabled
|
|
48
|
+
* - required env vars are missing (graceful fallback, logs warning)
|
|
49
|
+
*
|
|
50
|
+
* Throws when:
|
|
51
|
+
* - provider name is unknown
|
|
52
|
+
*/
|
|
53
|
+
export function resolveProxy(config) {
|
|
54
|
+
if (!config || !config.enabled)
|
|
55
|
+
return null;
|
|
56
|
+
const providerName = config.provider ?? process.env.SHOWRUN_PROXY_PROVIDER ?? 'oxylabs';
|
|
57
|
+
const provider = providers.get(providerName);
|
|
58
|
+
if (!provider) {
|
|
59
|
+
throw new Error(`Unknown proxy provider "${providerName}". Registered providers: ${listProxyProviders().join(', ')}`);
|
|
60
|
+
}
|
|
61
|
+
const credentials = readCredentialsFromEnv();
|
|
62
|
+
if (!credentials) {
|
|
63
|
+
console.warn(`[proxy] Proxy enabled but credentials not configured. ` +
|
|
64
|
+
`Set SHOWRUN_PROXY_USERNAME and SHOWRUN_PROXY_PASSWORD environment variables. ` +
|
|
65
|
+
`Running without proxy.`);
|
|
66
|
+
return null;
|
|
67
|
+
}
|
|
68
|
+
return provider.resolve(config, credentials);
|
|
69
|
+
}
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Proxy configuration types for browser and HTTP request proxying.
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Proxy mode determines how proxy sessions are managed:
|
|
6
|
+
* - 'session': Sticky IP for the duration of the session (default)
|
|
7
|
+
* - 'random': Rotating IP per request
|
|
8
|
+
*/
|
|
9
|
+
export type ProxyMode = 'session' | 'random';
|
|
10
|
+
/**
|
|
11
|
+
* Proxy configuration stored in taskpack.json under `browser.proxy`.
|
|
12
|
+
* Persisted by the agent's `set_proxy` tool.
|
|
13
|
+
*/
|
|
14
|
+
export interface ProxyConfig {
|
|
15
|
+
/** Whether proxy is enabled */
|
|
16
|
+
enabled: boolean;
|
|
17
|
+
/** Proxy mode: sticky session or random rotation */
|
|
18
|
+
mode?: ProxyMode;
|
|
19
|
+
/** Provider name (default: 'oxylabs') */
|
|
20
|
+
provider?: string;
|
|
21
|
+
/** Two-letter ISO country code for geo-targeting */
|
|
22
|
+
country?: string;
|
|
23
|
+
/** Session duration in minutes (for session mode) */
|
|
24
|
+
sessionDurationMinutes?: number;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* Resolved proxy credentials ready for Playwright / fetch.
|
|
28
|
+
* Maps directly to Playwright's `proxy` launch option.
|
|
29
|
+
*/
|
|
30
|
+
export interface ResolvedProxy {
|
|
31
|
+
server: string;
|
|
32
|
+
username: string;
|
|
33
|
+
password: string;
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Credentials supplied to a proxy provider (from env vars).
|
|
37
|
+
*/
|
|
38
|
+
export interface ProxyCredentials {
|
|
39
|
+
username: string;
|
|
40
|
+
password: string;
|
|
41
|
+
}
|
|
42
|
+
/**
|
|
43
|
+
* Interface for pluggable proxy providers.
|
|
44
|
+
* Implement this to add support for a new proxy service.
|
|
45
|
+
*/
|
|
46
|
+
export interface ProxyProvider {
|
|
47
|
+
/** Provider name (e.g. 'oxylabs') */
|
|
48
|
+
readonly name: string;
|
|
49
|
+
/**
|
|
50
|
+
* Resolve proxy config + credentials into a concrete proxy connection.
|
|
51
|
+
*/
|
|
52
|
+
resolve(config: ProxyConfig, credentials: ProxyCredentials): ResolvedProxy;
|
|
53
|
+
/**
|
|
54
|
+
* List of env var suffixes this provider needs (for display/validation).
|
|
55
|
+
* e.g. ['USERNAME', 'PASSWORD'] → expects SHOWRUN_PROXY_USERNAME, SHOWRUN_PROXY_PASSWORD
|
|
56
|
+
*/
|
|
57
|
+
requiredCredentialKeys(): string[];
|
|
58
|
+
}
|
|
59
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../src/proxy/types.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH;;;;GAIG;AACH,MAAM,MAAM,SAAS,GAAG,SAAS,GAAG,QAAQ,CAAC;AAE7C;;;GAGG;AACH,MAAM,WAAW,WAAW;IAC1B,+BAA+B;IAC/B,OAAO,EAAE,OAAO,CAAC;IACjB,oDAAoD;IACpD,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,yCAAyC;IACzC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,oDAAoD;IACpD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qDAAqD;IACrD,sBAAsB,CAAC,EAAE,MAAM,CAAC;CACjC;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE,MAAM,CAAC;IACf,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;GAEG;AACH,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;GAGG;AACH,MAAM,WAAW,aAAa;IAC5B,qCAAqC;IACrC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;OAEG;IACH,OAAO,CAAC,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,gBAAgB,GAAG,aAAa,CAAC;IAE3E;;;OAGG;IACH,sBAAsB,IAAI,MAAM,EAAE,CAAC;CACpC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Registry client for ShowRun pack registry.
|
|
3
|
+
*
|
|
4
|
+
* Communicates with the registry REST API to authenticate, publish, search,
|
|
5
|
+
* and install task packs. Uses the global token store for persistence and
|
|
6
|
+
* automatically refreshes access tokens before they expire.
|
|
7
|
+
*/
|
|
8
|
+
import type { IRegistryClient, DeviceCodeResponse, DevicePollResult, UserProfile, PublishParams, PublishResult, PackSummary, PaginatedResponse, SearchQuery } from './types.js';
|
|
9
|
+
export declare class RegistryError extends Error {
|
|
10
|
+
readonly status: number;
|
|
11
|
+
readonly body?: unknown | undefined;
|
|
12
|
+
constructor(message: string, status: number, body?: unknown | undefined);
|
|
13
|
+
}
|
|
14
|
+
export declare class RegistryClient implements IRegistryClient {
|
|
15
|
+
private readonly registryUrl;
|
|
16
|
+
constructor(registryUrl?: string);
|
|
17
|
+
startDeviceLogin(): Promise<DeviceCodeResponse>;
|
|
18
|
+
pollDeviceLogin(deviceCode: string): Promise<DevicePollResult>;
|
|
19
|
+
logout(): Promise<void>;
|
|
20
|
+
whoami(): Promise<UserProfile>;
|
|
21
|
+
isAuthenticated(): boolean;
|
|
22
|
+
publishPack(params: PublishParams): Promise<PublishResult>;
|
|
23
|
+
searchPacks(query: SearchQuery): Promise<PaginatedResponse<PackSummary>>;
|
|
24
|
+
installPack(slug: string, destDir: string, version?: string): Promise<void>;
|
|
25
|
+
private request;
|
|
26
|
+
/**
|
|
27
|
+
* Get a valid access token, refreshing if it's about to expire (within 60s).
|
|
28
|
+
* If refresh fails, clears tokens and throws.
|
|
29
|
+
*/
|
|
30
|
+
private getValidAccessToken;
|
|
31
|
+
private refreshAccessToken;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=client.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"client.d.ts","sourceRoot":"","sources":["../../src/registry/client.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAMH,OAAO,KAAK,EACV,eAAe,EACf,kBAAkB,EAClB,gBAAgB,EAGhB,WAAW,EACX,aAAa,EACb,aAAa,EACb,WAAW,EAEX,iBAAiB,EACjB,WAAW,EAEZ,MAAM,YAAY,CAAC;AAMpB,qBAAa,aAAc,SAAQ,KAAK;aAGpB,MAAM,EAAE,MAAM;aACd,IAAI,CAAC,EAAE,OAAO;gBAF9B,OAAO,EAAE,MAAM,EACC,MAAM,EAAE,MAAM,EACd,IAAI,CAAC,EAAE,OAAO,YAAA;CAKjC;AAwBD,qBAAa,cAAe,YAAW,eAAe;IACpD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAS;gBAEzB,WAAW,CAAC,EAAE,MAAM;IAc1B,gBAAgB,IAAI,OAAO,CAAC,kBAAkB,CAAC;IAS/C,eAAe,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,gBAAgB,CAAC;IA+C9D,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAIvB,MAAM,IAAI,OAAO,CAAC,WAAW,CAAC;IAIpC,eAAe,IAAI,OAAO;IAOpB,WAAW,CAAC,MAAM,EAAE,aAAa,GAAG,OAAO,CAAC,aAAa,CAAC;IAiE1D,WAAW,CAAC,KAAK,EAAE,WAAW,GAAG,OAAO,CAAC,iBAAiB,CAAC,WAAW,CAAC,CAAC;IAWxE,WAAW,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;YA0BnE,OAAO;IAwCrB;;;OAGG;YACW,mBAAmB;YAkCnB,kBAAkB;CAcjC"}
|