@tangle-network/browser-agent-driver 0.22.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +357 -126
- package/dist/brain/index.d.ts +27 -0
- package/dist/brain/index.d.ts.map +1 -1
- package/dist/brain/index.js +492 -41
- package/dist/brain/index.js.map +1 -1
- package/dist/browser-launch.d.ts +2 -0
- package/dist/browser-launch.d.ts.map +1 -1
- package/dist/browser-launch.js +12 -5
- package/dist/browser-launch.js.map +1 -1
- package/dist/captcha.d.ts +0 -7
- package/dist/captcha.d.ts.map +1 -1
- package/dist/captcha.js +113 -4
- package/dist/captcha.js.map +1 -1
- package/dist/cli.js +69 -13
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts +7 -0
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/drivers/cursor-overlay.d.ts +1 -1
- package/dist/drivers/cursor-overlay.d.ts.map +1 -1
- package/dist/drivers/cursor-overlay.js +21 -9
- package/dist/drivers/cursor-overlay.js.map +1 -1
- package/dist/drivers/extract-with-index.d.ts +45 -0
- package/dist/drivers/extract-with-index.d.ts.map +1 -0
- package/dist/drivers/extract-with-index.js +141 -0
- package/dist/drivers/extract-with-index.js.map +1 -0
- package/dist/drivers/playwright.d.ts +7 -0
- package/dist/drivers/playwright.d.ts.map +1 -1
- package/dist/drivers/playwright.js +232 -1
- package/dist/drivers/playwright.js.map +1 -1
- package/dist/drivers/som-overlay.d.ts +36 -0
- package/dist/drivers/som-overlay.d.ts.map +1 -0
- package/dist/drivers/som-overlay.js +109 -0
- package/dist/drivers/som-overlay.js.map +1 -0
- package/dist/drivers/types.d.ts +2 -0
- package/dist/drivers/types.d.ts.map +1 -1
- package/dist/memory/knowledge.d.ts +6 -0
- package/dist/memory/knowledge.d.ts.map +1 -1
- package/dist/memory/knowledge.js +15 -0
- package/dist/memory/knowledge.js.map +1 -1
- package/dist/recovery.d.ts.map +1 -1
- package/dist/recovery.js +2 -0
- package/dist/recovery.js.map +1 -1
- package/dist/run-state.d.ts +27 -1
- package/dist/run-state.d.ts.map +1 -1
- package/dist/run-state.js +40 -1
- package/dist/run-state.js.map +1 -1
- package/dist/runner/goal-decomposer.d.ts +38 -0
- package/dist/runner/goal-decomposer.d.ts.map +1 -0
- package/dist/runner/goal-decomposer.js +125 -0
- package/dist/runner/goal-decomposer.js.map +1 -0
- package/dist/runner/parallel-runner.d.ts +61 -0
- package/dist/runner/parallel-runner.d.ts.map +1 -0
- package/dist/runner/parallel-runner.js +133 -0
- package/dist/runner/parallel-runner.js.map +1 -0
- package/dist/runner/pattern-extractor.d.ts +40 -0
- package/dist/runner/pattern-extractor.d.ts.map +1 -0
- package/dist/runner/pattern-extractor.js +122 -0
- package/dist/runner/pattern-extractor.js.map +1 -0
- package/dist/runner/runner.d.ts +21 -0
- package/dist/runner/runner.d.ts.map +1 -1
- package/dist/runner/runner.js +315 -18
- package/dist/runner/runner.js.map +1 -1
- package/dist/supervisor/critic.d.ts.map +1 -1
- package/dist/supervisor/critic.js +1 -0
- package/dist/supervisor/critic.js.map +1 -1
- package/dist/supervisor/policy.js +10 -0
- package/dist/supervisor/policy.js.map +1 -1
- package/dist/test-runner.d.ts.map +1 -1
- package/dist/test-runner.js +8 -3
- package/dist/test-runner.js.map +1 -1
- package/dist/types.d.ts +94 -1
- package/dist/types.d.ts.map +1 -1
- package/package.json +3 -1
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Parallel Runner — executes compound goals across multiple browser tabs.
|
|
3
|
+
*
|
|
4
|
+
* Gen 21: takes decomposed sub-goals from GoalDecomposer, creates one
|
|
5
|
+
* BrowserAgent per sub-goal in separate Pages (shared BrowserContext),
|
|
6
|
+
* runs them concurrently, and merges results via EvidenceMerger.
|
|
7
|
+
*
|
|
8
|
+
* Uses the same PlaywrightDriver + BrowserAgent stack as single-page runs.
|
|
9
|
+
* The parallel execution layer is thin — the intelligence is in the
|
|
10
|
+
* GoalDecomposer (split) and EvidenceMerger (combine).
|
|
11
|
+
*/
|
|
12
|
+
import { PlaywrightDriver } from '../drivers/playwright.js';
|
|
13
|
+
import { BrowserAgent } from './runner.js';
|
|
14
|
+
/**
|
|
15
|
+
* Run sub-goals in parallel across separate browser tabs.
|
|
16
|
+
*
|
|
17
|
+
* Creates one Page + PlaywrightDriver + BrowserAgent per sub-goal,
|
|
18
|
+
* runs them concurrently with per-agent timeouts, and collects results.
|
|
19
|
+
*/
|
|
20
|
+
export async function runParallel(options) {
|
|
21
|
+
const startTime = Date.now();
|
|
22
|
+
const { context, config, subGoals, scenario, onTurn } = options;
|
|
23
|
+
// Per-sub-agent timeout: split the total timeout proportionally
|
|
24
|
+
const totalTimeout = options.timeoutMs || 600_000;
|
|
25
|
+
const perAgentTimeout = Math.floor(totalTimeout * 0.85); // 85% of total, leave room for merge
|
|
26
|
+
// Create and run sub-agents in parallel
|
|
27
|
+
const subPromises = subGoals.map(async (subGoal, index) => {
|
|
28
|
+
const label = `sub-${index}`;
|
|
29
|
+
let page;
|
|
30
|
+
try {
|
|
31
|
+
page = await context.newPage();
|
|
32
|
+
const driver = new PlaywrightDriver(page, {
|
|
33
|
+
...options.driverOptions,
|
|
34
|
+
showCursor: false, // no cursor overlay on parallel tabs
|
|
35
|
+
});
|
|
36
|
+
// Set up resource blocking if configured
|
|
37
|
+
const blocking = config.resourceBlocking;
|
|
38
|
+
if (blocking) {
|
|
39
|
+
await driver.setupResourceBlocking(blocking);
|
|
40
|
+
}
|
|
41
|
+
const subConfig = {
|
|
42
|
+
...config,
|
|
43
|
+
// Scale token budget by sub-goal's budget fraction
|
|
44
|
+
tokenBudget: options.totalTokenBudget
|
|
45
|
+
? Math.floor(options.totalTokenBudget * subGoal.budgetFraction)
|
|
46
|
+
: undefined,
|
|
47
|
+
};
|
|
48
|
+
const agent = new BrowserAgent({
|
|
49
|
+
driver,
|
|
50
|
+
config: subConfig,
|
|
51
|
+
onTurn: onTurn ? (turn) => onTurn(label, turn) : undefined,
|
|
52
|
+
projectStore: options.projectStore,
|
|
53
|
+
// extensions passed through if available
|
|
54
|
+
});
|
|
55
|
+
const subScenario = {
|
|
56
|
+
...scenario,
|
|
57
|
+
goal: subGoal.goal,
|
|
58
|
+
startUrl: subGoal.startUrl || scenario.startUrl,
|
|
59
|
+
// Scale max turns by budget fraction
|
|
60
|
+
maxTurns: Math.max(10, Math.floor((scenario.maxTurns || 30) * subGoal.budgetFraction * 1.5)),
|
|
61
|
+
};
|
|
62
|
+
const result = await Promise.race([
|
|
63
|
+
agent.run(subScenario),
|
|
64
|
+
new Promise((_, reject) => setTimeout(() => reject(new Error(`Sub-agent ${label} timed out`)), perAgentTimeout)),
|
|
65
|
+
]);
|
|
66
|
+
return { subGoal, result };
|
|
67
|
+
}
|
|
68
|
+
catch (err) {
|
|
69
|
+
// Sub-agent failed — return a failure result
|
|
70
|
+
return {
|
|
71
|
+
subGoal,
|
|
72
|
+
result: {
|
|
73
|
+
success: false,
|
|
74
|
+
reason: `Sub-agent error: ${err instanceof Error ? err.message : String(err)}`,
|
|
75
|
+
turns: [],
|
|
76
|
+
totalMs: Date.now() - startTime,
|
|
77
|
+
},
|
|
78
|
+
};
|
|
79
|
+
}
|
|
80
|
+
finally {
|
|
81
|
+
// Close the page but not the context (shared with other sub-agents)
|
|
82
|
+
if (page && !page.isClosed()) {
|
|
83
|
+
await page.close().catch(() => { });
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
const subResults = await Promise.all(subPromises);
|
|
88
|
+
// Merge results
|
|
89
|
+
const mergedResult = mergeEvidence(options.originalGoal, subResults);
|
|
90
|
+
const totalTokens = subResults.reduce((sum, sr) => {
|
|
91
|
+
const tokens = sr.result.totalTokensUsed || 0;
|
|
92
|
+
return sum + tokens;
|
|
93
|
+
}, 0);
|
|
94
|
+
return {
|
|
95
|
+
mergedResult: mergedResult.text,
|
|
96
|
+
success: mergedResult.success,
|
|
97
|
+
subResults,
|
|
98
|
+
totalTokens,
|
|
99
|
+
totalMs: Date.now() - startTime,
|
|
100
|
+
};
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Merge evidence from parallel sub-agents into one coherent answer.
|
|
104
|
+
*
|
|
105
|
+
* For now: deterministic merge (concatenate results with labels).
|
|
106
|
+
* Future: LLM-based synthesis for complex comparisons.
|
|
107
|
+
*/
|
|
108
|
+
function mergeEvidence(originalGoal, subResults) {
|
|
109
|
+
const successCount = subResults.filter(sr => sr.result.success).length;
|
|
110
|
+
const allSucceeded = successCount === subResults.length;
|
|
111
|
+
const anySucceeded = successCount > 0;
|
|
112
|
+
// Build merged result text
|
|
113
|
+
const parts = [`Goal: ${originalGoal}`, ''];
|
|
114
|
+
for (const { subGoal, result } of subResults) {
|
|
115
|
+
const status = result.success ? 'COMPLETED' : 'FAILED';
|
|
116
|
+
parts.push(`[${status}] ${subGoal.goal}:`);
|
|
117
|
+
if (result.reason) {
|
|
118
|
+
parts.push(result.reason);
|
|
119
|
+
}
|
|
120
|
+
parts.push('');
|
|
121
|
+
}
|
|
122
|
+
if (!anySucceeded) {
|
|
123
|
+
parts.push('None of the sub-goals could be completed.');
|
|
124
|
+
}
|
|
125
|
+
else if (!allSucceeded) {
|
|
126
|
+
parts.push(`${successCount}/${subResults.length} sub-goals completed.`);
|
|
127
|
+
}
|
|
128
|
+
return {
|
|
129
|
+
text: parts.join('\n'),
|
|
130
|
+
success: anySucceeded, // succeed if at least one sub-goal worked
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
//# sourceMappingURL=parallel-runner.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parallel-runner.js","sourceRoot":"","sources":["../../src/runner/parallel-runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAA;AAE3D,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAA;AA4C1C;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAAC,OAA2B;IAC3D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IAC5B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,OAAO,CAAA;IAE/D,gEAAgE;IAChE,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,IAAI,OAAO,CAAA;IACjD,MAAM,eAAe,GAAG,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,IAAI,CAAC,CAAA,CAAC,qCAAqC;IAE7F,wCAAwC;IACxC,MAAM,WAAW,GAAG,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE;QACxD,MAAM,KAAK,GAAG,OAAO,KAAK,EAAE,CAAA;QAC5B,IAAI,IAA2C,CAAA;QAE/C,IAAI,CAAC;YACH,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,EAAE,CAAA;YAE9B,MAAM,MAAM,GAAG,IAAI,gBAAgB,CAAC,IAAI,EAAE;gBACxC,GAAG,OAAO,CAAC,aAAa;gBACxB,UAAU,EAAE,KAAK,EAAE,qCAAqC;aACzD,CAAC,CAAA;YAEF,yCAAyC;YACzC,MAAM,QAAQ,GAAI,MAAuF,CAAC,gBAAgB,CAAA;YAC1H,IAAI,QAAQ,EAAE,CAAC;gBACb,MAAM,MAAM,CAAC,qBAAqB,CAAC,QAAQ,CAAC,CAAA;YAC9C,CAAC;YAED,MAAM,SAAS,GAAgB;gBAC7B,GAAG,MAAM;gBACT,mDAAmD;gBACnD,WAAW,EAAE,OAAO,CAAC,gBAAgB;oBACnC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,gBAAgB,GAAG,OAAO,CAAC,cAAc,CAAC;oBAC/D,CAAC,CAAC,SAAS;aACd,CAAA;YAED,MAAM,KAAK,GAAG,IAAI,YAAY,CAAC;gBAC7B,MAAM;gBACN,MAAM,EAAE,SAAS;gBACjB,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,IAAU,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;gBAChE,YAAY,EAAE,OAAO,CAAC,YAAY;gBAClC,yCAAyC;aAC1C,CAAC,CAAA;YAEF,MAAM,WAAW,GAAa;gBAC5B,GAAG,QAAQ;gBACX,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,QAAQ,EAAE,OAAO,CAAC,QAAQ,IAAI,QAAQ,CAAC,QAAQ;gBAC/C,qCAAqC;gBACrC,QAAQ,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,QAAQ,CAAC,QAAQ,IAAI,EAAE,CAAC,GAAG,OAAO,CAAC,cAAc,GAAG,GAAG,CAAC,CAAC;aAC7F,CAAA;YAED,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;gBAChC,KAAK,CAAC,GAAG,CAAC,WAAW,CAAC;gBACtB,IAAI,OAAO,CAAc,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CACrC,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,aAAa,KAAK,YAAY,CAAC,CAAC,EAAE,eAAe,CAAC,CACrF;aACF,CAAC,CAAA;YAEF,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,CAAA;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,6CAA6C;YAC7C,OAAO;gBACL,OAAO;gBACP,MAAM,EAAE;oBACN,OAAO,EAAE,KAAK;oBACd,MAAM,EAAE,oBAAoB,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE;oBAC9E,KAAK,EAAE,EAAE;oBACT,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;iBACjB;aACjB,CAAA;QACH,CAAC;gBAAS,CAAC;YACT,oEAAoE;YACpE,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,EAAE,CAAC;gBAC7B,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;YACpC,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,MAAM,UAAU,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,CAAA;IAEjD,gBAAgB;IAChB,MAAM,YAAY,GAAG,aAAa,CAAC,OAAO,CAAC,YAAY,EAAE,UAAU,CAAC,CAAA;IACpE,MAAM,WAAW,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,EAAE,EAAE;QAChD,MAAM,MAAM,GAAI,EAAE,CAAC,MAAuC,CAAC,eAAe,IAAI,CAAC,CAAA;QAC/E,OAAO,GAAG,GAAG,MAAM,CAAA;IACrB,CAAC,EAAE,CAAC,CAAC,CAAA;IAEL,OAAO;QACL,YAAY,EAAE,YAAY,CAAC,IAAI;QAC/B,OAAO,EAAE,YAAY,CAAC,OAAO;QAC7B,UAAU;QACV,WAAW;QACX,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAChC,CAAA;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CACpB,YAAoB,EACpB,UAA4D;IAE5D,MAAM,YAAY,GAAG,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAA;IACtE,MAAM,YAAY,GAAG,YAAY,KAAK,UAAU,CAAC,MAAM,CAAA;IACvD,MAAM,YAAY,GAAG,YAAY,GAAG,CAAC,CAAA;IAErC,2BAA2B;IAC3B,MAAM,KAAK,GAAa,CAAC,SAAS,YAAY,EAAE,EAAE,EAAE,CAAC,CAAA;IAErD,KAAK,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,IAAI,UAAU,EAAE,CAAC;QAC7C,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAA;QACtD,KAAK,CAAC,IAAI,CAAC,IAAI,MAAM,KAAK,OAAO,CAAC,IAAI,GAAG,CAAC,CAAA;QAC1C,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;QAC3B,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IAChB,CAAC;IAED,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAA;IACzD,CAAC;SAAM,IAAI,CAAC,YAAY,EAAE,CAAC;QACzB,KAAK,CAAC,IAAI,CAAC,GAAG,YAAY,IAAI,UAAU,CAAC,MAAM,uBAAuB,CAAC,CAAA;IACzE,CAAC;IAED,OAAO;QACL,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC;QACtB,OAAO,EAAE,YAAY,EAAE,0CAA0C;KAClE,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pattern Extractor — learns reusable navigation patterns from completed runs.
|
|
3
|
+
*
|
|
4
|
+
* Gen 26b: after a successful run, mechanically extract domain-level patterns
|
|
5
|
+
* from the turn log and record them as AppKnowledge facts. No LLM call needed —
|
|
6
|
+
* patterns are detected by observing action/state sequences.
|
|
7
|
+
*
|
|
8
|
+
* Extracted patterns:
|
|
9
|
+
* - Cookie/consent banner dismissal (which action dismissed it, on which turn)
|
|
10
|
+
* - Page load timing (how long the site takes to settle)
|
|
11
|
+
* - Form structure (which refs are used for key form fields)
|
|
12
|
+
* - Navigation paths (effective URL patterns for search/results)
|
|
13
|
+
* - Blockers encountered (modals, auth walls, rate limits)
|
|
14
|
+
*
|
|
15
|
+
* Design constraints:
|
|
16
|
+
* - No bloat: only records patterns with clear signal (not every action)
|
|
17
|
+
* - Cleanable: all facts have confidence scores; low-confidence facts auto-prune
|
|
18
|
+
* - Workspace-isolated: patterns stored per-domain in the knowledge store
|
|
19
|
+
* - Smart: confirms patterns on repeat observation, decays on contradiction
|
|
20
|
+
*/
|
|
21
|
+
import type { Turn } from '../types.js';
|
|
22
|
+
import type { AppKnowledge, Fact } from '../memory/knowledge.js';
|
|
23
|
+
interface ExtractedPattern {
|
|
24
|
+
type: Fact['type'];
|
|
25
|
+
key: string;
|
|
26
|
+
value: string;
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Extract reusable patterns from a completed run's turns.
|
|
30
|
+
* Only extracts from SUCCESSFUL runs — failed runs produce unreliable patterns.
|
|
31
|
+
*/
|
|
32
|
+
export declare function extractPatterns(turns: Turn[], domain: string, success: boolean): ExtractedPattern[];
|
|
33
|
+
/**
|
|
34
|
+
* Record extracted patterns into the knowledge store.
|
|
35
|
+
* Respects the existing confidence system — repeated patterns gain confidence,
|
|
36
|
+
* contradicted patterns decay.
|
|
37
|
+
*/
|
|
38
|
+
export declare function recordPatterns(knowledge: AppKnowledge, patterns: ExtractedPattern[]): void;
|
|
39
|
+
export {};
|
|
40
|
+
//# sourceMappingURL=pattern-extractor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pattern-extractor.d.ts","sourceRoot":"","sources":["../../src/runner/pattern-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,KAAK,EAAE,IAAI,EAAE,MAAM,aAAa,CAAA;AACvC,OAAO,KAAK,EAAE,YAAY,EAAE,IAAI,EAAE,MAAM,wBAAwB,CAAA;AAEhE,UAAU,gBAAgB;IACxB,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAA;IAClB,GAAG,EAAE,MAAM,CAAA;IACX,KAAK,EAAE,MAAM,CAAA;CACd;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAC7B,KAAK,EAAE,IAAI,EAAE,EACb,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,OAAO,GACf,gBAAgB,EAAE,CA6FpB;AAED;;;;GAIG;AACH,wBAAgB,cAAc,CAC5B,SAAS,EAAE,YAAY,EACvB,QAAQ,EAAE,gBAAgB,EAAE,GAC3B,IAAI,CAIN"}
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Pattern Extractor — learns reusable navigation patterns from completed runs.
|
|
3
|
+
*
|
|
4
|
+
* Gen 26b: after a successful run, mechanically extract domain-level patterns
|
|
5
|
+
* from the turn log and record them as AppKnowledge facts. No LLM call needed —
|
|
6
|
+
* patterns are detected by observing action/state sequences.
|
|
7
|
+
*
|
|
8
|
+
* Extracted patterns:
|
|
9
|
+
* - Cookie/consent banner dismissal (which action dismissed it, on which turn)
|
|
10
|
+
* - Page load timing (how long the site takes to settle)
|
|
11
|
+
* - Form structure (which refs are used for key form fields)
|
|
12
|
+
* - Navigation paths (effective URL patterns for search/results)
|
|
13
|
+
* - Blockers encountered (modals, auth walls, rate limits)
|
|
14
|
+
*
|
|
15
|
+
* Design constraints:
|
|
16
|
+
* - No bloat: only records patterns with clear signal (not every action)
|
|
17
|
+
* - Cleanable: all facts have confidence scores; low-confidence facts auto-prune
|
|
18
|
+
* - Workspace-isolated: patterns stored per-domain in the knowledge store
|
|
19
|
+
* - Smart: confirms patterns on repeat observation, decays on contradiction
|
|
20
|
+
*/
|
|
21
|
+
/**
|
|
22
|
+
* Extract reusable patterns from a completed run's turns.
|
|
23
|
+
* Only extracts from SUCCESSFUL runs — failed runs produce unreliable patterns.
|
|
24
|
+
*/
|
|
25
|
+
export function extractPatterns(turns, domain, success) {
|
|
26
|
+
if (!success || turns.length < 2)
|
|
27
|
+
return [];
|
|
28
|
+
const patterns = [];
|
|
29
|
+
// 1. Cookie/consent banner dismissal
|
|
30
|
+
// Look for early turns where the agent clicked something that looks like
|
|
31
|
+
// cookie/consent/accept and then proceeded normally
|
|
32
|
+
for (let i = 0; i < Math.min(turns.length, 5); i++) {
|
|
33
|
+
const turn = turns[i];
|
|
34
|
+
const action = turn.action;
|
|
35
|
+
if (action.action === 'click' || action.action === 'clickAt' || action.action === 'clickLabel') {
|
|
36
|
+
const reasoning = turn.reasoning?.toLowerCase() || '';
|
|
37
|
+
const snapshot = turn.state?.snapshot?.toLowerCase() || '';
|
|
38
|
+
if (/cookie|consent|accept.*cookie|gdpr|privacy/i.test(reasoning + snapshot)) {
|
|
39
|
+
const selector = 'selector' in action ? action.selector : `clickAt(${action.x},${action.y})`;
|
|
40
|
+
patterns.push({
|
|
41
|
+
type: 'pattern',
|
|
42
|
+
key: 'cookie-dismiss',
|
|
43
|
+
value: `Turn ${i + 1}: ${action.action} ${selector}`,
|
|
44
|
+
});
|
|
45
|
+
break; // only record the first one
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
// 2. Page load timing — how many turns before first meaningful action
|
|
50
|
+
const firstMeaningfulTurn = turns.findIndex(t => t.action.action !== 'wait' && t.action.action !== 'scroll' &&
|
|
51
|
+
!t.error && t.action.action !== 'navigate');
|
|
52
|
+
if (firstMeaningfulTurn >= 2) {
|
|
53
|
+
patterns.push({
|
|
54
|
+
type: 'timing',
|
|
55
|
+
key: 'first-meaningful-action',
|
|
56
|
+
value: `Turn ${firstMeaningfulTurn + 1} (${firstMeaningfulTurn} setup turns)`,
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
// 3. Effective search/navigation URL pattern
|
|
60
|
+
// If the agent used navigate with URL params, record the pattern
|
|
61
|
+
for (const turn of turns) {
|
|
62
|
+
if (turn.action.action === 'navigate' && turn.action.url) {
|
|
63
|
+
try {
|
|
64
|
+
const url = new URL(turn.action.url);
|
|
65
|
+
if (url.hostname.includes(domain) && url.search.length > 5) {
|
|
66
|
+
// Generalize: replace specific values with placeholders
|
|
67
|
+
const pattern = url.pathname + url.search
|
|
68
|
+
.replace(/=[^&]+/g, '={value}')
|
|
69
|
+
.slice(0, 100);
|
|
70
|
+
patterns.push({
|
|
71
|
+
type: 'pattern',
|
|
72
|
+
key: 'search-url',
|
|
73
|
+
value: pattern,
|
|
74
|
+
});
|
|
75
|
+
break; // only record the first effective search URL
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
catch { /* invalid URL, skip */ }
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
// 4. Turn efficiency — how many turns the task took
|
|
82
|
+
patterns.push({
|
|
83
|
+
type: 'timing',
|
|
84
|
+
key: 'typical-turns',
|
|
85
|
+
value: `${turns.length} turns`,
|
|
86
|
+
});
|
|
87
|
+
// 5. Form fields used — record which refs were used for form filling
|
|
88
|
+
const fillTurns = turns.filter(t => t.action.action === 'fill' || t.action.action === 'type');
|
|
89
|
+
if (fillTurns.length >= 2) {
|
|
90
|
+
const fields = fillTurns
|
|
91
|
+
.map(t => {
|
|
92
|
+
if (t.action.action === 'fill' && 'fields' in t.action && t.action.fields) {
|
|
93
|
+
return Object.keys(t.action.fields).join(',');
|
|
94
|
+
}
|
|
95
|
+
if (t.action.action === 'type' && 'selector' in t.action) {
|
|
96
|
+
return t.action.selector;
|
|
97
|
+
}
|
|
98
|
+
return '';
|
|
99
|
+
})
|
|
100
|
+
.filter(Boolean)
|
|
101
|
+
.slice(0, 5);
|
|
102
|
+
if (fields.length > 0) {
|
|
103
|
+
patterns.push({
|
|
104
|
+
type: 'selector',
|
|
105
|
+
key: 'form-fields',
|
|
106
|
+
value: fields.join(' → '),
|
|
107
|
+
});
|
|
108
|
+
}
|
|
109
|
+
}
|
|
110
|
+
return patterns;
|
|
111
|
+
}
|
|
112
|
+
/**
|
|
113
|
+
* Record extracted patterns into the knowledge store.
|
|
114
|
+
* Respects the existing confidence system — repeated patterns gain confidence,
|
|
115
|
+
* contradicted patterns decay.
|
|
116
|
+
*/
|
|
117
|
+
export function recordPatterns(knowledge, patterns) {
|
|
118
|
+
for (const p of patterns) {
|
|
119
|
+
knowledge.recordFact(p.type, p.key, p.value);
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
//# sourceMappingURL=pattern-extractor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pattern-extractor.js","sourceRoot":"","sources":["../../src/runner/pattern-extractor.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAWH;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,KAAa,EACb,MAAc,EACd,OAAgB;IAEhB,IAAI,CAAC,OAAO,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAA;IAE3C,MAAM,QAAQ,GAAuB,EAAE,CAAA;IAEvC,qCAAqC;IACrC,yEAAyE;IACzE,oDAAoD;IACpD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QACnD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACrB,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,CAAA;QAC1B,IAAI,MAAM,CAAC,MAAM,KAAK,OAAO,IAAI,MAAM,CAAC,MAAM,KAAK,SAAS,IAAI,MAAM,CAAC,MAAM,KAAK,YAAY,EAAE,CAAC;YAC/F,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;YACrD,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,IAAI,EAAE,CAAA;YAC1D,IAAI,6CAA6C,CAAC,IAAI,CAAC,SAAS,GAAG,QAAQ,CAAC,EAAE,CAAC;gBAC7E,MAAM,QAAQ,GAAG,UAAU,IAAI,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAY,MAAyB,CAAC,CAAC,IAAK,MAAyB,CAAC,CAAC,GAAG,CAAA;gBACpI,QAAQ,CAAC,IAAI,CAAC;oBACZ,IAAI,EAAE,SAAS;oBACf,GAAG,EAAE,gBAAgB;oBACrB,KAAK,EAAE,QAAQ,CAAC,GAAG,CAAC,KAAK,MAAM,CAAC,MAAM,IAAI,QAAQ,EAAE;iBACrD,CAAC,CAAA;gBACF,MAAK,CAAC,4BAA4B;YACpC,CAAC;QACH,CAAC;IACH,CAAC;IAED,sEAAsE;IACtE,MAAM,mBAAmB,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAC9C,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,QAAQ;QAC1D,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,CAC3C,CAAA;IACD,IAAI,mBAAmB,IAAI,CAAC,EAAE,CAAC;QAC7B,QAAQ,CAAC,IAAI,CAAC;YACZ,IAAI,EAAE,QAAQ;YACd,GAAG,EAAE,yBAAyB;YAC9B,KAAK,EAAE,QAAQ,mBAAmB,GAAG,CAAC,KAAK,mBAAmB,eAAe;SAC9E,CAAC,CAAA;IACJ,CAAC;IAED,6CAA6C;IAC7C,iEAAiE;IACjE,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,CAAC,MAAM,CAAC,MAAM,KAAK,UAAU,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;YACzD,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;gBACpC,IAAI,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;oBAC3D,wDAAwD;oBACxD,MAAM,OAAO,GAAG,GAAG,CAAC,QAAQ,GAAG,GAAG,CAAC,MAAM;yBACtC,OAAO,CAAC,SAAS,EAAE,UAAU,CAAC;yBAC9B,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAA;oBAChB,QAAQ,CAAC,IAAI,CAAC;wBACZ,IAAI,EAAE,SAAS;wBACf,GAAG,EAAE,YAAY;wBACjB,KAAK,EAAE,OAAO;qBACf,CAAC,CAAA;oBACF,MAAK,CAAC,6CAA6C;gBACrD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC,CAAC,uBAAuB,CAAC,CAAC;QACrC,CAAC;IACH,CAAC;IAED,oDAAoD;IACpD,QAAQ,CAAC,IAAI,CAAC;QACZ,IAAI,EAAE,QAAQ;QACd,GAAG,EAAE,eAAe;QACpB,KAAK,EAAE,GAAG,KAAK,CAAC,MAAM,QAAQ;KAC/B,CAAC,CAAA;IAEF,qEAAqE;IACrE,MAAM,SAAS,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,CAAC,CAAA;IAC7F,IAAI,SAAS,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1B,MAAM,MAAM,GAAG,SAAS;aACrB,GAAG,CAAC,CAAC,CAAC,EAAE;YACP,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,QAAQ,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;gBAC1E,OAAO,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;YAC/C,CAAC;YACD,IAAI,CAAC,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,UAAU,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;gBACzD,OAAO,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAA;YAC1B,CAAC;YACD,OAAO,EAAE,CAAA;QACX,CAAC,CAAC;aACD,MAAM,CAAC,OAAO,CAAC;aACf,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAA;QACd,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,QAAQ,CAAC,IAAI,CAAC;gBACZ,IAAI,EAAE,UAAU;gBAChB,GAAG,EAAE,aAAa;gBAClB,KAAK,EAAE,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;aAC1B,CAAC,CAAA;QACJ,CAAC;IACH,CAAC;IAED,OAAO,QAAQ,CAAA;AACjB,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAC5B,SAAuB,EACvB,QAA4B;IAE5B,KAAK,MAAM,CAAC,IAAI,QAAQ,EAAE,CAAC;QACzB,SAAS,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAA;IAC9C,CAAC;AACH,CAAC"}
|
package/dist/runner/runner.d.ts
CHANGED
|
@@ -88,6 +88,27 @@ export interface BrowserAgentOptions {
|
|
|
88
88
|
* actual data (no nulls, no placeholder markers) passes through unchanged.
|
|
89
89
|
*/
|
|
90
90
|
export declare function hasPlaceholderPattern(text: string): boolean;
|
|
91
|
+
/**
|
|
92
|
+
* Gen 9 — runtime two-pass extraction. When the planner emits a single
|
|
93
|
+
* runScript step (per Gen 7.2 rule #7) and that script returns null /
|
|
94
|
+
* empty / whitespace / `{x: null}` / a placeholder pattern, the auto-
|
|
95
|
+
* complete-from-runScript path should NOT fire. Instead the runner should
|
|
96
|
+
* mark the plan as deviated and fall through to the per-action loop where
|
|
97
|
+
* Brain.decide can re-observe the loaded page and emit a smarter action
|
|
98
|
+
* (different selector, click+wait, scroll, etc.).
|
|
99
|
+
*
|
|
100
|
+
* This addresses the failure mode the Gen 8 head-to-head gauntlet
|
|
101
|
+
* surfaced: bad's planner-only path lost to browser-use's per-action loop
|
|
102
|
+
* on tasks where the first runScript pick was wrong (npm, mdn signature,
|
|
103
|
+
* w3c, github, wikipedia variance). Two-pass gives bad's per-action loop
|
|
104
|
+
* the same recovery surface browser-use uses, with the planner's speed
|
|
105
|
+
* advantage on the cases where runScript succeeds first try.
|
|
106
|
+
*
|
|
107
|
+
* "Meaningful" means: not empty/whitespace, not the literal string `null`
|
|
108
|
+
* or `undefined`, and not matching `hasPlaceholderPattern` (which already
|
|
109
|
+
* detects JSON null fields, "<from prior step>" markers, etc.).
|
|
110
|
+
*/
|
|
111
|
+
export declare function isMeaningfulRunScriptOutput(output: string | null | undefined): boolean;
|
|
91
112
|
export declare class BrowserAgent {
|
|
92
113
|
private driver;
|
|
93
114
|
private brain;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAkC,MAAM,aAAa,CAAC;AAKvH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;
|
|
1
|
+
{"version":3,"file":"runner.d.ts","sourceRoot":"","sources":["../../src/runner/runner.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAGH,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,KAAK,EAAE,QAAQ,EAAE,WAAW,EAAE,WAAW,EAAE,IAAI,EAAE,SAAS,EAAkC,MAAM,aAAa,CAAC;AAKvH,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAuB/D,OAAO,EAAE,WAAW,EAAE,MAAM,2BAA2B,CAAC;AACxD,OAAO,EAAE,YAAY,EAAa,MAAM,aAAa,CAAC;AAGtD,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,wBAAwB,CAAC;AAEjE;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AACH,wBAAgB,0BAA0B,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,GAAG,MAAM,GAAG,IAAI,CAiDzF;AA4CD,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,6BAA6B;IAC7B,MAAM,CAAC,EAAE,CAAC,IAAI,EAAE,IAAI,KAAK,IAAI,CAAC;IAC9B,wDAAwD;IACxD,aAAa,CAAC,EAAE,CAAC,KAAK,EAAE,UAAU,GAAG,SAAS,GAAG,QAAQ,GAAG,SAAS,EAAE,UAAU,EAAE,MAAM,KAAK,IAAI,CAAC;IACnG,wDAAwD;IACxD,mBAAmB,CAAC,EAAE,MAAM,CAAC;IAC7B,sEAAsE;IACtE,YAAY,CAAC,EAAE,YAAY,CAAC;IAC5B,sDAAsD;IACtD,WAAW,CAAC,EAAE,WAAW,CAAC;IAC1B;;;;;;;OAOG;IACH,QAAQ,CAAC,EAAE,YAAY,CAAC;IACxB;;;;;OAKG;IACH,UAAU,CAAC,EAAE,kBAAkB,CAAC;CACjC;AAED;;;;;;;;;;;;;;;;GAgBG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAgB3D;AAED;;;;;;;;;;;;;;;;;;;GAmBG;AACH,wBAAgB,2BAA2B,CAAC,MAAM,EAAE,MAAM,GAAG,IAAI,GAAG,SAAS,GAAG,OAAO,CA2BtF;AAED,qBAAa,YAAY;IACvB,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,KAAK,CAAQ;IACrB,OAAO,CAAC,MAAM,CAAc;IAC5B,OAAO,CAAC,MAAM,CAAC,CAAuB;IACtC,OAAO,CAAC,aAAa,CAAC,CAAqF;IAC3G,OAAO,CAAC,mBAAmB,CAAC,CAAS;IACrC,OAAO,CAAC,YAAY,CAAC,CAAe;IACpC,OAAO,CAAC,WAAW,CAAC,CAAc;IAClC,OAAO,CAAC,SAAS,CAAC,CAAe;IACjC,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,eAAe,CAAwB;IAC/C,OAAO,CAAC,GAAG,CAAe;IAC1B,OAAO,CAAC,YAAY,CAAM;IAK1B,OAAO,CAAC,aAAa,CAAC,CAAgB;IACtC,OAAO,CAAC,UAAU,CAAC,CAAqB;gBAE5B,OAAO,EAAE,mBAAmB;IAyBlC,GAAG,CAAC,QAAQ,EAAE,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC;IAm4DnD,OAAO,CAAC,qBAAqB;IA2B7B,qEAAqE;IACrE,OAAO,CAAC,UAAU;IAkClB;;;;;;;;;;;;OAYG;IACH;;;;;;;;;;;;;;;;;;;OAmBG;YACW,WAAW;YAofX,YAAY;YAkCZ,+BAA+B;YAgE/B,mCAAmC;YAwCnC,6BAA6B;YA4C7B,qCAAqC;YAqBrC,4BAA4B;YA4B5B,4BAA4B;YAkC5B,wBAAwB;CASvC;AAED,2BAA2B;AAC3B,wBAAsB,eAAe,CACnC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,QAAQ,EAClB,OAAO,CAAC,EAAE,IAAI,CAAC,mBAAmB,EAAE,QAAQ,CAAC,GAC5C,OAAO,CAAC,WAAW,CAAC,CAGtB"}
|