chrome-devtools-frontend 1.0.1642845 → 1.0.1642899
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SECURITY.md +1 -0
- package/front_end/core/host/UserMetrics.ts +2 -1
- package/front_end/core/sdk/CSSMatchedStyles.ts +55 -26
- package/front_end/core/sdk/CSSRule.ts +1 -0
- package/front_end/core/sdk/DebuggerModel.ts +5 -0
- package/front_end/entrypoints/greendev_floaty/FloatyEntrypoint.ts +4 -3
- package/front_end/entrypoints/greendev_floaty/greendev_floaty.ts +4 -3
- package/front_end/models/ai_assistance/AiAgent2.ts +80 -16
- package/front_end/models/ai_assistance/AiConversation.ts +3 -2
- package/front_end/models/ai_assistance/README.md +8 -0
- package/front_end/models/ai_assistance/agents/AccessibilityAgent.ts +50 -35
- package/front_end/models/ai_assistance/agents/AiAgent.ts +16 -0
- package/front_end/models/ai_assistance/agents/ContextSelectionAgent.ts +2 -2
- package/front_end/models/ai_assistance/agents/PerformanceAgent.ts +195 -147
- package/front_end/models/ai_assistance/agents/StylingAgent.snapshot.txt +0 -25
- package/front_end/models/ai_assistance/agents/StylingAgent.ts +24 -305
- package/front_end/models/ai_assistance/ai_assistance.ts +8 -0
- package/front_end/models/ai_assistance/contexts/DOMNodeContext.snapshot.txt +51 -0
- package/front_end/models/ai_assistance/contexts/DOMNodeContext.ts +200 -0
- package/front_end/models/ai_assistance/skills/styling.md +36 -2
- package/front_end/models/ai_assistance/tools/GetStyles.ts +137 -0
- package/front_end/models/ai_assistance/tools/Tool.ts +55 -0
- package/front_end/models/ai_assistance/tools/ToolRegistry.ts +34 -0
- package/front_end/models/lighthouse/LighthouseReporterTypes.ts +5 -0
- package/front_end/models/live-metrics/LiveMetrics.ts +24 -13
- package/front_end/models/stack_trace/DetailedErrorStackParser.ts +2 -2
- package/front_end/models/stack_trace/StackTrace.ts +4 -1
- package/front_end/models/stack_trace/StackTraceImpl.ts +9 -2
- package/front_end/models/stack_trace/StackTraceModel.ts +17 -4
- package/front_end/models/stack_trace/Trie.ts +1 -1
- package/front_end/panels/ai_assistance/AiAssistancePanel.ts +19 -15
- package/front_end/panels/ai_assistance/ai_assistance-meta.ts +16 -0
- package/front_end/panels/ai_assistance/components/ChatInput.ts +2 -2
- package/front_end/panels/application/DOMStorageItemsView.ts +4 -0
- package/front_end/panels/application/KeyValueStorageItemsView.ts +39 -7
- package/front_end/panels/common/ExtensionServer.ts +26 -15
- package/front_end/panels/elements/StandaloneStylesContainer.ts +1 -1
- package/front_end/panels/elements/StylePropertiesSection.ts +8 -0
- package/front_end/panels/elements/StylePropertyHighlighter.ts +4 -2
- package/front_end/panels/elements/StylePropertyTreeElement.ts +6 -5
- package/front_end/panels/elements/StylesContainer.ts +1 -1
- package/front_end/panels/elements/StylesSidebarPane.ts +4 -4
- package/front_end/panels/layer_viewer/PaintProfilerView.ts +106 -132
- package/front_end/panels/lighthouse/LighthousePanel.ts +4 -3
- package/front_end/panels/network/NetworkLogView.ts +3 -0
- package/front_end/panels/network/networkLogView.css +0 -15
- package/front_end/ui/legacy/components/cookie_table/CookiesTable.ts +36 -3
- package/front_end/ui/legacy/components/data_grid/dataGridAiButton.css +20 -0
- package/front_end/ui/legacy/components/utils/Linkifier.ts +19 -4
- package/front_end/ui/visual_logging/KnownContextValues.ts +1 -0
- package/package.json +1 -1
package/SECURITY.md
CHANGED
|
@@ -22,6 +22,7 @@ Chrome DevTools is a privileged web app running in a sandboxed renderer process,
|
|
|
22
22
|
* **Legitimate Data Exposure:** Displaying local user data, cookies, tokens, or auth headers within UI panels.
|
|
23
23
|
* **Local Data Persistence:** Saving user-initiated traces, heaps, profiles, or logs to disk.
|
|
24
24
|
* **Correctness & Availability:** Stale, misleading, or missing UI information (classified as functional bugs).
|
|
25
|
+
* **Denial of Service:** Chrome extensions or debug targets that disrupt DevTools' availability, e.g. through performance degradation.
|
|
25
26
|
* **Disabled-by-default Experimental Features:** Experimental features or capabilities that are disabled by default, e.g. requiring a command-line flag to be activated, or a specific runtime setting within DevTools to be manually turned on.
|
|
26
27
|
|
|
27
28
|
## Specific Severity Classification Rules (S1–S4)
|
|
@@ -496,7 +496,8 @@ export enum Action {
|
|
|
496
496
|
AiCodeGenerationRequestTriggeredFromSources = 205,
|
|
497
497
|
AiCodeCompletionFreCompletedFromConsole = 206,
|
|
498
498
|
AiCodeCompletionFreCompletedFromSources = 207,
|
|
499
|
-
|
|
499
|
+
AiAssistanceOpenedFromStoragePanelFloatingButton = 208,
|
|
500
|
+
MAX_VALUE = 209,
|
|
500
501
|
/* eslint-enable @typescript-eslint/naming-convention */
|
|
501
502
|
}
|
|
502
503
|
|
|
@@ -196,6 +196,36 @@ function queryMatches(style: CSSStyleDeclaration): boolean {
|
|
|
196
196
|
return true;
|
|
197
197
|
}
|
|
198
198
|
|
|
199
|
+
function treeScopeDistance(node: DOMNode, property: CSSProperty): number {
|
|
200
|
+
if (!property.ownerStyle.parentRule && property.ownerStyle.type !== Type.Inline) {
|
|
201
|
+
return -1;
|
|
202
|
+
}
|
|
203
|
+
const root = node.getTreeRoot();
|
|
204
|
+
const nodeId = property.ownerStyle.parentRule?.treeScope ?? root?.backendNodeId();
|
|
205
|
+
if (nodeId === undefined) {
|
|
206
|
+
return -1;
|
|
207
|
+
}
|
|
208
|
+
return distanceToTreeScope(node, nodeId);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Distance from node to parent that is the tree scope. Returns -1 if tree scope is not found.
|
|
213
|
+
*
|
|
214
|
+
* @param node The child node to start from.
|
|
215
|
+
* @param treeScope The tree scope node ID.
|
|
216
|
+
* @returns The distance to the tree scope node, or -1 if not found.
|
|
217
|
+
*/
|
|
218
|
+
export function distanceToTreeScope(node: DOMNode, treeScope: Protocol.DOM.BackendNodeId): number {
|
|
219
|
+
let distance = 0;
|
|
220
|
+
for (let ancestor: DOMNode|null = node; ancestor; ancestor = ancestor.parentNode) {
|
|
221
|
+
if (ancestor.backendNodeId() === treeScope) {
|
|
222
|
+
return distance;
|
|
223
|
+
}
|
|
224
|
+
distance++;
|
|
225
|
+
}
|
|
226
|
+
return -1;
|
|
227
|
+
}
|
|
228
|
+
|
|
199
229
|
export interface CSSMatchedStylesPayload {
|
|
200
230
|
cssModel: CSSModel;
|
|
201
231
|
node: DOMNode;
|
|
@@ -307,7 +337,7 @@ export class CSSMatchedStyles {
|
|
|
307
337
|
#customHighlightPseudoDOMCascades?: Map<string, DOMInheritanceCascade>;
|
|
308
338
|
#functionRules: CSSFunctionRule[];
|
|
309
339
|
#atRules: CSSAtRule[];
|
|
310
|
-
#functionRuleMap = new Map<string, CSSFunctionRule>();
|
|
340
|
+
#functionRuleMap = new Map<string, CSSFunctionRule[]>();
|
|
311
341
|
#environmentVariables: Record<string, string> = {};
|
|
312
342
|
|
|
313
343
|
static async create(payload: CSSMatchedStylesPayload): Promise<CSSMatchedStyles> {
|
|
@@ -381,7 +411,9 @@ export class CSSMatchedStyles {
|
|
|
381
411
|
}
|
|
382
412
|
|
|
383
413
|
for (const rule of this.#functionRules) {
|
|
384
|
-
this.#functionRuleMap.
|
|
414
|
+
const rules = this.#functionRuleMap.get(rule.functionName().text) ?? [];
|
|
415
|
+
rules.push(rule);
|
|
416
|
+
this.#functionRuleMap.set(rule.functionName().text, rules);
|
|
385
417
|
}
|
|
386
418
|
}
|
|
387
419
|
|
|
@@ -790,9 +822,25 @@ export class CSSMatchedStyles {
|
|
|
790
822
|
return this.#registeredPropertyMap.get(name);
|
|
791
823
|
}
|
|
792
824
|
|
|
793
|
-
getRegisteredFunction(name: string):
|
|
794
|
-
|
|
795
|
-
|
|
825
|
+
getRegisteredFunction(name: string, sourceProperty: CSSProperty):
|
|
826
|
+
{treeScopeDistance: number, registeredFunction?: string} {
|
|
827
|
+
const minTreeScopeDistance = treeScopeDistance(this.#node, sourceProperty);
|
|
828
|
+
const functionRules = this.#functionRuleMap.get(name) ?? [];
|
|
829
|
+
let result: {treeScopeDistance: number, registeredFunction?: string} = {treeScopeDistance: -1};
|
|
830
|
+
for (const functionRule of functionRules) {
|
|
831
|
+
if (!functionRule.treeScope) {
|
|
832
|
+
continue;
|
|
833
|
+
}
|
|
834
|
+
const distance = distanceToTreeScope(this.#node, functionRule.treeScope);
|
|
835
|
+
if (distance === -1 || distance < minTreeScopeDistance) {
|
|
836
|
+
continue;
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
if (result.treeScopeDistance === -1 || distance < result.treeScopeDistance) {
|
|
840
|
+
result = {registeredFunction: functionRule.nameWithParameters(), treeScopeDistance: distance};
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
return result;
|
|
796
844
|
}
|
|
797
845
|
|
|
798
846
|
functionRules(): CSSFunctionRule[] {
|
|
@@ -1045,26 +1093,6 @@ class NodeCascade {
|
|
|
1045
1093
|
}
|
|
1046
1094
|
}
|
|
1047
1095
|
|
|
1048
|
-
#treeScopeDistance(property: CSSProperty): number {
|
|
1049
|
-
if (!property.ownerStyle.parentRule && property.ownerStyle.type !== Type.Inline) {
|
|
1050
|
-
return -1;
|
|
1051
|
-
}
|
|
1052
|
-
const root = this.#node.getTreeRoot();
|
|
1053
|
-
const nodeId = property.ownerStyle.parentRule?.treeScope ?? root?.backendNodeId();
|
|
1054
|
-
if (nodeId === undefined) {
|
|
1055
|
-
return -1;
|
|
1056
|
-
}
|
|
1057
|
-
|
|
1058
|
-
let distance = 0;
|
|
1059
|
-
for (let ancestor: DOMNode|null = this.#node; ancestor; ancestor = ancestor.parentNode) {
|
|
1060
|
-
if (ancestor.backendNodeId() === nodeId) {
|
|
1061
|
-
return distance;
|
|
1062
|
-
}
|
|
1063
|
-
distance++;
|
|
1064
|
-
}
|
|
1065
|
-
return -1;
|
|
1066
|
-
}
|
|
1067
|
-
|
|
1068
1096
|
#needsCascadeContextStep(): boolean {
|
|
1069
1097
|
if (!this.#node.isInShadowTree()) {
|
|
1070
1098
|
return false;
|
|
@@ -1084,7 +1112,8 @@ class NodeCascade {
|
|
|
1084
1112
|
const activeProperty = this.activeProperties.get(canonicalName);
|
|
1085
1113
|
if (activeProperty?.important && !propertyWithHigherSpecificity.important ||
|
|
1086
1114
|
activeProperty && this.#needsCascadeContextStep() &&
|
|
1087
|
-
this.#
|
|
1115
|
+
treeScopeDistance(this.#node, activeProperty) >
|
|
1116
|
+
treeScopeDistance(this.#node, propertyWithHigherSpecificity)) {
|
|
1088
1117
|
this.propertiesState.set(propertyWithHigherSpecificity, PropertyState.OVERLOADED);
|
|
1089
1118
|
return;
|
|
1090
1119
|
}
|
|
@@ -418,6 +418,7 @@ export class CSSFunctionRule extends CSSRule {
|
|
|
418
418
|
styleSheetId: payload.styleSheetId
|
|
419
419
|
},
|
|
420
420
|
header: styleSheetHeaderForRule(cssModel, payload),
|
|
421
|
+
originTreeScopeNodeId: payload.originTreeScopeNodeId
|
|
421
422
|
});
|
|
422
423
|
this.#name = new CSSValue(payload.name);
|
|
423
424
|
this.#parameters = payload.parameters.map(({name}) => name);
|
|
@@ -535,6 +535,11 @@ export class DebuggerModel extends SDKModel<EventTypes> {
|
|
|
535
535
|
return this.#scripts.get(scriptId) || null;
|
|
536
536
|
}
|
|
537
537
|
|
|
538
|
+
isWasm(scriptId: string): boolean {
|
|
539
|
+
const script = this.scriptForId(scriptId);
|
|
540
|
+
return script ? script.isWasm() : false;
|
|
541
|
+
}
|
|
542
|
+
|
|
538
543
|
/**
|
|
539
544
|
* Returns all `Script` objects with the same provided `sourceURL`. The
|
|
540
545
|
* resulting array is sorted by time with the newest `Script` in the front.
|
|
@@ -41,7 +41,7 @@ class GreenDevFloaty {
|
|
|
41
41
|
#playButton!: HTMLButtonElement;
|
|
42
42
|
#node?: SDK.DOMModel.DOMNode;
|
|
43
43
|
#agent?: AiAssistance.GreenDevAgent.GreenDevAgent|AiAssistance.StylingAgent.StylingAgent;
|
|
44
|
-
#nodeContext?: AiAssistance.
|
|
44
|
+
#nodeContext?: AiAssistance.DOMNodeContext.DOMNodeContext;
|
|
45
45
|
#backendNodeId?: Protocol.DOM.BackendNodeId;
|
|
46
46
|
#syncChannel: BroadcastChannel;
|
|
47
47
|
#isFloatyWindow: boolean;
|
|
@@ -341,7 +341,7 @@ class GreenDevFloaty {
|
|
|
341
341
|
});
|
|
342
342
|
} else {
|
|
343
343
|
this.#agent = new AiAssistance.StylingAgent.StylingAgent({aidaClient});
|
|
344
|
-
this.#nodeContext = new AiAssistance.
|
|
344
|
+
this.#nodeContext = new AiAssistance.DOMNodeContext.DOMNodeContext(this.#node);
|
|
345
345
|
}
|
|
346
346
|
}
|
|
347
347
|
|
|
@@ -428,7 +428,8 @@ class GreenDevFloaty {
|
|
|
428
428
|
'Could not get the backendNodeId for the selected element.';
|
|
429
429
|
|
|
430
430
|
// --- Get some context information about the selected node ---
|
|
431
|
-
const
|
|
431
|
+
const domNodeContext = new AiAssistance.DOMNodeContext.DOMNodeContext(this.#node);
|
|
432
|
+
const elementContext = await domNodeContext.describe();
|
|
432
433
|
|
|
433
434
|
// Now construct the full context.
|
|
434
435
|
const context = `# Page URL
|
|
@@ -20,7 +20,8 @@ import * as AiAssistance from '../../models/ai_assistance/ai_assistance.js';
|
|
|
20
20
|
|
|
21
21
|
const {AidaClient} = Host.AidaClient;
|
|
22
22
|
const {ResponseType} = AiAssistance.AiAgent;
|
|
23
|
-
const {
|
|
23
|
+
const {DOMNodeContext} = AiAssistance.DOMNodeContext;
|
|
24
|
+
const {StylingAgent} = AiAssistance.StylingAgent;
|
|
24
25
|
|
|
25
26
|
class GreenDevFloaty {
|
|
26
27
|
#chatContainer: HTMLDivElement;
|
|
@@ -28,7 +29,7 @@ class GreenDevFloaty {
|
|
|
28
29
|
#playButton: HTMLButtonElement;
|
|
29
30
|
#node?: SDK.DOMModel.DOMNode;
|
|
30
31
|
#agent?: StylingAgent;
|
|
31
|
-
#nodeContext?:
|
|
32
|
+
#nodeContext?: DOMNodeContext;
|
|
32
33
|
#backendNodeId?: Protocol.DOM.BackendNodeId;
|
|
33
34
|
// Switching this to false can help while investigating tool conflicts.
|
|
34
35
|
#highlightNodeOnWindowFocus = false;
|
|
@@ -170,7 +171,7 @@ class GreenDevFloaty {
|
|
|
170
171
|
if (!this.#agent) {
|
|
171
172
|
const aidaClient = new AidaClient();
|
|
172
173
|
this.#agent = new StylingAgent({aidaClient});
|
|
173
|
-
this.#nodeContext = new
|
|
174
|
+
this.#nodeContext = new DOMNodeContext(this.#node);
|
|
174
175
|
}
|
|
175
176
|
|
|
176
177
|
this.#addMessage(query, true);
|
|
@@ -9,12 +9,19 @@ import {
|
|
|
9
9
|
AiAgent,
|
|
10
10
|
type ContextResponse,
|
|
11
11
|
type ConversationContext,
|
|
12
|
+
type MultimodalInputType,
|
|
12
13
|
type RequestOptions,
|
|
13
14
|
ResponseType
|
|
14
15
|
} from './agents/AiAgent.js';
|
|
15
16
|
import {debugLog} from './debug.js';
|
|
16
17
|
import type {Skill, SkillName} from './skills/Skill.js';
|
|
17
18
|
import {SKILLS} from './skills/SkillRegistry.js';
|
|
19
|
+
import type {Tool} from './tools/Tool.js';
|
|
20
|
+
import {ToolRegistry} from './tools/ToolRegistry.js';
|
|
21
|
+
|
|
22
|
+
const SKILL_DISPLAY_NAMES: Record<SkillName, string> = {
|
|
23
|
+
styling: 'CSS and styling',
|
|
24
|
+
};
|
|
18
25
|
|
|
19
26
|
export class AiAgent2 extends AiAgent<unknown> {
|
|
20
27
|
// TODO: The static preamble is a placeholder and will eventually live server-side.
|
|
@@ -29,9 +36,11 @@ export class AiAgent2 extends AiAgent<unknown> {
|
|
|
29
36
|
}
|
|
30
37
|
|
|
31
38
|
readonly #activeSkills = new Set<SkillName>();
|
|
39
|
+
readonly #declaredTools = new Set<string>();
|
|
32
40
|
|
|
33
41
|
constructor(opts: AgentOptions) {
|
|
34
42
|
super(opts);
|
|
43
|
+
this.#declaredTools.add('learnSkills');
|
|
35
44
|
const skillsList = Object.keys(SKILLS).join(', ');
|
|
36
45
|
this.declareFunction<{skills: SkillName[]}>('learnSkills', {
|
|
37
46
|
description: `Load skills to help with the task. Available skills: ${skillsList}.`,
|
|
@@ -51,8 +60,12 @@ export class AiAgent2 extends AiAgent<unknown> {
|
|
|
51
60
|
required: ['skills'],
|
|
52
61
|
},
|
|
53
62
|
displayInfoFromArgs: args => {
|
|
63
|
+
const isSingular = args.skills.length === 1;
|
|
64
|
+
const prefix = isSingular ? 'Learning skill' : 'Learning skills';
|
|
65
|
+
const names = args.skills.map(name => SKILL_DISPLAY_NAMES[name] ?? name).join(', ');
|
|
54
66
|
return {
|
|
55
|
-
title:
|
|
67
|
+
title: `${prefix}: ${names}`,
|
|
68
|
+
action: `learnSkills(${args.skills.map(name => `'${name}'`).join(', ')})`,
|
|
56
69
|
};
|
|
57
70
|
},
|
|
58
71
|
handler: async args => {
|
|
@@ -62,33 +75,58 @@ export class AiAgent2 extends AiAgent<unknown> {
|
|
|
62
75
|
});
|
|
63
76
|
}
|
|
64
77
|
|
|
65
|
-
override async enhanceQuery(
|
|
78
|
+
override async enhanceQuery(
|
|
79
|
+
query: string,
|
|
80
|
+
selected: ConversationContext<unknown>|null = null,
|
|
81
|
+
// TODO: support multimodal input in AiAgent2.
|
|
82
|
+
_multimodalInputType?: MultimodalInputType,
|
|
83
|
+
): Promise<string> {
|
|
84
|
+
let enhancedQuery = query;
|
|
85
|
+
if (selected) {
|
|
86
|
+
const promptDetails = await selected.getPromptDetails();
|
|
87
|
+
if (promptDetails) {
|
|
88
|
+
enhancedQuery = `${promptDetails}
|
|
89
|
+
|
|
90
|
+
# User request
|
|
91
|
+
|
|
92
|
+
QUERY: ${query}`;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
66
96
|
if (this.#skillsInjected) {
|
|
67
|
-
return
|
|
97
|
+
return enhancedQuery;
|
|
68
98
|
}
|
|
69
99
|
this.#skillsInjected = true;
|
|
70
|
-
const skillsManifest =
|
|
100
|
+
const skillsManifest =
|
|
101
|
+
Object.entries(this.getSkills()).map(([name, skill]) => `- ${name}: ${skill.description}`).join('\n');
|
|
71
102
|
return `Available skills:
|
|
72
103
|
${skillsManifest}
|
|
73
104
|
|
|
74
105
|
You must call \`learnSkills\` to load a skill before you can use it.
|
|
75
106
|
|
|
76
|
-
User query: ${
|
|
107
|
+
User query: ${enhancedQuery}`;
|
|
77
108
|
}
|
|
78
109
|
|
|
79
|
-
async *
|
|
80
|
-
handleContextDetails(
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
details
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
110
|
+
override async *
|
|
111
|
+
handleContextDetails(selected: ConversationContext<unknown>|null): AsyncGenerator<ContextResponse, void, void> {
|
|
112
|
+
if (selected) {
|
|
113
|
+
const details = await selected.getUserFacingDetails();
|
|
114
|
+
if (details) {
|
|
115
|
+
yield {
|
|
116
|
+
type: ResponseType.CONTEXT,
|
|
117
|
+
details,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
getSkills(): Record<SkillName, Skill> {
|
|
124
|
+
return SKILLS;
|
|
88
125
|
}
|
|
89
126
|
|
|
90
127
|
async learnSkill(names: SkillName[]): Promise<string> {
|
|
91
128
|
let response = '';
|
|
129
|
+
const skills = this.getSkills();
|
|
92
130
|
for (const name of names) {
|
|
93
131
|
debugLog(`AiAgent2: Attempting to load skill ${name}`);
|
|
94
132
|
if (this.#activeSkills.has(name)) {
|
|
@@ -97,19 +135,45 @@ User query: ${query}`;
|
|
|
97
135
|
continue;
|
|
98
136
|
}
|
|
99
137
|
|
|
100
|
-
const skillObj: Skill =
|
|
138
|
+
const skillObj: Skill = skills[name];
|
|
101
139
|
if (skillObj) {
|
|
102
140
|
this.#activeSkills.add(name);
|
|
103
141
|
debugLog(`AiAgent2: Skill ${name} loaded successfully`);
|
|
104
142
|
response += `Skill ${name} loaded. Instructions:\n${skillObj.instructions}\n`;
|
|
143
|
+
for (const toolName of skillObj.allowedTools) {
|
|
144
|
+
const tool = ToolRegistry.get(toolName);
|
|
145
|
+
if (tool) {
|
|
146
|
+
this.#declareTool(tool);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
105
149
|
} else {
|
|
106
150
|
debugLog(`AiAgent2: Failed to load skill ${name}`);
|
|
107
|
-
response += `Failed to load skill ${name}. Valid skills are: ${Object.keys(
|
|
151
|
+
response += `Failed to load skill ${name}. Valid skills are: ${Object.keys(skills).join(', ')}.\n`;
|
|
108
152
|
}
|
|
109
153
|
}
|
|
110
154
|
return response.trim();
|
|
111
155
|
}
|
|
112
156
|
|
|
157
|
+
/**
|
|
158
|
+
* Declares a tool to be available to the agent model, verifying first that
|
|
159
|
+
* it hasn't already been declared to prevent duplicate declaration errors.
|
|
160
|
+
*/
|
|
161
|
+
#declareTool(tool: Tool): void {
|
|
162
|
+
if (this.#declaredTools.has(tool.name)) {
|
|
163
|
+
debugLog(`AiAgent2: Tool ${tool.name} is already declared`);
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
this.#declaredTools.add(tool.name);
|
|
167
|
+
this.declareFunction(tool.name, {
|
|
168
|
+
description: tool.description,
|
|
169
|
+
parameters: tool.parameters,
|
|
170
|
+
displayInfoFromArgs: tool.displayInfoFromArgs,
|
|
171
|
+
handler: args => tool.handler(args, {
|
|
172
|
+
conversationContext: this.context ?? null,
|
|
173
|
+
}),
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
|
|
113
177
|
get activeSkills(): Set<SkillName> {
|
|
114
178
|
return this.#activeSkills;
|
|
115
179
|
}
|
|
@@ -29,10 +29,11 @@ import {FileAgent, FileContext} from './agents/FileAgent.js';
|
|
|
29
29
|
import {NetworkAgent, RequestContext} from './agents/NetworkAgent.js';
|
|
30
30
|
import {PerformanceAgent, PerformanceTraceContext} from './agents/PerformanceAgent.js';
|
|
31
31
|
import {StorageAgent, StorageContext} from './agents/StorageAgent.js';
|
|
32
|
-
import {
|
|
32
|
+
import {StylingAgent} from './agents/StylingAgent.js';
|
|
33
33
|
import {AiAgent2} from './AiAgent2.js';
|
|
34
34
|
import {AiHistoryStorage, ConversationType, type SerializedConversation} from './AiHistoryStorage.js';
|
|
35
35
|
import type {ChangeManager} from './ChangeManager.js';
|
|
36
|
+
import {DOMNodeContext} from './contexts/DOMNodeContext.js';
|
|
36
37
|
|
|
37
38
|
export const NOT_FOUND_IMAGE_DATA = '';
|
|
38
39
|
export const CONTEXT_TITLE = 'Analyzing data';
|
|
@@ -182,7 +183,7 @@ export class AiConversation {
|
|
|
182
183
|
if (isAiAssistanceContextSelectionAgentEnabled()) {
|
|
183
184
|
if (updateContext instanceof FileContext) {
|
|
184
185
|
this.#updateAgent(ConversationType.FILE);
|
|
185
|
-
} else if (updateContext instanceof
|
|
186
|
+
} else if (updateContext instanceof DOMNodeContext) {
|
|
186
187
|
this.#updateAgent(ConversationType.STYLING);
|
|
187
188
|
} else if (updateContext instanceof RequestContext) {
|
|
188
189
|
this.#updateAgent(ConversationType.NETWORK);
|
|
@@ -38,6 +38,14 @@ This work is currently in progress and behind a feature flag.
|
|
|
38
38
|
To support dynamic loading of skills, we generate JavaScript files from Markdown files containing skill definitions.
|
|
39
39
|
We use a nested `BUILD.gn` file in the `skills/` subdirectory specifically for this purpose. This ensures that GN's `target_gen_dir` points to `gen/front_end/models/ai_assistance/skills/`, placing the generated `.skill.js` files in the same relative structure as their source `.md` files. This allows TypeScript files in the `skills/` directory (like `map.ts`) to import the generated files using relative paths (e.g., `./styling.skill.js`) seamlessly.
|
|
40
40
|
|
|
41
|
+
### Tools and ToolRegistry
|
|
42
|
+
|
|
43
|
+
To support skills requiring execution of code or fetching page state (like computed styles), the architecture defines **Tools** in the `tools/` directory.
|
|
44
|
+
|
|
45
|
+
- **BaseTool**: A non-generic base interface capturing tool metadata (`name`, `description`, `parameters`). This acts as the type-erased representation for generic registry storage and fallback string lookups.
|
|
46
|
+
- **Tool**: A generic interface extending `BaseTool` that binds argument types (`Args`) to the schema `parameters` and the `handler()` method. This ensures compile-time safety and alignment inside each tool implementation.
|
|
47
|
+
- **ToolRegistry**: A static registry (`ToolRegistry`) storing instantiated tools. It uses TypeScript function overloading and generic lookups (`static get<K extends keyof typeof TOOLS>(name: K): typeof TOOLS[K]`) to return the precise class type of each tool, preventing type-erasure and escape-hatches (such as `any` or `as unknown` type assertions) at integration points like `AiAgent.ts`.
|
|
48
|
+
|
|
41
49
|
## Performance specific documentation
|
|
42
50
|
|
|
43
51
|
### `TimelineUtils.AIContext.AgentFocus`
|
|
@@ -81,6 +81,12 @@ If the user asks a question that requires an investigation of a problem, use thi
|
|
|
81
81
|
- [Suggestion 2]
|
|
82
82
|
`;
|
|
83
83
|
|
|
84
|
+
const SECURITY_WARNING = `**CRITICAL CONSTRAINT**: This Lighthouse report was imported from a file and is static.
|
|
85
|
+
You do NOT have access to the inspected page.
|
|
86
|
+
Tools like \`executeJavaScript\`, \`getStyles\`, or \`runAccessibilityAudits\` are disabled.
|
|
87
|
+
Do NOT attempt to use them or instruct the user that you will use them.
|
|
88
|
+
Rely ONLY on the static report data below.`;
|
|
89
|
+
|
|
84
90
|
export class AccessibilityContext extends ConversationContext<LHModel.ReporterTypes.ReportJSON> {
|
|
85
91
|
#lh: LHModel.ReporterTypes.ReportJSON;
|
|
86
92
|
|
|
@@ -235,85 +241,91 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
|
|
|
235
241
|
}
|
|
236
242
|
|
|
237
243
|
#declareFunctions(): void {
|
|
238
|
-
this.
|
|
244
|
+
const isImported = this.context?.getItem().isImported;
|
|
239
245
|
|
|
240
|
-
this.declareFunction<{
|
|
246
|
+
this.declareFunction<{categoryId: LHModel.RunTypes.CategoryId}, {audits: string}>('getLighthouseAudits', {
|
|
241
247
|
description:
|
|
242
|
-
'
|
|
248
|
+
'Returns the audits for a specific Lighthouse category. Use this to get more information about the performance, accessibility, best-practices, or seo audits.',
|
|
243
249
|
parameters: {
|
|
244
250
|
type: Host.AidaClient.ParametersTypes.OBJECT,
|
|
245
251
|
description: '',
|
|
246
252
|
nullable: false,
|
|
247
253
|
properties: {
|
|
248
|
-
|
|
254
|
+
categoryId: {
|
|
249
255
|
type: Host.AidaClient.ParametersTypes.STRING,
|
|
250
|
-
description:
|
|
256
|
+
description:
|
|
257
|
+
'The category of audits to retrieve. Valid values are "performance", "accessibility", "best-practices", "seo".',
|
|
251
258
|
nullable: false,
|
|
252
259
|
},
|
|
253
260
|
},
|
|
254
|
-
required: ['
|
|
261
|
+
required: ['categoryId'],
|
|
255
262
|
},
|
|
256
263
|
displayInfoFromArgs: params => {
|
|
257
264
|
return {
|
|
258
|
-
title: i18n.i18n.lockedString(
|
|
259
|
-
|
|
260
|
-
action: 'runAccessibilityAudits()'
|
|
265
|
+
title: i18n.i18n.lockedString(`Getting Lighthouse audits for ${params.categoryId}`),
|
|
266
|
+
action: `getLighthouseAudits('${params.categoryId}')`
|
|
261
267
|
};
|
|
262
268
|
},
|
|
263
269
|
handler: async params => {
|
|
264
|
-
debugLog('Function call:
|
|
265
|
-
|
|
266
|
-
return {error: 'Lighthouse recording is not available.'};
|
|
267
|
-
}
|
|
268
|
-
const report = await this.#lighthouseRecording({
|
|
269
|
-
mode: 'snapshot',
|
|
270
|
-
categoryIds: ['accessibility'],
|
|
271
|
-
isAIControlled: true,
|
|
272
|
-
});
|
|
270
|
+
debugLog('Function call: getLighthouseAudits', params);
|
|
271
|
+
const report = this.context?.getItem();
|
|
273
272
|
if (!report) {
|
|
274
|
-
return {error: '
|
|
273
|
+
return {error: 'No Lighthouse report available.'};
|
|
275
274
|
}
|
|
276
|
-
const audits = new LighthouseFormatter().audits(report,
|
|
275
|
+
const audits = new LighthouseFormatter().audits(report, params.categoryId);
|
|
277
276
|
return {
|
|
278
277
|
result: {audits},
|
|
279
|
-
widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report
|
|
278
|
+
widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report}}],
|
|
280
279
|
};
|
|
281
280
|
}
|
|
282
281
|
});
|
|
283
282
|
|
|
284
|
-
|
|
283
|
+
if (isImported) {
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
this.declareFunction('executeJavaScript', executeJavaScriptFunction(this.#javascriptExecutor));
|
|
288
|
+
|
|
289
|
+
this.declareFunction<{explanation: string}, {audits: string}>('runAccessibilityAudits', {
|
|
285
290
|
description:
|
|
286
|
-
'
|
|
291
|
+
'Triggers new Lighthouse accessibility audits in snapshot mode. Use this if the user has made changes to the page and you want to re-evaluate the accessibility audits.',
|
|
287
292
|
parameters: {
|
|
288
293
|
type: Host.AidaClient.ParametersTypes.OBJECT,
|
|
289
294
|
description: '',
|
|
290
295
|
nullable: false,
|
|
291
296
|
properties: {
|
|
292
|
-
|
|
297
|
+
explanation: {
|
|
293
298
|
type: Host.AidaClient.ParametersTypes.STRING,
|
|
294
|
-
description:
|
|
295
|
-
'The category of audits to retrieve. Valid values are "performance", "accessibility", "best-practices", "seo".',
|
|
299
|
+
description: 'Explain why you want to run new audits.',
|
|
296
300
|
nullable: false,
|
|
297
301
|
},
|
|
298
302
|
},
|
|
299
|
-
required: ['
|
|
303
|
+
required: ['explanation'],
|
|
300
304
|
},
|
|
301
305
|
displayInfoFromArgs: params => {
|
|
302
306
|
return {
|
|
303
|
-
title: i18n.i18n.lockedString(
|
|
304
|
-
|
|
307
|
+
title: i18n.i18n.lockedString('Running accessibility audits'),
|
|
308
|
+
thought: params.explanation,
|
|
309
|
+
action: 'runAccessibilityAudits()'
|
|
305
310
|
};
|
|
306
311
|
},
|
|
307
312
|
handler: async params => {
|
|
308
|
-
debugLog('Function call:
|
|
309
|
-
|
|
313
|
+
debugLog('Function call: runAccessibilityAudits', params);
|
|
314
|
+
if (!this.#lighthouseRecording) {
|
|
315
|
+
return {error: 'Lighthouse recording is not available.'};
|
|
316
|
+
}
|
|
317
|
+
const report = await this.#lighthouseRecording({
|
|
318
|
+
mode: 'snapshot',
|
|
319
|
+
categoryIds: ['accessibility'],
|
|
320
|
+
isAIControlled: true,
|
|
321
|
+
});
|
|
310
322
|
if (!report) {
|
|
311
|
-
return {error: '
|
|
323
|
+
return {error: 'Failed to run accessibility audits.'};
|
|
312
324
|
}
|
|
313
|
-
const audits = new LighthouseFormatter().audits(report,
|
|
325
|
+
const audits = new LighthouseFormatter().audits(report, 'accessibility');
|
|
314
326
|
return {
|
|
315
327
|
result: {audits},
|
|
316
|
-
widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report}}],
|
|
328
|
+
widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report, snapshotReport: true}}],
|
|
317
329
|
};
|
|
318
330
|
}
|
|
319
331
|
});
|
|
@@ -510,7 +522,10 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
|
|
|
510
522
|
if (lhr) {
|
|
511
523
|
this.#declareFunctions();
|
|
512
524
|
}
|
|
513
|
-
|
|
525
|
+
let enhancedQuery = lhr ? `${this.#getInitialPayload(lhr)}\n# User request:\n\n` : '';
|
|
526
|
+
if (lhr?.getItem().isImported) {
|
|
527
|
+
enhancedQuery = `${SECURITY_WARNING}\n\n${enhancedQuery}`;
|
|
528
|
+
}
|
|
514
529
|
return `${enhancedQuery}${query}`;
|
|
515
530
|
}
|
|
516
531
|
|
|
@@ -226,6 +226,22 @@ export abstract class ConversationContext<T> {
|
|
|
226
226
|
async getSuggestions(): Promise<ConversationSuggestions|undefined> {
|
|
227
227
|
return;
|
|
228
228
|
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Returns a detailed description of the context item for inclusion in the AI model prompt.
|
|
232
|
+
* Currently only used by AiAgent2.
|
|
233
|
+
*/
|
|
234
|
+
async getPromptDetails(): Promise<string|null> {
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Returns a list of context details to display to the user in the UI.
|
|
240
|
+
* Currently only used by AiAgent2.
|
|
241
|
+
*/
|
|
242
|
+
async getUserFacingDetails(): Promise<[ContextDetail, ...ContextDetail[]]|null> {
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
229
245
|
}
|
|
230
246
|
|
|
231
247
|
export interface ComputedStyleAiWidget {
|
|
@@ -13,6 +13,7 @@ import * as NetworkTimeCalculator from '../../network_time_calculator/network_ti
|
|
|
13
13
|
import type * as Trace from '../../trace/trace.js';
|
|
14
14
|
import * as Workspace from '../../workspace/workspace.js';
|
|
15
15
|
import {isOpaqueOrigin} from '../AiOrigins.js';
|
|
16
|
+
import {DOMNodeContext} from '../contexts/DOMNodeContext.js';
|
|
16
17
|
import {debugLog} from '../debug.js';
|
|
17
18
|
|
|
18
19
|
import {AccessibilityContext} from './AccessibilityAgent.js';
|
|
@@ -26,7 +27,6 @@ import {
|
|
|
26
27
|
import {FileContext} from './FileAgent.js';
|
|
27
28
|
import {RequestContext} from './NetworkAgent.js';
|
|
28
29
|
import {PerformanceTraceContext} from './PerformanceAgent.js';
|
|
29
|
-
import {NodeContext} from './StylingAgent.js';
|
|
30
30
|
|
|
31
31
|
const lockedString = i18n.i18n.lockedString;
|
|
32
32
|
/**
|
|
@@ -469,7 +469,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
|
|
|
469
469
|
const node = await this.#onInspectElement();
|
|
470
470
|
if (node) {
|
|
471
471
|
return {
|
|
472
|
-
context: new
|
|
472
|
+
context: new DOMNodeContext(node),
|
|
473
473
|
description: 'User selected an element',
|
|
474
474
|
};
|
|
475
475
|
}
|