chrome-devtools-frontend 1.0.1642246 → 1.0.1642899
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/SECURITY.md +1 -0
- package/front_end/core/host/UserMetrics.ts +2 -1
- package/front_end/core/sdk/CSSMatchedStyles.ts +55 -26
- package/front_end/core/sdk/CSSRule.ts +1 -0
- package/front_end/core/sdk/DebuggerModel.ts +5 -0
- package/front_end/entrypoints/greendev_floaty/FloatyEntrypoint.ts +4 -3
- package/front_end/entrypoints/greendev_floaty/greendev_floaty.ts +4 -3
- package/front_end/entrypoints/heap_snapshot_worker/HeapSnapshot.ts +11 -0
- package/front_end/models/ai_assistance/AiAgent2.ts +80 -16
- package/front_end/models/ai_assistance/AiConversation.ts +3 -2
- package/front_end/models/ai_assistance/README.md +8 -0
- package/front_end/models/ai_assistance/agents/AccessibilityAgent.ts +50 -39
- package/front_end/models/ai_assistance/agents/AiAgent.ts +17 -6
- package/front_end/models/ai_assistance/agents/ContextSelectionAgent.ts +2 -2
- package/front_end/models/ai_assistance/agents/PerformanceAgent.ts +195 -147
- package/front_end/models/ai_assistance/agents/StylingAgent.snapshot.txt +1 -26
- package/front_end/models/ai_assistance/agents/StylingAgent.ts +24 -309
- package/front_end/models/ai_assistance/ai_assistance.ts +8 -0
- package/front_end/models/ai_assistance/contexts/DOMNodeContext.snapshot.txt +51 -0
- package/front_end/models/ai_assistance/contexts/DOMNodeContext.ts +200 -0
- package/front_end/models/ai_assistance/data_formatters/NetworkRequestFormatter.ts +13 -12
- package/front_end/models/ai_assistance/skills/styling.md +36 -2
- package/front_end/models/ai_assistance/tools/GetStyles.ts +137 -0
- package/front_end/models/ai_assistance/tools/Tool.ts +55 -0
- package/front_end/models/ai_assistance/tools/ToolRegistry.ts +34 -0
- package/front_end/models/heap_snapshot/HeapSnapshotProxy.ts +4 -0
- package/front_end/models/javascript_metadata/NativeFunctions.js +16 -0
- package/front_end/models/lighthouse/LighthouseReporterTypes.ts +5 -0
- package/front_end/models/live-metrics/LiveMetrics.ts +24 -13
- package/front_end/models/stack_trace/DetailedErrorStackParser.ts +2 -2
- package/front_end/models/stack_trace/StackTrace.ts +4 -1
- package/front_end/models/stack_trace/StackTraceImpl.ts +9 -2
- package/front_end/models/stack_trace/StackTraceModel.ts +17 -4
- package/front_end/models/stack_trace/Trie.ts +1 -1
- package/front_end/panels/ai_assistance/AiAssistancePanel.ts +19 -15
- package/front_end/panels/ai_assistance/ai_assistance-meta.ts +16 -0
- package/front_end/panels/ai_assistance/components/ChatInput.ts +2 -2
- package/front_end/panels/application/DOMStorageItemsView.ts +4 -0
- package/front_end/panels/application/KeyValueStorageItemsView.ts +39 -7
- package/front_end/panels/common/ExtensionServer.ts +26 -15
- package/front_end/panels/console/SymbolizedErrorWidget.ts +96 -2
- package/front_end/panels/console/symbolizedErrorWidget.css +48 -0
- package/front_end/panels/elements/StandaloneStylesContainer.ts +1 -1
- package/front_end/panels/elements/StylePropertiesSection.ts +8 -0
- package/front_end/panels/elements/StylePropertyHighlighter.ts +4 -2
- package/front_end/panels/elements/StylePropertyTreeElement.ts +6 -5
- package/front_end/panels/elements/StylesContainer.ts +1 -1
- package/front_end/panels/elements/StylesSidebarPane.ts +4 -4
- package/front_end/panels/layer_viewer/PaintProfilerView.ts +106 -132
- package/front_end/panels/lighthouse/LighthousePanel.ts +4 -3
- package/front_end/panels/network/NetworkLogView.ts +3 -0
- package/front_end/panels/network/networkLogView.css +0 -15
- package/front_end/third_party/chromium/README.chromium +1 -1
- package/front_end/ui/legacy/components/cookie_table/CookiesTable.ts +36 -3
- package/front_end/ui/legacy/components/data_grid/dataGridAiButton.css +20 -0
- package/front_end/ui/legacy/components/utils/Linkifier.ts +19 -4
- package/front_end/ui/visual_logging/KnownContextValues.ts +1 -0
- package/front_end/ui/visual_logging/LoggingDriver.ts +13 -0
- package/package.json +1 -1
package/SECURITY.md
CHANGED
|
@@ -22,6 +22,7 @@ Chrome DevTools is a privileged web app running in a sandboxed renderer process,
|
|
|
22
22
|
* **Legitimate Data Exposure:** Displaying local user data, cookies, tokens, or auth headers within UI panels.
|
|
23
23
|
* **Local Data Persistence:** Saving user-initiated traces, heaps, profiles, or logs to disk.
|
|
24
24
|
* **Correctness & Availability:** Stale, misleading, or missing UI information (classified as functional bugs).
|
|
25
|
+
* **Denial of Service:** Chrome extensions or debug targets that disrupt DevTools' availability, e.g. through performance degradation.
|
|
25
26
|
* **Disabled-by-default Experimental Features:** Experimental features or capabilities that are disabled by default, e.g. requiring a command-line flag to be activated, or a specific runtime setting within DevTools to be manually turned on.
|
|
26
27
|
|
|
27
28
|
## Specific Severity Classification Rules (S1–S4)
|
|
@@ -496,7 +496,8 @@ export enum Action {
|
|
|
496
496
|
AiCodeGenerationRequestTriggeredFromSources = 205,
|
|
497
497
|
AiCodeCompletionFreCompletedFromConsole = 206,
|
|
498
498
|
AiCodeCompletionFreCompletedFromSources = 207,
|
|
499
|
-
|
|
499
|
+
AiAssistanceOpenedFromStoragePanelFloatingButton = 208,
|
|
500
|
+
MAX_VALUE = 209,
|
|
500
501
|
/* eslint-enable @typescript-eslint/naming-convention */
|
|
501
502
|
}
|
|
502
503
|
|
|
@@ -196,6 +196,36 @@ function queryMatches(style: CSSStyleDeclaration): boolean {
|
|
|
196
196
|
return true;
|
|
197
197
|
}
|
|
198
198
|
|
|
199
|
+
function treeScopeDistance(node: DOMNode, property: CSSProperty): number {
|
|
200
|
+
if (!property.ownerStyle.parentRule && property.ownerStyle.type !== Type.Inline) {
|
|
201
|
+
return -1;
|
|
202
|
+
}
|
|
203
|
+
const root = node.getTreeRoot();
|
|
204
|
+
const nodeId = property.ownerStyle.parentRule?.treeScope ?? root?.backendNodeId();
|
|
205
|
+
if (nodeId === undefined) {
|
|
206
|
+
return -1;
|
|
207
|
+
}
|
|
208
|
+
return distanceToTreeScope(node, nodeId);
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Distance from node to parent that is the tree scope. Returns -1 if tree scope is not found.
|
|
213
|
+
*
|
|
214
|
+
* @param node The child node to start from.
|
|
215
|
+
* @param treeScope The tree scope node ID.
|
|
216
|
+
* @returns The distance to the tree scope node, or -1 if not found.
|
|
217
|
+
*/
|
|
218
|
+
export function distanceToTreeScope(node: DOMNode, treeScope: Protocol.DOM.BackendNodeId): number {
|
|
219
|
+
let distance = 0;
|
|
220
|
+
for (let ancestor: DOMNode|null = node; ancestor; ancestor = ancestor.parentNode) {
|
|
221
|
+
if (ancestor.backendNodeId() === treeScope) {
|
|
222
|
+
return distance;
|
|
223
|
+
}
|
|
224
|
+
distance++;
|
|
225
|
+
}
|
|
226
|
+
return -1;
|
|
227
|
+
}
|
|
228
|
+
|
|
199
229
|
export interface CSSMatchedStylesPayload {
|
|
200
230
|
cssModel: CSSModel;
|
|
201
231
|
node: DOMNode;
|
|
@@ -307,7 +337,7 @@ export class CSSMatchedStyles {
|
|
|
307
337
|
#customHighlightPseudoDOMCascades?: Map<string, DOMInheritanceCascade>;
|
|
308
338
|
#functionRules: CSSFunctionRule[];
|
|
309
339
|
#atRules: CSSAtRule[];
|
|
310
|
-
#functionRuleMap = new Map<string, CSSFunctionRule>();
|
|
340
|
+
#functionRuleMap = new Map<string, CSSFunctionRule[]>();
|
|
311
341
|
#environmentVariables: Record<string, string> = {};
|
|
312
342
|
|
|
313
343
|
static async create(payload: CSSMatchedStylesPayload): Promise<CSSMatchedStyles> {
|
|
@@ -381,7 +411,9 @@ export class CSSMatchedStyles {
|
|
|
381
411
|
}
|
|
382
412
|
|
|
383
413
|
for (const rule of this.#functionRules) {
|
|
384
|
-
this.#functionRuleMap.
|
|
414
|
+
const rules = this.#functionRuleMap.get(rule.functionName().text) ?? [];
|
|
415
|
+
rules.push(rule);
|
|
416
|
+
this.#functionRuleMap.set(rule.functionName().text, rules);
|
|
385
417
|
}
|
|
386
418
|
}
|
|
387
419
|
|
|
@@ -790,9 +822,25 @@ export class CSSMatchedStyles {
|
|
|
790
822
|
return this.#registeredPropertyMap.get(name);
|
|
791
823
|
}
|
|
792
824
|
|
|
793
|
-
getRegisteredFunction(name: string):
|
|
794
|
-
|
|
795
|
-
|
|
825
|
+
getRegisteredFunction(name: string, sourceProperty: CSSProperty):
|
|
826
|
+
{treeScopeDistance: number, registeredFunction?: string} {
|
|
827
|
+
const minTreeScopeDistance = treeScopeDistance(this.#node, sourceProperty);
|
|
828
|
+
const functionRules = this.#functionRuleMap.get(name) ?? [];
|
|
829
|
+
let result: {treeScopeDistance: number, registeredFunction?: string} = {treeScopeDistance: -1};
|
|
830
|
+
for (const functionRule of functionRules) {
|
|
831
|
+
if (!functionRule.treeScope) {
|
|
832
|
+
continue;
|
|
833
|
+
}
|
|
834
|
+
const distance = distanceToTreeScope(this.#node, functionRule.treeScope);
|
|
835
|
+
if (distance === -1 || distance < minTreeScopeDistance) {
|
|
836
|
+
continue;
|
|
837
|
+
}
|
|
838
|
+
|
|
839
|
+
if (result.treeScopeDistance === -1 || distance < result.treeScopeDistance) {
|
|
840
|
+
result = {registeredFunction: functionRule.nameWithParameters(), treeScopeDistance: distance};
|
|
841
|
+
}
|
|
842
|
+
}
|
|
843
|
+
return result;
|
|
796
844
|
}
|
|
797
845
|
|
|
798
846
|
functionRules(): CSSFunctionRule[] {
|
|
@@ -1045,26 +1093,6 @@ class NodeCascade {
|
|
|
1045
1093
|
}
|
|
1046
1094
|
}
|
|
1047
1095
|
|
|
1048
|
-
#treeScopeDistance(property: CSSProperty): number {
|
|
1049
|
-
if (!property.ownerStyle.parentRule && property.ownerStyle.type !== Type.Inline) {
|
|
1050
|
-
return -1;
|
|
1051
|
-
}
|
|
1052
|
-
const root = this.#node.getTreeRoot();
|
|
1053
|
-
const nodeId = property.ownerStyle.parentRule?.treeScope ?? root?.backendNodeId();
|
|
1054
|
-
if (nodeId === undefined) {
|
|
1055
|
-
return -1;
|
|
1056
|
-
}
|
|
1057
|
-
|
|
1058
|
-
let distance = 0;
|
|
1059
|
-
for (let ancestor: DOMNode|null = this.#node; ancestor; ancestor = ancestor.parentNode) {
|
|
1060
|
-
if (ancestor.backendNodeId() === nodeId) {
|
|
1061
|
-
return distance;
|
|
1062
|
-
}
|
|
1063
|
-
distance++;
|
|
1064
|
-
}
|
|
1065
|
-
return -1;
|
|
1066
|
-
}
|
|
1067
|
-
|
|
1068
1096
|
#needsCascadeContextStep(): boolean {
|
|
1069
1097
|
if (!this.#node.isInShadowTree()) {
|
|
1070
1098
|
return false;
|
|
@@ -1084,7 +1112,8 @@ class NodeCascade {
|
|
|
1084
1112
|
const activeProperty = this.activeProperties.get(canonicalName);
|
|
1085
1113
|
if (activeProperty?.important && !propertyWithHigherSpecificity.important ||
|
|
1086
1114
|
activeProperty && this.#needsCascadeContextStep() &&
|
|
1087
|
-
this.#
|
|
1115
|
+
treeScopeDistance(this.#node, activeProperty) >
|
|
1116
|
+
treeScopeDistance(this.#node, propertyWithHigherSpecificity)) {
|
|
1088
1117
|
this.propertiesState.set(propertyWithHigherSpecificity, PropertyState.OVERLOADED);
|
|
1089
1118
|
return;
|
|
1090
1119
|
}
|
|
@@ -418,6 +418,7 @@ export class CSSFunctionRule extends CSSRule {
|
|
|
418
418
|
styleSheetId: payload.styleSheetId
|
|
419
419
|
},
|
|
420
420
|
header: styleSheetHeaderForRule(cssModel, payload),
|
|
421
|
+
originTreeScopeNodeId: payload.originTreeScopeNodeId
|
|
421
422
|
});
|
|
422
423
|
this.#name = new CSSValue(payload.name);
|
|
423
424
|
this.#parameters = payload.parameters.map(({name}) => name);
|
|
@@ -535,6 +535,11 @@ export class DebuggerModel extends SDKModel<EventTypes> {
|
|
|
535
535
|
return this.#scripts.get(scriptId) || null;
|
|
536
536
|
}
|
|
537
537
|
|
|
538
|
+
isWasm(scriptId: string): boolean {
|
|
539
|
+
const script = this.scriptForId(scriptId);
|
|
540
|
+
return script ? script.isWasm() : false;
|
|
541
|
+
}
|
|
542
|
+
|
|
538
543
|
/**
|
|
539
544
|
* Returns all `Script` objects with the same provided `sourceURL`. The
|
|
540
545
|
* resulting array is sorted by time with the newest `Script` in the front.
|
|
@@ -41,7 +41,7 @@ class GreenDevFloaty {
|
|
|
41
41
|
#playButton!: HTMLButtonElement;
|
|
42
42
|
#node?: SDK.DOMModel.DOMNode;
|
|
43
43
|
#agent?: AiAssistance.GreenDevAgent.GreenDevAgent|AiAssistance.StylingAgent.StylingAgent;
|
|
44
|
-
#nodeContext?: AiAssistance.
|
|
44
|
+
#nodeContext?: AiAssistance.DOMNodeContext.DOMNodeContext;
|
|
45
45
|
#backendNodeId?: Protocol.DOM.BackendNodeId;
|
|
46
46
|
#syncChannel: BroadcastChannel;
|
|
47
47
|
#isFloatyWindow: boolean;
|
|
@@ -341,7 +341,7 @@ class GreenDevFloaty {
|
|
|
341
341
|
});
|
|
342
342
|
} else {
|
|
343
343
|
this.#agent = new AiAssistance.StylingAgent.StylingAgent({aidaClient});
|
|
344
|
-
this.#nodeContext = new AiAssistance.
|
|
344
|
+
this.#nodeContext = new AiAssistance.DOMNodeContext.DOMNodeContext(this.#node);
|
|
345
345
|
}
|
|
346
346
|
}
|
|
347
347
|
|
|
@@ -428,7 +428,8 @@ class GreenDevFloaty {
|
|
|
428
428
|
'Could not get the backendNodeId for the selected element.';
|
|
429
429
|
|
|
430
430
|
// --- Get some context information about the selected node ---
|
|
431
|
-
const
|
|
431
|
+
const domNodeContext = new AiAssistance.DOMNodeContext.DOMNodeContext(this.#node);
|
|
432
|
+
const elementContext = await domNodeContext.describe();
|
|
432
433
|
|
|
433
434
|
// Now construct the full context.
|
|
434
435
|
const context = `# Page URL
|
|
@@ -20,7 +20,8 @@ import * as AiAssistance from '../../models/ai_assistance/ai_assistance.js';
|
|
|
20
20
|
|
|
21
21
|
const {AidaClient} = Host.AidaClient;
|
|
22
22
|
const {ResponseType} = AiAssistance.AiAgent;
|
|
23
|
-
const {
|
|
23
|
+
const {DOMNodeContext} = AiAssistance.DOMNodeContext;
|
|
24
|
+
const {StylingAgent} = AiAssistance.StylingAgent;
|
|
24
25
|
|
|
25
26
|
class GreenDevFloaty {
|
|
26
27
|
#chatContainer: HTMLDivElement;
|
|
@@ -28,7 +29,7 @@ class GreenDevFloaty {
|
|
|
28
29
|
#playButton: HTMLButtonElement;
|
|
29
30
|
#node?: SDK.DOMModel.DOMNode;
|
|
30
31
|
#agent?: StylingAgent;
|
|
31
|
-
#nodeContext?:
|
|
32
|
+
#nodeContext?: DOMNodeContext;
|
|
32
33
|
#backendNodeId?: Protocol.DOM.BackendNodeId;
|
|
33
34
|
// Switching this to false can help while investigating tool conflicts.
|
|
34
35
|
#highlightNodeOnWindowFocus = false;
|
|
@@ -170,7 +171,7 @@ class GreenDevFloaty {
|
|
|
170
171
|
if (!this.#agent) {
|
|
171
172
|
const aidaClient = new AidaClient();
|
|
172
173
|
this.#agent = new StylingAgent({aidaClient});
|
|
173
|
-
this.#nodeContext = new
|
|
174
|
+
this.#nodeContext = new DOMNodeContext(this.#node);
|
|
174
175
|
}
|
|
175
176
|
|
|
176
177
|
this.#addMessage(query, true);
|
|
@@ -1068,6 +1068,17 @@ export abstract class HeapSnapshot {
|
|
|
1068
1068
|
this.#progress.updateStatus('Finished processing.');
|
|
1069
1069
|
}
|
|
1070
1070
|
|
|
1071
|
+
nodeIndexForId(nodeId: number): number|undefined {
|
|
1072
|
+
const nodesLength = this.nodes.length;
|
|
1073
|
+
const {nodes, nodeFieldCount, nodeIdOffset} = this;
|
|
1074
|
+
for (let nodeIndex = 0; nodeIndex < nodesLength; nodeIndex += nodeFieldCount) {
|
|
1075
|
+
if (nodes.getValue(nodeIndex + nodeIdOffset) === nodeId) {
|
|
1076
|
+
return nodeIndex;
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
return undefined;
|
|
1080
|
+
}
|
|
1081
|
+
|
|
1071
1082
|
private startInitStep1InSecondThread(secondWorker: PlatformApi.HostRuntime.WorkerMessagePort):
|
|
1072
1083
|
Promise<ResultsFromSecondWorker> {
|
|
1073
1084
|
const resultsFromSecondWorker = new Promise<ResultsFromSecondWorker>((resolve, reject) => {
|
|
@@ -9,12 +9,19 @@ import {
|
|
|
9
9
|
AiAgent,
|
|
10
10
|
type ContextResponse,
|
|
11
11
|
type ConversationContext,
|
|
12
|
+
type MultimodalInputType,
|
|
12
13
|
type RequestOptions,
|
|
13
14
|
ResponseType
|
|
14
15
|
} from './agents/AiAgent.js';
|
|
15
16
|
import {debugLog} from './debug.js';
|
|
16
17
|
import type {Skill, SkillName} from './skills/Skill.js';
|
|
17
18
|
import {SKILLS} from './skills/SkillRegistry.js';
|
|
19
|
+
import type {Tool} from './tools/Tool.js';
|
|
20
|
+
import {ToolRegistry} from './tools/ToolRegistry.js';
|
|
21
|
+
|
|
22
|
+
const SKILL_DISPLAY_NAMES: Record<SkillName, string> = {
|
|
23
|
+
styling: 'CSS and styling',
|
|
24
|
+
};
|
|
18
25
|
|
|
19
26
|
export class AiAgent2 extends AiAgent<unknown> {
|
|
20
27
|
// TODO: The static preamble is a placeholder and will eventually live server-side.
|
|
@@ -29,9 +36,11 @@ export class AiAgent2 extends AiAgent<unknown> {
|
|
|
29
36
|
}
|
|
30
37
|
|
|
31
38
|
readonly #activeSkills = new Set<SkillName>();
|
|
39
|
+
readonly #declaredTools = new Set<string>();
|
|
32
40
|
|
|
33
41
|
constructor(opts: AgentOptions) {
|
|
34
42
|
super(opts);
|
|
43
|
+
this.#declaredTools.add('learnSkills');
|
|
35
44
|
const skillsList = Object.keys(SKILLS).join(', ');
|
|
36
45
|
this.declareFunction<{skills: SkillName[]}>('learnSkills', {
|
|
37
46
|
description: `Load skills to help with the task. Available skills: ${skillsList}.`,
|
|
@@ -51,8 +60,12 @@ export class AiAgent2 extends AiAgent<unknown> {
|
|
|
51
60
|
required: ['skills'],
|
|
52
61
|
},
|
|
53
62
|
displayInfoFromArgs: args => {
|
|
63
|
+
const isSingular = args.skills.length === 1;
|
|
64
|
+
const prefix = isSingular ? 'Learning skill' : 'Learning skills';
|
|
65
|
+
const names = args.skills.map(name => SKILL_DISPLAY_NAMES[name] ?? name).join(', ');
|
|
54
66
|
return {
|
|
55
|
-
title:
|
|
67
|
+
title: `${prefix}: ${names}`,
|
|
68
|
+
action: `learnSkills(${args.skills.map(name => `'${name}'`).join(', ')})`,
|
|
56
69
|
};
|
|
57
70
|
},
|
|
58
71
|
handler: async args => {
|
|
@@ -62,33 +75,58 @@ export class AiAgent2 extends AiAgent<unknown> {
|
|
|
62
75
|
});
|
|
63
76
|
}
|
|
64
77
|
|
|
65
|
-
override async enhanceQuery(
|
|
78
|
+
override async enhanceQuery(
|
|
79
|
+
query: string,
|
|
80
|
+
selected: ConversationContext<unknown>|null = null,
|
|
81
|
+
// TODO: support multimodal input in AiAgent2.
|
|
82
|
+
_multimodalInputType?: MultimodalInputType,
|
|
83
|
+
): Promise<string> {
|
|
84
|
+
let enhancedQuery = query;
|
|
85
|
+
if (selected) {
|
|
86
|
+
const promptDetails = await selected.getPromptDetails();
|
|
87
|
+
if (promptDetails) {
|
|
88
|
+
enhancedQuery = `${promptDetails}
|
|
89
|
+
|
|
90
|
+
# User request
|
|
91
|
+
|
|
92
|
+
QUERY: ${query}`;
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
66
96
|
if (this.#skillsInjected) {
|
|
67
|
-
return
|
|
97
|
+
return enhancedQuery;
|
|
68
98
|
}
|
|
69
99
|
this.#skillsInjected = true;
|
|
70
|
-
const skillsManifest =
|
|
100
|
+
const skillsManifest =
|
|
101
|
+
Object.entries(this.getSkills()).map(([name, skill]) => `- ${name}: ${skill.description}`).join('\n');
|
|
71
102
|
return `Available skills:
|
|
72
103
|
${skillsManifest}
|
|
73
104
|
|
|
74
105
|
You must call \`learnSkills\` to load a skill before you can use it.
|
|
75
106
|
|
|
76
|
-
User query: ${
|
|
107
|
+
User query: ${enhancedQuery}`;
|
|
77
108
|
}
|
|
78
109
|
|
|
79
|
-
async *
|
|
80
|
-
handleContextDetails(
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
details
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
110
|
+
override async *
|
|
111
|
+
handleContextDetails(selected: ConversationContext<unknown>|null): AsyncGenerator<ContextResponse, void, void> {
|
|
112
|
+
if (selected) {
|
|
113
|
+
const details = await selected.getUserFacingDetails();
|
|
114
|
+
if (details) {
|
|
115
|
+
yield {
|
|
116
|
+
type: ResponseType.CONTEXT,
|
|
117
|
+
details,
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
getSkills(): Record<SkillName, Skill> {
|
|
124
|
+
return SKILLS;
|
|
88
125
|
}
|
|
89
126
|
|
|
90
127
|
async learnSkill(names: SkillName[]): Promise<string> {
|
|
91
128
|
let response = '';
|
|
129
|
+
const skills = this.getSkills();
|
|
92
130
|
for (const name of names) {
|
|
93
131
|
debugLog(`AiAgent2: Attempting to load skill ${name}`);
|
|
94
132
|
if (this.#activeSkills.has(name)) {
|
|
@@ -97,19 +135,45 @@ User query: ${query}`;
|
|
|
97
135
|
continue;
|
|
98
136
|
}
|
|
99
137
|
|
|
100
|
-
const skillObj: Skill =
|
|
138
|
+
const skillObj: Skill = skills[name];
|
|
101
139
|
if (skillObj) {
|
|
102
140
|
this.#activeSkills.add(name);
|
|
103
141
|
debugLog(`AiAgent2: Skill ${name} loaded successfully`);
|
|
104
142
|
response += `Skill ${name} loaded. Instructions:\n${skillObj.instructions}\n`;
|
|
143
|
+
for (const toolName of skillObj.allowedTools) {
|
|
144
|
+
const tool = ToolRegistry.get(toolName);
|
|
145
|
+
if (tool) {
|
|
146
|
+
this.#declareTool(tool);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
105
149
|
} else {
|
|
106
150
|
debugLog(`AiAgent2: Failed to load skill ${name}`);
|
|
107
|
-
response += `Failed to load skill ${name}. Valid skills are: ${Object.keys(
|
|
151
|
+
response += `Failed to load skill ${name}. Valid skills are: ${Object.keys(skills).join(', ')}.\n`;
|
|
108
152
|
}
|
|
109
153
|
}
|
|
110
154
|
return response.trim();
|
|
111
155
|
}
|
|
112
156
|
|
|
157
|
+
/**
|
|
158
|
+
* Declares a tool to be available to the agent model, verifying first that
|
|
159
|
+
* it hasn't already been declared to prevent duplicate declaration errors.
|
|
160
|
+
*/
|
|
161
|
+
#declareTool(tool: Tool): void {
|
|
162
|
+
if (this.#declaredTools.has(tool.name)) {
|
|
163
|
+
debugLog(`AiAgent2: Tool ${tool.name} is already declared`);
|
|
164
|
+
return;
|
|
165
|
+
}
|
|
166
|
+
this.#declaredTools.add(tool.name);
|
|
167
|
+
this.declareFunction(tool.name, {
|
|
168
|
+
description: tool.description,
|
|
169
|
+
parameters: tool.parameters,
|
|
170
|
+
displayInfoFromArgs: tool.displayInfoFromArgs,
|
|
171
|
+
handler: args => tool.handler(args, {
|
|
172
|
+
conversationContext: this.context ?? null,
|
|
173
|
+
}),
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
|
|
113
177
|
get activeSkills(): Set<SkillName> {
|
|
114
178
|
return this.#activeSkills;
|
|
115
179
|
}
|
|
@@ -29,10 +29,11 @@ import {FileAgent, FileContext} from './agents/FileAgent.js';
|
|
|
29
29
|
import {NetworkAgent, RequestContext} from './agents/NetworkAgent.js';
|
|
30
30
|
import {PerformanceAgent, PerformanceTraceContext} from './agents/PerformanceAgent.js';
|
|
31
31
|
import {StorageAgent, StorageContext} from './agents/StorageAgent.js';
|
|
32
|
-
import {
|
|
32
|
+
import {StylingAgent} from './agents/StylingAgent.js';
|
|
33
33
|
import {AiAgent2} from './AiAgent2.js';
|
|
34
34
|
import {AiHistoryStorage, ConversationType, type SerializedConversation} from './AiHistoryStorage.js';
|
|
35
35
|
import type {ChangeManager} from './ChangeManager.js';
|
|
36
|
+
import {DOMNodeContext} from './contexts/DOMNodeContext.js';
|
|
36
37
|
|
|
37
38
|
export const NOT_FOUND_IMAGE_DATA = '';
|
|
38
39
|
export const CONTEXT_TITLE = 'Analyzing data';
|
|
@@ -182,7 +183,7 @@ export class AiConversation {
|
|
|
182
183
|
if (isAiAssistanceContextSelectionAgentEnabled()) {
|
|
183
184
|
if (updateContext instanceof FileContext) {
|
|
184
185
|
this.#updateAgent(ConversationType.FILE);
|
|
185
|
-
} else if (updateContext instanceof
|
|
186
|
+
} else if (updateContext instanceof DOMNodeContext) {
|
|
186
187
|
this.#updateAgent(ConversationType.STYLING);
|
|
187
188
|
} else if (updateContext instanceof RequestContext) {
|
|
188
189
|
this.#updateAgent(ConversationType.NETWORK);
|
|
@@ -38,6 +38,14 @@ This work is currently in progress and behind a feature flag.
|
|
|
38
38
|
To support dynamic loading of skills, we generate JavaScript files from Markdown files containing skill definitions.
|
|
39
39
|
We use a nested `BUILD.gn` file in the `skills/` subdirectory specifically for this purpose. This ensures that GN's `target_gen_dir` points to `gen/front_end/models/ai_assistance/skills/`, placing the generated `.skill.js` files in the same relative structure as their source `.md` files. This allows TypeScript files in the `skills/` directory (like `map.ts`) to import the generated files using relative paths (e.g., `./styling.skill.js`) seamlessly.
|
|
40
40
|
|
|
41
|
+
### Tools and ToolRegistry
|
|
42
|
+
|
|
43
|
+
To support skills requiring execution of code or fetching page state (like computed styles), the architecture defines **Tools** in the `tools/` directory.
|
|
44
|
+
|
|
45
|
+
- **BaseTool**: A non-generic base interface capturing tool metadata (`name`, `description`, `parameters`). This acts as the type-erased representation for generic registry storage and fallback string lookups.
|
|
46
|
+
- **Tool**: A generic interface extending `BaseTool` that binds argument types (`Args`) to the schema `parameters` and the `handler()` method. This ensures compile-time safety and alignment inside each tool implementation.
|
|
47
|
+
- **ToolRegistry**: A static registry (`ToolRegistry`) storing instantiated tools. It uses TypeScript function overloading and generic lookups (`static get<K extends keyof typeof TOOLS>(name: K): typeof TOOLS[K]`) to return the precise class type of each tool, preventing type-erasure and escape-hatches (such as `any` or `as unknown` type assertions) at integration points like `AiAgent.ts`.
|
|
48
|
+
|
|
41
49
|
## Performance specific documentation
|
|
42
50
|
|
|
43
51
|
### `TimelineUtils.AIContext.AgentFocus`
|
|
@@ -81,6 +81,12 @@ If the user asks a question that requires an investigation of a problem, use thi
|
|
|
81
81
|
- [Suggestion 2]
|
|
82
82
|
`;
|
|
83
83
|
|
|
84
|
+
const SECURITY_WARNING = `**CRITICAL CONSTRAINT**: This Lighthouse report was imported from a file and is static.
|
|
85
|
+
You do NOT have access to the inspected page.
|
|
86
|
+
Tools like \`executeJavaScript\`, \`getStyles\`, or \`runAccessibilityAudits\` are disabled.
|
|
87
|
+
Do NOT attempt to use them or instruct the user that you will use them.
|
|
88
|
+
Rely ONLY on the static report data below.`;
|
|
89
|
+
|
|
84
90
|
export class AccessibilityContext extends ConversationContext<LHModel.ReporterTypes.ReportJSON> {
|
|
85
91
|
#lh: LHModel.ReporterTypes.ReportJSON;
|
|
86
92
|
|
|
@@ -161,10 +167,6 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
|
|
|
161
167
|
};
|
|
162
168
|
}
|
|
163
169
|
|
|
164
|
-
override preambleFeatures(): string[] {
|
|
165
|
-
return ['function_calling'];
|
|
166
|
-
}
|
|
167
|
-
|
|
168
170
|
protected override async preRun(): Promise<void> {
|
|
169
171
|
this.#currentTurnId++;
|
|
170
172
|
const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget();
|
|
@@ -239,85 +241,91 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
|
|
|
239
241
|
}
|
|
240
242
|
|
|
241
243
|
#declareFunctions(): void {
|
|
242
|
-
this.
|
|
244
|
+
const isImported = this.context?.getItem().isImported;
|
|
243
245
|
|
|
244
|
-
this.declareFunction<{
|
|
246
|
+
this.declareFunction<{categoryId: LHModel.RunTypes.CategoryId}, {audits: string}>('getLighthouseAudits', {
|
|
245
247
|
description:
|
|
246
|
-
'
|
|
248
|
+
'Returns the audits for a specific Lighthouse category. Use this to get more information about the performance, accessibility, best-practices, or seo audits.',
|
|
247
249
|
parameters: {
|
|
248
250
|
type: Host.AidaClient.ParametersTypes.OBJECT,
|
|
249
251
|
description: '',
|
|
250
252
|
nullable: false,
|
|
251
253
|
properties: {
|
|
252
|
-
|
|
254
|
+
categoryId: {
|
|
253
255
|
type: Host.AidaClient.ParametersTypes.STRING,
|
|
254
|
-
description:
|
|
256
|
+
description:
|
|
257
|
+
'The category of audits to retrieve. Valid values are "performance", "accessibility", "best-practices", "seo".',
|
|
255
258
|
nullable: false,
|
|
256
259
|
},
|
|
257
260
|
},
|
|
258
|
-
required: ['
|
|
261
|
+
required: ['categoryId'],
|
|
259
262
|
},
|
|
260
263
|
displayInfoFromArgs: params => {
|
|
261
264
|
return {
|
|
262
|
-
title: i18n.i18n.lockedString(
|
|
263
|
-
|
|
264
|
-
action: 'runAccessibilityAudits()'
|
|
265
|
+
title: i18n.i18n.lockedString(`Getting Lighthouse audits for ${params.categoryId}`),
|
|
266
|
+
action: `getLighthouseAudits('${params.categoryId}')`
|
|
265
267
|
};
|
|
266
268
|
},
|
|
267
269
|
handler: async params => {
|
|
268
|
-
debugLog('Function call:
|
|
269
|
-
|
|
270
|
-
return {error: 'Lighthouse recording is not available.'};
|
|
271
|
-
}
|
|
272
|
-
const report = await this.#lighthouseRecording({
|
|
273
|
-
mode: 'snapshot',
|
|
274
|
-
categoryIds: ['accessibility'],
|
|
275
|
-
isAIControlled: true,
|
|
276
|
-
});
|
|
270
|
+
debugLog('Function call: getLighthouseAudits', params);
|
|
271
|
+
const report = this.context?.getItem();
|
|
277
272
|
if (!report) {
|
|
278
|
-
return {error: '
|
|
273
|
+
return {error: 'No Lighthouse report available.'};
|
|
279
274
|
}
|
|
280
|
-
const audits = new LighthouseFormatter().audits(report,
|
|
275
|
+
const audits = new LighthouseFormatter().audits(report, params.categoryId);
|
|
281
276
|
return {
|
|
282
277
|
result: {audits},
|
|
283
|
-
widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report
|
|
278
|
+
widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report}}],
|
|
284
279
|
};
|
|
285
280
|
}
|
|
286
281
|
});
|
|
287
282
|
|
|
288
|
-
|
|
283
|
+
if (isImported) {
|
|
284
|
+
return;
|
|
285
|
+
}
|
|
286
|
+
|
|
287
|
+
this.declareFunction('executeJavaScript', executeJavaScriptFunction(this.#javascriptExecutor));
|
|
288
|
+
|
|
289
|
+
this.declareFunction<{explanation: string}, {audits: string}>('runAccessibilityAudits', {
|
|
289
290
|
description:
|
|
290
|
-
'
|
|
291
|
+
'Triggers new Lighthouse accessibility audits in snapshot mode. Use this if the user has made changes to the page and you want to re-evaluate the accessibility audits.',
|
|
291
292
|
parameters: {
|
|
292
293
|
type: Host.AidaClient.ParametersTypes.OBJECT,
|
|
293
294
|
description: '',
|
|
294
295
|
nullable: false,
|
|
295
296
|
properties: {
|
|
296
|
-
|
|
297
|
+
explanation: {
|
|
297
298
|
type: Host.AidaClient.ParametersTypes.STRING,
|
|
298
|
-
description:
|
|
299
|
-
'The category of audits to retrieve. Valid values are "performance", "accessibility", "best-practices", "seo".',
|
|
299
|
+
description: 'Explain why you want to run new audits.',
|
|
300
300
|
nullable: false,
|
|
301
301
|
},
|
|
302
302
|
},
|
|
303
|
-
required: ['
|
|
303
|
+
required: ['explanation'],
|
|
304
304
|
},
|
|
305
305
|
displayInfoFromArgs: params => {
|
|
306
306
|
return {
|
|
307
|
-
title: i18n.i18n.lockedString(
|
|
308
|
-
|
|
307
|
+
title: i18n.i18n.lockedString('Running accessibility audits'),
|
|
308
|
+
thought: params.explanation,
|
|
309
|
+
action: 'runAccessibilityAudits()'
|
|
309
310
|
};
|
|
310
311
|
},
|
|
311
312
|
handler: async params => {
|
|
312
|
-
debugLog('Function call:
|
|
313
|
-
|
|
313
|
+
debugLog('Function call: runAccessibilityAudits', params);
|
|
314
|
+
if (!this.#lighthouseRecording) {
|
|
315
|
+
return {error: 'Lighthouse recording is not available.'};
|
|
316
|
+
}
|
|
317
|
+
const report = await this.#lighthouseRecording({
|
|
318
|
+
mode: 'snapshot',
|
|
319
|
+
categoryIds: ['accessibility'],
|
|
320
|
+
isAIControlled: true,
|
|
321
|
+
});
|
|
314
322
|
if (!report) {
|
|
315
|
-
return {error: '
|
|
323
|
+
return {error: 'Failed to run accessibility audits.'};
|
|
316
324
|
}
|
|
317
|
-
const audits = new LighthouseFormatter().audits(report,
|
|
325
|
+
const audits = new LighthouseFormatter().audits(report, 'accessibility');
|
|
318
326
|
return {
|
|
319
327
|
result: {audits},
|
|
320
|
-
widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report}}],
|
|
328
|
+
widgets: [{name: 'LIGHTHOUSE_REPORT', data: {report, snapshotReport: true}}],
|
|
321
329
|
};
|
|
322
330
|
}
|
|
323
331
|
});
|
|
@@ -514,7 +522,10 @@ export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON
|
|
|
514
522
|
if (lhr) {
|
|
515
523
|
this.#declareFunctions();
|
|
516
524
|
}
|
|
517
|
-
|
|
525
|
+
let enhancedQuery = lhr ? `${this.#getInitialPayload(lhr)}\n# User request:\n\n` : '';
|
|
526
|
+
if (lhr?.getItem().isImported) {
|
|
527
|
+
enhancedQuery = `${SECURITY_WARNING}\n\n${enhancedQuery}`;
|
|
528
|
+
}
|
|
518
529
|
return `${enhancedQuery}${query}`;
|
|
519
530
|
}
|
|
520
531
|
|
|
@@ -226,6 +226,22 @@ export abstract class ConversationContext<T> {
|
|
|
226
226
|
async getSuggestions(): Promise<ConversationSuggestions|undefined> {
|
|
227
227
|
return;
|
|
228
228
|
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Returns a detailed description of the context item for inclusion in the AI model prompt.
|
|
232
|
+
* Currently only used by AiAgent2.
|
|
233
|
+
*/
|
|
234
|
+
async getPromptDetails(): Promise<string|null> {
|
|
235
|
+
return null;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Returns a list of context details to display to the user in the UI.
|
|
240
|
+
* Currently only used by AiAgent2.
|
|
241
|
+
*/
|
|
242
|
+
async getUserFacingDetails(): Promise<[ContextDetail, ...ContextDetail[]]|null> {
|
|
243
|
+
return null;
|
|
244
|
+
}
|
|
229
245
|
}
|
|
230
246
|
|
|
231
247
|
export interface ComputedStyleAiWidget {
|
|
@@ -530,10 +546,6 @@ export abstract class AiAgent<T> {
|
|
|
530
546
|
return undefined;
|
|
531
547
|
}
|
|
532
548
|
|
|
533
|
-
preambleFeatures(): string[] {
|
|
534
|
-
return [];
|
|
535
|
-
}
|
|
536
|
-
|
|
537
549
|
buildRequest(
|
|
538
550
|
part: Host.AidaClient.Part|Host.AidaClient.Part[],
|
|
539
551
|
role: Host.AidaClient.Role.USER|Host.AidaClient.Role.ROLE_UNSPECIFIED): Host.AidaClient.DoConversationRequest {
|
|
@@ -577,8 +589,7 @@ export abstract class AiAgent<T> {
|
|
|
577
589
|
disable_user_content_logging: !(this.#serverSideLoggingEnabled ?? false),
|
|
578
590
|
string_session_id: this.#sessionId,
|
|
579
591
|
user_tier: userTier,
|
|
580
|
-
client_version:
|
|
581
|
-
Root.Runtime.getChromeVersion() + this.preambleFeatures().map(feature => `+${feature}`).join(''),
|
|
592
|
+
client_version: Root.Runtime.getChromeVersion(),
|
|
582
593
|
},
|
|
583
594
|
|
|
584
595
|
functionality_type: enableAidaFunctionCalling ? Host.AidaClient.FunctionalityType.AGENTIC_CHAT :
|