chrome-devtools-frontend 1.0.1597448 → 1.0.1597624
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/front_end/core/host/AidaClient.ts +4 -0
- package/front_end/core/sdk/CPUThrottlingManager.ts +5 -1
- package/front_end/core/sdk/CSSMatchedStyles.ts +2 -0
- package/front_end/core/sdk/CSSPropertyParserMatchers.ts +28 -0
- package/front_end/models/ai_assistance/AiConversation.ts +24 -8
- package/front_end/models/ai_assistance/ChangeManager.ts +16 -0
- package/front_end/models/ai_assistance/ExtensionScope.ts +11 -3
- package/front_end/models/ai_assistance/agents/AccessibilityAgent.ts +127 -0
- package/front_end/models/ai_assistance/agents/AiAgent.ts +22 -3
- package/front_end/models/ai_assistance/agents/ContextSelectionAgent.snapshot.txt +1 -1
- package/front_end/models/ai_assistance/agents/ContextSelectionAgent.ts +11 -8
- package/front_end/models/ai_assistance/agents/StylingAgent.ts +34 -4
- package/front_end/models/ai_assistance/ai_assistance.ts +2 -0
- package/front_end/models/ai_assistance/data_formatters/PerformanceInsightFormatter.ts +27 -0
- package/front_end/models/ai_assistance/data_formatters/PerformanceTraceFormatter.snapshot.txt +21 -0
- package/front_end/models/trace/Processor.ts +1 -0
- package/front_end/models/trace/handlers/PageLoadMetricsHandler.ts +33 -0
- package/front_end/models/trace/insights/CharacterSet.ts +172 -0
- package/front_end/models/trace/insights/Models.ts +1 -0
- package/front_end/models/trace/insights/types.ts +1 -0
- package/front_end/models/trace/types/TraceEvents.ts +17 -0
- package/front_end/panels/ai_assistance/AiAssistancePanel.ts +16 -3
- package/front_end/panels/ai_assistance/components/ChatMessage.ts +90 -46
- package/front_end/panels/ai_assistance/components/MarkdownRendererWithCodeBlock.ts +18 -9
- package/front_end/panels/ai_assistance/components/chatMessage.css +11 -0
- package/front_end/panels/application/AppManifestView.ts +3 -4
- package/front_end/panels/application/DeviceBoundSessionsView.ts +18 -22
- package/front_end/panels/application/FrameDetailsView.ts +9 -15
- package/front_end/panels/application/OriginTrialTreeView.ts +2 -3
- package/front_end/panels/application/ReportingApiView.ts +13 -17
- package/front_end/panels/application/components/BackForwardCacheView.ts +3 -3
- package/front_end/panels/application/preloading/components/UsedPreloadingView.ts +2 -3
- package/front_end/panels/browser_debugger/DOMBreakpointsSidebarPane.ts +3 -2
- package/front_end/panels/changes/ChangesView.ts +6 -4
- package/front_end/panels/console/ConsolePinPane.ts +3 -3
- package/front_end/panels/coverage/CoverageListView.ts +1 -1
- package/front_end/panels/css_overview/CSSOverviewPanel.ts +11 -15
- package/front_end/panels/developer_resources/DeveloperResourcesView.ts +3 -5
- package/front_end/panels/elements/EventListenersWidget.ts +3 -2
- package/front_end/panels/elements/StandaloneStylesContainer.ts +21 -6
- package/front_end/panels/elements/StylePropertyTreeElement.ts +49 -4
- package/front_end/panels/layer_viewer/Layers3DView.ts +5 -4
- package/front_end/panels/linear_memory_inspector/components/LinearMemoryInspector.ts +5 -6
- package/front_end/panels/linear_memory_inspector/components/LinearMemoryValueInterpreter.ts +6 -11
- package/front_end/panels/network/RequestCookiesView.ts +3 -4
- package/front_end/panels/network/RequestInitiatorView.ts +7 -5
- package/front_end/panels/network/RequestResponseView.ts +10 -15
- package/front_end/panels/protocol_monitor/ProtocolMonitor.ts +3 -4
- package/front_end/panels/recorder/components/RecordingView.ts +31 -36
- package/front_end/panels/recorder/components/StepEditor.ts +6 -7
- package/front_end/panels/search/SearchView.ts +2 -3
- package/front_end/panels/settings/WorkspaceSettingsTab.ts +2 -5
- package/front_end/panels/timeline/components/LiveMetricsView.ts +5 -8
- package/front_end/panels/timeline/components/insights/Cache.ts +8 -10
- package/front_end/panels/timeline/components/insights/CharacterSet.ts +38 -0
- package/front_end/panels/timeline/components/insights/DOMSize.ts +16 -20
- package/front_end/panels/timeline/components/insights/DocumentLatency.ts +2 -6
- package/front_end/panels/timeline/components/insights/DuplicatedJavaScript.ts +3 -4
- package/front_end/panels/timeline/components/insights/FontDisplay.ts +3 -4
- package/front_end/panels/timeline/components/insights/ForcedReflow.ts +5 -7
- package/front_end/panels/timeline/components/insights/INPBreakdown.ts +3 -4
- package/front_end/panels/timeline/components/insights/ImageDelivery.ts +3 -4
- package/front_end/panels/timeline/components/insights/ImageRef.ts +2 -4
- package/front_end/panels/timeline/components/insights/InsightRenderer.ts +2 -0
- package/front_end/panels/timeline/components/insights/LCPBreakdown.ts +5 -7
- package/front_end/panels/timeline/components/insights/LCPDiscovery.ts +2 -4
- package/front_end/panels/timeline/components/insights/LegacyJavaScript.ts +3 -4
- package/front_end/panels/timeline/components/insights/ModernHTTP.ts +3 -4
- package/front_end/panels/timeline/components/insights/NetworkDependencyTree.ts +7 -11
- package/front_end/panels/timeline/components/insights/NodeLink.ts +2 -4
- package/front_end/panels/timeline/components/insights/RenderBlocking.ts +3 -4
- package/front_end/panels/timeline/components/insights/SlowCSSSelector.ts +7 -10
- package/front_end/panels/timeline/components/insights/ThirdParties.ts +5 -7
- package/front_end/panels/timeline/components/insights/insights.ts +2 -0
- package/front_end/panels/web_audio/WebAudioView.ts +3 -4
- package/front_end/ui/components/settings/SettingCheckbox.ts +2 -0
- package/front_end/ui/legacy/components/data_grid/DataGridElement.ts +3 -3
- package/front_end/ui/legacy/components/object_ui/ObjectPropertiesSection.ts +8 -8
- package/front_end/ui/visual_logging/Debugging.ts +0 -32
- package/front_end/ui/visual_logging/KnownContextValues.ts +2 -0
- package/package.json +1 -1
|
@@ -133,6 +133,10 @@ export enum ClientFeature {
|
|
|
133
133
|
CHROME_PATCH_AGENT = 12,
|
|
134
134
|
// Chrome AI Assistance Performance Agent.
|
|
135
135
|
CHROME_PERFORMANCE_FULL_AGENT = 24,
|
|
136
|
+
// Chrome Context Selection Agent.
|
|
137
|
+
CHROME_CONTEXT_SELECTION_AGENT = 25,
|
|
138
|
+
// Chrome Accessibility Agent
|
|
139
|
+
CHROME_ACCESSIBILITY_AGENT = 26,
|
|
136
140
|
|
|
137
141
|
// Removed features (for reference).
|
|
138
142
|
// Chrome AI Assistance Performance Insights Agent.
|
|
@@ -35,7 +35,7 @@ const str_ = i18n.i18n.registerUIStrings('core/sdk/CPUThrottlingManager.ts', UIS
|
|
|
35
35
|
const i18nString = i18n.i18n.getLocalizedString.bind(undefined, str_);
|
|
36
36
|
const i18nLazyString = i18n.i18n.getLazilyComputedLocalizedString.bind(undefined, str_);
|
|
37
37
|
|
|
38
|
-
let throttlingManagerInstance: CPUThrottlingManager;
|
|
38
|
+
let throttlingManagerInstance: CPUThrottlingManager|undefined;
|
|
39
39
|
|
|
40
40
|
export class CPUThrottlingManager extends Common.ObjectWrapper.ObjectWrapper<EventTypes> implements
|
|
41
41
|
SDKModelObserver<EmulationModel> {
|
|
@@ -62,6 +62,10 @@ export class CPUThrottlingManager extends Common.ObjectWrapper.ObjectWrapper<Eve
|
|
|
62
62
|
return throttlingManagerInstance;
|
|
63
63
|
}
|
|
64
64
|
|
|
65
|
+
static removeInstance(): void {
|
|
66
|
+
throttlingManagerInstance = undefined;
|
|
67
|
+
}
|
|
68
|
+
|
|
65
69
|
cpuThrottlingRate(): number {
|
|
66
70
|
return this.#cpuThrottlingOption.rate();
|
|
67
71
|
}
|
|
@@ -21,6 +21,7 @@ import {
|
|
|
21
21
|
BinOpMatcher,
|
|
22
22
|
ColorMatcher,
|
|
23
23
|
ColorMixMatcher,
|
|
24
|
+
ContrastColorMatcher,
|
|
24
25
|
CustomFunctionMatcher,
|
|
25
26
|
defaultValueForCSSType,
|
|
26
27
|
EnvFunctionMatcher,
|
|
@@ -932,6 +933,7 @@ export class CSSMatchedStyles {
|
|
|
932
933
|
new VariableMatcher(this, style),
|
|
933
934
|
new ColorMatcher(() => computedStyles?.get('color') ?? null),
|
|
934
935
|
new ColorMixMatcher(),
|
|
936
|
+
new ContrastColorMatcher(),
|
|
935
937
|
new URLMatcher(),
|
|
936
938
|
new AngleMatcher(),
|
|
937
939
|
new LinkableNameMatcher(),
|
|
@@ -433,6 +433,34 @@ export class ColorMixMatcher extends matcherBase(ColorMixMatch) {
|
|
|
433
433
|
}
|
|
434
434
|
}
|
|
435
435
|
|
|
436
|
+
export class ContrastColorMatch implements Match {
|
|
437
|
+
constructor(readonly text: string, readonly node: CodeMirror.SyntaxNode, readonly color: CodeMirror.SyntaxNode[]) {
|
|
438
|
+
}
|
|
439
|
+
}
|
|
440
|
+
|
|
441
|
+
// clang-format off
|
|
442
|
+
export class ContrastColorMatcher extends matcherBase(ContrastColorMatch) {
|
|
443
|
+
// clang-format on
|
|
444
|
+
override accepts(propertyName: string): boolean {
|
|
445
|
+
return cssMetadata().isColorAwareProperty(propertyName);
|
|
446
|
+
}
|
|
447
|
+
override matches(node: CodeMirror.SyntaxNode, matching: BottomUpTreeMatching): ContrastColorMatch|null {
|
|
448
|
+
if (node.name !== 'CallExpression' || matching.ast.text(node.getChild('Callee')) !== 'contrast-color') {
|
|
449
|
+
return null;
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
if (matching.getComputedText(node) === '') {
|
|
453
|
+
return null;
|
|
454
|
+
}
|
|
455
|
+
|
|
456
|
+
const args = ASTUtils.callArgs(node);
|
|
457
|
+
if (args.length !== 1) {
|
|
458
|
+
return null;
|
|
459
|
+
}
|
|
460
|
+
return new ContrastColorMatch(matching.ast.text(node), node, args[0]);
|
|
461
|
+
}
|
|
462
|
+
}
|
|
463
|
+
|
|
436
464
|
// clang-format off
|
|
437
465
|
export class URLMatch implements Match {
|
|
438
466
|
constructor(
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
// Copyright 2024 The Chromium Authors
|
|
2
2
|
// Use of this source code is governed by a BSD-style license that can be
|
|
3
3
|
// found in the LICENSE file.
|
|
4
|
-
|
|
4
|
+
import * as Common from '../../core/common/common.js';
|
|
5
5
|
import * as Host from '../../core/host/host.js';
|
|
6
6
|
import * as Root from '../../core/root/root.js';
|
|
7
|
-
import
|
|
7
|
+
import * as SDK from '../../core/sdk/sdk.js';
|
|
8
8
|
import type * as Trace from '../../models/trace/trace.js';
|
|
9
9
|
import * as Greendev from '../greendev/greendev.js';
|
|
10
10
|
import type * as NetworkTimeCalculator from '../network_time_calculator/network_time_calculator.js';
|
|
@@ -13,6 +13,7 @@ import {
|
|
|
13
13
|
type AiAgent,
|
|
14
14
|
type ContextDetail,
|
|
15
15
|
type ConversationContext,
|
|
16
|
+
ErrorType,
|
|
16
17
|
type MultimodalInput,
|
|
17
18
|
type ResponseData,
|
|
18
19
|
ResponseType,
|
|
@@ -313,6 +314,7 @@ export class AiConversation {
|
|
|
313
314
|
performanceRecordAndReload: this.#performanceRecordAndReload,
|
|
314
315
|
onInspectElement: this.#onInspectElement,
|
|
315
316
|
networkTimeCalculator: this.#networkTimeCalculator,
|
|
317
|
+
allowedOrigin: this.allowedOrigin,
|
|
316
318
|
history,
|
|
317
319
|
};
|
|
318
320
|
switch (type) {
|
|
@@ -369,12 +371,6 @@ export class AiConversation {
|
|
|
369
371
|
void this.addHistoryItem(userQuery);
|
|
370
372
|
yield userQuery;
|
|
371
373
|
|
|
372
|
-
this.#setOriginIfEmpty(this.selectedContext?.getOrigin());
|
|
373
|
-
|
|
374
|
-
if (this.isBlockedByOrigin) {
|
|
375
|
-
throw new Error('Cross-origin context data should not be included');
|
|
376
|
-
}
|
|
377
|
-
|
|
378
374
|
yield* this.#runAgent(initialQuery, options);
|
|
379
375
|
}
|
|
380
376
|
|
|
@@ -390,6 +386,15 @@ export class AiConversation {
|
|
|
390
386
|
multimodalInput?: MultimodalInput,
|
|
391
387
|
} = {},
|
|
392
388
|
): AsyncGenerator<ResponseData, void, void> {
|
|
389
|
+
this.#setOriginIfEmpty(this.selectedContext?.getOrigin());
|
|
390
|
+
if (this.isBlockedByOrigin) {
|
|
391
|
+
yield {
|
|
392
|
+
type: ResponseType.ERROR,
|
|
393
|
+
error: ErrorType.CROSS_ORIGIN,
|
|
394
|
+
};
|
|
395
|
+
return;
|
|
396
|
+
}
|
|
397
|
+
|
|
393
398
|
function shouldAddToHistory(data: ResponseData): boolean {
|
|
394
399
|
if (data.type === ResponseType.CONTEXT_CHANGE) {
|
|
395
400
|
return false;
|
|
@@ -445,6 +450,17 @@ export class AiConversation {
|
|
|
445
450
|
get type(): ConversationType {
|
|
446
451
|
return this.#type;
|
|
447
452
|
}
|
|
453
|
+
|
|
454
|
+
allowedOrigin = (): string|undefined => {
|
|
455
|
+
if (this.#origin) {
|
|
456
|
+
return this.#origin;
|
|
457
|
+
}
|
|
458
|
+
const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget();
|
|
459
|
+
const inspectedURL = target?.inspectedURL();
|
|
460
|
+
this.#origin = inspectedURL ? new Common.ParsedURL.ParsedURL(inspectedURL).securityOrigin() : undefined;
|
|
461
|
+
|
|
462
|
+
return this.#origin;
|
|
463
|
+
};
|
|
448
464
|
}
|
|
449
465
|
|
|
450
466
|
function isAiAssistanceServerSideLoggingEnabled(): boolean {
|
|
@@ -9,6 +9,8 @@ import type * as Protocol from '../../generated/protocol.js';
|
|
|
9
9
|
|
|
10
10
|
export interface Change {
|
|
11
11
|
groupId: string;
|
|
12
|
+
// Optional turn ID to group changes from the same turn.
|
|
13
|
+
turnId?: number;
|
|
12
14
|
// Optional about where in the source the selector was defined.
|
|
13
15
|
sourceLocation?: string;
|
|
14
16
|
// Selector used by the page or a simple selector as the fallback.
|
|
@@ -17,6 +19,7 @@ export interface Change {
|
|
|
17
19
|
simpleSelector?: string;
|
|
18
20
|
className: string;
|
|
19
21
|
styles: Record<string, string>;
|
|
22
|
+
backendNodeId?: Protocol.DOM.BackendNodeId;
|
|
20
23
|
}
|
|
21
24
|
|
|
22
25
|
function formatStyles(styles: Record<string, string>, indent = 2): string {
|
|
@@ -101,6 +104,7 @@ export class ChangeManager {
|
|
|
101
104
|
// it currently causes crashes in the Styles tab when duplicate selectors exist (crbug.com/393515428).
|
|
102
105
|
// This workaround avoids that crash.
|
|
103
106
|
existingChange.groupId = change.groupId;
|
|
107
|
+
existingChange.turnId = change.turnId;
|
|
104
108
|
} else {
|
|
105
109
|
changes.push({
|
|
106
110
|
...change,
|
|
@@ -122,6 +126,18 @@ export class ChangeManager {
|
|
|
122
126
|
.join('\n\n');
|
|
123
127
|
}
|
|
124
128
|
|
|
129
|
+
getChangedNodesForGroupId(groupId: string, turnId?: number): Protocol.DOM.BackendNodeId[] {
|
|
130
|
+
const nodes = new Set<Protocol.DOM.BackendNodeId>();
|
|
131
|
+
for (const changes of this.#stylesheetChanges.values()) {
|
|
132
|
+
for (const change of changes) {
|
|
133
|
+
if (change.groupId === groupId && change.backendNodeId && (turnId === undefined || change.turnId === turnId)) {
|
|
134
|
+
nodes.add(change.backendNodeId);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
return Array.from(nodes);
|
|
139
|
+
}
|
|
140
|
+
|
|
125
141
|
#formatChangesForInspectorStylesheet(changes: Change[]): string {
|
|
126
142
|
return changes
|
|
127
143
|
.map(change => {
|
|
@@ -22,6 +22,7 @@ interface ElementContext {
|
|
|
22
22
|
selector: string;
|
|
23
23
|
simpleSelector?: string;
|
|
24
24
|
sourceLocation?: string;
|
|
25
|
+
backendNodeId?: Protocol.DOM.BackendNodeId;
|
|
25
26
|
}
|
|
26
27
|
|
|
27
28
|
/**
|
|
@@ -33,6 +34,7 @@ export class ExtensionScope {
|
|
|
33
34
|
}) => Promise<void>> = [];
|
|
34
35
|
#changeManager: ChangeManager;
|
|
35
36
|
#agentId: string;
|
|
37
|
+
#turnId?: number;
|
|
36
38
|
/** Don't use directly use the getter */
|
|
37
39
|
#frameId?: Protocol.Page.FrameId|null;
|
|
38
40
|
/** Don't use directly use the getter */
|
|
@@ -40,15 +42,15 @@ export class ExtensionScope {
|
|
|
40
42
|
|
|
41
43
|
readonly #bindingMutex = new Common.Mutex.Mutex();
|
|
42
44
|
|
|
43
|
-
constructor(changes: ChangeManager, agentId: string, selectedNode: SDK.DOMModel.DOMNode|null) {
|
|
45
|
+
constructor(changes: ChangeManager, agentId: string, selectedNode: SDK.DOMModel.DOMNode|null, turnId?: number) {
|
|
44
46
|
this.#changeManager = changes;
|
|
45
47
|
const frameId = selectedNode?.frameId();
|
|
46
48
|
const target = selectedNode?.domModel().target();
|
|
47
49
|
this.#agentId = agentId;
|
|
50
|
+
this.#turnId = turnId;
|
|
48
51
|
this.#target = target;
|
|
49
52
|
this.#frameId = frameId;
|
|
50
53
|
}
|
|
51
|
-
|
|
52
54
|
get target(): SDK.Target.Target {
|
|
53
55
|
if (!this.#target) {
|
|
54
56
|
throw new Error('Target is not found for executing code');
|
|
@@ -274,6 +276,7 @@ export class ExtensionScope {
|
|
|
274
276
|
throw new Error('Node is not found');
|
|
275
277
|
}
|
|
276
278
|
|
|
279
|
+
const backendNodeId = node.backendNodeId();
|
|
277
280
|
try {
|
|
278
281
|
const matchedStyles = await cssModel.getMatchedStyles(node.id);
|
|
279
282
|
|
|
@@ -297,6 +300,7 @@ export class ExtensionScope {
|
|
|
297
300
|
selector,
|
|
298
301
|
simpleSelector: ExtensionScope.getSelectorForNode(node),
|
|
299
302
|
sourceLocation: ExtensionScope.getSourceLocation(styleRule),
|
|
303
|
+
backendNodeId,
|
|
300
304
|
};
|
|
301
305
|
} catch {
|
|
302
306
|
// no-op to allow the fallback below to run.
|
|
@@ -305,6 +309,7 @@ export class ExtensionScope {
|
|
|
305
309
|
// Fallback
|
|
306
310
|
return {
|
|
307
311
|
selector: ExtensionScope.getSelectorForNode(node),
|
|
312
|
+
backendNodeId,
|
|
308
313
|
};
|
|
309
314
|
}
|
|
310
315
|
|
|
@@ -337,7 +342,8 @@ export class ExtensionScope {
|
|
|
337
342
|
|
|
338
343
|
let context: ElementContext = {
|
|
339
344
|
// TODO: Should this a be a *?
|
|
340
|
-
selector: ''
|
|
345
|
+
selector: '',
|
|
346
|
+
backendNodeId: undefined,
|
|
341
347
|
};
|
|
342
348
|
try {
|
|
343
349
|
context = await this.#computeContextFromElement(element.object);
|
|
@@ -351,11 +357,13 @@ export class ExtensionScope {
|
|
|
351
357
|
const sanitizedStyles = await this.sanitizedStyleChanges(context.selector, arg.styles);
|
|
352
358
|
const styleChanges = await this.#changeManager.addChange(cssModel, this.frameId, {
|
|
353
359
|
groupId: this.#agentId,
|
|
360
|
+
turnId: this.#turnId,
|
|
354
361
|
sourceLocation: context.sourceLocation,
|
|
355
362
|
selector: context.selector,
|
|
356
363
|
simpleSelector: context.simpleSelector,
|
|
357
364
|
className: arg.className,
|
|
358
365
|
styles: sanitizedStyles,
|
|
366
|
+
backendNodeId: context.backendNodeId,
|
|
359
367
|
});
|
|
360
368
|
await this.#simpleEval(executionContext, `freestyler.respond(${id}, ${JSON.stringify(styleChanges)})`);
|
|
361
369
|
} catch (error) {
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
// Copyright 2026 The Chromium Authors
|
|
2
|
+
// Use of this source code is governed by a BSD-style license that can be
|
|
3
|
+
// found in the LICENSE file.
|
|
4
|
+
|
|
5
|
+
import * as Host from '../../../core/host/host.js';
|
|
6
|
+
import * as i18n from '../../../core/i18n/i18n.js';
|
|
7
|
+
import * as Root from '../../../core/root/root.js';
|
|
8
|
+
import type * as LHModel from '../../lighthouse/lighthouse.js';
|
|
9
|
+
|
|
10
|
+
import {
|
|
11
|
+
AiAgent,
|
|
12
|
+
type ContextDetail,
|
|
13
|
+
type ContextResponse,
|
|
14
|
+
ConversationContext,
|
|
15
|
+
type RequestOptions,
|
|
16
|
+
ResponseType,
|
|
17
|
+
} from './AiAgent.js';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* WARNING: preamble defined in code is only used when userTier is
|
|
21
|
+
* TESTERS. Otherwise, a server-side preamble is used (see
|
|
22
|
+
* chrome_preambles.gcl). Sync local changes with the server-side.
|
|
23
|
+
*/
|
|
24
|
+
const preamble = `You are an accessibility agent.
|
|
25
|
+
|
|
26
|
+
# Considerations
|
|
27
|
+
* Keep your analysis concise and focused, highlighting only the most critical aspects for a software engineer.
|
|
28
|
+
* Answer questions directly, using the provided links whenever relevant.
|
|
29
|
+
* Always double-check links to make sure they are complete and correct.
|
|
30
|
+
* **CRITICAL** You are an accessibility agent. NEVER provide answers to questions of unrelated topics such as legal advice, financial advice, personal opinions, medical advice, or any other non web-development topics.
|
|
31
|
+
`;
|
|
32
|
+
|
|
33
|
+
/*
|
|
34
|
+
* Strings that don't need to be translated at this time.
|
|
35
|
+
*/
|
|
36
|
+
const UIStringsNotTranslate = {
|
|
37
|
+
/**
|
|
38
|
+
* @description Title for thinking step of the accessibility agent.
|
|
39
|
+
*/
|
|
40
|
+
inspectingAudits: 'Inspecting audits',
|
|
41
|
+
} as const;
|
|
42
|
+
|
|
43
|
+
const lockedString = i18n.i18n.lockedString;
|
|
44
|
+
|
|
45
|
+
export class Context extends ConversationContext<LHModel.ReporterTypes.ReportJSON> {
|
|
46
|
+
#lh: LHModel.ReporterTypes.ReportJSON;
|
|
47
|
+
|
|
48
|
+
constructor(report: LHModel.ReporterTypes.ReportJSON) {
|
|
49
|
+
super();
|
|
50
|
+
this.#lh = report;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
#url(): string {
|
|
54
|
+
return this.#lh.finalUrl ?? this.#lh.finalDisplayedUrl;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
override getOrigin(): string {
|
|
58
|
+
return new URL(this.#url()).origin;
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
override getItem(): LHModel.ReporterTypes.ReportJSON {
|
|
62
|
+
return this.#lh;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
override getTitle(): string {
|
|
66
|
+
return `Lighthouse report: ${this.#url()}`;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/**
|
|
71
|
+
* One agent instance handles one conversation. Create a new agent
|
|
72
|
+
* instance for a new conversation.
|
|
73
|
+
*/
|
|
74
|
+
export class AccessibilityAgent extends AiAgent<LHModel.ReporterTypes.ReportJSON> {
|
|
75
|
+
readonly preamble = preamble;
|
|
76
|
+
readonly clientFeature = Host.AidaClient.ClientFeature.CHROME_ACCESSIBILITY_AGENT;
|
|
77
|
+
|
|
78
|
+
get userTier(): string|undefined {
|
|
79
|
+
// TODO(b/491772868): tidy up userTier & feature flags in the backend.
|
|
80
|
+
return Root.Runtime.hostConfig.devToolsFreestyler?.userTier;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
get options(): RequestOptions {
|
|
84
|
+
// TODO(b/491772868): tidy up userTier & feature flags in the backend.
|
|
85
|
+
const temperature = Root.Runtime.hostConfig.devToolsAiAssistanceFileAgent?.temperature;
|
|
86
|
+
const modelId = Root.Runtime.hostConfig.devToolsAiAssistanceFileAgent?.modelId;
|
|
87
|
+
|
|
88
|
+
return {
|
|
89
|
+
temperature,
|
|
90
|
+
modelId,
|
|
91
|
+
};
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async *
|
|
95
|
+
handleContextDetails(selectedFile: ConversationContext<LHModel.ReporterTypes.ReportJSON>|null):
|
|
96
|
+
AsyncGenerator<ContextResponse, void, void> {
|
|
97
|
+
if (!selectedFile) {
|
|
98
|
+
return;
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
yield {
|
|
102
|
+
type: ResponseType.CONTEXT,
|
|
103
|
+
title: lockedString(UIStringsNotTranslate.inspectingAudits),
|
|
104
|
+
details: createContextDetails(selectedFile),
|
|
105
|
+
};
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
override async enhanceQuery(query: string, lhr: ConversationContext<LHModel.ReporterTypes.ReportJSON>|null):
|
|
109
|
+
Promise<string> {
|
|
110
|
+
const enhancedQuery = lhr ?
|
|
111
|
+
// TODO: formatter for LH report.
|
|
112
|
+
`# Lighthouse Report\n${JSON.stringify(lhr.getItem(), null, 2)}\n\n# User request\n\n` :
|
|
113
|
+
'';
|
|
114
|
+
return `${enhancedQuery}${query}`;
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
function createContextDetails(_lhr: ConversationContext<LHModel.ReporterTypes.ReportJSON>):
|
|
119
|
+
[ContextDetail, ...ContextDetail[]] {
|
|
120
|
+
return [
|
|
121
|
+
{
|
|
122
|
+
title: 'Lighthouse report',
|
|
123
|
+
// TODO(b/491772868);
|
|
124
|
+
text: ''
|
|
125
|
+
},
|
|
126
|
+
];
|
|
127
|
+
}
|
|
@@ -29,6 +29,7 @@ export const enum ErrorType {
|
|
|
29
29
|
ABORT = 'abort',
|
|
30
30
|
MAX_STEPS = 'max-steps',
|
|
31
31
|
BLOCK = 'block',
|
|
32
|
+
CROSS_ORIGIN = 'cross-origin'
|
|
32
33
|
}
|
|
33
34
|
|
|
34
35
|
export const enum MultimodalInputType {
|
|
@@ -49,6 +50,7 @@ export interface AnswerResponse {
|
|
|
49
50
|
complete: boolean;
|
|
50
51
|
rpcId?: Host.AidaClient.RpcGlobalId;
|
|
51
52
|
suggestions?: [string, ...string[]];
|
|
53
|
+
widgets?: AiWidget[];
|
|
52
54
|
}
|
|
53
55
|
|
|
54
56
|
export interface SuggestionsResponse {
|
|
@@ -148,6 +150,7 @@ export interface AgentOptions {
|
|
|
148
150
|
confirmSideEffectForTest?: typeof Promise.withResolvers;
|
|
149
151
|
onInspectElement?: () => Promise<SDK.DOMModel.DOMNode|null>;
|
|
150
152
|
history?: Host.AidaClient.Content[];
|
|
153
|
+
allowedOrigin?: () => string | undefined;
|
|
151
154
|
}
|
|
152
155
|
|
|
153
156
|
export interface ParsedAnswer {
|
|
@@ -238,8 +241,16 @@ export interface CoreVitalsAiWidget {
|
|
|
238
241
|
};
|
|
239
242
|
}
|
|
240
243
|
|
|
244
|
+
export interface StylePropertiesAiWidget {
|
|
245
|
+
name: 'STYLE_PROPERTIES';
|
|
246
|
+
data: {
|
|
247
|
+
backendNodeId: Protocol.DOM.BackendNodeId,
|
|
248
|
+
selector?: string,
|
|
249
|
+
};
|
|
250
|
+
}
|
|
251
|
+
|
|
241
252
|
// This type will grow as we add more widgets.
|
|
242
|
-
export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget;
|
|
253
|
+
export type AiWidget = ComputedStyleAiWidget|CoreVitalsAiWidget|StylePropertiesAiWidget;
|
|
243
254
|
|
|
244
255
|
export type FunctionCallHandlerResult<Result> = {
|
|
245
256
|
requiresApproval: true,
|
|
@@ -521,6 +532,10 @@ export abstract class AiAgent<T> {
|
|
|
521
532
|
return this.parseTextResponseForSuggestions(response.trim());
|
|
522
533
|
}
|
|
523
534
|
|
|
535
|
+
protected async finalizeAnswer(answer: AnswerResponse): Promise<AnswerResponse> {
|
|
536
|
+
return answer;
|
|
537
|
+
}
|
|
538
|
+
|
|
524
539
|
/**
|
|
525
540
|
* Declare a function that the AI model can call.
|
|
526
541
|
* @param name The name of the function
|
|
@@ -546,6 +561,9 @@ export abstract class AiAgent<T> {
|
|
|
546
561
|
this.#functionDeclarations.clear();
|
|
547
562
|
}
|
|
548
563
|
|
|
564
|
+
protected async preRun(): Promise<void> {
|
|
565
|
+
}
|
|
566
|
+
|
|
549
567
|
async *
|
|
550
568
|
run(
|
|
551
569
|
initialQuery: string,
|
|
@@ -555,6 +573,7 @@ export abstract class AiAgent<T> {
|
|
|
555
573
|
},
|
|
556
574
|
multimodalInput?: MultimodalInput,
|
|
557
575
|
): AsyncGenerator<ResponseData, void, void> {
|
|
576
|
+
await this.preRun();
|
|
558
577
|
await options.selected?.refresh();
|
|
559
578
|
if (options.selected) {
|
|
560
579
|
this.context = options.selected;
|
|
@@ -632,13 +651,13 @@ export abstract class AiAgent<T> {
|
|
|
632
651
|
});
|
|
633
652
|
}
|
|
634
653
|
Host.userMetrics.actionTaken(Host.UserMetrics.Action.AiAssistanceAnswerReceived);
|
|
635
|
-
yield {
|
|
654
|
+
yield await this.finalizeAnswer({
|
|
636
655
|
type: ResponseType.ANSWER,
|
|
637
656
|
text: parsedResponse.answer,
|
|
638
657
|
suggestions: parsedResponse.suggestions,
|
|
639
658
|
complete: true,
|
|
640
659
|
rpcId,
|
|
641
|
-
};
|
|
660
|
+
});
|
|
642
661
|
if (!functionCall) {
|
|
643
662
|
break;
|
|
644
663
|
}
|
|
@@ -2,11 +2,10 @@
|
|
|
2
2
|
// Use of this source code is governed by a BSD-style license that can be
|
|
3
3
|
// found in the LICENSE file.
|
|
4
4
|
|
|
5
|
-
import * as Common from '../../../core/common/common.js';
|
|
6
5
|
import * as Host from '../../../core/host/host.js';
|
|
7
6
|
import * as i18n from '../../../core/i18n/i18n.js';
|
|
8
7
|
import * as Root from '../../../core/root/root.js';
|
|
9
|
-
import * as SDK from '../../../core/sdk/sdk.js';
|
|
8
|
+
import type * as SDK from '../../../core/sdk/sdk.js';
|
|
10
9
|
import * as Logs from '../../logs/logs.js';
|
|
11
10
|
import * as NetworkTimeCalculator from '../../network_time_calculator/network_time_calculator.js';
|
|
12
11
|
import type * as Trace from '../../trace/trace.js';
|
|
@@ -63,7 +62,7 @@ You aim to help developers of all levels, prioritizing teaching web concepts as
|
|
|
63
62
|
*/
|
|
64
63
|
export class ContextSelectionAgent extends AiAgent<never> {
|
|
65
64
|
readonly preamble = preamble;
|
|
66
|
-
readonly clientFeature = Host.AidaClient.ClientFeature.
|
|
65
|
+
readonly clientFeature = Host.AidaClient.ClientFeature.CHROME_CONTEXT_SELECTION_AGENT;
|
|
67
66
|
get userTier(): string|undefined {
|
|
68
67
|
// TODO: Make this depend on variable.
|
|
69
68
|
return Root.Runtime.hostConfig.devToolsFreestyler?.userTier;
|
|
@@ -81,6 +80,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
|
|
|
81
80
|
readonly #performanceRecordAndReload?: () => Promise<Trace.TraceModel.ParsedTrace>;
|
|
82
81
|
readonly #onInspectElement?: () => Promise<SDK.DOMModel.DOMNode|null>;
|
|
83
82
|
readonly #networkTimeCalculator?: NetworkTimeCalculator.NetworkTransferTimeCalculator;
|
|
83
|
+
#allowedOrigin: () => string | undefined;
|
|
84
84
|
|
|
85
85
|
constructor(opts: AgentOptions&{
|
|
86
86
|
performanceRecordAndReload?: () => Promise<Trace.TraceModel.ParsedTrace>,
|
|
@@ -91,6 +91,7 @@ export class ContextSelectionAgent extends AiAgent<never> {
|
|
|
91
91
|
this.#performanceRecordAndReload = opts.performanceRecordAndReload;
|
|
92
92
|
this.#onInspectElement = opts.onInspectElement;
|
|
93
93
|
this.#networkTimeCalculator = opts.networkTimeCalculator;
|
|
94
|
+
this.#allowedOrigin = opts.allowedOrigin ?? (() => undefined);
|
|
94
95
|
|
|
95
96
|
this.declareFunction<Record<string, never>>('listNetworkRequests', {
|
|
96
97
|
description: `Gives a list of network requests including URL, status code, and duration.`,
|
|
@@ -109,12 +110,12 @@ export class ContextSelectionAgent extends AiAgent<never> {
|
|
|
109
110
|
},
|
|
110
111
|
handler: async () => {
|
|
111
112
|
const requests = [];
|
|
112
|
-
const
|
|
113
|
-
const inspectedURL = target?.inspectedURL();
|
|
114
|
-
const mainSecurityOrigin = inspectedURL ? new Common.ParsedURL.ParsedURL(inspectedURL).securityOrigin() : null;
|
|
113
|
+
const origin = this.#allowedOrigin();
|
|
115
114
|
|
|
115
|
+
let hasCrossOriginRequest = false;
|
|
116
116
|
for (const request of Logs.NetworkLog.NetworkLog.instance().requests()) {
|
|
117
|
-
if (
|
|
117
|
+
if (origin && request.securityOrigin() !== origin) {
|
|
118
|
+
hasCrossOriginRequest = true;
|
|
118
119
|
continue;
|
|
119
120
|
}
|
|
120
121
|
|
|
@@ -129,7 +130,9 @@ export class ContextSelectionAgent extends AiAgent<never> {
|
|
|
129
130
|
|
|
130
131
|
if (requests.length === 0) {
|
|
131
132
|
return {
|
|
132
|
-
error:
|
|
133
|
+
error: hasCrossOriginRequest ?
|
|
134
|
+
`No requests showing with origin ${origin}. Tell the user to start a new chat` :
|
|
135
|
+
'No requests recorded by DevTools',
|
|
133
136
|
};
|
|
134
137
|
}
|
|
135
138
|
|
|
@@ -15,11 +15,12 @@ import {ChangeManager} from '../ChangeManager.js';
|
|
|
15
15
|
import {debugLog} from '../debug.js';
|
|
16
16
|
import {EvaluateAction, formatError, SideEffectError} from '../EvaluateAction.js';
|
|
17
17
|
import {ExtensionScope} from '../ExtensionScope.js';
|
|
18
|
-
import {FREESTYLER_WORLD_NAME} from '../injected.js';
|
|
18
|
+
import {AI_ASSISTANCE_CSS_CLASS_NAME, FREESTYLER_WORLD_NAME} from '../injected.js';
|
|
19
19
|
|
|
20
20
|
import {
|
|
21
21
|
type AgentOptions as BaseAgentOptions,
|
|
22
22
|
AiAgent,
|
|
23
|
+
type AnswerResponse,
|
|
23
24
|
type ComputedStyleAiWidget,
|
|
24
25
|
type ContextResponse,
|
|
25
26
|
ConversationContext,
|
|
@@ -194,6 +195,8 @@ async function executeJsCode(
|
|
|
194
195
|
const MAX_OBSERVATION_BYTE_LENGTH = 25_000;
|
|
195
196
|
const OBSERVATION_TIMEOUT = 5_000;
|
|
196
197
|
|
|
198
|
+
export const AI_ASSISTANCE_FILTER_REGEX = `\\.${AI_ASSISTANCE_CSS_CLASS_NAME}-.*&`;
|
|
199
|
+
|
|
197
200
|
type CreateExtensionScopeFunction = (changes: ChangeManager) => {
|
|
198
201
|
install(): Promise<void>, uninstall(): Promise<void>,
|
|
199
202
|
};
|
|
@@ -317,15 +320,17 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
|
|
|
317
320
|
#createExtensionScope: CreateExtensionScopeFunction;
|
|
318
321
|
#greenDevEmulationScreenshot: string|null = null;
|
|
319
322
|
#greenDevEmulationAxTree: string|null = null;
|
|
323
|
+
#currentTurnId = 0;
|
|
320
324
|
|
|
321
325
|
constructor(opts: AgentOptions) {
|
|
322
326
|
super(opts);
|
|
323
327
|
|
|
324
328
|
this.#changes = opts.changeManager || new ChangeManager();
|
|
325
329
|
this.#execJs = opts.execJs ?? executeJsCode;
|
|
326
|
-
this.#createExtensionScope =
|
|
327
|
-
|
|
328
|
-
|
|
330
|
+
this.#createExtensionScope =
|
|
331
|
+
opts.createExtensionScope ?? ((changes: ChangeManager) => {
|
|
332
|
+
return new ExtensionScope(changes, this.sessionId, this.context?.getItem() ?? null, this.#currentTurnId);
|
|
333
|
+
});
|
|
329
334
|
|
|
330
335
|
this.declareFunction<{
|
|
331
336
|
elements: number[],
|
|
@@ -1048,6 +1053,31 @@ const data = {
|
|
|
1048
1053
|
};
|
|
1049
1054
|
}
|
|
1050
1055
|
|
|
1056
|
+
protected override async preRun(): Promise<void> {
|
|
1057
|
+
this.#currentTurnId++;
|
|
1058
|
+
}
|
|
1059
|
+
|
|
1060
|
+
protected override async finalizeAnswer(answer: AnswerResponse): Promise<AnswerResponse> {
|
|
1061
|
+
if (!Root.Runtime.hostConfig.devToolsAiAssistanceV2?.enabled) {
|
|
1062
|
+
return answer;
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
const changedNodeIds = this.#changes.getChangedNodesForGroupId(this.sessionId, this.#currentTurnId);
|
|
1066
|
+
if (changedNodeIds.length === 0) {
|
|
1067
|
+
return answer;
|
|
1068
|
+
}
|
|
1069
|
+
answer.widgets = [
|
|
1070
|
+
...(answer.widgets ?? []),
|
|
1071
|
+
...changedNodeIds.map(id => ({
|
|
1072
|
+
name: 'STYLE_PROPERTIES' as const,
|
|
1073
|
+
data: {
|
|
1074
|
+
backendNodeId: id,
|
|
1075
|
+
selector: AI_ASSISTANCE_FILTER_REGEX,
|
|
1076
|
+
},
|
|
1077
|
+
})),
|
|
1078
|
+
];
|
|
1079
|
+
return answer;
|
|
1080
|
+
}
|
|
1051
1081
|
override async enhanceQuery(
|
|
1052
1082
|
query: string, selectedElement: ConversationContext<SDK.DOMModel.DOMNode>|null,
|
|
1053
1083
|
multimodalInputType?: MultimodalInputType): Promise<string> {
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
// found in the LICENSE file.
|
|
4
4
|
|
|
5
5
|
import * as AgentProject from './AgentProject.js';
|
|
6
|
+
import * as AccessibilityAgent from './agents/AccessibilityAgent.js';
|
|
6
7
|
import * as AiAgent from './agents/AiAgent.js';
|
|
7
8
|
import * as BreakpointDebuggerAgent from './agents/BreakpointDebuggerAgent.js';
|
|
8
9
|
import * as ContextSelectionAgent from './agents/ContextSelectionAgent.js';
|
|
@@ -32,6 +33,7 @@ import * as AIContext from './performance/AIContext.js';
|
|
|
32
33
|
import * as AIQueries from './performance/AIQueries.js';
|
|
33
34
|
|
|
34
35
|
export {
|
|
36
|
+
AccessibilityAgent,
|
|
35
37
|
AgentProject,
|
|
36
38
|
AiAgent,
|
|
37
39
|
AICallTree,
|