chrome-devtools-frontend 1.0.1654411 → 1.0.1656897
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/skills/ui-eng-vision-local-lit-renderer/SKILL.md +116 -0
- package/.agents/skills/ui-eng-vision-logic-consolidator/SKILL.md +82 -0
- package/.agents/skills/ui-eng-vision-orchestrator/SKILL.md +123 -0
- package/.agents/skills/ui-eng-vision-test-scaffolder/SKILL.md +125 -0
- package/.agents/skills/ui-eng-vision-widget-promoter/SKILL.md +75 -0
- package/AUTHORS +1 -0
- package/config/owner/COMMON_OWNERS +0 -4
- package/docs/playbook.md +1 -1
- package/front_end/Tests.js +1 -0
- package/front_end/core/common/Settings.ts +47 -30
- package/front_end/core/sdk/CPUThrottlingManager.ts +10 -159
- package/front_end/core/sdk/ConsoleModel.ts +13 -11
- package/front_end/core/sdk/DOMDebuggerModel.ts +11 -5
- package/front_end/core/sdk/DOMModel.ts +7 -5
- package/front_end/core/sdk/EventBreakpointsModel.ts +9 -5
- package/front_end/core/sdk/IsolateManager.ts +6 -6
- package/front_end/core/sdk/ResourceTreeModel.ts +5 -3
- package/front_end/core/sdk/SourceMapCache.ts +2 -4
- package/front_end/core/sdk/SourceMapManager.ts +8 -7
- package/front_end/core/sdk/TargetManager.ts +17 -0
- package/front_end/entrypoints/devtools_app/devtools_app.ts +0 -1
- package/front_end/entrypoints/main/MainImpl.ts +12 -36
- package/front_end/entrypoints/main/SimpleApp.ts +0 -13
- package/front_end/entrypoints/main/main-meta.ts +1 -1
- package/front_end/foundation/Universe.ts +110 -3
- package/front_end/generated/InspectorBackendCommands.ts +5 -1
- package/front_end/generated/SupportedCSSProperties.js +2 -6
- package/front_end/generated/protocol-mapping.d.ts +8 -0
- package/front_end/generated/protocol-proxy-api.d.ts +15 -0
- package/front_end/generated/protocol.ts +36 -1
- package/front_end/models/ai_assistance/AiConversation.ts +0 -6
- package/front_end/models/ai_assistance/agents/NetworkAgent.snapshot.txt +1 -1
- package/front_end/models/ai_assistance/agents/PerformanceAgent.ts +2 -156
- package/front_end/models/ai_assistance/agents/StylingAgent.snapshot.txt +0 -24
- package/front_end/models/ai_assistance/agents/StylingAgent.ts +2 -358
- package/front_end/models/ai_assistance/ai_assistance.ts +0 -6
- package/front_end/models/ai_assistance/contexts/RequestContext.snapshot.txt +0 -1
- package/front_end/models/ai_assistance/data_formatters/NetworkRequestFormatter.ts +0 -2
- package/front_end/models/ai_assistance/data_formatters/PerformanceTraceFormatter.ts +1 -14
- package/front_end/models/autofill_manager/AutofillManager.ts +1 -2
- package/front_end/models/crux-manager/CrUXManager.ts +16 -11
- package/front_end/models/javascript_metadata/NativeFunctions.js +1 -1
- package/front_end/models/persistence/AutomaticFileSystemManager.ts +14 -13
- package/front_end/models/persistence/AutomaticFileSystemWorkspaceBinding.ts +12 -9
- package/front_end/models/persistence/IsolatedFileSystemManager.ts +18 -9
- package/front_end/models/project_settings/ProjectSettingsModel.ts +12 -11
- package/front_end/models/trace/handlers/ScreenshotsHandler.ts +14 -59
- package/front_end/panels/ai_assistance/AiAssistancePanel.ts +3 -29
- package/front_end/panels/application/DeviceBoundSessionsView.ts +15 -3
- package/front_end/panels/application/FrameDetailsView.ts +2 -1
- package/front_end/panels/application/ServiceWorkersView.ts +193 -161
- package/front_end/panels/application/serviceWorkersView.css +8 -0
- package/front_end/panels/common/CPUThrottlingOption.ts +142 -0
- package/front_end/panels/common/ThrottlingUtils.ts +9 -4
- package/front_end/panels/common/common.ts +1 -1
- package/front_end/panels/console/ConsoleViewMessage.ts +45 -0
- package/front_end/panels/elements/ElementsPanel.ts +35 -102
- package/front_end/panels/elements/ElementsTreeOutline.ts +0 -4
- package/front_end/panels/elements/elements-meta.ts +3 -2
- package/front_end/panels/emulation/DeviceModeToolbar.ts +46 -45
- package/front_end/panels/mobile_throttling/CalibrationController.ts +6 -5
- package/front_end/panels/mobile_throttling/MobileThrottlingSelector.ts +1 -1
- package/front_end/panels/mobile_throttling/ThrottlingManager.ts +40 -13
- package/front_end/panels/mobile_throttling/ThrottlingPresets.ts +13 -17
- package/front_end/panels/mobile_throttling/ThrottlingSettingsTab.ts +15 -13
- package/front_end/panels/network/NetworkLogView.ts +0 -9
- package/front_end/panels/network/NetworkPanel.ts +1 -63
- package/front_end/panels/network/RequestInitiatorView.ts +29 -4
- package/front_end/panels/network/components/HeaderSectionRow.css +9 -7
- package/front_end/panels/network/components/HeaderSectionRow.ts +20 -1
- package/front_end/panels/network/components/RequestHeaderSection.css +5 -3
- package/front_end/panels/network/components/RequestHeaderSection.ts +1 -2
- package/front_end/panels/network/components/RequestHeadersView.css +1 -1
- package/front_end/panels/security/SecurityPanel.ts +36 -23
- package/front_end/panels/settings/SettingsScreen.ts +18 -103
- package/front_end/panels/settings/settings-meta.ts +0 -24
- package/front_end/panels/timeline/TimelineController.ts +3 -2
- package/front_end/panels/timeline/TimelinePanel.ts +20 -3
- package/front_end/panels/timeline/components/CPUThrottlingSelector.ts +15 -14
- package/front_end/panels/timeline/timelineFlamechartPopover.css +2 -0
- package/front_end/panels/utils/utils.ts +25 -24
- package/front_end/third_party/chromium/README.chromium +1 -1
- package/front_end/third_party/puppeteer/README.chromium +2 -2
- package/front_end/third_party/puppeteer/package/lib/es5-iife/puppeteer-core-browser.d.ts +34 -55
- package/front_end/third_party/puppeteer/package/lib/es5-iife/puppeteer-core-browser.js +96 -83
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/Browser.d.ts +10 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/Browser.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/Browser.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/BrowserContext.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/BrowserContext.js +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/BrowserContext.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/CDPSession.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/CDPSession.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/HTTPResponse.d.ts +3 -2
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/HTTPResponse.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/HTTPResponse.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/Realm.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/Realm.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/Target.d.ts +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/Target.js +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/locators/locators.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/api/locators/locators.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/bidi/HTTPResponse.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/bidi/HTTPResponse.js +2 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/bidi/HTTPResponse.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/bidi/core/Realm.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/bidi/core/Realm.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/Browser.d.ts +2 -2
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/Browser.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/Browser.js +5 -2
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/Browser.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/Frame.js +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/Frame.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/FrameManager.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/FrameManager.js +23 -13
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/FrameManager.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/HTTPResponse.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/HTTPResponse.js +4 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/HTTPResponse.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/Input.js +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/LifecycleWatcher.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/LifecycleWatcher.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/Target.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/Target.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/WebMCP.d.ts +3 -41
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/WebMCP.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/cdp/WebMCP.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/common/ConnectOptions.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/node/BrowserLauncher.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/node/BrowserLauncher.js +4 -2
- package/front_end/third_party/puppeteer/package/lib/puppeteer/node/BrowserLauncher.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/node/LaunchOptions.d.ts +5 -0
- package/front_end/third_party/puppeteer/package/lib/puppeteer/node/LaunchOptions.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/revisions.d.ts +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/revisions.js +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/util/httpUtils.d.ts +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/util/httpUtils.d.ts.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/util/httpUtils.js +5 -5
- package/front_end/third_party/puppeteer/package/lib/puppeteer/util/httpUtils.js.map +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/util/version.d.ts +1 -1
- package/front_end/third_party/puppeteer/package/lib/puppeteer/util/version.js +1 -1
- package/front_end/third_party/puppeteer/package/lib/types.d.ts +34 -55
- package/front_end/third_party/puppeteer/package/package.json +4 -4
- package/front_end/third_party/puppeteer/package/src/api/Browser.ts +13 -1
- package/front_end/third_party/puppeteer/package/src/api/BrowserContext.ts +2 -3
- package/front_end/third_party/puppeteer/package/src/api/CDPSession.ts +3 -1
- package/front_end/third_party/puppeteer/package/src/api/HTTPResponse.ts +3 -2
- package/front_end/third_party/puppeteer/package/src/api/Realm.ts +1 -2
- package/front_end/third_party/puppeteer/package/src/api/Target.ts +1 -1
- package/front_end/third_party/puppeteer/package/src/api/locators/locators.ts +1 -2
- package/front_end/third_party/puppeteer/package/src/bidi/HTTPResponse.ts +3 -1
- package/front_end/third_party/puppeteer/package/src/bidi/core/Realm.ts +1 -3
- package/front_end/third_party/puppeteer/package/src/cdp/Browser.ts +13 -3
- package/front_end/third_party/puppeteer/package/src/cdp/Frame.ts +1 -1
- package/front_end/third_party/puppeteer/package/src/cdp/FrameManager.ts +24 -13
- package/front_end/third_party/puppeteer/package/src/cdp/HTTPResponse.ts +4 -1
- package/front_end/third_party/puppeteer/package/src/cdp/Input.ts +1 -1
- package/front_end/third_party/puppeteer/package/src/cdp/LifecycleWatcher.ts +1 -4
- package/front_end/third_party/puppeteer/package/src/cdp/Target.ts +2 -4
- package/front_end/third_party/puppeteer/package/src/cdp/WebMCP.ts +7 -76
- package/front_end/third_party/puppeteer/package/src/common/ConnectOptions.ts +1 -4
- package/front_end/third_party/puppeteer/package/src/node/BrowserLauncher.ts +4 -1
- package/front_end/third_party/puppeteer/package/src/node/LaunchOptions.ts +5 -0
- package/front_end/third_party/puppeteer/package/src/revisions.ts +1 -1
- package/front_end/third_party/puppeteer/package/src/util/httpUtils.ts +5 -5
- package/front_end/third_party/puppeteer/package/src/util/version.ts +1 -1
- package/front_end/third_party/puppeteer/third_party/parsel/README.chromium +1 -1
- package/front_end/ui/components/dialogs/Dialog.ts +3 -0
- package/front_end/ui/components/markdown_view/MarkdownLinksMap.ts +0 -7
- package/front_end/ui/helpers/OpenInNewTab.ts +15 -7
- package/front_end/ui/legacy/TabbedPane.ts +1 -108
- package/front_end/ui/legacy/components/data_grid/DataGrid.ts +9 -0
- package/front_end/ui/legacy/components/data_grid/DataGridElement.ts +172 -123
- package/front_end/ui/legacy/components/data_grid/DataGridExporter.ts +89 -0
- package/front_end/ui/legacy/components/data_grid/dataGrid.css +10 -0
- package/front_end/ui/legacy/components/data_grid/data_grid.ts +3 -1
- package/front_end/ui/legacy/components/perf_ui/FlameChart.ts +89 -40
- package/front_end/ui/visual_logging/KnownContextValues.ts +2 -0
- package/package.json +1 -1
- package/front_end/Images/navigationControls.png +0 -0
- package/front_end/Images/navigationControls_2x.png +0 -0
- package/front_end/Images/popoverArrows.png +0 -0
- package/front_end/entrypoints/greendev_floaty/FloatyEntrypoint.ts +0 -746
- package/front_end/entrypoints/greendev_floaty/floaty.css +0 -301
- package/front_end/entrypoints/greendev_floaty/floaty.html +0 -38
- package/front_end/entrypoints/greendev_floaty/greendev_floaty.ts +0 -415
- package/front_end/models/ai_assistance/agents/GreenDevAgent.ts +0 -834
- package/front_end/models/ai_assistance/agents/GreenDevAgentAntigravityCliSocketClient.ts +0 -53
- package/front_end/models/ai_assistance/agents/GreenDevAgentGeminiCliSocketClient.ts +0 -117
- package/front_end/models/annotations/AnnotationRepository.ts +0 -238
- package/front_end/models/annotations/AnnotationType.ts +0 -10
- package/front_end/models/annotations/README.md +0 -7
- package/front_end/models/annotations/annotations.ts +0 -6
- package/front_end/models/greendev/Prototypes.ts +0 -68
- package/front_end/models/greendev/README.md +0 -5
- package/front_end/models/greendev/greendev.ts +0 -5
- package/front_end/panels/common/Annotation.ts +0 -184
- package/front_end/panels/common/AnnotationManager.ts +0 -208
- package/front_end/panels/common/annotation.css +0 -40
- package/front_end/panels/greendev/GreenDevPanel.css +0 -282
- package/front_end/panels/greendev/GreenDevPanel.ts +0 -364
- package/front_end/panels/greendev/GreenDevShared.ts +0 -13
- package/front_end/panels/greendev/greendev-meta.ts +0 -52
- package/front_end/panels/greendev/greendev.ts +0 -9
|
@@ -10,7 +10,6 @@ import type * as Platform from '../../../core/platform/platform.js';
|
|
|
10
10
|
import * as Root from '../../../core/root/root.js';
|
|
11
11
|
import * as SDK from '../../../core/sdk/sdk.js';
|
|
12
12
|
import * as Tracing from '../../../services/tracing/tracing.js';
|
|
13
|
-
import * as Annotations from '../../annotations/annotations.js';
|
|
14
13
|
import * as Logs from '../../logs/logs.js';
|
|
15
14
|
import * as TextUtils from '../../text_utils/text_utils.js';
|
|
16
15
|
import * as Trace from '../../trace/trace.js';
|
|
@@ -62,36 +61,6 @@ export type MainThreadSectionLabel = 'nav-to-lcp'|'lcp-ttfb'|'lcp-render-delay'|
|
|
|
62
61
|
* chrome_preambles.gcl). Sync local changes with the server-side.
|
|
63
62
|
*/
|
|
64
63
|
|
|
65
|
-
const GREEN_DEV_ANNOTATIONS_INSTRUCTIONS = `
|
|
66
|
-
- CRITICAL: You also have access to functions called addElementAnnotation and addNeworkRequestAnnotation,
|
|
67
|
-
which should be used to highlight elements and network requests (respectively).
|
|
68
|
-
|
|
69
|
-
- CRITICAL: Each time an element or a network request is mentioned, you MUST ALSO call the functions
|
|
70
|
-
addElementAnnotation (for an element) or addNeworkRequestAnnotation (for a network request).
|
|
71
|
-
- CRITICAL: Don't add more than one annotation per element or network request.
|
|
72
|
-
- These functions should be called as soon as you identify the entity that needs to be highlighted.
|
|
73
|
-
- In addition to this, the addElementAnnotation function should always be called for the LCP element, if known.
|
|
74
|
-
- The annotationMessage should be descriptive and relevant to why the element or network request is being highlighted.
|
|
75
|
-
`;
|
|
76
|
-
|
|
77
|
-
const GREEN_DEV_FRESH_TRACE_ANNOTATIONS_INSTRUCTIONS = `
|
|
78
|
-
When referring to an element for which you know the nodeId, always call the function addElementAnnotation, specifying
|
|
79
|
-
the id and an annotation reason.
|
|
80
|
-
When referring to a network request for which you know the eventKey for, always call the function
|
|
81
|
-
addNetworkRequestAnnotation, specifying the id and an annotation reason.
|
|
82
|
-
- CRITICAL: Each time you add an annotating link you MUST ALSO call the function addElementAnnotation.
|
|
83
|
-
- CRITICAL: Each time you describe an element or network request as being problematic you MUST call the function
|
|
84
|
-
addElementAnnotation and specify an annotation reason.
|
|
85
|
-
- CRITICAL: Each time you describe a network request as being problematic you MUST call the function
|
|
86
|
-
addNetworkRequestAnnotation and specify an annotation reason.
|
|
87
|
-
- CRITICAL: If you spot ANY of the following problems:
|
|
88
|
-
- Render-blocking elements/network requests.
|
|
89
|
-
- Significant long task (especially on main thread).
|
|
90
|
-
- Layout shifts (e.g. due to unsized images).
|
|
91
|
-
... then you MUST call addNetworkRequestAnnotation for ALL network requests and addaddElementAnnotation for all
|
|
92
|
-
elements described in your conclusion.
|
|
93
|
-
`;
|
|
94
|
-
|
|
95
64
|
/**
|
|
96
65
|
* Preamble clocks in at ~1341 tokens.
|
|
97
66
|
* The prose is around 4.5 chars per token.
|
|
@@ -292,14 +261,7 @@ export class PerformanceAgent extends AiAgent<AgentFocus> {
|
|
|
292
261
|
text: freshTracePreamble,
|
|
293
262
|
metadata: {source: 'devtools', score: ScorePriority.CRITICAL}
|
|
294
263
|
};
|
|
295
|
-
|
|
296
|
-
text: GREEN_DEV_ANNOTATIONS_INSTRUCTIONS,
|
|
297
|
-
metadata: {source: 'devtools', score: ScorePriority.CRITICAL}
|
|
298
|
-
};
|
|
299
|
-
#greenDevFreshTraceAnnotationsFact: Host.AidaClient.RequestFact = {
|
|
300
|
-
text: GREEN_DEV_FRESH_TRACE_ANNOTATIONS_INSTRUCTIONS,
|
|
301
|
-
metadata: {source: 'devtools', score: ScorePriority.CRITICAL}
|
|
302
|
-
};
|
|
264
|
+
|
|
303
265
|
#networkDataDescriptionFact: Host.AidaClient.RequestFact = {
|
|
304
266
|
text: PerformanceTraceFormatter.networkDataFormatDescription,
|
|
305
267
|
metadata: {source: 'devtools', score: ScorePriority.CRITICAL}
|
|
@@ -319,8 +281,6 @@ export class PerformanceAgent extends AiAgent<AgentFocus> {
|
|
|
319
281
|
this.#networkDataDescriptionFact,
|
|
320
282
|
this.#freshTraceExtraPreambleFact,
|
|
321
283
|
this.#notExternalExtraPreambleFact,
|
|
322
|
-
this.#greenDevAnnotationsFact,
|
|
323
|
-
this.#greenDevFreshTraceAnnotationsFact,
|
|
324
284
|
]);
|
|
325
285
|
|
|
326
286
|
/**
|
|
@@ -334,9 +294,7 @@ export class PerformanceAgent extends AiAgent<AgentFocus> {
|
|
|
334
294
|
return Host.AidaClient.ClientFeature.CHROME_PERFORMANCE_FULL_AGENT;
|
|
335
295
|
}
|
|
336
296
|
get userTier(): string|undefined {
|
|
337
|
-
return
|
|
338
|
-
'TESTERS' :
|
|
339
|
-
Root.Runtime.hostConfig.devToolsAiAssistancePerformanceAgent?.userTier;
|
|
297
|
+
return Root.Runtime.hostConfig.devToolsAiAssistancePerformanceAgent?.userTier;
|
|
340
298
|
}
|
|
341
299
|
get options(): RequestOptions {
|
|
342
300
|
const temperature = Root.Runtime.hostConfig.devToolsAiAssistancePerformanceAgent?.temperature;
|
|
@@ -708,17 +666,9 @@ export class PerformanceAgent extends AiAgent<AgentFocus> {
|
|
|
708
666
|
|
|
709
667
|
this.addFact(this.#notExternalExtraPreambleFact);
|
|
710
668
|
|
|
711
|
-
const annotationsEnabled = Annotations.AnnotationRepository.annotationsEnabled();
|
|
712
|
-
if (annotationsEnabled) {
|
|
713
|
-
this.addFact(this.#greenDevAnnotationsFact);
|
|
714
|
-
}
|
|
715
|
-
|
|
716
669
|
const isFresh = Tracing.FreshRecording.Tracker.instance().recordingIsFresh(focus.parsedTrace);
|
|
717
670
|
if (isFresh) {
|
|
718
671
|
this.addFact(this.#freshTraceExtraPreambleFact);
|
|
719
|
-
if (annotationsEnabled) {
|
|
720
|
-
this.addFact(this.#greenDevFreshTraceAnnotationsFact);
|
|
721
|
-
}
|
|
722
672
|
}
|
|
723
673
|
|
|
724
674
|
this.addFact(this.#callFrameDataDescriptionFact);
|
|
@@ -1149,69 +1099,6 @@ export class PerformanceAgent extends AiAgent<AgentFocus> {
|
|
|
1149
1099
|
|
|
1150
1100
|
});
|
|
1151
1101
|
|
|
1152
|
-
if (Annotations.AnnotationRepository.annotationsEnabled()) {
|
|
1153
|
-
this.declareFunction<{
|
|
1154
|
-
elementId: string,
|
|
1155
|
-
annotationMessage: string,
|
|
1156
|
-
}>('addElementAnnotation', {
|
|
1157
|
-
description:
|
|
1158
|
-
'Adds a visual annotation in the Elements panel, attached to a node with the specific UID provided. Use it to highlight nodes in the Elements panel and provide contextual suggestions to the user related to their queries.',
|
|
1159
|
-
parameters: {
|
|
1160
|
-
type: Host.AidaClient.ParametersTypes.OBJECT,
|
|
1161
|
-
description: '',
|
|
1162
|
-
nullable: false,
|
|
1163
|
-
properties: {
|
|
1164
|
-
elementId: {
|
|
1165
|
-
type: Host.AidaClient.ParametersTypes.STRING,
|
|
1166
|
-
description: 'The UID of the element to annotate.',
|
|
1167
|
-
nullable: false,
|
|
1168
|
-
},
|
|
1169
|
-
annotationMessage: {
|
|
1170
|
-
type: Host.AidaClient.ParametersTypes.STRING,
|
|
1171
|
-
description: 'The message the annotation should show to the user.',
|
|
1172
|
-
nullable: false,
|
|
1173
|
-
},
|
|
1174
|
-
|
|
1175
|
-
},
|
|
1176
|
-
required: ['elementId', 'annotationMessage']
|
|
1177
|
-
},
|
|
1178
|
-
handler: async params => {
|
|
1179
|
-
return await this.addElementAnnotation(params.elementId, params.annotationMessage);
|
|
1180
|
-
},
|
|
1181
|
-
});
|
|
1182
|
-
|
|
1183
|
-
this.declareFunction<{
|
|
1184
|
-
eventKey: string,
|
|
1185
|
-
annotationMessage: string,
|
|
1186
|
-
}>('addNetworkRequestAnnotation', {
|
|
1187
|
-
description:
|
|
1188
|
-
'Adds a visual annotation in the Network panel, attached to the request with the specific UID provided. ' +
|
|
1189
|
-
'Use it to highlight requests in the Network panel and provide contextual suggestions to the user ' +
|
|
1190
|
-
'related to their queries.',
|
|
1191
|
-
parameters: {
|
|
1192
|
-
type: Host.AidaClient.ParametersTypes.OBJECT,
|
|
1193
|
-
description: '',
|
|
1194
|
-
nullable: false,
|
|
1195
|
-
properties: {
|
|
1196
|
-
eventKey: {
|
|
1197
|
-
type: Host.AidaClient.ParametersTypes.STRING,
|
|
1198
|
-
description: 'The event key of the network request to annotate.',
|
|
1199
|
-
nullable: false,
|
|
1200
|
-
},
|
|
1201
|
-
annotationMessage: {
|
|
1202
|
-
type: Host.AidaClient.ParametersTypes.STRING,
|
|
1203
|
-
description: 'The message the annotation should show to the user.',
|
|
1204
|
-
nullable: false,
|
|
1205
|
-
},
|
|
1206
|
-
},
|
|
1207
|
-
required: ['eventKey', 'annotationMessage']
|
|
1208
|
-
},
|
|
1209
|
-
handler: async params => {
|
|
1210
|
-
return await this.addNetworkRequestAnnotation(params.eventKey, params.annotationMessage);
|
|
1211
|
-
},
|
|
1212
|
-
});
|
|
1213
|
-
}
|
|
1214
|
-
|
|
1215
1102
|
this.declareFunction<{scriptUrl: UrlString, line: number, column: number}, {result: string}>('getFunctionCode', {
|
|
1216
1103
|
description:
|
|
1217
1104
|
'Returns the code for a function defined at the given location. The result is annotated with the runtime performance of each line of code.',
|
|
@@ -1475,47 +1362,6 @@ export class PerformanceAgent extends AiAgent<AgentFocus> {
|
|
|
1475
1362
|
return null;
|
|
1476
1363
|
}
|
|
1477
1364
|
|
|
1478
|
-
async addElementAnnotation(elementId: string, annotationMessage: string):
|
|
1479
|
-
Promise<FunctionCallHandlerResult<unknown>> {
|
|
1480
|
-
if (!Annotations.AnnotationRepository.annotationsEnabled()) {
|
|
1481
|
-
console.warn('Received agent request to add element annotation with annotations disabled');
|
|
1482
|
-
return {error: 'Annotations are not currently enabled'};
|
|
1483
|
-
}
|
|
1484
|
-
|
|
1485
|
-
// eslint-disable-next-line no-console
|
|
1486
|
-
console.log(`AI AGENT EVENT: Performance Agent adding annotation for element ${elementId}: '${annotationMessage}'`);
|
|
1487
|
-
Annotations.AnnotationRepository.instance().addElementsAnnotation(annotationMessage, elementId);
|
|
1488
|
-
return {result: {success: true}};
|
|
1489
|
-
}
|
|
1490
|
-
|
|
1491
|
-
async addNetworkRequestAnnotation(eventKey: string, annotationMessage: string):
|
|
1492
|
-
Promise<FunctionCallHandlerResult<unknown>> {
|
|
1493
|
-
if (!Annotations.AnnotationRepository.annotationsEnabled()) {
|
|
1494
|
-
console.warn('Received agent request to add network request annotation with annotations disabled');
|
|
1495
|
-
return {error: 'Annotations are not currently enabled'};
|
|
1496
|
-
}
|
|
1497
|
-
|
|
1498
|
-
// eslint-disable-next-line no-console
|
|
1499
|
-
console.log(
|
|
1500
|
-
`AI AGENT EVENT: Performance Agent adding annotation for network request ${eventKey}: '${annotationMessage}'`);
|
|
1501
|
-
|
|
1502
|
-
let requestId = undefined;
|
|
1503
|
-
const focus = this.context?.getItem();
|
|
1504
|
-
if (focus) {
|
|
1505
|
-
const event = focus.lookupEvent(eventKey);
|
|
1506
|
-
if (event && Trace.Types.Events.isSyntheticNetworkRequest(event)) {
|
|
1507
|
-
requestId = event.args.data.requestId;
|
|
1508
|
-
}
|
|
1509
|
-
}
|
|
1510
|
-
|
|
1511
|
-
if (!requestId) {
|
|
1512
|
-
console.warn('Unable to lookup requestId for request with event key', eventKey);
|
|
1513
|
-
}
|
|
1514
|
-
|
|
1515
|
-
Annotations.AnnotationRepository.instance().addNetworkRequestAnnotation(annotationMessage, requestId);
|
|
1516
|
-
return {result: {success: true}};
|
|
1517
|
-
}
|
|
1518
|
-
|
|
1519
1365
|
async #getNetworkRequestImageData(lcpRequest: Trace.Types.Events.SyntheticNetworkRequest):
|
|
1520
1366
|
Promise<TextUtils.ContentData.ContentData|undefined> {
|
|
1521
1367
|
const target = SDK.TargetManager.TargetManager.instance().primaryPageTarget();
|
|
@@ -95,30 +95,6 @@ Content:
|
|
|
95
95
|
"title"
|
|
96
96
|
]
|
|
97
97
|
}
|
|
98
|
-
},
|
|
99
|
-
{
|
|
100
|
-
"name": "activateDeviceEmulation",
|
|
101
|
-
"description": "Sets emulation viewing mode for a specific device and optionally enables vision deficiency emulation.",
|
|
102
|
-
"parameters": {
|
|
103
|
-
"type": 6,
|
|
104
|
-
"description": "",
|
|
105
|
-
"nullable": false,
|
|
106
|
-
"properties": {
|
|
107
|
-
"deviceName": {
|
|
108
|
-
"type": 1,
|
|
109
|
-
"description": "The name of the device to emulate. Allowed values: Pixel 3 XL, Pixel 7, Samsung Galaxy S8+, Samsung Galaxy S20 Ultra, Surface Pro 7, Surface Duo, Galaxy Z Fold 5, Asus Zenbook Fold, Samsung Galaxy A51/71, Nest Hub Max, Nest Hub, iPhone 4, iPhone 5/SE, iPhone 6/7/8, iPhone SE, iPhone XR, iPhone 12 Pro, iPhone 14 Pro Max, iPad Mini, iPad Air, iPad Pro.",
|
|
110
|
-
"nullable": false
|
|
111
|
-
},
|
|
112
|
-
"visionDeficiency": {
|
|
113
|
-
"type": 1,
|
|
114
|
-
"description": "Optional vision deficiency to emulate. Allowed values: blurredVision, reducedContrast, achromatopsia, deuteranopia, protanopia, tritanopia.",
|
|
115
|
-
"nullable": true
|
|
116
|
-
}
|
|
117
|
-
},
|
|
118
|
-
"required": [
|
|
119
|
-
"deviceName"
|
|
120
|
-
]
|
|
121
|
-
}
|
|
122
98
|
}
|
|
123
99
|
],
|
|
124
100
|
"options": {
|
|
@@ -5,10 +5,6 @@
|
|
|
5
5
|
import * as Host from '../../../core/host/host.js';
|
|
6
6
|
import * as Root from '../../../core/root/root.js';
|
|
7
7
|
import * as SDK from '../../../core/sdk/sdk.js';
|
|
8
|
-
import * as Protocol from '../../../generated/protocol.js';
|
|
9
|
-
import * as Greendev from '../../../models/greendev/greendev.js';
|
|
10
|
-
import * as Annotations from '../../annotations/annotations.js';
|
|
11
|
-
import * as Emulation from '../../emulation/emulation.js';
|
|
12
8
|
import {ChangeManager} from '../ChangeManager.js';
|
|
13
9
|
import {ExtensionScope} from '../ExtensionScope.js';
|
|
14
10
|
import {AI_ASSISTANCE_CSS_CLASS_NAME} from '../injected.js';
|
|
@@ -19,8 +15,6 @@ import {
|
|
|
19
15
|
AiAgent,
|
|
20
16
|
type ContextResponse,
|
|
21
17
|
type ConversationContext,
|
|
22
|
-
type FunctionCallHandlerResult,
|
|
23
|
-
type MultimodalInput,
|
|
24
18
|
MultimodalInputType,
|
|
25
19
|
type RequestOptions,
|
|
26
20
|
ResponseType
|
|
@@ -69,40 +63,6 @@ If the user asks a question that requires an investigation of a problem, use thi
|
|
|
69
63
|
- [Suggestion 1]
|
|
70
64
|
- [Suggestion 2]`;
|
|
71
65
|
|
|
72
|
-
const emulationInstructions = `
|
|
73
|
-
# Emulation and Screenshots
|
|
74
|
-
|
|
75
|
-
* If asked to verify whether the page is visually broken or if there are display problems with specific devices, use the \`activateDeviceEmulation\` tool. This tool will activate emulation for a specified device and capture a screenshot.
|
|
76
|
-
* **DEVICE SELECTION**: You must choose the most closely related device match from the allowed list.
|
|
77
|
-
* If the user asks about a specific device (e.g., "iPhone 6"), choose the closest match (e.g., "iPhone 6/7/8").
|
|
78
|
-
* If the user specifies a generic category (e.g., "Android phone", "iPhone", "Samsung"), choose the device with the highest version number available in that category (e.g., "Pixel 7" or "Samsung Galaxy S20" for Android, "iPhone 14 Pro Max" for iPhone).
|
|
79
|
-
* **VISION DEFICIENCY**: If the user asks about checking for color blindness or vision issues, you can pass an optional \`visionDeficiency\` parameter to \`activateDeviceEmulation\`. Allowed values are: 'blurredVision', 'reducedContrast', 'achromatopsia', 'deuteranopia', 'protanopia', 'tritanopia'.
|
|
80
|
-
* **IMPORTANT**: This is a **TWO-STEP** process.
|
|
81
|
-
* **STEP 1**: Call \`activateDeviceEmulation\`. After calling this tool, YOU MUST STOP and tell the user that the screenshot has been captured and ask them whether they would like you to focus on specific sections of the screenshot or review it all for possible problems.
|
|
82
|
-
* **STEP 2**: The captured screenshot will be automatically attached to the user's **NEXT** query.
|
|
83
|
-
* **CRITICAL**: DO NOT try to investigate/analyze the page state or element visibility automatically. But, after the user has requested to analyze the page, you can prompt the user to select one of the problematic elements if they want to diagnose further.
|
|
84
|
-
* **CRITICAL**: The output of the analysis should only be in json form (no supplemental text) and the json should list the problems found on the device, with a short description of the problem. If identical problems are identified acress multiple devices, feel free to combine sections.
|
|
85
|
-
* **CRITICAL**: ALWAYS escape single and double quotes within the json output strings (\' and \").
|
|
86
|
-
*
|
|
87
|
-
* Example (with no duplication):
|
|
88
|
-
|
|
89
|
-
[
|
|
90
|
-
{
|
|
91
|
-
"Problem": "Element not resizing",
|
|
92
|
-
"Element": "Hero banner",
|
|
93
|
-
"NodeId": "23",
|
|
94
|
-
"Details": "The \"hero\" element is not resizing because... etc etc."
|
|
95
|
-
}
|
|
96
|
-
]
|
|
97
|
-
|
|
98
|
-
# Additional notes:
|
|
99
|
-
|
|
100
|
-
When referring to an element for which you know the nodeId, annotate your output using markdown link syntax:
|
|
101
|
-
- For example, if nodeId is 23: ([link](#node-23))
|
|
102
|
-
- Always prefix the nodeId with the 'node-' prefix when using the markdown syntax.
|
|
103
|
-
- This link will reveal the element in the Elements panel
|
|
104
|
-
- Never mention node or nodeId when referring to the element, and especially not in the link text.`;
|
|
105
|
-
|
|
106
66
|
/* clang-format on */
|
|
107
67
|
|
|
108
68
|
const promptForScreenshot =
|
|
@@ -144,8 +104,7 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
|
|
|
144
104
|
readonly preamble = preamble;
|
|
145
105
|
readonly clientFeature = Host.AidaClient.ClientFeature.CHROME_STYLING_AGENT;
|
|
146
106
|
get userTier(): string|undefined {
|
|
147
|
-
|
|
148
|
-
return greenDevEmulationEnabled ? 'TESTERS' : Root.Runtime.hostConfig.devToolsFreestyler?.userTier;
|
|
107
|
+
return Root.Runtime.hostConfig.devToolsFreestyler?.userTier;
|
|
149
108
|
}
|
|
150
109
|
get executionMode(): Root.Runtime.HostConfigFreestylerExecutionMode {
|
|
151
110
|
return Root.Runtime.hostConfig.devToolsFreestyler?.executionMode ??
|
|
@@ -170,9 +129,6 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
|
|
|
170
129
|
|
|
171
130
|
#changes: ChangeManager;
|
|
172
131
|
#createExtensionScope: CreateExtensionScopeFunction;
|
|
173
|
-
#greenDevEmulationScreenshot: string|null = null;
|
|
174
|
-
#greenDevEmulationAxTree: string|null = null;
|
|
175
|
-
#hasAddedEmulationInstructions = false;
|
|
176
132
|
|
|
177
133
|
constructor(opts: ExecuteJsAgentOptions) {
|
|
178
134
|
super(opts);
|
|
@@ -226,314 +182,12 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
|
|
|
226
182
|
),
|
|
227
183
|
});
|
|
228
184
|
|
|
229
|
-
if (Annotations.AnnotationRepository.annotationsEnabled()) {
|
|
230
|
-
this.declareFunction<{
|
|
231
|
-
elementId: string,
|
|
232
|
-
annotationMessage: string,
|
|
233
|
-
}>('addElementAnnotation', {
|
|
234
|
-
description: 'Adds a visual annotation in the Elements panel, attached to a node with ' +
|
|
235
|
-
'the specific UID provided. Use it to highlight nodes in the Elements panel ' +
|
|
236
|
-
'and provide contextual suggestions to the user related to their queries.',
|
|
237
|
-
parameters: {
|
|
238
|
-
type: Host.AidaClient.ParametersTypes.OBJECT,
|
|
239
|
-
description: '',
|
|
240
|
-
nullable: false,
|
|
241
|
-
properties: {
|
|
242
|
-
elementId: {
|
|
243
|
-
type: Host.AidaClient.ParametersTypes.STRING,
|
|
244
|
-
description: 'The UID of the element to annotate.',
|
|
245
|
-
nullable: false,
|
|
246
|
-
},
|
|
247
|
-
annotationMessage: {
|
|
248
|
-
type: Host.AidaClient.ParametersTypes.STRING,
|
|
249
|
-
description: 'The message the annotation should show to the user.',
|
|
250
|
-
nullable: false,
|
|
251
|
-
},
|
|
252
|
-
},
|
|
253
|
-
required: ['elementId', 'annotationMessage']
|
|
254
|
-
},
|
|
255
|
-
handler: async params => {
|
|
256
|
-
return await this.addElementAnnotation(params.elementId, params.annotationMessage);
|
|
257
|
-
},
|
|
258
|
-
});
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
this.declareFunction<{
|
|
262
|
-
deviceName: string,
|
|
263
|
-
visionDeficiency?: string,
|
|
264
|
-
}>('activateDeviceEmulation', {
|
|
265
|
-
description:
|
|
266
|
-
'Sets emulation viewing mode for a specific device and optionally enables vision deficiency emulation.',
|
|
267
|
-
parameters: {
|
|
268
|
-
type: Host.AidaClient.ParametersTypes.OBJECT,
|
|
269
|
-
description: '',
|
|
270
|
-
nullable: false,
|
|
271
|
-
properties: {
|
|
272
|
-
deviceName: {
|
|
273
|
-
type: Host.AidaClient.ParametersTypes.STRING,
|
|
274
|
-
description:
|
|
275
|
-
'The name of the device to emulate. Allowed values: Pixel 3 XL, Pixel 7, Samsung Galaxy S8+, Samsung Galaxy S20 Ultra, Surface Pro 7, Surface Duo, Galaxy Z Fold 5, Asus Zenbook Fold, Samsung Galaxy A51/71, Nest Hub Max, Nest Hub, iPhone 4, iPhone 5/SE, iPhone 6/7/8, iPhone SE, iPhone XR, iPhone 12 Pro, iPhone 14 Pro Max, iPad Mini, iPad Air, iPad Pro.',
|
|
276
|
-
nullable: false,
|
|
277
|
-
},
|
|
278
|
-
visionDeficiency: {
|
|
279
|
-
type: Host.AidaClient.ParametersTypes.STRING,
|
|
280
|
-
description:
|
|
281
|
-
'Optional vision deficiency to emulate. Allowed values: blurredVision, reducedContrast, achromatopsia, deuteranopia, protanopia, tritanopia.',
|
|
282
|
-
nullable: true,
|
|
283
|
-
},
|
|
284
|
-
},
|
|
285
|
-
required: ['deviceName']
|
|
286
|
-
},
|
|
287
|
-
handler: async params => {
|
|
288
|
-
return await this.activateDeviceEmulation(params.deviceName, params.visionDeficiency);
|
|
289
|
-
},
|
|
290
|
-
});
|
|
291
|
-
}
|
|
292
|
-
|
|
293
|
-
/**
|
|
294
|
-
* Clears styling-agent-specific caches and state.
|
|
295
|
-
* Resets cached emulation data (screenshots, accessibility tree) and the
|
|
296
|
-
* instructions flag to ensure they are re-evaluated in subsequent queries.
|
|
297
|
-
*/
|
|
298
|
-
override clearCache(): void {
|
|
299
|
-
super.clearCache();
|
|
300
|
-
// Reset emulation state so that subsequent queries will re-initialize
|
|
301
|
-
// emulation details and fetch fresh data.
|
|
302
|
-
this.#greenDevEmulationScreenshot = null;
|
|
303
|
-
this.#greenDevEmulationAxTree = null;
|
|
304
|
-
this.#hasAddedEmulationInstructions = false;
|
|
305
185
|
}
|
|
306
186
|
|
|
307
187
|
override preambleFeatures(): string[] {
|
|
308
188
|
return ['function_calling'];
|
|
309
189
|
}
|
|
310
190
|
|
|
311
|
-
#getSelectedNode(): SDK.DOMModel.DOMNode|null {
|
|
312
|
-
return this.context?.getItem() ?? null;
|
|
313
|
-
}
|
|
314
|
-
|
|
315
|
-
async addElementAnnotation(elementId: string, annotationMessage: string):
|
|
316
|
-
Promise<FunctionCallHandlerResult<unknown>> {
|
|
317
|
-
if (!Annotations.AnnotationRepository.annotationsEnabled()) {
|
|
318
|
-
console.warn('Received agent request to add annotation with annotations disabled');
|
|
319
|
-
return {error: 'Annotations are not currently enabled'};
|
|
320
|
-
}
|
|
321
|
-
|
|
322
|
-
// eslint-disable-next-line no-console
|
|
323
|
-
console.log(
|
|
324
|
-
`AI AGENT EVENT: Styling Agent adding annotation for element ${elementId} with message '${annotationMessage}'`);
|
|
325
|
-
const selectedNode = this.#getSelectedNode();
|
|
326
|
-
if (!selectedNode) {
|
|
327
|
-
return {error: 'Error: Unable to find currently selected element.'};
|
|
328
|
-
}
|
|
329
|
-
const domModel = selectedNode.domModel();
|
|
330
|
-
const backendNodeId = Number(elementId) as Protocol.DOM.BackendNodeId;
|
|
331
|
-
const nodeMap = await domModel.pushNodesByBackendIdsToFrontend(new Set([backendNodeId]));
|
|
332
|
-
const node = nodeMap?.get(backendNodeId);
|
|
333
|
-
|
|
334
|
-
if (!node) {
|
|
335
|
-
return {error: `Error: Could not find the element with backendNodeId=${elementId}`};
|
|
336
|
-
}
|
|
337
|
-
|
|
338
|
-
Annotations.AnnotationRepository.instance().addElementsAnnotation(annotationMessage, node);
|
|
339
|
-
|
|
340
|
-
return {
|
|
341
|
-
result: `Annotation added for element ${elementId}: ${annotationMessage}`,
|
|
342
|
-
};
|
|
343
|
-
}
|
|
344
|
-
|
|
345
|
-
async #compressScreenshot(base64Data: string): Promise<string> {
|
|
346
|
-
return await new Promise((resolve, reject) => {
|
|
347
|
-
const img = new Image();
|
|
348
|
-
img.onload = () => {
|
|
349
|
-
// eslint-disable-next-line @devtools/no-imperative-dom-api
|
|
350
|
-
const canvas = document.createElement('canvas');
|
|
351
|
-
const maxDimension = 2000;
|
|
352
|
-
let scale = 1;
|
|
353
|
-
if (img.width > maxDimension || img.height > maxDimension) {
|
|
354
|
-
scale = maxDimension / Math.max(img.width, img.height);
|
|
355
|
-
}
|
|
356
|
-
canvas.width = img.width * scale;
|
|
357
|
-
canvas.height = img.height * scale;
|
|
358
|
-
|
|
359
|
-
const ctx = canvas.getContext('2d');
|
|
360
|
-
if (!ctx) {
|
|
361
|
-
reject(new Error('Could not get canvas context'));
|
|
362
|
-
return;
|
|
363
|
-
}
|
|
364
|
-
ctx.imageSmoothingEnabled = true;
|
|
365
|
-
ctx.imageSmoothingQuality = 'high';
|
|
366
|
-
ctx.drawImage(img, 0, 0, canvas.width, canvas.height);
|
|
367
|
-
const dataUrl = canvas.toDataURL('image/jpeg', 0.9);
|
|
368
|
-
resolve(dataUrl.split(',')[1]);
|
|
369
|
-
};
|
|
370
|
-
img.onerror = e => reject(new Error('Image load error: ' + e));
|
|
371
|
-
img.src = 'data:image/png;base64,' + base64Data;
|
|
372
|
-
});
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
async activateDeviceEmulation(deviceName: string, visionDeficiency?: string):
|
|
376
|
-
Promise<FunctionCallHandlerResult<unknown>> {
|
|
377
|
-
const greenDevEmulationEnabled = Greendev.Prototypes.instance().isEnabled('emulationCapabilities');
|
|
378
|
-
if (!greenDevEmulationEnabled) {
|
|
379
|
-
return {error: `GreenDev emulation capabilities not enabled`};
|
|
380
|
-
}
|
|
381
|
-
|
|
382
|
-
// eslint-disable-next-line no-console
|
|
383
|
-
console.log('activateDeviceEmulation called with device:', deviceName, 'visionDeficiency:', visionDeficiency);
|
|
384
|
-
|
|
385
|
-
this.#greenDevEmulationScreenshot = null;
|
|
386
|
-
this.#greenDevEmulationAxTree = null;
|
|
387
|
-
|
|
388
|
-
const emulatedDevicesList = Emulation.EmulatedDevices.EmulatedDevicesList.instance();
|
|
389
|
-
const device = emulatedDevicesList.standard().find(d => d.title === deviceName);
|
|
390
|
-
|
|
391
|
-
if (!device) {
|
|
392
|
-
return {
|
|
393
|
-
error: `Could not find device "${deviceName}" in the list of emulated devices.`,
|
|
394
|
-
};
|
|
395
|
-
}
|
|
396
|
-
|
|
397
|
-
const deviceModeModel = Emulation.DeviceModeModel.DeviceModeModel.instance();
|
|
398
|
-
|
|
399
|
-
const verticalMode = device.modesForOrientation(Emulation.EmulatedDevices.Vertical)[0];
|
|
400
|
-
if (!verticalMode) {
|
|
401
|
-
return {
|
|
402
|
-
error: `Could not find vertical mode for "${deviceName}".`,
|
|
403
|
-
};
|
|
404
|
-
}
|
|
405
|
-
deviceModeModel.emulate(Emulation.DeviceModeModel.Type.Device, device, verticalMode);
|
|
406
|
-
|
|
407
|
-
// Get the selected node early to use for both vision deficiency and wait mechanism.
|
|
408
|
-
const selectedNode = this.#getSelectedNode();
|
|
409
|
-
|
|
410
|
-
// Apply vision deficiency if provided (and turn it off when not provided).
|
|
411
|
-
try {
|
|
412
|
-
if (selectedNode) {
|
|
413
|
-
const target = selectedNode.domModel().target();
|
|
414
|
-
const emulationModel = target.model(SDK.EmulationModel.EmulationModel);
|
|
415
|
-
if (emulationModel) {
|
|
416
|
-
let type = Protocol.Emulation.SetEmulatedVisionDeficiencyRequestType.None;
|
|
417
|
-
if (visionDeficiency && visionDeficiency !== 'none') {
|
|
418
|
-
type = visionDeficiency as Protocol.Emulation.SetEmulatedVisionDeficiencyRequestType;
|
|
419
|
-
}
|
|
420
|
-
await target.emulationAgent().invoke_setEmulatedVisionDeficiency({type});
|
|
421
|
-
}
|
|
422
|
-
} else {
|
|
423
|
-
console.error('No selected node context to retrieve EmulationModel.');
|
|
424
|
-
}
|
|
425
|
-
} catch {
|
|
426
|
-
return {
|
|
427
|
-
error: `Unable to apply vision deficiency "${visionDeficiency}".`,
|
|
428
|
-
};
|
|
429
|
-
}
|
|
430
|
-
|
|
431
|
-
// Wait for the layout to settle after emulation changes.
|
|
432
|
-
// We use a double requestAnimationFrame to ensure at least one frame is rendered.
|
|
433
|
-
if (selectedNode) {
|
|
434
|
-
try {
|
|
435
|
-
const code = 'await new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))';
|
|
436
|
-
// We use throwOnSideEffect: false because this is a benign wait, not a modification of the page state relevant to the user.
|
|
437
|
-
await this.#execJs(code, {throwOnSideEffect: false, contextNode: selectedNode});
|
|
438
|
-
} catch (e) {
|
|
439
|
-
console.error('Failed to wait for layout settle:', e);
|
|
440
|
-
}
|
|
441
|
-
}
|
|
442
|
-
|
|
443
|
-
const orientation = device.orientationByName(Emulation.EmulatedDevices.Vertical);
|
|
444
|
-
const width = orientation.width;
|
|
445
|
-
|
|
446
|
-
// TODO(finnur): Investigate better screen capture alternatives (that can do the whole page).
|
|
447
|
-
let documentHeight = 2000;
|
|
448
|
-
if (selectedNode) {
|
|
449
|
-
try {
|
|
450
|
-
const heightJs = 'document.body.scrollHeight';
|
|
451
|
-
const result = await this.#execJs(heightJs, {throwOnSideEffect: false, contextNode: selectedNode});
|
|
452
|
-
const parsedHeight = Number(result);
|
|
453
|
-
if (!isNaN(parsedHeight)) {
|
|
454
|
-
documentHeight = Math.min(parsedHeight, 2000);
|
|
455
|
-
}
|
|
456
|
-
} catch (e) {
|
|
457
|
-
console.error('Failed to get document height:', e);
|
|
458
|
-
}
|
|
459
|
-
}
|
|
460
|
-
|
|
461
|
-
// Specify a clip capping the height to the top 5000px.
|
|
462
|
-
const clip: Protocol.Page.Viewport = {
|
|
463
|
-
x: 0,
|
|
464
|
-
y: 0,
|
|
465
|
-
width,
|
|
466
|
-
height: documentHeight,
|
|
467
|
-
scale: 1,
|
|
468
|
-
};
|
|
469
|
-
|
|
470
|
-
// Capture using the clip. fullSize must be false when clip is used.
|
|
471
|
-
const screenshot = await deviceModeModel.captureScreenshot(false, clip);
|
|
472
|
-
|
|
473
|
-
if (!screenshot) {
|
|
474
|
-
return {
|
|
475
|
-
error: `Emulation for ${deviceName} activated, but failed to capture screenshot.`,
|
|
476
|
-
};
|
|
477
|
-
}
|
|
478
|
-
|
|
479
|
-
try {
|
|
480
|
-
this.#greenDevEmulationScreenshot = await this.#compressScreenshot(screenshot);
|
|
481
|
-
} catch (e) {
|
|
482
|
-
console.error('Screenshot compression failed, using original', e);
|
|
483
|
-
this.#greenDevEmulationScreenshot = screenshot;
|
|
484
|
-
}
|
|
485
|
-
|
|
486
|
-
try {
|
|
487
|
-
if (selectedNode) {
|
|
488
|
-
const accessibilityModel = selectedNode.domModel().target().model(SDK.AccessibilityModel.AccessibilityModel);
|
|
489
|
-
if (accessibilityModel) {
|
|
490
|
-
await accessibilityModel.resumeModel();
|
|
491
|
-
const axResponse = await accessibilityModel.agent.invoke_getFullAXTree({});
|
|
492
|
-
if (!axResponse.getError()) {
|
|
493
|
-
this.#greenDevEmulationAxTree = JSON.stringify(axResponse.nodes);
|
|
494
|
-
} else {
|
|
495
|
-
console.error('Failed to capture Accessibility Tree:', axResponse.getError());
|
|
496
|
-
}
|
|
497
|
-
}
|
|
498
|
-
}
|
|
499
|
-
} catch (e) {
|
|
500
|
-
console.error('Exception capturing Accessibility Tree:', e);
|
|
501
|
-
}
|
|
502
|
-
|
|
503
|
-
let resultMsg = `Emulation for ${deviceName} activated and screenshot has been captured.`;
|
|
504
|
-
if (visionDeficiency) {
|
|
505
|
-
resultMsg += ` Vision deficiency "${visionDeficiency}" was also applied.`;
|
|
506
|
-
}
|
|
507
|
-
resultMsg += ' Ready for analysis.';
|
|
508
|
-
|
|
509
|
-
return {
|
|
510
|
-
result: resultMsg,
|
|
511
|
-
};
|
|
512
|
-
}
|
|
513
|
-
|
|
514
|
-
override popPendingMultimodalInput(): MultimodalInput|undefined {
|
|
515
|
-
const greenDevEmulationEnabled = Greendev.Prototypes.instance().isEnabled('emulationCapabilities');
|
|
516
|
-
if (!greenDevEmulationEnabled) {
|
|
517
|
-
return undefined;
|
|
518
|
-
}
|
|
519
|
-
|
|
520
|
-
if (this.#greenDevEmulationScreenshot) {
|
|
521
|
-
const data = this.#greenDevEmulationScreenshot;
|
|
522
|
-
this.#greenDevEmulationScreenshot = null;
|
|
523
|
-
return {
|
|
524
|
-
type: MultimodalInputType.SCREENSHOT,
|
|
525
|
-
input: {
|
|
526
|
-
inlineData: {
|
|
527
|
-
data,
|
|
528
|
-
mimeType: 'image/jpeg',
|
|
529
|
-
},
|
|
530
|
-
},
|
|
531
|
-
id: crypto.randomUUID(),
|
|
532
|
-
};
|
|
533
|
-
}
|
|
534
|
-
return undefined;
|
|
535
|
-
}
|
|
536
|
-
|
|
537
191
|
override async *
|
|
538
192
|
handleContextDetails(selectedElement: ConversationContext<SDK.DOMModel.DOMNode>|null):
|
|
539
193
|
AsyncGenerator<ContextResponse, void, void> {
|
|
@@ -551,19 +205,9 @@ export class StylingAgent extends AiAgent<SDK.DOMModel.DOMNode> {
|
|
|
551
205
|
override async enhanceQuery(
|
|
552
206
|
query: string, selectedElement: ConversationContext<SDK.DOMModel.DOMNode>|null,
|
|
553
207
|
multimodalInputType?: MultimodalInputType): Promise<string> {
|
|
554
|
-
|
|
208
|
+
const multimodalInputEnhancementQuery =
|
|
555
209
|
this.multimodalInputEnabled && multimodalInputType ? MULTIMODAL_ENHANCEMENT_PROMPTS[multimodalInputType] : '';
|
|
556
210
|
|
|
557
|
-
if (this.#greenDevEmulationAxTree) {
|
|
558
|
-
multimodalInputEnhancementQuery += '\n# Accessibility Tree\n\n' + this.#greenDevEmulationAxTree;
|
|
559
|
-
this.#greenDevEmulationAxTree = null;
|
|
560
|
-
}
|
|
561
|
-
|
|
562
|
-
if (Greendev.Prototypes.instance().isEnabled('emulationCapabilities') && !this.#hasAddedEmulationInstructions) {
|
|
563
|
-
multimodalInputEnhancementQuery = emulationInstructions + '\n' + multimodalInputEnhancementQuery;
|
|
564
|
-
this.#hasAddedEmulationInstructions = true;
|
|
565
|
-
}
|
|
566
|
-
|
|
567
211
|
const promptDetails = selectedElement ? await selectedElement.getPromptDetails() : null;
|
|
568
212
|
const elementEnchancementQuery = promptDetails ? `${promptDetails}\n\n# User request\n\n` : '';
|
|
569
213
|
return `${multimodalInputEnhancementQuery}${elementEnchancementQuery}QUERY: ${query}`;
|
|
@@ -7,9 +7,6 @@ import * as AccessibilityAgent from './agents/AccessibilityAgent.js';
|
|
|
7
7
|
import * as AiAgent from './agents/AiAgent.js';
|
|
8
8
|
import * as ContextSelectionAgent from './agents/ContextSelectionAgent.js';
|
|
9
9
|
import * as FileAgent from './agents/FileAgent.js';
|
|
10
|
-
import * as GreenDevAgent from './agents/GreenDevAgent.js';
|
|
11
|
-
import * as GreenDevAgentAntigravityCliSocketClient from './agents/GreenDevAgentAntigravityCliSocketClient.js';
|
|
12
|
-
import * as GreenDevAgentGeminiCliSocketClient from './agents/GreenDevAgentGeminiCliSocketClient.js';
|
|
13
10
|
import * as NetworkAgent from './agents/NetworkAgent.js';
|
|
14
11
|
import * as PatchAgent from './agents/PatchAgent.js';
|
|
15
12
|
import * as PerformanceAgent from './agents/PerformanceAgent.js';
|
|
@@ -82,9 +79,6 @@ export {
|
|
|
82
79
|
GetLighthouseAudits,
|
|
83
80
|
GetNetworkRequestDetails,
|
|
84
81
|
GetStyles,
|
|
85
|
-
GreenDevAgent,
|
|
86
|
-
GreenDevAgentAntigravityCliSocketClient,
|
|
87
|
-
GreenDevAgentGeminiCliSocketClient,
|
|
88
82
|
Injected,
|
|
89
83
|
LighthouseFormatter,
|
|
90
84
|
ListNetworkRequests,
|