chrome-devtools-frontend 1.0.1559913 → 1.0.1561528

package/front_end/panels/ai_assistance/AiAssistancePanel.ts

@@ -10,7 +10,6 @@ import * as i18n from '../../core/i18n/i18n.js';
 import * as Platform from '../../core/platform/platform.js';
 import * as Root from '../../core/root/root.js';
 import * as SDK from '../../core/sdk/sdk.js';
-import * as Protocol from '../../generated/protocol.js';
 import * as AiAssistanceModel from '../../models/ai_assistance/ai_assistance.js';
 import * as Annotations from '../../models/annotations/annotations.js';
 import * as Badges from '../../models/badges/badges.js';
@@ -30,28 +29,30 @@ import * as TimelinePanel from '../timeline/timeline.js';
 import aiAssistancePanelStyles from './aiAssistancePanel.css.js';
 import {ArtifactsViewer} from './components/ArtifactsViewer.js';
 import {
-  type AnswerPart,
-  type ChatMessage,
-  ChatMessageEntity,
   ChatView,
-  type ImageInputData,
-  type ModelChatMessage,
   type Props as ChatViewProps,
-  type Step
 } from './components/ChatView.js';
 import {DisabledWidget} from './components/DisabledWidget.js';
 import {ExploreWidget} from './components/ExploreWidget.js';
 import {MarkdownRendererWithCodeBlock} from './components/MarkdownRendererWithCodeBlock.js';
 import {PerformanceAgentMarkdownRenderer} from './components/PerformanceAgentMarkdownRenderer.js';
+import {
+  type AnswerPart,
+  type ChatMessage,
+  ChatMessageEntity,
+  type ModelChatMessage,
+  type Step,
+} from './components/UserActionRow.js';
 import {isAiAssistancePatchingEnabled} from './PatchWidget.js';
 
+// FIXME: this export is temporary to avoid rewriting tests.
+export {ChatMessageEntity} from './components/UserActionRow.js';
+export type {AnswerPart, ModelChatMessage, StepPart} from './components/UserActionRow.js';
+
 const {html} = Lit;
 
 const AI_ASSISTANCE_SEND_FEEDBACK = 'https://crbug.com/364805393' as Platform.DevToolsPath.UrlString;
 const AI_ASSISTANCE_HELP = 'https://developer.chrome.com/docs/devtools/ai-assistance';
-const SCREENSHOT_QUALITY = 100;
-const SHOW_LOADING_STATE_TIMEOUT = 100;
-const JPEG_MIME_TYPE = 'image/jpeg';
 
 const UIStrings = {
   /**
@@ -204,14 +205,6 @@ const UIStringsNotTranslate = {
    */
   inputDisclaimerForPerformanceEnterpriseNoLogging:
       'Chat messages and data from your performance trace are sent to Google. The content you submit and that is generated by this feature will not be used to improve Google’s AI models. This is an experimental AI feature and won’t always get it right.',
-  /**
-   * @description Message displayed in toast in case of any failures while taking a screenshot of the page.
-   */
-  screenshotFailureMessage: 'Failed to take a screenshot. Please try again.',
-  /**
-   * @description Message displayed in toast in case of any failures while uploading an image file as input.
-   */
-  uploadImageFailureMessage: 'Failed to upload image. Please try again.',
 } as const;
 
 const str_ = i18n.i18n.registerUIStrings('panels/ai_assistance/AiAssistancePanel.ts', UIStrings);
@@ -536,9 +529,6 @@ export class AiAssistancePanel extends UI.Panel.Panel {
     accountImage?: string,
     accountFullName?: string,
   };
-  #imageInput?: ImageInputData;
-  // Used to disable send button when there is not text input.
-  #isTextInputEmpty = true;
   #timelinePanelInstance: TimelinePanel.TimelinePanel.TimelinePanel|null = null;
   #runAbortController = new AbortController();
   #additionalContextItemsFromFloaty: UI.Floaty.FloatyContextSelection[] = [];
@@ -603,12 +593,10 @@ export class AiAssistancePanel extends UI.Panel.Panel {
           canShowFeedbackForm: this.#serverSideLoggingEnabled,
           multimodalInputEnabled: isAiAssistanceMultimodalInputEnabled() &&
               this.#conversation.type === AiAssistanceModel.AiHistoryStorage.ConversationType.STYLING,
-          imageInput: this.#imageInput,
           isTextInputDisabled: this.#isTextInputDisabled(),
           emptyStateSuggestions,
           inputPlaceholder: this.#getChatInputPlaceholder(),
           disclaimerText: this.#getDisclaimerText(),
-          isTextInputEmpty: this.#isTextInputEmpty,
           changeManager: this.#changeManager,
           uploadImageInputEnabled: isAiAssistanceMultimodalUploadInputEnabled() &&
               this.#conversation.type === AiAssistanceModel.AiHistoryStorage.ConversationType.STYLING,
@@ -617,8 +605,6 @@ export class AiAssistancePanel extends UI.Panel.Panel {
           onTextSubmit: async (
               text: string, imageInput?: Host.AidaClient.Part,
               multimodalInputType?: AiAssistanceModel.AiAgent.MultimodalInputType) => {
-            this.#imageInput = undefined;
-            this.#isTextInputEmpty = true;
             Host.userMetrics.actionTaken(Host.UserMetrics.Action.AiAssistanceQuerySubmitted);
             await this.#startConversation(text, imageInput, multimodalInputType);
           },
@@ -627,12 +613,7 @@ export class AiAssistancePanel extends UI.Panel.Panel {
           onCancelClick: this.#cancel.bind(this),
           onContextClick: this.#handleContextClick.bind(this),
           onNewConversation: this.#handleNewChatRequest.bind(this),
-          onTakeScreenshot: isAiAssistanceMultimodalInputEnabled() ? this.#handleTakeScreenshot.bind(this) : undefined,
-          onRemoveImageInput: isAiAssistanceMultimodalInputEnabled() ? this.#handleRemoveImageInput.bind(this) :
-                                                                       undefined,
           onCopyResponseClick: this.#onCopyResponseClick.bind(this),
-          onTextInputChange: this.#handleTextInputChange.bind(this),
-          onLoadImage: isAiAssistanceMultimodalUploadInputEnabled() ? this.#handleLoadImage.bind(this) : undefined,
         }
       };
     }
@@ -806,10 +787,6 @@ export class AiAssistancePanel extends UI.Panel.Panel {
     SDK.TargetManager.TargetManager.instance().addModelListener(
         SDK.DOMModel.DOMModel, SDK.DOMModel.Events.AttrRemoved, this.#handleDOMNodeAttrChange, this);
 
-    SDK.TargetManager.TargetManager.instance().addModelListener(
-        SDK.ResourceTreeModel.ResourceTreeModel, SDK.ResourceTreeModel.Events.PrimaryPageChanged,
-        this.#onPrimaryPageChanged, this);
-
     // Listen to changes in the Timeline Panel state. We also call the
     // function immediately in case the Performance panel is already shown
     // when AI Assistance is loaded.
@@ -863,9 +840,6 @@ export class AiAssistancePanel extends UI.Panel.Panel {
         this.#handleDOMNodeAttrChange,
         this,
     );
-    SDK.TargetManager.TargetManager.instance().removeModelListener(
-        SDK.ResourceTreeModel.ResourceTreeModel, SDK.ResourceTreeModel.Events.PrimaryPageChanged,
-        this.#onPrimaryPageChanged, this);
 
     if (this.#timelinePanelInstance) {
       this.#timelinePanelInstance.removeEventListener(
@@ -944,15 +918,6 @@ export class AiAssistancePanel extends UI.Panel.Panel {
         this.#updateConversationState(this.#conversation);
       };
 
-  #onPrimaryPageChanged(): void {
-    if (!this.#imageInput) {
-      return;
-    }
-
-    this.#imageInput = undefined;
-    this.requestUpdate();
-  }
-
   #getChangeSummary(): string|undefined {
     if (!isAiAssistancePatchingEnabled() || !this.#conversation || this.#conversation?.isReadOnly) {
       return;
@@ -1224,8 +1189,6 @@ export class AiAssistancePanel extends UI.Panel.Panel {
         return;
       }
 
-      this.#imageInput = undefined;
-      this.#isTextInputEmpty = true;
       Host.userMetrics.actionTaken(Host.UserMetrics.Action.AiAssistanceQuerySubmitted);
       if (this.#conversation && this.#conversation.isBlockedByOrigin) {
         this.#handleNewChatRequest();
@@ -1322,100 +1285,6 @@ export class AiAssistancePanel extends UI.Panel.Panel {
     }
   }
 
-  async #handleTakeScreenshot(): Promise<void> {
-    const mainTarget = SDK.TargetManager.TargetManager.instance().primaryPageTarget();
-    if (!mainTarget) {
-      throw new Error('Could not find main target');
-    }
-    const model = mainTarget.model(SDK.ScreenCaptureModel.ScreenCaptureModel);
-    if (!model) {
-      throw new Error('Could not find model');
-    }
-    const showLoadingTimeout = setTimeout(() => {
-      this.#imageInput = {isLoading: true};
-      this.requestUpdate();
-    }, SHOW_LOADING_STATE_TIMEOUT);
-    const bytes = await model.captureScreenshot(
-        Protocol.Page.CaptureScreenshotRequestFormat.Jpeg,
-        SCREENSHOT_QUALITY,
-        SDK.ScreenCaptureModel.ScreenshotMode.FROM_VIEWPORT,
-    );
-    clearTimeout(showLoadingTimeout);
-    if (bytes) {
-      this.#imageInput = {
-        isLoading: false,
-        data: bytes,
-        mimeType: JPEG_MIME_TYPE,
-        inputType: AiAssistanceModel.AiAgent.MultimodalInputType.SCREENSHOT
-      };
-      this.requestUpdate();
-      void this.updateComplete.then(() => {
-        this.#viewOutput.chatView?.focusTextInput();
-      });
-    } else {
-      this.#imageInput = undefined;
-      this.requestUpdate();
-      Snackbars.Snackbar.Snackbar.show({
-        message: lockedString(UIStringsNotTranslate.screenshotFailureMessage),
-      });
-    }
-  }
-
-  #handleRemoveImageInput(): void {
-    this.#imageInput = undefined;
-    this.requestUpdate();
-    void this.updateComplete.then(() => {
-      this.#viewOutput.chatView?.focusTextInput();
-    });
-  }
-
-  #handleTextInputChange(value: string): void {
-    const disableSubmit = !value;
-    if (disableSubmit !== this.#isTextInputEmpty) {
-      this.#isTextInputEmpty = disableSubmit;
-      void this.requestUpdate();
-    }
-  }
-
-  async #handleLoadImage(file: File): Promise<void> {
-    const showLoadingTimeout = setTimeout(() => {
-      this.#imageInput = {isLoading: true};
-      this.requestUpdate();
-    }, SHOW_LOADING_STATE_TIMEOUT);
-    try {
-      const reader = new FileReader();
-      const dataUrl = await new Promise<string>((resolve, reject) => {
-        reader.onload = () => {
-          if (typeof reader.result === 'string') {
-            resolve(reader.result);
-          } else {
-            reject(new Error('FileReader result was not a string.'));
-          }
-        };
-        reader.readAsDataURL(file);
-      });
-      const commaIndex = dataUrl.indexOf(',');
-      const bytes = dataUrl.substring(commaIndex + 1);
-      this.#imageInput = {
-        isLoading: false,
-        data: bytes,
-        mimeType: file.type,
-        inputType: AiAssistanceModel.AiAgent.MultimodalInputType.UPLOADED_IMAGE
-      };
-    } catch {
-      this.#imageInput = undefined;
-      Snackbars.Snackbar.Snackbar.show({
-        message: lockedString(UIStringsNotTranslate.uploadImageFailureMessage),
-      });
-    }
-
-    clearTimeout(showLoadingTimeout);
-    this.requestUpdate();
-    void this.updateComplete.then(() => {
-      this.#viewOutput.chatView?.focusTextInput();
-    });
-  }
-
   #cancel(): void {
     this.#runAbortController.abort();
     this.#runAbortController = new AbortController();

package/front_end/panels/ai_assistance/PatchWidget.ts

@@ -12,6 +12,7 @@ import * as i18n from '../../core/i18n/i18n.js';
 import * as Platform from '../../core/platform/platform.js';
 import * as Root from '../../core/root/root.js';
 import * as AiAssistanceModel from '../../models/ai_assistance/ai_assistance.js';
+import * as GreenDev from '../../models/greendev/greendev.js';
 import * as Persistence from '../../models/persistence/persistence.js';
 import * as Workspace from '../../models/workspace/workspace.js';
 import * as WorkspaceDiff from '../../models/workspace_diff/workspace_diff.js';
@@ -186,41 +187,42 @@ export interface ViewOutput {
 }
 
 type View = (input: ViewInput, output: ViewOutput, target: HTMLElement) => void;
-const DEFAULT_VIEW: View =
-    (input, output, target) => {
-      if (!input.changeSummary && input.patchSuggestionState === PatchSuggestionState.INITIAL) {
-        return;
-      }
+const DEFAULT_VIEW: View = (input, output, target) => {
+  if (!input.changeSummary && input.patchSuggestionState === PatchSuggestionState.INITIAL) {
+    return;
+  }
 
-      output.changeRef = output.changeRef ?? Directives.createRef<HTMLElement>();
-      output.summaryRef = output.summaryRef ?? Directives.createRef<HTMLElement>();
+  output.changeRef = output.changeRef ?? Directives.createRef<HTMLElement>();
+  output.summaryRef = output.summaryRef ?? Directives.createRef<HTMLElement>();
 
-      function renderSourcesLink(): LitTemplate {
-        if (!input.sources) {
-          return nothing;
-        }
+  function renderSourcesLink(): LitTemplate {
+    if (!input.sources) {
+      return nothing;
+    }
 
-        return html`<x-link
+    return html`<x-link
           class="link"
           title="${UIStringsNotTranslate.viewUploadedFiles} ${UIStringsNotTranslate.opensInNewTab}"
           href="data:text/plain;charset=utf-8,${encodeURIComponent(input.sources)}"
-          jslog=${VisualLogging.link('files-used-in-patching').track({click: true})}>
+          jslog=${VisualLogging.link('files-used-in-patching').track({
+      click: true
+    })}>
           ${UIStringsNotTranslate.viewUploadedFiles}
         </x-link>`;
-      }
+  }
 
-      function renderHeader(): LitTemplate {
-        if (input.savedToDisk) {
-          return html`
+  function renderHeader(): LitTemplate {
+    if (input.savedToDisk) {
+      return html`
             <devtools-icon class="green-bright-icon summary-badge" name="check-circle"></devtools-icon>
             <span class="header-text">
               ${lockedString(UIStringsNotTranslate.savedToDisk)}
             </span>
           `;
-        }
+    }
 
-        if (input.patchSuggestionState === PatchSuggestionState.SUCCESS) {
-          return html`
+    if (input.patchSuggestionState === PatchSuggestionState.SUCCESS) {
+      return html`
             <devtools-icon class="on-tonal-icon summary-badge" name="difference"></devtools-icon>
             <span class="header-text">
               ${lockedString(`File changes in ${input.projectName}`)}
@@ -230,9 +232,9 @@ const DEFAULT_VIEW: View =
               name="chevron-down"
             ></devtools-icon>
           `;
-        }
+    }
 
-        return html`
+    return html`
           <devtools-icon class="on-tonal-icon summary-badge" name="pen-spark"></devtools-icon>
           <span class="header-text">
             ${lockedString(UIStringsNotTranslate.unsavedChanges)}
@@ -242,49 +244,63 @@ const DEFAULT_VIEW: View =
             name="chevron-down"
           ></devtools-icon>
         `;
-      }
+  }
 
-      function renderContent(): LitTemplate {
-        if ((!input.changeSummary && input.patchSuggestionState === PatchSuggestionState.INITIAL) || input.savedToDisk) {
-          return nothing;
-        }
+  function renderContent(): LitTemplate {
+    if ((!input.changeSummary && input.patchSuggestionState === PatchSuggestionState.INITIAL) || input.savedToDisk) {
+      return nothing;
+    }
 
-        if (input.patchSuggestionState === PatchSuggestionState.SUCCESS) {
-          return html`<devtools-widget .widgetConfig=${UI.Widget.widgetConfig(ChangesPanel.CombinedDiffView.CombinedDiffView, {
+    if (input.patchSuggestionState === PatchSuggestionState.SUCCESS) {
+      return html`<devtools-widget .widgetConfig=${
+          UI.Widget.widgetConfig(ChangesPanel.CombinedDiffView.CombinedDiffView, {
             workspaceDiff: input.workspaceDiff,
             // Ignore user creates inspector-stylesheets
             ignoredUrls: ['inspector://']
           })}></devtools-widget>`;
-        }
+    }
 
-        return html`<devtools-code-block
+    return html`<devtools-code-block
           .code=${input.changeSummary ?? ''}
           .codeLang=${'css'}
           .displayNotice=${true}
         ></devtools-code-block>
-        ${input.patchSuggestionState === PatchSuggestionState.ERROR
-          ? html`<div class="error-container">
+        ${
+        input.patchSuggestionState === PatchSuggestionState.ERROR ?
+            html`<div class="error-container">
               <devtools-icon name="cross-circle-filled"></devtools-icon>${
-              lockedString(UIStringsNotTranslate.genericErrorMessage)
-              } ${renderSourcesLink()}
-            </div>`
-          : nothing
-        }`;
-      }
+                lockedString(UIStringsNotTranslate.genericErrorMessage)} ${renderSourcesLink()}
+            </div>` :
+            nothing}`;
+  }
 
-      function renderFooter(): LitTemplate {
-        if (input.savedToDisk) {
-          return nothing;
-        }
+  function renderCopyPrompt(changedCode?: string): LitTemplate {
+    if (!GreenDev.Prototypes.instance().isEnabled('copyToGemini') || !changedCode) {
+      return nothing;
+    }
+
+    // clang-format off
+    return html`<devtools-widget class="copy-to-prompt"
+      .widgetConfig=${UI.Widget.widgetConfig(PanelCommon.CopyChangesToPrompt, {
+        workspaceDiff: input.workspaceDiff,
+        patchAgentCSSChange: changedCode,
+      })}></devtools-widget>`;
+    // clang-format on
+  }
 
-        if (input.patchSuggestionState === PatchSuggestionState.SUCCESS) {
-          return html`
+  function renderFooter(): LitTemplate {
+    if (input.savedToDisk) {
+      return nothing;
+    }
+
+    if (input.patchSuggestionState === PatchSuggestionState.SUCCESS) {
+      return html`
           <div class="footer">
             <div class="left-side">
               <x-link class="link disclaimer-link" href="https://support.google.com/legal/answer/13505487" jslog=${
-                VisualLogging.link('code-disclaimer').track({
-                  click: true,
-                })}>
+          VisualLogging.link('code-disclaimer').track({
+            click: true,
+          })}>
                 ${lockedString(UIStringsNotTranslate.codeDisclaimer)}
               </x-link>
               ${renderSourcesLink()}
@@ -305,18 +321,20 @@ const DEFAULT_VIEW: View =
             </div>
           </div>
           `;
-        }
+    }
 
-        const iconName = input.projectType === SelectedProjectType.AUTOMATIC_DISCONNECTED ? 'folder-off' : input.projectType === SelectedProjectType.AUTOMATIC_CONNECTED ? 'folder-asterisk' : 'folder';
-        return html`
+    const iconName = input.projectType === SelectedProjectType.AUTOMATIC_DISCONNECTED ? 'folder-off' :
+        input.projectType === SelectedProjectType.AUTOMATIC_CONNECTED                 ? 'folder-asterisk' :
+                                                                                        'folder';
+    return html`
         <div class="footer">
           ${
-            input.projectName ? html`
+        input.projectName ? html`
             <div class="change-workspace" jslog=${VisualLogging.section('patch-widget.workspace')}>
                 <devtools-icon .name=${iconName}></devtools-icon>
                 <span class="folder-name" title=${input.projectPath}>${input.projectName}</span>
               ${
-                                    input.onChangeWorkspaceClick ? html`
+                                input.onChangeWorkspaceClick ? html`
                 <devtools-button
                   @click=${input.onChangeWorkspaceClick}
                   .jslogContext=${'change-workspace'}
@@ -326,23 +344,24 @@ const DEFAULT_VIEW: View =
                   ${Directives.ref(output.changeRef)}
                 >${lockedString(UIStringsNotTranslate.change)}</devtools-button>
               ` :
-                                                                   nothing}
+                                                               nothing}
             </div>
           ` :
-                                nothing}
+                            nothing}
           <div class="apply-to-workspace-container" aria-live="polite">
             ${
-            input.patchSuggestionState === PatchSuggestionState.LOADING ?
-                html`
+        input.patchSuggestionState === PatchSuggestionState.LOADING ?
+            html`
               <div class="loading-text-container" jslog=${
-                    VisualLogging.section('patch-widget.apply-to-workspace-loading')}>
+                VisualLogging.section('patch-widget.apply-to-workspace-loading')}>
                 <devtools-spinner></devtools-spinner>
                 <span>
                   ${lockedString(UIStringsNotTranslate.applyingToWorkspace)}
                 </span>
               </div>
             ` :
-                html`
+            html`
+               ${renderCopyPrompt(input.changeSummary)}
                 <devtools-button
                 @click=${input.onApplyToWorkspace}
                 .jslogContext=${'patch-widget.apply-to-workspace'}
@@ -351,13 +370,13 @@ const DEFAULT_VIEW: View =
               </devtools-button>
             `}
             ${
-            input.patchSuggestionState === PatchSuggestionState.LOADING ? html`<devtools-button
+        input.patchSuggestionState === PatchSuggestionState.LOADING ? html`<devtools-button
               @click=${input.onCancel}
               .jslogContext=${'cancel'}
               .variant=${Buttons.Button.Variant.OUTLINED}>
               ${lockedString(UIStringsNotTranslate.cancel)}
             </devtools-button>` :
-                                                                          nothing}
+                                                                      nothing}
             <devtools-button
               aria-details="info-tooltip"
               .jslogContext=${'patch-widget.info-tooltip-trigger'}
@@ -374,26 +393,25 @@ const DEFAULT_VIEW: View =
                  class="link tooltip-link"
                  role="link"
                  jslog=${VisualLogging.link('open-ai-settings').track({
-          click: true,
-        })}
+      click: true,
+    })}
                  @click=${input.onLearnMoreTooltipClick}
                >${lockedString(UIStringsNotTranslate.learnMore)}</button>
              </div>
             </devtools-tooltip>
           </div>
         </div>`;
-      }
+  }
 
-      // Use a simple div for the "Saved to disk" state as it's not expandable,
-      // otherwise use the interactive <details> element.
-      const template = input.savedToDisk
-        ? html`
+  // Use a simple div for the "Saved to disk" state as it's not expandable,
+  // otherwise use the interactive <details> element.
+  const template = input.savedToDisk ? html`
           <div class="change-summary saved-to-disk" role="status" aria-live="polite">
             <div class="header-container">
              ${renderHeader()}
              </div>
-          </div>`
-        : html`
+          </div>` :
+                                       html`
           <details class="change-summary" jslog=${VisualLogging.section('patch-widget')}>
             <summary class="header-container" ${Directives.ref(output.summaryRef)}>
               ${renderHeader()}
@@ -403,8 +421,8 @@ const DEFAULT_VIEW: View =
           </details>
         `;
 
-      render(template, target);
-    };
+  render(template, target);
+};
 
 export class PatchWidget extends UI.Widget.Widget {
   changeSummary = '';

package/front_end/panels/ai_assistance/ai_assistance.ts

@@ -4,6 +4,7 @@
 
 export * from './AiAssistancePanel.js';
 export * from './components/ChatView.js';
+export * as ChatInput from './components/ChatInput.js';
 export * from './components/MarkdownRendererWithCodeBlock.js';
 export * from './components/PerformanceAgentFlameChart.js';
 export * from './SelectWorkspaceDialog.js';