android-midscene-automation 0.2.4 → 1.1.1

package/artemis-main/apps/showcase_ui/src/app/components/floating-video-player/floating-video-player.component.html

@@ -0,0 +1,433 @@
+<!--
+Copyright 2026 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+<!-- Backdrop overlay only when in theater / modal mode -->
+@if (agentService.isVideoWindowOpen() && isTheaterMode()) {
+  <div
+    class="theater-backdrop"
+    (click)="isTheaterMode.set(false)"
+  ></div>
+}
+
+@if (agentService.isVideoWindowOpen()) {
+  <div
+    class="floating-video-wrapper"
+    [class.minimized]="agentService.isVideoMinimized()"
+    [class.theater]="isTheaterMode()"
+    [style.left.px]="!isTheaterMode() ? posX() : null"
+    [style.top.px]="!isTheaterMode() ? posY() : null"
+    >
+    <!-- Window Header (Draggable Handle) -->
+    <header class="window-header" (mousedown)="startDrag($event)">
+      <div class="header-left">
+        <span class="material-symbols-outlined header-icon">
+          {{ agentService.recordingPlaybackStatus() === 'live' ? 'sensors' : (agentService.playerMode() === 'steps' ? 'slideshow' : (agentService.recordingPlaybackStatus() === 'processing' ? 'progress_activity' : 'smart_display')) }}
+        </span>
+        <span class="header-title" [title]="agentService.activeVideoTitle()">
+          {{ agentService.recordingPlaybackStatus() === 'live' ? ('Live Device Screen' | t) : (agentService.playerMode() === 'steps' ? (('Step Replay (' | t) + (totalStepFrames() > 0 ? (activeStepIndex() + 1) + '/' + totalStepFrames() : '0') + ')') : (agentService.recordingPlaybackStatus() === 'processing' ? ('Preparing Recording' | t) : (agentService.activeVideoTitle() || ('Screen Recording' | t)))) }}
+        </span>
+      </div>
+      <div class="header-actions" (mousedown)="$event.stopPropagation()">
+        <!-- Mode Toggle Group: Video vs. Step Replay -->
+        @if (agentService.recordingPlaybackStatus() === 'ready' && agentService.hasCurrentSessionStepFrames() && !agentService.isVideoMinimized()) {
+          <div class="mode-toggle-group">
+            <button
+              class="mode-btn"
+              [class.active]="agentService.playerMode() === 'video'"
+              (click)="agentService.setPlayerMode('video')"
+              [title]="'Video Recording' | t"
+              >
+              <span class="material-symbols-outlined mode-icon">videocam</span>
+              <span>{{ 'Video' | t }}</span>
+            </button>
+            <button
+              class="mode-btn"
+              [class.active]="agentService.playerMode() === 'steps'"
+              (click)="agentService.setPlayerMode('steps')"
+              [title]="'Step Screenshot Replay' | t"
+              >
+              <span class="material-symbols-outlined mode-icon">slideshow</span>
+              <span>{{ 'Steps' | t }}</span>
+            </button>
+          </div>
+        }
+        <!-- Minimize / Restore Toggle -->
+        <button
+          class="header-btn"
+          (click)="toggleMinimize()"
+          [title]="agentService.isVideoMinimized() ? ('Restore' | t) : ('Minimize' | t)"
+          [attr.aria-label]="'Minimize' | t"
+          >
+          <span class="material-symbols-outlined">{{ agentService.isVideoMinimized() ? 'expand_less' : 'remove' }}</span>
+        </button>
+        <!-- Theater / Fullscreen Toggle -->
+        @if (!agentService.isVideoMinimized()) {
+          <button
+            class="header-btn"
+            (click)="toggleTheater()"
+            [title]="isTheaterMode() ? ('Exit Theater' | t) : ('Theater Mode' | t)"
+            [attr.aria-label]="'Theater' | t"
+            >
+            <span class="material-symbols-outlined">{{ isTheaterMode() ? 'close_fullscreen' : 'open_in_full' }}</span>
+          </button>
+        }
+        <!-- Close Window -->
+        <button
+          class="header-btn close-btn"
+          (click)="agentService.closeVideoPlayer()"
+          [title]="'Close' | t"
+          [attr.aria-label]="'Close' | t"
+          >
+          <span class="material-symbols-outlined">close</span>
+        </button>
+      </div>
+    </header>
+    <!-- Video Main Content Area -->
+    @if (!agentService.isVideoMinimized()) {
+      <div class="video-body">
+        <!-- 1. LIVE SCREEN STREAM (AUTOMATICALLY ACTIVE DURING RUNNING TEST) -->
+        @if (agentService.recordingPlaybackStatus() === 'live') {
+          @if (!liveStreamError()) {
+            <div class="live-stream-container">
+              <img
+                class="live-screen-img"
+                [src]="liveStreamUrl()"
+                (error)="onLiveStreamError()"
+                [alt]="'Live Device Screen' | t"
+                />
+                <div class="live-status-overlay">
+                  <div class="live-badge">
+                    <span class="pulse-dot"></span>
+                    <span>{{ 'LIVE' | t }}</span>
+                  </div>
+                  <button class="live-refresh-btn" (click)="retryLiveStream()" [title]="'Refresh Live Stream' | t">
+                    <span class="material-symbols-outlined">refresh</span>
+                  </button>
+                </div>
+              </div>
+            }
+            @if (liveStreamError()) {
+              <div class="empty-video-fallback">
+                <div class="fallback-icon-wrap">
+                  <span class="material-symbols-outlined fallback-icon">phonelink_off</span>
+                </div>
+                <h4 class="fallback-title">{{ 'Device Stream Connecting' | t }}</h4>
+                <p class="fallback-desc">{{ 'Waiting for ADB device screen stream...' | t }}</p>
+                <button class="retry-btn" (click)="retryLiveStream()">
+                  <span class="material-symbols-outlined">refresh</span> {{ 'Reconnect' | t }}
+                </button>
+              </div>
+            }
+          }
+          <!-- 2. RECORDING PLAYBACK / STEP REPLAY (AUTOMATICALLY ACTIVE WHEN TEST FINISHED) -->
+          @if (agentService.recordingPlaybackStatus() !== 'live') {
+            <!-- Loading State (when in video mode and processing) -->
+            @if (agentService.playerMode() === 'video' && (agentService.recordingPlaybackStatus() === 'processing' || agentService.isVideoLoading())) {
+              <div class="video-loading-state">
+                <span class="material-symbols-outlined spinning-icon">progress_activity</span>
+                <span>{{ agentService.recordingPlaybackMessage() || ('Preparing screen recording...' | t) }}</span>
+              </div>
+            }
+            <!-- Option A: STEP REPLAY VIEW (ACTIVE IN STEP MODE) -->
+            @if (agentService.playerMode() === 'steps' && totalStepFrames() > 0) {
+              <div class="step-replay-container">
+                <!-- Fallback info banner if recording had failed -->
+                @if (agentService.recordingPlaybackStatus() === 'failed' || videoLoadError()) {
+                  <div class="step-replay-notice">
+                    <span class="material-symbols-outlined notice-icon">info</span>
+                    <span>{{ 'Screen recording unavailable. Replaying step-by-step screenshots.' | t }}</span>
+                  </div>
+                }
+                <!-- Step Image Viewport with Action Coordinates Overlay -->
+                <div
+                  class="step-image-viewport"
+                  (mouseenter)="onViewportMouseEnter()"
+                  (mouseleave)="onViewportMouseLeave()"
+                  >
+                  <!-- Loading Skeleton (Clean Light Spinner) -->
+                  @if (isStepImageLoading()) {
+                    <div class="step-image-loading-state">
+                      <span class="material-symbols-outlined spinning-icon">progress_activity</span>
+                      <span>{{ 'Loading screenshot...' | t }}</span>
+                    </div>
+                  }
+                  <!-- Error / Missing Image Fallback -->
+                  @if (stepImageError() && !isStepImageLoading()) {
+                    <div class="step-image-error-state">
+                      <span class="material-symbols-outlined broken-icon">broken_image</span>
+                      <span>{{ 'Screenshot preview unavailable for this step' | t }}</span>
+                    </div>
+                  }
+                  <!-- Device Screen Frame & Action Coordinates Overlay -->
+                  @if (!stepImageError()) {
+                    <div class="step-image-frame-wrap">
+                      <img
+                        #stepImgRef
+                        class="step-frame-img"
+                        [src]="currentStepImageUrl()"
+                        [alt]="$safeNavigationMigration(currentStepFrame()?.title)"
+                        (load)="onStepImageLoad()"
+                        (error)="onStepImageError()"
+                        (click)="openImageInNewTab()"
+                        [title]="'Click to view image in new tab' | t"
+                        />
+                        <!-- Action Visual Coordinates Overlay (Touch point, swipe arrow, sequence path) -->
+                        <div
+                          #stepOverlayRef
+                          class="overlay-layer"
+                          [style.display]="viewMode() === 'post' ? 'none' : 'block'"
+                        ></div>
+                      </div>
+                    }
+                    <!-- Hideable Floating Detail Card (Appears on hover or when pinned) -->
+                    @if (currentStepFrame(); as frame) {
+                      <div
+                        class="step-frame-badge"
+                        [class.visible]="isCardVisible()"
+                        (click)="$event.stopPropagation()"
+                        >
+                        <div class="badge-content-left">
+                          <span class="badge-step-no">{{ 'Step' | t }} {{ frame.stepNumber }}</span>
+                          <span class="badge-action-desc" [title]="frame.actionText || frame.title">
+                            @if (frame.action) {
+                              <span class="material-symbols-outlined action-type-icon">
+                                {{ getStepActionIcon(frame.action) }}
+                              </span>
+                            }
+                            {{ frame.actionText || frame.title }}
+                          </span>
+                        </div>
+                        <div class="badge-actions-right">
+                          <!-- Pre/Post Screenshot Switcher (Only if post-screenshot is available) -->
+                          @if (frame.postImageUrl) {
+                            <div class="pre-post-toggle">
+                              <button
+                                class="toggle-btn"
+                                [class.active]="viewMode() === 'pre'"
+                                (click)="setStepViewMode('pre', $event)"
+                                [title]="'View screen before action (with touch coordinates)' | t"
+                                >
+                                {{ 'Pre' | t }}
+                              </button>
+                              <button
+                                class="toggle-btn"
+                                [class.active]="viewMode() === 'post'"
+                                (click)="setStepViewMode('post', $event)"
+                                [title]="'View resulting screen after action executed' | t"
+                                >
+                                {{ 'Post' | t }}
+                              </button>
+                            </div>
+                          }
+                          <!-- Pin Card Button -->
+                          <button
+                            class="pin-card-btn"
+                            [class.pinned]="isCardPinned()"
+                            (click)="toggleCardPin($event)"
+                            [title]="isCardPinned() ? ('Unpin details card (auto-hide on mouse leave)' | t) : ('Pin details card (keep always visible)' | t)"
+                            >
+                            <span class="material-symbols-outlined">{{ isCardPinned() ? 'push_pin' : 'keep' }}</span>
+                          </button>
+                        </div>
+                      </div>
+                    }
+                  </div>
+                  <!-- Step Replay Controls Footer -->
+                  <div class="video-controls-footer">
+                    <!-- Timeline Scrubber Bar with Step Ticks -->
+                    <div class="scrubber-bar-container">
+                      <input
+                        type="range"
+                        class="scrubber-range"
+                        [min]="0"
+                        [max]="totalStepFrames() - 1"
+                        [step]="1"
+                        [value]="activeStepIndex()"
+                        (input)="onStepScrub($event)"
+                        />
+                        <div class="scrubber-progress-fill" [style.width.%]="stepProgressPercent()"></div>
+                        <!-- Step Tick Markers -->
+                        @if (totalStepFrames() > 1) {
+                          <div class="step-ticks-wrapper">
+                            @for (f of stepFrames(); track trackStepFrame(i, f); let i = $index) {
+                              <span
+                                class="step-tick"
+                                [class.active]="i === activeStepIndex()"
+                                [class.passed]="i < activeStepIndex()"
+                                [style.left.%]="(i / (totalStepFrames() - 1)) * 100"
+                                (click)="seekStepFrame(i); $event.stopPropagation()"
+                                [title]="('Step ' | t) + (i + 1) + (': ' | t) + (f.actionText || f.title)"
+                              ></span>
+                            }
+                          </div>
+                        }
+                      </div>
+                      <!-- Bottom Controls Row -->
+                      <div class="controls-row">
+                        <div class="controls-left">
+                          <!-- Play / Pause Button -->
+                          <button class="ctrl-btn play-btn" (click)="toggleStepPlay()" [title]="isStepPlaying() ? ('Pause' | t) : ('Play Slideshow' | t)">
+                            <span class="material-symbols-outlined">{{ isStepPlaying() ? 'pause' : 'play_arrow' }}</span>
+                          </button>
+                          <!-- Previous Frame Button -->
+                          <button class="ctrl-btn" (click)="prevStepFrame()" [disabled]="activeStepIndex() <= 0" [title]="'Previous Step' | t">
+                            <span class="material-symbols-outlined">skip_previous</span>
+                          </button>
+                          <!-- Next Frame Button -->
+                          <button class="ctrl-btn" (click)="nextStepFrame()" [disabled]="activeStepIndex() >= totalStepFrames() - 1" [title]="'Next Step' | t">
+                            <span class="material-symbols-outlined">skip_next</span>
+                          </button>
+                          <!-- Step Replay / Reset -->
+                          <button class="ctrl-btn" (click)="restartStep()" [title]="'Replay from start' | t">
+                            <span class="material-symbols-outlined">replay</span>
+                          </button>
+                          <!-- Step Counter Display -->
+                          <span class="time-display">
+                            {{ 'Step' | t }} {{ activeStepIndex() + 1 }} / {{ totalStepFrames() }}
+                          </span>
+                        </div>
+                        <div class="controls-right">
+                          <!-- Speed Button (Cycle 0.5x, 1x, 1.5x, 2x, 4x) -->
+                          <button
+                            class="ctrl-btn speed-pill-btn"
+                            (click)="cycleSpeed()"
+                            [title]="('Speed: ' | t) + playbackRate() + ('x (Click to cycle)' | t)"
+                            >
+                            <span class="speed-label">{{ playbackRate() }}{{ 'x' | t }}</span>
+                          </button>
+                          <!-- Loop Toggle -->
+                          <button
+                            class="ctrl-btn"
+                            [class.active]="isLooping()"
+                            (click)="toggleLoop()"
+                            [title]="isLooping() ? ('Loop enabled' | t) : ('Enable loop' | t)"
+                            >
+                            <span class="material-symbols-outlined">repeat</span>
+                          </button>
+                          <!-- Open in new tab -->
+                          <button class="ctrl-btn" (click)="openImageInNewTab()" [title]="'Open image in new tab' | t">
+                            <span class="material-symbols-outlined">open_in_new</span>
+                          </button>
+                        </div>
+                      </div>
+                    </div>
+                  </div>
+                }
+                <!-- Option B: ACTIVE VIDEO PLAYBACK VIEW -->
+                @if (agentService.playerMode() === 'video' && agentService.recordingPlaybackStatus() === 'ready' && !agentService.isVideoLoading() && agentService.activeVideoUrl(); as videoUrl) {
+                  @if (!videoLoadError()) {
+                    <div class="video-container">
+                      <video
+                        #videoRef
+                        [src]="currentVideoUrl()"
+                        playsinline
+                        (loadedmetadata)="onLoadedMetadata()"
+                        (timeupdate)="onTimeUpdate()"
+                        (play)="onPlay()"
+                        (pause)="onPause()"
+                        (ended)="onEnded()"
+                        (error)="onError()"
+                        (click)="togglePlay()"
+                      ></video>
+                    </div>
+                  }
+                  <!-- Dedicated Bottom Controls Bar for Video -->
+                  @if (!videoLoadError()) {
+                    <div class="video-controls-footer">
+                      <!-- Timeline Scrubber Bar -->
+                      <div class="scrubber-bar-container">
+                        <input
+                          type="range"
+                          class="scrubber-range"
+                          [min]="0"
+                          [max]="duration() || 100"
+                          [step]="0.1"
+                          [value]="currentTime()"
+                          (input)="onScrub($event)"
+                          />
+                          <div class="scrubber-progress-fill" [style.width.%]="progressPercent()"></div>
+                        </div>
+                        <!-- Bottom Controls Row -->
+                        <div class="controls-row">
+                          <div class="controls-left">
+                            <!-- Play / Pause Button -->
+                            <button class="ctrl-btn play-btn" (click)="togglePlay()" [title]="isPlaying() ? ('Pause' | t) : ('Play' | t)">
+                              <span class="material-symbols-outlined">{{ isPlaying() ? 'pause' : 'play_arrow' }}</span>
+                            </button>
+                            <!-- Replay / Reset -->
+                            <button class="ctrl-btn" (click)="restart()" [title]="'Replay from start' | t">
+                              <span class="material-symbols-outlined">replay</span>
+                            </button>
+                            <!-- Time Display -->
+                            <span class="time-display">
+                              {{ formatTime(currentTime()) }} / {{ formatTime(duration()) }}
+                            </span>
+                          </div>
+                          <div class="controls-right">
+                            <!-- Speed Button (Cycle 0.5x, 1x, 1.5x, 2x, 4x) -->
+                            <button
+                              class="ctrl-btn speed-pill-btn"
+                              (click)="cycleSpeed()"
+                              [title]="('Speed: ' | t) + playbackRate() + ('x (Click to cycle)' | t)"
+                              >
+                              <span class="speed-label">{{ playbackRate() }}{{ 'x' | t }}</span>
+                            </button>
+                            <!-- Loop Toggle -->
+                            <button
+                              class="ctrl-btn"
+                              [class.active]="isLooping()"
+                              (click)="toggleLoop()"
+                              [title]="isLooping() ? ('Loop enabled' | t) : ('Enable loop' | t)"
+                              >
+                              <span class="material-symbols-outlined">repeat</span>
+                            </button>
+                            <!-- Mute / Unmute Toggle -->
+                            <button class="ctrl-btn" (click)="toggleMute()" [title]="isMuted() ? ('Unmute' | t) : ('Mute' | t)">
+                              <span class="material-symbols-outlined">{{ isMuted() ? 'volume_off' : 'volume_up' }}</span>
+                            </button>
+                            <!-- Open in new tab -->
+                            <button class="ctrl-btn" (click)="openInNewTab()" [title]="'Open video in new tab' | t">
+                              <span class="material-symbols-outlined">open_in_new</span>
+                            </button>
+                          </div>
+                        </div>
+                      </div>
+                    }
+                  }
+                  <!-- Option C: EMPTY / FALLBACK STATE (Only when no video AND no step frames) -->
+                  @if (!agentService.isVideoLoading() && (agentService.playerMode() === 'video' || totalStepFrames() === 0) && (agentService.recordingPlaybackStatus() === 'failed' || agentService.recordingPlaybackStatus() === 'unavailable' || videoLoadError()) && totalStepFrames() === 0) {
+                    <div class="empty-video-fallback">
+                      <div class="fallback-icon-wrap">
+                        <span class="material-symbols-outlined fallback-icon">videocam_off</span>
+                      </div>
+                      <h4 class="fallback-title">
+                        {{ agentService.recordingPlaybackStatus() === 'failed' || videoLoadError() ? ('Recording Playback Failed' | t) : ('No Video Recording Available' | t) }}
+                      </h4>
+                      <p class="fallback-desc">
+                        {{ videoLoadError() ? ('The finalized recording could not be played.' | t) : (agentService.recordingPlaybackMessage() || ('No screen recording was found for this task.' | t)) }}
+                      </p>
+                      @if (agentService.recordingPlaybackStatus() === 'failed' || videoLoadError()) {
+                        <button class="retry-btn" (click)="videoLoadError.set(false); agentService.retryVideoRecording()">
+                          <span class="material-symbols-outlined">refresh</span> {{ 'Retry' | t }}
+                        </button>
+                      }
+                    </div>
+                  }
+                }
+              </div>
+            }
+          </div>
+        }