android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/apps/showcase_ui/package.json

@@ -0,0 +1,39 @@
+{
+  "name": "artemis-showcase-ui",
+  "version": "1.0.0",
+  "scripts": {
+    "ng": "ng",
+    "start": "ng serve --proxy-config proxy.conf.json",
+    "build": "ng build",
+    "watch": "ng build --watch --configuration development",
+    "test": "ng test"
+  },
+  "private": true,
+  "dependencies": {
+    "@angular/cdk": "^22.1.5",
+    "@angular/common": "^22.1.4",
+    "@angular/compiler": "^22.1.4",
+    "@angular/core": "^22.1.4",
+    "@angular/forms": "^22.1.4",
+    "@angular/platform-browser": "^22.1.4",
+    "@angular/platform-browser-dynamic": "^22.1.4",
+    "@angular/router": "^22.1.4",
+    "rxjs": "~7.8.0",
+    "tslib": "^2.3.0",
+    "zone.js": "~0.15.0"
+  },
+  "devDependencies": {
+    "@angular/build": "^22.1.6",
+    "@angular/cli": "^22.1.6",
+    "@angular/compiler-cli": "^22.1.4",
+    "@types/jasmine": "~5.1.0",
+    "istanbul-lib-instrument": "^6.0.3",
+    "jasmine-core": "~5.6.0",
+    "karma": "~6.4.0",
+    "karma-chrome-launcher": "~3.2.0",
+    "karma-coverage": "~2.2.0",
+    "karma-jasmine": "~5.1.0",
+    "karma-jasmine-html-reporter": "~2.1.0",
+    "typescript": "~6.0.3"
+  }
+}

package/artemis-main/apps/showcase_ui/proxy.conf.json

@@ -0,0 +1,23 @@
+{
+  "/api": {
+    "target": "http://localhost:8000",
+    "secure": false,
+    "logLevel": "debug"
+  },
+  "/images": {
+    "target": "http://localhost:8000",
+    "secure": false,
+    "logLevel": "debug"
+  },
+  "/videos": {
+    "target": "http://localhost:8000",
+    "secure": false,
+    "logLevel": "debug"
+  },
+  "/local_file": {
+    "target": "http://localhost:8000",
+    "secure": false,
+    "logLevel": "debug"
+  }
+}
+

package/artemis-main/apps/showcase_ui/public/logo.svg

@@ -0,0 +1,56 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 240 240" width="240" height="240">
+  <defs>
+    <!-- Radial Glows for Mesh effect -->
+    <radialGradient id="redGlow" cx="50%" cy="50%" r="50%">
+      <stop offset="0%" stop-color="#EA4335" stop-opacity="1" />
+      <stop offset="40%" stop-color="#EA4335" stop-opacity="0.95" />
+      <stop offset="100%" stop-color="#EA4335" stop-opacity="0" />
+    </radialGradient>
+    <radialGradient id="yellowGlow" cx="50%" cy="50%" r="50%">
+      <stop offset="0%" stop-color="#FBBC05" stop-opacity="1" />
+      <stop offset="40%" stop-color="#FBBC05" stop-opacity="0.95" />
+      <stop offset="100%" stop-color="#FBBC05" stop-opacity="0" />
+    </radialGradient>
+    <radialGradient id="greenGlow" cx="50%" cy="50%" r="50%">
+      <stop offset="0%" stop-color="#34A853" stop-opacity="1" />
+      <stop offset="40%" stop-color="#34A853" stop-opacity="0.95" />
+      <stop offset="100%" stop-color="#34A853" stop-opacity="0" />
+    </radialGradient>
+
+    <!-- Robot Silhouette Mask -->
+    <mask id="robotMeshMask">
+      <rect width="240" height="240" fill="black" />
+
+      <!-- Antennae -->
+      <path d="M 90 60 Q 75 35 65 28" stroke="white" stroke-width="10" stroke-linecap="round" />
+      <circle cx="62" cy="24" r="10" fill="white" />
+      <path d="M 150 60 Q 165 35 175 28" stroke="white" stroke-width="10" stroke-linecap="round" />
+      <circle cx="178" cy="24" r="10" fill="white" />
+
+      <!-- Ears -->
+      <rect x="28" y="100" width="16" height="48" rx="8" fill="white" />
+      <rect x="196" y="100" width="16" height="48" rx="8" fill="white" />
+
+      <!-- Head -->
+      <rect x="44" y="58" width="152" height="132" rx="40" fill="white" />
+
+      <!-- Visor Cutout -->
+      <rect x="64" y="88" width="112" height="52" rx="22" fill="black" />
+
+      <!-- Eyes (Inside Visor) -->
+      <path d="M 82 118 Q 92 106 102 118" stroke="white" stroke-width="6" stroke-linecap="round" fill="none" />
+      <path d="M 138 118 Q 148 106 158 118" stroke="white" stroke-width="6" stroke-linecap="round" fill="none" />
+
+      <!-- Smile Cutout -->
+      <path d="M 104 158 Q 120 172 136 158" stroke="black" stroke-width="6" stroke-linecap="round" fill="none" />
+    </mask>
+  </defs>
+
+  <!-- Mesh Gradient applied to Mask -->
+  <g mask="url(#robotMeshMask)">
+    <rect width="240" height="240" fill="#4285F4" />
+    <circle cx="160" cy="50" r="100" fill="url(#redGlow)" />
+    <circle cx="60" cy="120" r="95" fill="url(#yellowGlow)" />
+    <circle cx="110" cy="200" r="95" fill="url(#greenGlow)" />
+  </g>
+</svg>

package/artemis-main/apps/showcase_ui/src/app/app.component.html

@@ -0,0 +1,24 @@
+<!--
+ Copyright 2026 Google LLC
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<app-nav-switcher></app-nav-switcher>
+<div class="language-switch" role="group" aria-label="界面语言 / Interface language">
+  <button type="button" lang="zh-CN" (click)="language.setLanguage('zh')"
+    [class.active]="language.language() === 'zh'" [attr.aria-pressed]="language.language() === 'zh'">中文</button>
+  <button type="button" lang="en" (click)="language.setLanguage('en')"
+    [class.active]="language.language() === 'en'" [attr.aria-pressed]="language.language() === 'en'">English</button>
+</div>
+<router-outlet></router-outlet>

package/artemis-main/apps/showcase_ui/src/app/app.component.scss

@@ -0,0 +1,49 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+.language-switch {
+  position: fixed;
+  top: 12px;
+  right: 24px;
+  z-index: 1200;
+  display: flex;
+  gap: 2px;
+  padding: 3px;
+  background: #fff;
+  border: 1px solid #dce3ed;
+  border-radius: 20px;
+  box-shadow: 0 2px 8px #182b4910;
+  button {
+    min-height: 30px;
+    padding: 4px 12px;
+    border: 0;
+    border-radius: 16px;
+    background: transparent;
+    color: #475569;
+    font-family: inherit;
+    font-size: 12px;
+    font-weight: 500;
+    line-height: 20px;
+    cursor: pointer;
+    &.active { background: #e8f0fe; color: #174ea6; }
+    &:focus-visible { outline: 2px solid #1a73e8; outline-offset: 2px; }
+  }
+}
+@media (max-width: 600px) {
+  .language-switch { right: 8px; }
+  .language-switch button { padding: 4px 8px; }
+}

package/artemis-main/apps/showcase_ui/src/app/app.component.spec.ts

@@ -0,0 +1,42 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { TestBed } from '@angular/core/testing';
+import { provideRouter } from '@angular/router';
+import { provideHttpClient, withXhr } from '@angular/common/http';
+import { AppComponent } from './app.component';
+
+describe('AppComponent', () => {
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      imports: [AppComponent],
+      providers: [provideRouter([]), provideHttpClient(withXhr())]
+    }).compileComponents();
+  });
+
+  it('should create the app', () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.componentInstance;
+    expect(app).toBeTruthy();
+  });
+
+  it(`should have the 'frontend' title`, () => {
+    const fixture = TestBed.createComponent(AppComponent);
+    const app = fixture.componentInstance;
+    expect(app.title).toEqual('frontend');
+  });
+
+});

package/artemis-main/apps/showcase_ui/src/app/app.component.ts

@@ -0,0 +1,32 @@
+import { LanguageService } from './core/i18n/language.service';
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Component, ChangeDetectionStrategy, inject } from '@angular/core';
+import { RouterOutlet } from '@angular/router';
+import { NavSwitcherComponent } from './components/nav-switcher/nav-switcher.component';
+
+@Component({
+  selector: 'app-root',
+  imports: [RouterOutlet, NavSwitcherComponent],
+  templateUrl: './app.component.html',
+  changeDetection: ChangeDetectionStrategy.Eager,
+  styleUrl: './app.component.scss'
+})
+export class AppComponent {
+  title = 'frontend';
+  readonly language = inject(LanguageService);
+}

package/artemis-main/apps/showcase_ui/src/app/app.config.ts

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ApplicationConfig, provideZoneChangeDetection } from '@angular/core';
+import { provideRouter } from '@angular/router';
+import { provideHttpClient, withXhr } from '@angular/common/http';
+
+import { routes } from './app.routes';
+
+export const appConfig: ApplicationConfig = {
+  providers: [
+    provideZoneChangeDetection({ eventCoalescing: true }),
+    provideRouter(routes),
+    provideHttpClient(withXhr())
+  ]
+};

package/artemis-main/apps/showcase_ui/src/app/app.routes.ts

@@ -0,0 +1,27 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { Routes } from '@angular/router';
+import { HomeComponent } from './pages/home/home.component';
+import { WorkspaceComponent } from './pages/workspace/workspace.component';
+import { LegacyWorkspaceComponent } from './pages/legacy-workspace/legacy-workspace.component';
+
+export const routes: Routes = [
+  { path: '', component: HomeComponent },
+  { path: 'workspace', component: WorkspaceComponent },
+  { path: 'check', component: LegacyWorkspaceComponent },
+  { path: '**', redirectTo: '' }
+];

package/artemis-main/apps/showcase_ui/src/app/components/agent-stream/agent-stream-startup.spec.ts

@@ -0,0 +1,108 @@
+import { StartupProgressEvent } from '../../services/agent.service';
+import { buildStartupWorkItems } from './agent-stream.component';
+
+describe('startup Work block', () => {
+  it('shows only the three device preparation operations', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'submitting', message: 'Submitting the task', timestamp: 100 },
+      { stage: 'queued', message: 'Task received and queued', timestamp: 100.9 },
+      { stage: 'launching', message: 'Starting the execution process', timestamp: 101 },
+      { stage: 'configuration', message: 'Loading the run configuration', timestamp: 103.1 },
+      { stage: 'device_check', message: 'Checking the Android device', timestamp: 103.2 },
+      { stage: 'device_ready', message: 'Android device connected', timestamp: 123 },
+      { stage: 'model_warmup', message: 'Warming the model connection', timestamp: 123 },
+      { stage: 'uiautomator', message: 'Connecting to UI Automator', timestamp: 124 },
+      { stage: 'uiautomator_ready', message: 'UI Automator is ready', timestamp: 128 },
+      { stage: 'environment', message: 'Preparing the device environment', timestamp: 128 },
+      { stage: 'environment_ready', message: 'Device environment is ready', timestamp: 131 }
+    ];
+
+    const items = buildStartupWorkItems(events, 131, false, true);
+
+    expect(items.map((item) => item.stage)).toEqual([
+      'device_ready',
+      'uiautomator_ready',
+      'environment_ready'
+    ]);
+    expect(items.map((item) => item.elapsed)).toEqual(['20s', '4.0s', '3.0s']);
+    expect(items.every((item) => !item.isActive)).toBeTrue();
+  });
+
+  it('keeps the current preparation operation live without exposing first-action text', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'device_check', message: 'Checking the Android device', timestamp: 10 }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 11.4, false, true);
+
+    expect(item.message).toBe('Checking the Android device');
+    expect(item.elapsed).toBe('1.4s');
+    expect(item.isActive).toBeTrue();
+  });
+
+  it('shows the helper install sub-step while the hierarchy service is connecting', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
+      {
+        stage: 'helper_install',
+        message: 'Installing the Artemis accessibility helper on this device for the first time (v1.1.3, about 3 seconds)',
+        timestamp: 10.2
+      }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 12, false, true);
+
+    expect(item.isActive).toBeTrue();
+    expect(item.message).toContain('Installing the Artemis accessibility helper');
+    expect(item.elapsed).toBe('2.0s');
+  });
+
+  it('uses the hierarchy source line as the completion message of the second step', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
+      { stage: 'helper_install', message: 'Installing ...', timestamp: 10.2 },
+      { stage: 'uiautomator_ready', message: 'UI hierarchy service is ready (helper)', timestamp: 13 },
+      {
+        stage: 'hierarchy_backend',
+        message: 'UI hierarchy source: Artemis accessibility helper v1.1.3',
+        timestamp: 13.1
+      },
+      // A later mid-run switch must not rewrite the startup line.
+      {
+        stage: 'hierarchy_backend_changed',
+        message: 'UI hierarchy source switched from Artemis accessibility helper to UIAutomator2 because ...',
+        timestamp: 90
+      }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 100, true, true);
+
+    expect(item.isActive).toBeFalse();
+    expect(item.message).toBe('UI hierarchy source: Artemis accessibility helper v1.1.3');
+    expect(item.elapsed).toBe('3.0s');
+  });
+
+  it('falls back to the generic completion line for runs without a source event', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'uiautomator', message: 'Connecting to the UI hierarchy service', timestamp: 10 },
+      { stage: 'environment', message: 'Preparing the device environment', timestamp: 14 }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 20, false, true);
+
+    expect(item.message).toBe('UI hierarchy service is ready');
+  });
+
+  it('uses old first_response events only as a hidden completion boundary', () => {
+    const events: StartupProgressEvent[] = [
+      { stage: 'environment', message: 'Preparing the device environment', timestamp: 20 },
+      { stage: 'first_response', message: 'Generating the first action', timestamp: 23 }
+    ];
+
+    const [item] = buildStartupWorkItems(events, 24, false, true);
+
+    expect(item.message).toBe('Device environment is ready');
+    expect(item.elapsed).toBe('3.0s');
+    expect(item.isActive).toBeFalse();
+  });
+});