android-midscene-automation 0.2.4 → 1.1.0

package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.spec.ts

@@ -0,0 +1,70 @@
+import { parseNoteLines, parseNote, renderMarkdownToHtml } from './markdown-parser.util';
+
+describe('markdown-parser.util verification & check lines', () => {
+  it('should parse - verify: line as type verify with checkKind verify', () => {
+    const markdown = `- [x] Parent task
+  - [x] Subtask 1
+  - verify: Commute duration is recorded in note \`eta_details\``;
+
+    const lines = parseNoteLines(markdown);
+    expect(lines.length).toBe(3);
+    expect(lines[0].type).toBe('checked');
+    expect(lines[1].type).toBe('checked');
+
+    const verifyLine = lines[2];
+    expect(verifyLine.type).toBe('verify');
+    expect(verifyLine.checkKind).toBe('verify');
+    expect(verifyLine.atEnd).toBe(false);
+    expect(verifyLine.segments.map(s => s.text).join('')).toBe('Commute duration is recorded in note eta_details');
+    expect(verifyLine.segments.find(s => s.code)?.text).toBe('eta_details');
+  });
+
+  it('should parse - assert: and - assert@end: lines', () => {
+    const markdown = `- [ ] Open app
+  - assert: the welcome screen shows up
+- assert@end: final status is completed`;
+
+    const lines = parseNoteLines(markdown);
+    expect(lines.length).toBe(3);
+
+    expect(lines[1].type).toBe('verify');
+    expect(lines[1].checkKind).toBe('assert');
+    expect(lines[1].atEnd).toBe(false);
+
+    expect(lines[2].type).toBe('verify');
+    expect(lines[2].checkKind).toBe('assert');
+    expect(lines[2].atEnd).toBe(true);
+  });
+
+  it('should parse - finding: lines', () => {
+    const markdown = `- [x] Step 1
+  - finding: Unresolved verify failure`;
+
+    const lines = parseNoteLines(markdown);
+    expect(lines[1].type).toBe('finding');
+    expect(lines[1].checkKind).toBe('finding');
+    expect(lines[1].segments[0].text).toBe('Unresolved verify failure');
+  });
+
+  it('should group verify lines under parent milestone checks in parseNote', () => {
+    const note = `- [x] Open Maps
+  - [x] Search destination
+  - verify: Arrival time is visible in note \`eta\``;
+
+    const parsed = parseNote(note);
+    expect(parsed.milestones.length).toBe(1);
+    expect(parsed.milestones[0].subSteps.length).toBe(1);
+    expect(parsed.milestones[0].subSteps[0].type).toBe('checked');
+    expect(parsed.milestones[0].checks.length).toBe(1);
+    expect(parsed.milestones[0].checks[0].type).toBe('verify');
+    expect(parsed.milestones[0].checks[0].checkKind).toBe('verify');
+  });
+
+  it('should render verify badges in renderMarkdownToHtml', () => {
+    const text = `- verify: status is ok`;
+    const html = renderMarkdownToHtml(text);
+    expect(html).toContain('class="md-verify-item"');
+    expect(html).toContain('<span class="verify-badge">verify</span>');
+    expect(html).toContain('status is ok');
+  });
+});

package/artemis-main/apps/showcase_ui/src/app/utils/markdown-parser.util.ts

@@ -0,0 +1,383 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { MarkdownSegment, MarkdownLine, NoteMilestone, ParsedNote } from '../core/models/markdown.model';
+import { CheckerResult } from '../core/models/stream.model';
+
+/**
+ * Split text into bold, code, and plain segments
+ */
+export function parseMarkdownSegments(text: string): MarkdownSegment[] {
+  if (!text) return [];
+  const parts = text.split(/(\*\*.*?\*\*|`.*?`)/g);
+  return parts
+    .filter(part => part !== '')
+    .map(part => {
+      if (part.startsWith('**') && part.endsWith('**')) {
+        return {
+          text: part.slice(2, -2),
+          bold: true,
+          code: false
+        };
+      } else if (part.startsWith('`') && part.endsWith('`')) {
+        return {
+          text: part.slice(1, -1),
+          bold: false,
+          code: true
+        };
+      } else {
+        return {
+          text: part,
+          bold: false,
+          code: false
+        };
+      }
+    });
+}
+
+/**
+ * Parse markdown note content into structured lines (checklists, headers, list items)
+ */
+export function parseNoteLines(content: string): MarkdownLine[] {
+  if (!content) return [];
+  return content.split('\n').map(line => {
+    const trimmed = line.trim();
+    const indent = line.length - line.trimStart().length;
+
+    // Checkbox [x]
+    if (trimmed.startsWith('- [x] ') || trimmed.startsWith('- [X] ')) {
+      const rawText = trimmed.substring(6);
+      return { type: 'checked', segments: parseMarkdownSegments(rawText), indent };
+    }
+    // Active checkbox [/]
+    if (trimmed.startsWith('- [/] ')) {
+      const rawText = trimmed.substring(6);
+      return { type: 'progress', segments: parseMarkdownSegments(rawText), indent };
+    }
+    // Unchecked checkbox [ ]
+    if (trimmed.startsWith('- [ ] ')) {
+      const rawText = trimmed.substring(6);
+      return { type: 'unchecked', segments: parseMarkdownSegments(rawText), indent };
+    }
+    // Check lines (- verify: / - assert: / - verify@end: / - assert@end:)
+    const checkMatch = trimmed.match(/^[-*]\s*(verify|assert)(@end)?\s*:\s*(.*)$/i);
+    if (checkMatch) {
+      const kind = checkMatch[1].toLowerCase() as 'verify' | 'assert';
+      const atEnd = Boolean(checkMatch[2]);
+      const rawText = checkMatch[3].trim();
+      return {
+        type: 'verify',
+        checkKind: kind,
+        atEnd,
+        segments: parseMarkdownSegments(rawText),
+        indent
+      };
+    }
+    // System findings (- finding:)
+    const findingMatch = trimmed.match(/^[-*]\s*finding\s*:\s*(.*)$/i);
+    if (findingMatch) {
+      const rawText = findingMatch[1].trim();
+      return {
+        type: 'finding',
+        checkKind: 'finding',
+        segments: parseMarkdownSegments(rawText),
+        indent
+      };
+    }
+    // Regular list item
+    if (trimmed.startsWith('- ') || trimmed.startsWith('* ')) {
+      const rawText = trimmed.substring(2);
+      return { type: 'list-item', segments: parseMarkdownSegments(rawText), indent };
+    }
+    // Headers
+    if (trimmed.startsWith('# ')) {
+      const rawText = trimmed.substring(2);
+      return { type: 'h1', segments: parseMarkdownSegments(rawText), indent };
+    }
+    if (trimmed.startsWith('## ')) {
+      const rawText = trimmed.substring(3);
+      return { type: 'h2', segments: parseMarkdownSegments(rawText), indent };
+    }
+    if (trimmed.startsWith('### ')) {
+      const rawText = trimmed.substring(4);
+      return { type: 'h3', segments: parseMarkdownSegments(rawText), indent };
+    }
+    // Default line
+    return trimmed ? { type: 'text', segments: parseMarkdownSegments(line), indent } : { type: 'empty', segments: [], indent: 0 };
+  });
+}
+
+/**
+ * Group parsed markdown lines into title, milestones (checklists), and other lines
+ */
+export function parseNote(content: string): ParsedNote {
+  const lines = parseNoteLines(content);
+  const milestones: NoteMilestone[] = [];
+  const otherLines: MarkdownLine[] = [];
+  let title: string | null = null;
+  let currentMilestone: (NoteMilestone & { _indent?: number }) | null = null;
+  let milestoneCount = 0;
+
+  for (const line of lines) {
+    if (line.type === 'empty') {
+      continue;
+    }
+
+    if ((line.type === 'h1' || line.type === 'h2') && !title) {
+      title = line.segments.map(s => s.text).join('');
+      continue;
+    }
+
+    const isChecklistItem = ['checked', 'progress', 'unchecked'].includes(line.type);
+
+    if (isChecklistItem && (!currentMilestone || line.indent <= (currentMilestone._indent ?? 0))) {
+      milestoneCount++;
+      currentMilestone = {
+        index: milestoneCount,
+        type: line.type as 'checked' | 'progress' | 'unchecked',
+        segments: line.segments,
+        subSteps: [],
+        checks: [],
+        _indent: line.indent
+      };
+      milestones.push(currentMilestone);
+    } else if (currentMilestone) {
+      if (line.indent > (currentMilestone._indent ?? 0)) {
+        if (line.type === 'verify' || line.type === 'assert' || line.type === 'finding') {
+          currentMilestone.checks.push(line);
+        } else {
+          currentMilestone.subSteps.push(line);
+        }
+      } else {
+        otherLines.push(line);
+      }
+    } else {
+      otherLines.push(line);
+    }
+  }
+
+  return {
+    title,
+    milestones,
+    otherLines
+  };
+}
+
+/**
+ * Parse the Checker JSON response block
+ */
+export function extractCheckerResult(text: string): CheckerResult | null {
+  if (!text) return null;
+  let cleanText = text.trim();
+  if (cleanText.includes('```json')) {
+    const parts = cleanText.split('```json');
+    if (parts.length > 1) {
+      cleanText = parts[1].split('```')[0].trim();
+    }
+  } else if (cleanText.includes('```')) {
+    const parts = cleanText.split('```');
+    if (parts.length > 1) {
+      cleanText = parts[1].split('```')[0].trim();
+    }
+  }
+
+  try {
+    const parsed = JSON.parse(cleanText);
+    if (parsed && typeof parsed === 'object' && 'success' in parsed) {
+      return {
+        success: Boolean(parsed.success),
+        reason: parsed.reason || 'No reason provided.'
+      };
+    }
+  } catch {
+    // Ignore
+  }
+  return null;
+}
+
+/**
+ * Helper to format inline markdown segments (bold, italic, code, strikethrough)
+ */
+function formatInlineMarkdown(text: string): string {
+  if (!text) return '';
+  return text
+    // Inline code first (so asterisks inside code aren't formatted)
+    .replace(/`([^`]+)`/g, '<code class="inline-code">$1</code>')
+    // Bold + Italic
+    .replace(/\*\*\*([^*]+)\*\*\*/g, '<strong><em>$1</em></strong>')
+    .replace(/___([^_]+)___/g, '<strong><em>$1</em></strong>')
+    // Bold
+    .replace(/\*\*([^*]+)\*\*/g, '<strong>$1</strong>')
+    .replace(/__([^_]+)__/g, '<strong>$1</strong>')
+    // Italic
+    .replace(/\*([^*]+)\*/g, '<em>$1</em>')
+    .replace(/\b_([^_]+)_\b/g, '<em>$1</em>')
+    // Strikethrough
+    .replace(/~~([^~]+)~~/g, '<del>$1</del>');
+}
+
+/**
+ * Convert basic markdown text to safe HTML for agent logs/thinking
+ */
+export function renderMarkdownToHtml(text: string): string {
+  if (!text) return '';
+
+  // Clean up internal agent XML-like tags (e.g. <thought>, <short_term_memory>, etc.)
+  const cleanedText = text
+    .replace(/<\/?(thought|short_term_memory|strong_term_memory|reasoning|plan|task_plan|call_tool)[^\n>]*>?/gi, '')
+    .replace(/\r\n/g, '\n')
+    .replace(/\r/g, '\n')
+    .replace(/\n{3,}/g, '\n\n')
+    .trim();
+
+  // Escape HTML first to prevent XSS
+  const escaped = cleanedText
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;');
+
+  const rawLines = escaped.split('\n');
+  const result: string[] = [];
+  let currentListType: 'ul' | 'ol' | null = null;
+  let inCodeBlock = false;
+  let codeBlockLang = '';
+  let codeBlockLines: string[] = [];
+
+  const closeList = () => {
+    if (currentListType) {
+      result.push(`</${currentListType}>`);
+      currentListType = null;
+    }
+  };
+
+  for (let i = 0; i < rawLines.length; i++) {
+    const rawLine = rawLines[i];
+    const trimmed = rawLine.trim();
+
+    // 1. Code blocks: ```lang ... ```
+    if (trimmed.startsWith('```')) {
+      if (inCodeBlock) {
+        // End code block
+        const codeContent = codeBlockLines.join('\n');
+        result.push(`<pre class="code-block"><code class="lang-${codeBlockLang}">${codeContent}</code></pre>`);
+        inCodeBlock = false;
+        codeBlockLines = [];
+        codeBlockLang = '';
+      } else {
+        closeList();
+        inCodeBlock = true;
+        codeBlockLang = trimmed.substring(3).trim();
+        codeBlockLines = [];
+      }
+      continue;
+    }
+
+    if (inCodeBlock) {
+      codeBlockLines.push(rawLine);
+      continue;
+    }
+
+    // 2. Headings: # to ######
+    const headingMatch = trimmed.match(/^(#{1,6})\s+(.*)$/);
+    if (headingMatch) {
+      closeList();
+      const level = headingMatch[1].length;
+      const headingContent = formatInlineMarkdown(headingMatch[2]);
+      result.push(`<h${level}>${headingContent}</h${level}>`);
+      continue;
+    }
+
+    // 3. Horizontal Rule: ---, ***, ___
+    if (/^(\-{3,}|\*{3,}|_{3,})$/.test(trimmed)) {
+      closeList();
+      result.push('<hr class="md-hr" />');
+      continue;
+    }
+
+    // 4. Blockquotes: > quote
+    if (trimmed.startsWith('&gt; ') || trimmed.startsWith('&gt;')) {
+      closeList();
+      const quoteContent = formatInlineMarkdown(trimmed.replace(/^&gt;\s?/, ''));
+      result.push(`<blockquote>${quoteContent}</blockquote>`);
+      continue;
+    }
+
+    // 5. Unordered List: - item or * item
+    const ulMatch = rawLine.match(/^(\s*)([\*\-])\s+(.*)$/);
+    if (ulMatch) {
+      if (currentListType !== 'ul') {
+        closeList();
+        currentListType = 'ul';
+        result.push('<ul>');
+      }
+      const spaces = ulMatch[1].length;
+      const subLevel = Math.max(0, Math.floor((spaces - 2) / 2));
+      const indent = subLevel * 12;
+      const style = indent ? ` style="margin-left: ${indent}px"` : '';
+      const itemText = ulMatch[3];
+      const checkMatch = itemText.match(/^(verify|assert)(@end)?\s*:\s*(.*)$/i);
+      const findingMatch = itemText.match(/^finding\s*:\s*(.*)$/i);
+      if (checkMatch) {
+        const kind = checkMatch[1].toLowerCase();
+        const atEnd = checkMatch[2] ? '@end' : '';
+        const assertClass = kind === 'assert' ? ' assert' : '';
+        const itemContent = formatInlineMarkdown(checkMatch[3].trim());
+        result.push(`<li${style} class="md-verify-item"><span class="verify-badge${assertClass}">${kind}${atEnd}</span>${itemContent}</li>`);
+      } else if (findingMatch) {
+        const itemContent = formatInlineMarkdown(findingMatch[1].trim());
+        result.push(`<li${style} class="md-finding-item"><span class="finding-badge">finding</span>${itemContent}</li>`);
+      } else {
+        const itemContent = formatInlineMarkdown(itemText);
+        result.push(`<li${style}>${itemContent}</li>`);
+      }
+      continue;
+    }
+
+    // 6. Ordered List: 1. item
+    const olMatch = rawLine.match(/^(\s*)(\d+)\.\s+(.*)$/);
+    if (olMatch) {
+      if (currentListType !== 'ol') {
+        closeList();
+        currentListType = 'ol';
+        result.push('<ol>');
+      }
+      const spaces = olMatch[1].length;
+      const subLevel = Math.max(0, Math.floor((spaces - 2) / 2));
+      const indent = subLevel * 12;
+      const style = indent ? ` style="margin-left: ${indent}px"` : '';
+      const itemContent = formatInlineMarkdown(olMatch[3]);
+      result.push(`<li${style}>${itemContent}</li>`);
+      continue;
+    }
+
+    // 7. Regular paragraph / empty line
+    closeList();
+    if (trimmed === '') {
+      result.push('<div class="md-para-gap"></div>');
+    } else {
+      const lineContent = formatInlineMarkdown(rawLine);
+      result.push(`<div>${lineContent}</div>`);
+    }
+  }
+
+  if (inCodeBlock) {
+    const codeContent = codeBlockLines.join('\n');
+    result.push(`<pre class="code-block"><code class="lang-${codeBlockLang}">${codeContent}</code></pre>`);
+  }
+
+  closeList();
+  return result.join('\n');
+}

package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.spec.ts

@@ -0,0 +1,107 @@
+import {
+  hasSessionOffsets,
+  locateSessionTime,
+  locateTimelineTime,
+  sessionTimeToTimelineTime,
+  TimelineSegment
+} from './recording-timeline.util';
+
+describe('recording-timeline.util', () => {
+  // Recording started 1.2s after the session began. Segment 0 ran 10s, then
+  // scrcpy restarted for rotation (1.5s gap), segment 1 ran 5s, another 0.8s
+  // gap, and segment 2 ran 4s.
+  const v2Segments: TimelineSegment[] = [
+    { start: 0, duration: 10, offset_ms: 1200, duration_ms: 10000 },
+    { start: 10, duration: 5, offset_ms: 12700, duration_ms: 5000 },
+    { start: 15, duration: 4, offset_ms: 18500, duration_ms: 4000 }
+  ];
+
+  const legacySegments: TimelineSegment[] = [
+    { start: 0, duration: 10 },
+    { start: 10, duration: 5 }
+  ];
+
+  describe('hasSessionOffsets', () => {
+    it('detects manifest v2 segments and rejects legacy or mixed lists', () => {
+      expect(hasSessionOffsets(v2Segments)).toBeTrue();
+      expect(hasSessionOffsets(legacySegments)).toBeFalse();
+      expect(hasSessionOffsets([v2Segments[0], legacySegments[0]])).toBeFalse();
+      expect(hasSessionOffsets([])).toBeFalse();
+    });
+  });
+
+  describe('locateTimelineTime', () => {
+    it('maps back-to-back playlist times into segments', () => {
+      expect(locateTimelineTime(legacySegments, 3)).toEqual({ index: 0, localTime: 3, timelineTime: 3 });
+      expect(locateTimelineTime(legacySegments, 10)).toEqual({ index: 1, localTime: 0, timelineTime: 10 });
+      expect(locateTimelineTime(legacySegments, 12.5)).toEqual({ index: 1, localTime: 2.5, timelineTime: 12.5 });
+    });
+
+    it('clamps out-of-range times into the first or last segment', () => {
+      expect(locateTimelineTime(legacySegments, -4)).toEqual({ index: 0, localTime: 0, timelineTime: 0 });
+      expect(locateTimelineTime(legacySegments, 99)).toEqual({ index: 1, localTime: 5, timelineTime: 15 });
+      expect(locateTimelineTime([], 3)).toBeNull();
+    });
+  });
+
+  describe('locateSessionTime', () => {
+    it('maps a session time inside a segment using its real offset', () => {
+      // T+4.2s is 3.0s into segment 0 (which started at T+1.2s).
+      const inFirst = locateSessionTime(v2Segments, 4.2)!;
+      expect(inFirst.index).toBe(0);
+      expect(inFirst.localTime).toBeCloseTo(3.0, 6);
+      expect(inFirst.timelineTime).toBeCloseTo(3.0, 6);
+
+      // T+14.7s is 2.0s into segment 1 (started at T+12.7s), which sits at
+      // timeline 10s, so it must NOT drift to timeline 14.7s.
+      const inSecond = locateSessionTime(v2Segments, 14.7)!;
+      expect(inSecond.index).toBe(1);
+      expect(inSecond.localTime).toBeCloseTo(2.0, 6);
+      expect(inSecond.timelineTime).toBeCloseTo(12.0, 6);
+
+      const inThird = locateSessionTime(v2Segments, 20.5)!;
+      expect(inThird.index).toBe(2);
+      expect(inThird.localTime).toBeCloseTo(2.0, 6);
+      expect(inThird.timelineTime).toBeCloseTo(17.0, 6);
+    });
+
+    it('snaps a time inside a restart gap to the start of the next segment', () => {
+      // Segment 0 ends at T+11.2s, segment 1 starts at T+12.7s.
+      expect(locateSessionTime(v2Segments, 11.9)).toEqual({ index: 1, localTime: 0, timelineTime: 10 });
+      // Segment 1 ends at T+17.7s, segment 2 starts at T+18.5s.
+      expect(locateSessionTime(v2Segments, 18.0)).toEqual({ index: 2, localTime: 0, timelineTime: 15 });
+    });
+
+    it('treats segment boundaries as inclusive of the end frame', () => {
+      const atEnd = locateSessionTime(v2Segments, 11.2)!;
+      expect(atEnd.index).toBe(0);
+      expect(atEnd.localTime).toBeCloseTo(10, 6);
+      expect(locateSessionTime(v2Segments, 12.7)).toEqual({ index: 1, localTime: 0, timelineTime: 10 });
+    });
+
+    it('snaps times before the first frame and after the last frame', () => {
+      expect(locateSessionTime(v2Segments, 0)).toEqual({ index: 0, localTime: 0, timelineTime: 0 });
+      expect(locateSessionTime(v2Segments, 0.5)).toEqual({ index: 0, localTime: 0, timelineTime: 0 });
+      expect(locateSessionTime(v2Segments, 60)).toEqual({ index: 2, localTime: 4, timelineTime: 19 });
+      expect(locateSessionTime(v2Segments, Number.NaN)).toEqual({ index: 0, localTime: 0, timelineTime: 0 });
+    });
+
+    it('falls back to back-to-back timeline behaviour for legacy manifests', () => {
+      expect(locateSessionTime(legacySegments, 12.5)).toEqual({ index: 1, localTime: 2.5, timelineTime: 12.5 });
+      expect(locateSessionTime([], 5)).toBeNull();
+    });
+  });
+
+  describe('sessionTimeToTimelineTime', () => {
+    it('returns the playlist position for a session time', () => {
+      expect(sessionTimeToTimelineTime(v2Segments, 14.7)).toBeCloseTo(12.0, 6);
+      expect(sessionTimeToTimelineTime(v2Segments, 11.9)).toBe(10);
+      expect(sessionTimeToTimelineTime(legacySegments, 7)).toBe(7);
+    });
+
+    it('passes the time through when no playlist is available', () => {
+      expect(sessionTimeToTimelineTime([], 7)).toBe(7);
+      expect(sessionTimeToTimelineTime([], -2)).toBe(0);
+    });
+  });
+});

package/artemis-main/apps/showcase_ui/src/app/utils/recording-timeline.util.ts

@@ -0,0 +1,123 @@
+/**
+ * Copyright 2026 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Pure helpers that map times onto the multi-segment screen recording.
+ *
+ * A recording is split into segments whenever scrcpy restarts (device
+ * rotation, crash recovery). The player plays them back to back on a
+ * "timeline" axis (`start`/`duration`, no gaps), while step timestamps live
+ * on the "session" axis (seconds since the DataEngine session start). Manifest
+ * v2 segments carry `offset_ms`/`duration_ms` on the session axis, which lets
+ * a session time land in the right segment even after each restart gap.
+ */
+
+export interface TimelineSegment {
+  /** Back-to-back playlist start in seconds. */
+  start: number;
+  /** Segment length in seconds. */
+  duration: number;
+  /** Session-relative first-frame offset in milliseconds (manifest v2). */
+  offset_ms?: number;
+  /** Segment length in milliseconds (manifest v2). */
+  duration_ms?: number;
+}
+
+export interface SegmentLocation {
+  /** Index into the segment list. */
+  index: number;
+  /** Seconds into that segment's own media. */
+  localTime: number;
+  /** Equivalent position on the back-to-back playlist axis, in seconds. */
+  timelineTime: number;
+}
+
+const clamp = (value: number, min: number, max: number): number => Math.max(min, Math.min(max, value));
+
+/** True when every segment carries the session-relative v2 fields. */
+export function hasSessionOffsets(segments: readonly TimelineSegment[]): boolean {
+  return segments.length > 0 && segments.every(
+    (segment) => Number.isFinite(segment.offset_ms) && Number.isFinite(segment.duration_ms)
+  );
+}
+
+/**
+ * Locate a position on the back-to-back playlist axis (what the scrubber uses).
+ * Times past the end clamp into the last segment.
+ */
+export function locateTimelineTime(
+  segments: readonly TimelineSegment[],
+  timelineSeconds: number
+): SegmentLocation | null {
+  if (!segments.length) return null;
+  const total = segments.reduce((sum, segment) => sum + Math.max(0, segment.duration), 0);
+  const target = clamp(Number.isFinite(timelineSeconds) ? timelineSeconds : 0, 0, total);
+  let index = segments.findIndex((segment) => target < segment.start + segment.duration);
+  if (index < 0) index = segments.length - 1;
+  const segment = segments[index];
+  const localTime = clamp(target - segment.start, 0, Math.max(0, segment.duration));
+  return { index, localTime, timelineTime: segment.start + localTime };
+}
+
+/**
+ * Locate a session-relative time (seconds since session start, the axis step
+ * timestamps use). A time inside `[offset, offset + duration]` of a segment maps
+ * into that segment; a time in the gap between two segments (scrcpy restart)
+ * snaps to the start of the next segment; a time before the first frame snaps
+ * to the first frame; a time after the last frame clamps to the last frame.
+ *
+ * Segments without session offsets (legacy manifests) are treated as if the
+ * session axis were the back-to-back timeline axis.
+ */
+export function locateSessionTime(
+  segments: readonly TimelineSegment[],
+  sessionSeconds: number
+): SegmentLocation | null {
+  if (!segments.length) return null;
+  if (!hasSessionOffsets(segments)) {
+    return locateTimelineTime(segments, sessionSeconds);
+  }
+  const targetMs = Number.isFinite(sessionSeconds) ? sessionSeconds * 1000 : 0;
+  for (let index = 0; index < segments.length; index++) {
+    const segment = segments[index];
+    const offsetMs = segment.offset_ms as number;
+    const endMs = offsetMs + (segment.duration_ms as number);
+    if (targetMs < offsetMs) {
+      // Before this segment's first frame: either before the recording began
+      // or inside the restart gap after the previous segment. Snap forward.
+      return { index, localTime: 0, timelineTime: segment.start };
+    }
+    if (targetMs <= endMs) {
+      const localTime = clamp((targetMs - offsetMs) / 1000, 0, Math.max(0, segment.duration));
+      return { index, localTime, timelineTime: segment.start + localTime };
+    }
+  }
+  const lastIndex = segments.length - 1;
+  const last = segments[lastIndex];
+  const localTime = Math.max(0, last.duration);
+  return { index: lastIndex, localTime, timelineTime: last.start + localTime };
+}
+
+/**
+ * Convert a session-relative time into the back-to-back playlist time the
+ * player scrubber understands. Returns 0 when there are no segments.
+ */
+export function sessionTimeToTimelineTime(
+  segments: readonly TimelineSegment[],
+  sessionSeconds: number
+): number {
+  return locateSessionTime(segments, sessionSeconds)?.timelineTime ?? Math.max(0, sessionSeconds || 0);
+}