escribano 0.1.0

package/dist/services/activity-segmentation.js

@@ -0,0 +1,333 @@
+/**
+ * Escribano - Activity Segmentation Service
+ *
+ * Groups consecutive VLM observations by activity continuity.
+ * Replaces embedding-based clustering with VLM-driven segmentation.
+ */
+import { normalizeActivity } from './vlm-service.js';
+const DEFAULT_CONFIG = {
+    minSegmentDuration: 30,
+    gapTolerance: 5,
+};
+/**
+ * Extract activity type from VLM description.
+ * Uses prioritized activity detection with precise pattern matching.
+ */
+function extractActivityType(vlmDescription) {
+    if (!vlmDescription)
+        return 'other';
+    // Normalize the activity string
+    const normalized = vlmDescription.toLowerCase().trim();
+    // Check for known activity patterns (order matters - more specific first)
+    const activityPatterns = {
+        // Debugging - very specific technical terms
+        debugging: [
+            'debugging',
+            'troubleshooting',
+            'investigating error',
+            'reading error',
+            'stack trace',
+            'exception thrown',
+            'error message',
+            'fixing bug',
+        ],
+        // Coding/Development
+        coding: [
+            'writing code',
+            'implementing',
+            'developing',
+            'programming',
+            'refactoring',
+            'coding',
+        ],
+        // Code Review - specific workflow
+        review: ['reviewing pr', 'pull request', 'code review', 'reviewing code'],
+        // Meeting/Collaboration
+        meeting: [
+            'in zoom',
+            'in google meet',
+            'in slack huddle',
+            'video call',
+            'screen sharing',
+            'team meeting',
+        ],
+        // Research/Information gathering
+        research: [
+            'browsing',
+            'stack overflow',
+            'googling',
+            'researching',
+            'searching for',
+        ],
+        // Reading documentation
+        reading: [
+            'reading documentation',
+            'reading docs',
+            'reading manual',
+            'reading guide',
+        ],
+        // Terminal/CLI operations (only if explicitly mentioned)
+        terminal: [
+            'in terminal',
+            'in iterm',
+            'command line',
+            'running git',
+            'running npm',
+        ],
+    };
+    // Check each activity type in order (most specific patterns first)
+    for (const [activityType, patterns] of Object.entries(activityPatterns)) {
+        for (const pattern of patterns) {
+            if (normalized.includes(pattern)) {
+                return activityType;
+            }
+        }
+    }
+    // Check for single-word activities that appear at the start
+    const firstWord = normalized.split(/[\s,.]+/)[0];
+    const startPatterns = {
+        debugging: ['debug', 'fix'],
+        coding: ['writing', 'implementing', 'developing'],
+        reading: ['reading'],
+        research: ['researching', 'browsing'],
+    };
+    for (const [activityType, patterns] of Object.entries(startPatterns)) {
+        if (patterns.includes(firstWord)) {
+            return activityType;
+        }
+    }
+    return 'other';
+}
+/**
+ * Parse VLM description to extract apps and topics.
+ * Expects format like: "Debugging Python error in VSCode, working on escribano project"
+ */
+function extractContext(vlmDescription) {
+    if (!vlmDescription)
+        return { apps: [], topics: [] };
+    const apps = [];
+    const topics = [];
+    const text = vlmDescription.toLowerCase();
+    // Common app patterns
+    const appPatterns = [
+        /in (vscode|vs code|visual studio code)/i,
+        /in (terminal|iterm|alacritty|warp)/i,
+        /in (chrome|safari|firefox|browser)/i,
+        /in (slack|discord|teams|zoom)/i,
+        /in (github|gitlab|bitbucket)/i,
+        /in (intellij|webstorm|pycharm)/i,
+        /using (vscode|terminal|chrome|slack)/i,
+    ];
+    for (const pattern of appPatterns) {
+        const match = text.match(pattern);
+        if (match && match[1]) {
+            apps.push(match[1].toLowerCase().replace(' ', '_'));
+        }
+    }
+    // Extract potential project names (capitalized words after "working on" or "in")
+    const topicPatterns = [
+        /working on (?:the )?(\w+)/i,
+        /(?:in|for) (?:the )?(\w+) project/i,
+        /(?:implementing|fixing|debugging) (\w+)/i,
+    ];
+    for (const pattern of topicPatterns) {
+        const match = text.match(pattern);
+        if (match && match[1]) {
+            topics.push(match[1].toLowerCase());
+        }
+    }
+    return { apps: [...new Set(apps)], topics: [...new Set(topics)] };
+}
+const NOISY_APP_PATTERNS = [
+    /^and\s/i,
+    /^a\s/i,
+    /^the\s/i,
+    /\.\.\.$/,
+    /^\s*$/,
+    /^file manager$/i,
+    /^personal website$/i,
+];
+function cleanAppName(app) {
+    const cleaned = app.trim();
+    if (cleaned.length < 2 || cleaned.length > 50)
+        return null;
+    for (const pattern of NOISY_APP_PATTERNS) {
+        if (pattern.test(cleaned))
+            return null;
+    }
+    return cleaned;
+}
+function aggregateContextFromObservations(observations) {
+    const appsSet = new Set();
+    const topicsSet = new Set();
+    for (const obs of observations) {
+        if (obs.apps) {
+            try {
+                const appsArr = JSON.parse(obs.apps);
+                for (const app of appsArr) {
+                    const cleaned = cleanAppName(app);
+                    if (cleaned)
+                        appsSet.add(cleaned);
+                }
+            }
+            catch {
+                // Invalid JSON, skip
+            }
+        }
+        if (obs.topics) {
+            try {
+                const topicsArr = JSON.parse(obs.topics);
+                for (const topic of topicsArr) {
+                    const cleaned = topic.trim();
+                    if (cleaned && cleaned.length >= 2 && cleaned.length <= 50) {
+                        topicsSet.add(cleaned);
+                    }
+                }
+            }
+            catch {
+                // Invalid JSON, skip
+            }
+        }
+    }
+    return {
+        apps: [...appsSet].sort(),
+        topics: [...topicsSet].sort(),
+    };
+}
+/**
+ * Group consecutive observations by activity type.
+ *
+ * @param observations - Visual observations with VLM descriptions, sorted by timestamp
+ * @param config - Segmentation configuration
+ * @returns Array of segments grouped by activity continuity
+ */
+export function segmentByActivity(observations, config = {}) {
+    const cfg = { ...DEFAULT_CONFIG, ...config };
+    // Filter to visual observations only, sorted by timestamp
+    const visualObs = observations
+        .filter((o) => o.type === 'visual' && o.vlm_description)
+        .sort((a, b) => a.timestamp - b.timestamp);
+    if (visualObs.length === 0) {
+        return [];
+    }
+    // Group consecutive observations by activity type
+    const rawSegments = [];
+    let currentSegment = null;
+    for (const obs of visualObs) {
+        const rawActivity = obs.activity_type || extractActivityType(obs.vlm_description);
+        const activityType = normalizeActivity(rawActivity);
+        if (!currentSegment || currentSegment.activityType !== activityType) {
+            // Start new segment
+            if (currentSegment) {
+                rawSegments.push(currentSegment);
+            }
+            currentSegment = {
+                activityType,
+                startTime: obs.timestamp,
+                endTime: obs.end_timestamp ?? obs.timestamp,
+                observations: [obs],
+            };
+        }
+        else {
+            // Continue current segment
+            currentSegment.endTime = obs.end_timestamp ?? obs.timestamp;
+            currentSegment.observations.push(obs);
+        }
+    }
+    // Don't forget the last segment
+    if (currentSegment) {
+        rawSegments.push(currentSegment);
+    }
+    // Merge short segments into their longest neighbor
+    const mergedSegments = mergeShortSegments(rawSegments, cfg.minSegmentDuration);
+    // Convert to final Segment format
+    return mergedSegments.map((seg, index) => {
+        const aggregatedContext = aggregateContextFromObservations(seg.observations);
+        return {
+            id: `seg-${index}`,
+            recordingId: seg.observations[0]?.recording_id || '',
+            activityType: seg.activityType,
+            startTime: seg.startTime,
+            endTime: seg.endTime,
+            duration: seg.endTime - seg.startTime,
+            observationIds: seg.observations.map((o) => o.id),
+            keyDescription: seg.observations[0]?.vlm_description || '',
+            apps: aggregatedContext.apps,
+            topics: aggregatedContext.topics,
+        };
+    });
+}
+/**
+ * Merge segments shorter than minDuration into their longest neighbor.
+ *
+ * Strategy:
+ * 1. For each short segment, find the longer of (previous, next) neighbor
+ * 2. Merge into that neighbor (concatenate observations, extend time range)
+ * 3. If no neighbors exist (only segment), keep it as-is
+ */
+function mergeShortSegments(segments, minDuration) {
+    if (segments.length <= 1) {
+        return segments;
+    }
+    const result = [...segments];
+    let i = 0;
+    while (i < result.length) {
+        const seg = result[i];
+        const duration = seg.endTime - seg.startTime;
+        if (duration >= minDuration) {
+            // Segment is long enough, keep it
+            i++;
+            continue;
+        }
+        // Find neighbors
+        const prev = i > 0 ? result[i - 1] : null;
+        const next = i < result.length - 1 ? result[i + 1] : null;
+        if (!prev && !next) {
+            // Only segment, keep it
+            i++;
+            continue;
+        }
+        // Choose longer neighbor (never choose null)
+        let targetIndex;
+        if (!prev) {
+            targetIndex = i + 1;
+        }
+        else if (!next) {
+            targetIndex = i - 1;
+        }
+        else {
+            const prevDuration = prev.endTime - prev.startTime;
+            const nextDuration = next.endTime - next.startTime;
+            targetIndex = prevDuration >= nextDuration ? i - 1 : i + 1;
+        }
+        const target = result[targetIndex];
+        // Merge into target
+        target.observations = target.observations.concat(seg.observations);
+        target.startTime = Math.min(target.startTime, seg.startTime);
+        target.endTime = Math.max(target.endTime, seg.endTime);
+        // Remove short segment
+        result.splice(i, 1);
+        // If we merged into previous, stay at same index (since we removed current)
+        // If we merged into next, stay at same index
+        // No need to adjust i since we removed the current element
+    }
+    return result;
+}
+/**
+ * Get statistics about segments.
+ */
+export function getSegmentStats(segments) {
+    const activityTypeCounts = {};
+    let totalDuration = 0;
+    for (const seg of segments) {
+        activityTypeCounts[seg.activityType] =
+            (activityTypeCounts[seg.activityType] || 0) + 1;
+        totalDuration += seg.duration;
+    }
+    return {
+        totalSegments: segments.length,
+        totalDuration,
+        activityTypeCounts,
+        avgSegmentDuration: segments.length > 0 ? totalDuration / segments.length : 0,
+    };
+}

package/dist/services/activity-segmentation.test.js

@@ -0,0 +1,191 @@
+/**
+ * Tests for Activity Segmentation Service
+ */
+import { describe, expect, it } from 'vitest';
+import { getSegmentStats, segmentByActivity, } from './activity-segmentation.js';
+describe('segmentByActivity', () => {
+    const createObservation = (id, timestamp, vlmDescription, type = 'visual', duration = 10) => ({
+        id,
+        recording_id: 'test-recording',
+        type,
+        timestamp,
+        end_timestamp: timestamp + duration,
+        image_path: type === 'visual' ? `/path/${id}.png` : null,
+        ocr_text: null,
+        vlm_description: vlmDescription,
+        vlm_raw_response: null,
+        activity_type: null,
+        apps: null,
+        topics: null,
+        text: null,
+        audio_source: null,
+        audio_type: null,
+        embedding: null,
+        created_at: new Date().toISOString(),
+    });
+    it('should group consecutive same-activity observations', () => {
+        // Create longer observations (60s each) to avoid merging
+        const observations = [
+            createObservation('1', 0, 'Debugging Python error in VSCode', 'visual', 60),
+            createObservation('2', 60, 'Debugging stack trace in terminal', 'visual', 60),
+            createObservation('3', 120, 'Debugging the issue', 'visual', 60),
+            createObservation('4', 180, 'Writing code in VSCode', 'visual', 60),
+            createObservation('5', 240, 'Implementing new feature', 'visual', 60),
+            createObservation('6', 300, 'Reading documentation in Chrome', 'visual', 60),
+        ];
+        const segments = segmentByActivity(observations);
+        expect(segments).toHaveLength(3);
+        expect(segments[0].activityType).toBe('debugging');
+        expect(segments[0].observationIds).toEqual(['1', '2', '3']);
+        expect(segments[1].activityType).toBe('coding');
+        expect(segments[1].observationIds).toEqual(['4', '5']);
+        expect(segments[2].activityType).toBe('reading');
+        expect(segments[2].observationIds).toEqual(['6']);
+    });
+    it('should handle single observation segments', () => {
+        const observations = [
+            createObservation('1', 0, 'Debugging Python error'),
+            createObservation('2', 10, 'Writing new function'),
+            createObservation('3', 20, 'Reading documentation'),
+        ];
+        // Disable merging to test grouping logic directly
+        const segments = segmentByActivity(observations, { minSegmentDuration: 0 });
+        expect(segments).toHaveLength(3);
+        segments.forEach((seg, i) => {
+            expect(seg.observationIds).toHaveLength(1);
+            expect(seg.observationIds[0]).toBe(String(i + 1));
+        });
+    });
+    it('should merge short segments into longest neighbor', () => {
+        // Create a scenario where middle segment is too short
+        // First segment: 60s, Middle: 10s (will be merged), Last: 50s
+        const observations = [
+            createObservation('1', 0, 'Debugging error 1', 'visual', 10),
+            createObservation('2', 10, 'Debugging error 2', 'visual', 10),
+            createObservation('3', 20, 'Debugging error 3', 'visual', 10),
+            createObservation('4', 30, 'Debugging error 4', 'visual', 10),
+            createObservation('5', 40, 'Debugging error 5', 'visual', 10),
+            createObservation('6', 50, 'Debugging error 6', 'visual', 10),
+            // Short coding segment (10s) - will be merged into first debugging segment
+            createObservation('7', 60, 'Implementing new feature in VSCode', 'visual', 10),
+            // Second debugging segment is shorter (20s), should also be merged
+            createObservation('8', 70, 'Debugging more', 'visual', 10),
+            createObservation('9', 80, 'Debugging final', 'visual', 10),
+        ];
+        const segments = segmentByActivity(observations, {
+            minSegmentDuration: 30,
+        });
+        // The coding segment (60-70s) is only 10s
+        // After merging it into first debugging (now 70s), the second debugging (20s)
+        // The coding segment (60-70s) is only 10s
+        // After merging it into first debugging (now 70s), the second debugging (20s)
+        // becomes the new shortest and gets merged too
+        expect(segments).toHaveLength(1);
+        expect(segments[0].activityType).toBe('debugging');
+        // Should include all observations including the coding one
+        expect(segments[0].observationIds).toHaveLength(9);
+    });
+    it('should extract apps and topics from observation fields', () => {
+        const obs = createObservation('1', 0, 'Debugging Python error in VSCode, working on escribano project');
+        obs.apps = JSON.stringify(['vscode']);
+        obs.topics = JSON.stringify(['escribano']);
+        const segments = segmentByActivity([obs]);
+        expect(segments[0].apps).toContain('vscode');
+        expect(segments[0].topics).toContain('escribano');
+    });
+    it('should filter out audio observations', () => {
+        const observations = [
+            createObservation('1', 0, 'Debugging error', 'visual'),
+            createObservation('2', 10, 'This is audio transcript', 'audio'),
+            createObservation('3', 20, 'Writing code', 'visual'),
+        ];
+        // Disable merging to test filtering logic
+        const segments = segmentByActivity(observations, { minSegmentDuration: 0 });
+        expect(segments).toHaveLength(2);
+        expect(segments[0].activityType).toBe('debugging');
+        expect(segments[1].activityType).toBe('coding');
+    });
+    it('should handle observations without VLM descriptions', () => {
+        const observations = [
+            createObservation('1', 0, 'Debugging error', 'visual', 60),
+            // null observation is filtered out entirely
+            createObservation('2', 60, null, 'visual', 60),
+            createObservation('3', 120, 'Debugging more', 'visual', 60),
+        ];
+        // With longer durations and minSegmentDuration: 0, we get 2 debugging segments
+        // (filtered out null creates a gap, not a segment)
+        const segments = segmentByActivity(observations, { minSegmentDuration: 0 });
+        // null observations are filtered out, so we only have the 2 debugging observations
+        expect(segments.length).toBeGreaterThanOrEqual(1);
+        expect(segments[0].activityType).toBe('debugging');
+    });
+    it('should sort observations by timestamp', () => {
+        const observations = [
+            createObservation('1', 30, 'Reading documentation'),
+            createObservation('2', 0, 'Debugging error'),
+            createObservation('3', 10, 'Writing code'),
+        ];
+        // Disable merging to test sorting
+        const segments = segmentByActivity(observations, { minSegmentDuration: 0 });
+        expect(segments[0].startTime).toBe(0);
+        expect(segments[0].activityType).toBe('debugging');
+        expect(segments[1].startTime).toBe(10);
+        expect(segments[1].activityType).toBe('coding');
+        expect(segments[2].startTime).toBe(30);
+        expect(segments[2].activityType).toBe('reading');
+    });
+    it('should return empty array for empty observations', () => {
+        const segments = segmentByActivity([]);
+        expect(segments).toEqual([]);
+    });
+    it('should handle all other activities', () => {
+        const observations = [
+            createObservation('1', 0, 'Browsing Stack Overflow for solutions'),
+            createObservation('2', 10, 'In Zoom meeting discussing project'),
+            createObservation('3', 20, 'Reviewing pull request in GitHub'),
+            createObservation('4', 30, 'Some generic unknown activity'),
+        ];
+        // Disable merging to test activity detection
+        const segments = segmentByActivity(observations, { minSegmentDuration: 0 });
+        expect(segments).toHaveLength(4);
+        expect(segments[0].activityType).toBe('research');
+        expect(segments[1].activityType).toBe('meeting');
+        expect(segments[2].activityType).toBe('review');
+        expect(segments[3].activityType).toBe('other');
+    });
+});
+describe('getSegmentStats', () => {
+    const createMockSegment = (activityType, duration) => ({
+        id: `seg-${activityType}`,
+        recordingId: 'test',
+        activityType,
+        startTime: 0,
+        endTime: duration,
+        duration,
+        observationIds: [],
+        keyDescription: '',
+        apps: [],
+        topics: [],
+    });
+    it('should calculate correct statistics', () => {
+        const segments = [
+            createMockSegment('debugging', 60),
+            createMockSegment('coding', 120),
+            createMockSegment('debugging', 45),
+        ];
+        const stats = getSegmentStats(segments);
+        expect(stats.totalSegments).toBe(3);
+        expect(stats.totalDuration).toBe(225);
+        expect(stats.avgSegmentDuration).toBe(75);
+        expect(stats.activityTypeCounts).toEqual({
+            debugging: 2,
+            coding: 1,
+        });
+    });
+    it('should handle empty segments', () => {
+        const stats = getSegmentStats([]);
+        expect(stats.totalSegments).toBe(0);
+        expect(stats.totalDuration).toBe(0);
+        expect(stats.avgSegmentDuration).toBe(0);
+    });
+});