escribano 0.1.0

package/dist/services/app-normalization.js

@@ -0,0 +1,212 @@
+/**
+ * Escribano - App Name Normalization Service
+ *
+ * Normalizes and deduplicates app names extracted from VLM descriptions.
+ * Uses fuzzy matching and known alias maps to produce consistent app names.
+ */
+const KNOWN_ALIASES = {
+    ghosty: 'Ghostty',
+    ghosttie: 'Ghostty',
+    'ghostty terminal': 'Ghostty',
+    iterm: 'iTerm',
+    iterm2: 'iTerm',
+    'iterm 2': 'iTerm',
+    'vs code': 'VSCode',
+    'visual studio code': 'VSCode',
+    vscode: 'VSCode',
+    'visual studio': 'VSCode',
+    chrome: 'Google Chrome',
+    'google chrome': 'Google Chrome',
+    safari: 'Safari',
+    firefox: 'Firefox',
+    slack: 'Slack',
+    zoom: 'Zoom',
+    'google meet': 'Google Meet',
+    teams: 'Microsoft Teams',
+    'microsoft teams': 'Microsoft Teams',
+    discord: 'Discord',
+    whatsapp: 'WhatsApp',
+    telegram: 'Telegram',
+    notion: 'Notion',
+    figma: 'Figma',
+    github: 'GitHub',
+    gitlab: 'GitLab',
+    bitbucket: 'Bitbucket',
+    terminal: 'Terminal',
+    finder: 'Finder',
+    mail: 'Mail',
+    gmail: 'Gmail',
+    calendar: 'Calendar',
+    notes: 'Notes',
+    spotify: 'Spotify',
+    music: 'Music',
+    photos: 'Photos',
+    preview: 'Preview',
+    quicktime: 'QuickTime Player',
+    'quicktime player': 'QuickTime Player',
+    'activity monitor': 'Activity Monitor',
+    'system preferences': 'System Preferences',
+    settings: 'System Settings',
+    'system settings': 'System Settings',
+    tableplus: 'TablePlus',
+    postgres: 'PostgreSQL',
+    postgresql: 'PostgreSQL',
+    sqlite: 'SQLite',
+    sqlitebrowser: 'SQLite',
+};
+const NOISY_APP_NAMES = new Set([
+    'a',
+    'the',
+    'and',
+    'or',
+    'in',
+    'on',
+    'at',
+    'to',
+    'for',
+    'of',
+    'with',
+    'unknown',
+    'unidentified',
+    'application',
+    'app',
+    'program',
+    'software',
+    'window',
+    'screen',
+    'desktop',
+    'mac os',
+    'macos',
+    'os x',
+    'operating system',
+]);
+function levenshteinDistance(a, b) {
+    const matrix = [];
+    for (let i = 0; i <= b.length; i++) {
+        matrix[i] = [i];
+    }
+    for (let j = 0; j <= a.length; j++) {
+        matrix[0][j] = j;
+    }
+    for (let i = 1; i <= b.length; i++) {
+        for (let j = 1; j <= a.length; j++) {
+            if (b.charAt(i - 1) === a.charAt(j - 1)) {
+                matrix[i][j] = matrix[i - 1][j - 1];
+            }
+            else {
+                matrix[i][j] = Math.min(matrix[i - 1][j - 1] + 1, matrix[i][j - 1] + 1, matrix[i - 1][j] + 1);
+            }
+        }
+    }
+    return matrix[b.length][a.length];
+}
+function normalizeAppName(name) {
+    const normalized = name
+        .trim()
+        .toLowerCase()
+        .replace(/[^a-z0-9\s]/g, '')
+        .replace(/\s+/g, ' ')
+        .trim();
+    if (KNOWN_ALIASES[normalized]) {
+        return KNOWN_ALIASES[normalized];
+    }
+    for (const [alias, canonical] of Object.entries(KNOWN_ALIASES)) {
+        if (normalized.includes(alias)) {
+            return canonical;
+        }
+    }
+    return name.trim();
+}
+function isNoisyAppName(name) {
+    const normalized = name.toLowerCase().trim();
+    if (NOISY_APP_NAMES.has(normalized))
+        return true;
+    if (normalized.length < 2)
+        return true;
+    if (normalized.length > 50)
+        return true;
+    return false;
+}
+function fuzzyMatchAppName(name, knownApps) {
+    const normalized = normalizeAppName(name);
+    if (isNoisyAppName(normalized))
+        return null;
+    for (const known of knownApps) {
+        const distance = levenshteinDistance(normalized.toLowerCase(), known.toLowerCase());
+        const maxLen = Math.max(normalized.length, known.length);
+        const similarity = 1 - distance / maxLen;
+        if (similarity >= 0.85) {
+            return known;
+        }
+    }
+    return null;
+}
+export function normalizeAppNames(apps) {
+    if (apps.length === 0)
+        return [];
+    const normalizedApps = [];
+    const knownApps = new Set();
+    const sortedApps = [...apps].sort((a, b) => a.length - b.length);
+    for (const app of sortedApps) {
+        const trimmed = app.trim();
+        if (!trimmed)
+            continue;
+        let normalized = normalizeAppName(trimmed);
+        if (isNoisyAppName(normalized))
+            continue;
+        const fuzzyMatch = fuzzyMatchAppName(normalized, knownApps);
+        if (fuzzyMatch) {
+            normalized = fuzzyMatch;
+        }
+        else {
+            knownApps.add(normalized);
+        }
+        if (!normalizedApps.includes(normalized)) {
+            normalizedApps.push(normalized);
+        }
+    }
+    return normalizedApps.sort();
+}
+export function normalizeAppNamesInRecord(record) {
+    return {
+        ...record,
+        apps: normalizeAppNames(record.apps),
+    };
+}
+export function normalizeAppNamesInRecords(records) {
+    const allApps = records.flatMap((r) => r.apps);
+    const globalNormalized = normalizeAppNames(allApps);
+    const appMapping = new Map();
+    for (const app of allApps) {
+        const normalized = normalizeAppName(app);
+        const fuzzyMatch = fuzzyMatchAppName(normalized, new Set(globalNormalized));
+        appMapping.set(app, fuzzyMatch || normalized);
+    }
+    return records.map((record) => ({
+        ...record,
+        apps: record.apps
+            .map((app) => appMapping.get(app) || normalizeAppName(app))
+            .filter((app) => !isNoisyAppName(app))
+            .filter((app, index, arr) => arr.indexOf(app) === index)
+            .sort(),
+    }));
+}
+export function isPersonalApp(app) {
+    const normalized = app.toLowerCase().trim();
+    const personalApps = [
+        'whatsapp',
+        'instagram',
+        'tiktok',
+        'telegram',
+        'facebook',
+        'twitter',
+        'snapchat',
+        'discord',
+        'messenger',
+        'signal',
+        'facetime',
+        'imessage',
+        'messages',
+    ];
+    return personalApps.some((personal) => normalized.includes(personal));
+}

package/dist/services/cluster-merge.js

@@ -0,0 +1,69 @@
+/**
+ * Escribano - Cluster Merge Service
+ *
+ * Merges audio clusters with visual clusters based on classification similarity.
+ * Many-to-many: one audio cluster can merge with multiple visual clusters.
+ */
+const MERGE_THRESHOLD = 0.6; // Minimum similarity for merge
+/**
+ * Find all valid merges between visual and audio clusters.
+ * Returns many-to-many: each audio cluster can merge with multiple visual clusters.
+ */
+export function findClusterMerges(visualClusters, audioClusters, embeddingService) {
+    const merges = [];
+    for (const audio of audioClusters) {
+        for (const visual of visualClusters) {
+            const result = computeMerge(visual, audio, embeddingService);
+            if (result) {
+                merges.push(result);
+            }
+        }
+    }
+    return merges;
+}
+function computeMerge(visual, audio, embeddingService) {
+    // Check shared topics
+    const sharedTopics = visual.signals.topics.filter((t) => audio.signals.topics.some((at) => t.toLowerCase().includes(at.toLowerCase()) ||
+        at.toLowerCase().includes(t.toLowerCase())));
+    if (sharedTopics.length > 0) {
+        return {
+            visualClusterId: visual.cluster.id,
+            audioClusterId: audio.cluster.id,
+            similarityScore: 1.0,
+            mergeReason: 'shared_topic',
+        };
+    }
+    // Check shared apps
+    const sharedApps = visual.signals.apps.filter((a) => audio.signals.apps.includes(a));
+    if (sharedApps.length > 0) {
+        return {
+            visualClusterId: visual.cluster.id,
+            audioClusterId: audio.cluster.id,
+            similarityScore: 0.9,
+            mergeReason: 'shared_app',
+        };
+    }
+    // Check shared projects
+    const sharedProjects = visual.signals.projects.filter((p) => audio.signals.projects.includes(p));
+    if (sharedProjects.length > 0) {
+        return {
+            visualClusterId: visual.cluster.id,
+            audioClusterId: audio.cluster.id,
+            similarityScore: 0.85,
+            mergeReason: 'shared_project',
+        };
+    }
+    // Fallback: centroid similarity
+    if (visual.centroid.length > 0 && audio.centroid.length > 0) {
+        const similarity = embeddingService.similarity(visual.centroid, audio.centroid);
+        if (similarity >= MERGE_THRESHOLD) {
+            return {
+                visualClusterId: visual.cluster.id,
+                audioClusterId: audio.cluster.id,
+                similarityScore: similarity,
+                mergeReason: 'centroid_similarity',
+            };
+        }
+    }
+    return null;
+}

package/dist/services/clustering.js

@@ -0,0 +1,237 @@
+/**
+ * Escribano - Semantic Clustering Service
+ *
+ * ═══════════════════════════════════════════════════════════════════════════════
+ * ALGORITHM: Agglomerative Hierarchical Clustering with Time Constraints
+ * ═══════════════════════════════════════════════════════════════════════════════
+ *
+ * WHAT IT DOES:
+ * Groups observations into semantic clusters based on embedding similarity,
+ * while respecting temporal constraints (observations far apart in time
+ * shouldn't cluster together even if semantically similar).
+ *
+ * WHY AGGLOMERATIVE:
+ * - No need to specify number of clusters upfront (unlike K-means)
+ * - Natural hierarchical structure matches how work sessions evolve
+ * - Can stop at any similarity threshold
+ *
+ * HOW IT WORKS:
+ *
+ * 1. INITIALIZATION
+ *    - Start with N clusters, each containing one observation
+ *    - Pre-compute all pairwise distances (1 - cosine_similarity)
+ *    - Apply time constraint: if |timestamp_i - timestamp_j| > timeWindow,
+ *      set distance to Infinity (can never merge)
+ *
+ * 2. ITERATIVE MERGING
+ *    - Find the two closest clusters (single-linkage: min distance between any pair)
+ *    - If closest distance > threshold → STOP (clusters are distinct enough)
+ *    - Otherwise, merge them into one cluster
+ *    - Repeat until no more merges possible
+ *
+ * 3. POST-PROCESSING
+ *    - Small clusters (< minSize) are merged with their nearest neighbor
+ *    - Prevents fragmentation from noise or outliers
+ *
+ * EXAMPLE:
+ *
+ *   Input: 10 observations with embeddings
+ *
+ *   Step 1: [1] [2] [3] [4] [5] [6] [7] [8] [9] [10]  (10 clusters)
+ *   Step 2: [1,2] [3] [4] [5] [6] [7] [8] [9] [10]    (obs 1 & 2 merged)
+ *   Step 3: [1,2] [3,4] [5] [6] [7] [8] [9] [10]      (obs 3 & 4 merged)
+ *   ...
+ *   Final:  [1,2,3,4] [5,6,7] [8,9,10]                (3 clusters)
+ *
+ * TIME CONSTRAINT VISUALIZATION:
+ *
+ *   Time:    0min ──────────────────────────── 60min
+ *   Obs:     ●●●●●         ●●●●           ●●●●●●
+ *            └─────┘       └────┘         └──────┘
+ *            Cluster A     Cluster B      Cluster C
+ *
+ *   Even if B is semantically similar to A, they won't merge if
+ *   the time gap exceeds timeWindowSeconds.
+ *
+ * ═══════════════════════════════════════════════════════════════════════════════
+ */
+const DEFAULT_CONFIG = {
+    timeWindowSeconds: 600, // 10 minutes
+    distanceThreshold: 0.4, // 0.6 similarity threshold
+    minClusterSize: 3,
+};
+/**
+ * Main clustering function.
+ *
+ * @param observations - Observations to cluster (must have embeddings)
+ * @param embeddingService - Service for computing similarity
+ * @param config - Clustering parameters
+ * @returns Array of clusters, sorted by start timestamp
+ */
+export function clusterObservations(observations, embeddingService, config = {}) {
+    const cfg = { ...DEFAULT_CONFIG, ...config };
+    // Filter to observations with valid embeddings
+    const validObs = observations.filter((obs) => obs.embedding?.length);
+    if (validObs.length === 0)
+        return [];
+    // Parse embeddings from Buffer format
+    const embeddings = validObs.map((obs) => {
+        if (!obs.embedding) {
+            throw new Error(`Observation ${obs.id} has no embedding`);
+        }
+        return bufferToEmbedding(obs.embedding);
+    });
+    // STEP 1: Initialize - each observation is its own cluster
+    // Clusters are represented as arrays of indices into validObs
+    let clusters = validObs.map((_, index) => [index]);
+    // STEP 2: Pre-compute distance matrix with time constraints
+    const distances = computeDistanceMatrix(validObs, embeddings, embeddingService, cfg.timeWindowSeconds);
+    // STEP 3: Agglomerative merging
+    // Keep merging until no clusters are close enough
+    let mergeCount = 0;
+    const maxMerges = validObs.length; // Safety limit
+    while (mergeCount < maxMerges) {
+        const closest = findClosestClusterPair(clusters, distances);
+        // Exit condition: no clusters are close enough to merge
+        if (closest.distance > cfg.distanceThreshold) {
+            break;
+        }
+        // Merge the two closest clusters
+        clusters = mergeClusters(clusters, closest.indexA, closest.indexB);
+        mergeCount++;
+    }
+    // STEP 4: Post-process - absorb small clusters
+    clusters = absorbSmallClusters(clusters, distances, cfg.minClusterSize);
+    // STEP 5: Build result objects
+    return clusters
+        .map((indices) => buildClusterResult(indices, validObs, embeddings, embeddingService))
+        .sort((a, b) => a.startTimestamp - b.startTimestamp);
+}
+// ═══════════════════════════════════════════════════════════════════════════════
+// HELPER FUNCTIONS
+// ═══════════════════════════════════════════════════════════════════════════════
+/**
+ * Convert SQLite BLOB buffer to number array.
+ */
+function bufferToEmbedding(buffer) {
+    const float32 = new Float32Array(buffer.buffer, buffer.byteOffset, buffer.length / 4);
+    return Array.from(float32);
+}
+/**
+ * Compute NxN distance matrix.
+ * Distance = 1 - cosine_similarity (so 0 = identical, 1 = orthogonal)
+ * Time-violating pairs get Infinity distance.
+ */
+function computeDistanceMatrix(observations, embeddings, embeddingService, timeWindowSeconds) {
+    const n = observations.length;
+    // Initialize with Infinity (no connection)
+    const matrix = Array.from({ length: n }, () => Array.from({ length: n }, () => Infinity));
+    // Fill in distances for valid pairs
+    for (const [i, obsA] of observations.entries()) {
+        matrix[i][i] = 0; // Self-distance is 0
+        for (const [j, obsB] of observations.entries()) {
+            if (j <= i)
+                continue; // Only compute upper triangle
+            // Time constraint check
+            const timeDiff = Math.abs(obsA.timestamp - obsB.timestamp);
+            if (timeDiff > timeWindowSeconds) {
+                continue; // Leave as Infinity
+            }
+            // Compute semantic distance
+            const similarity = embeddingService.similarity(embeddings[i], embeddings[j]);
+            const distance = 1 - similarity;
+            // Symmetric matrix
+            matrix[i][j] = distance;
+            matrix[j][i] = distance;
+        }
+    }
+    return matrix;
+}
+/**
+ * Find the two clusters with minimum distance (single-linkage).
+ * Single-linkage = minimum distance between ANY pair of points from each cluster.
+ */
+function findClosestClusterPair(clusters, distances) {
+    let minDistance = Infinity;
+    let bestA = -1;
+    let bestB = -1;
+    for (const [i, clusterA] of clusters.entries()) {
+        for (const [j, clusterB] of clusters.entries()) {
+            if (j <= i)
+                continue; // Only check each pair once
+            const pairDistance = computeClusterDistance(clusterA, clusterB, distances);
+            if (pairDistance < minDistance) {
+                minDistance = pairDistance;
+                bestA = i;
+                bestB = j;
+            }
+        }
+    }
+    return { indexA: bestA, indexB: bestB, distance: minDistance };
+}
+/**
+ * Single-linkage distance between two clusters.
+ * Returns the minimum distance between any observation in A and any in B.
+ */
+function computeClusterDistance(clusterA, clusterB, distances) {
+    let minDist = Infinity;
+    for (const i of clusterA) {
+        for (const j of clusterB) {
+            if (distances[i][j] < minDist) {
+                minDist = distances[i][j];
+            }
+        }
+    }
+    return minDist;
+}
+/**
+ * Merge two clusters by combining their observation indices.
+ * Returns new cluster array with merged result.
+ */
+function mergeClusters(clusters, indexA, indexB) {
+    // Ensure indexA < indexB for consistent splicing
+    const [smaller, larger] = indexA < indexB ? [indexA, indexB] : [indexB, indexA];
+    const merged = [...clusters[smaller], ...clusters[larger]];
+    return clusters
+        .filter((_, index) => index !== smaller && index !== larger)
+        .concat([merged]);
+}
+/**
+ * Absorb clusters smaller than minSize into their nearest neighbor.
+ */
+function absorbSmallClusters(clusters, distances, minSize) {
+    const large = clusters.filter((c) => c.length >= minSize);
+    const small = clusters.filter((c) => c.length < minSize);
+    if (large.length === 0) {
+        // All clusters are small - just return them
+        return clusters;
+    }
+    // Merge each small cluster into nearest large cluster
+    for (const smallCluster of small) {
+        let nearestIndex = 0;
+        let nearestDistance = Infinity;
+        for (const [index, largeCluster] of large.entries()) {
+            const dist = computeClusterDistance(smallCluster, largeCluster, distances);
+            if (dist < nearestDistance) {
+                nearestDistance = dist;
+                nearestIndex = index;
+            }
+        }
+        large[nearestIndex] = [...large[nearestIndex], ...smallCluster];
+    }
+    return large;
+}
+/**
+ * Build a ClusterResult from observation indices.
+ */
+function buildClusterResult(indices, observations, embeddings, embeddingService) {
+    const clusterObs = indices.map((i) => observations[i]);
+    const clusterEmbeddings = indices.map((i) => embeddings[i]);
+    return {
+        clusterId: `cluster-${Date.now()}`, // Placeholder, replaced with UUIDv7 later
+        observations: clusterObs,
+        centroid: embeddingService.centroid(clusterEmbeddings),
+        startTimestamp: Math.min(...clusterObs.map((o) => o.timestamp)),
+        endTimestamp: Math.max(...clusterObs.map((o) => o.end_timestamp ?? o.timestamp)),
+    };
+}

package/dist/services/debug.js

@@ -0,0 +1,58 @@
+/**
+ * Escribano - Debug Utilities
+ * @deprecated No longer needed - all data is stored in the database.
+ * Kept for backward compatibility only.
+ *
+ * Utilities for saving debug artifacts (VLM responses, frame copies) during processing.
+ */
+import { copyFile, mkdir, writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import path, { dirname } from 'node:path';
+const DEBUG_ENABLED = process.env.ESCRIBANO_DEBUG_VLM === 'true';
+const DEBUG_DIR = path.join(homedir(), '.escribano', 'debug');
+/**
+ * Initialize debug directory for a recording.
+ * @deprecated
+ */
+export async function initDebugDir(recordingId) {
+    if (!DEBUG_ENABLED)
+        return '';
+    const debugPath = path.join(DEBUG_DIR, recordingId);
+    const responsesPath = path.join(debugPath, 'vlm-responses');
+    const framesPath = path.join(debugPath, 'frames');
+    await mkdir(responsesPath, { recursive: true });
+    await mkdir(framesPath, { recursive: true });
+    return debugPath;
+}
+/**
+ * Save a VLM response to disk.
+ * @deprecated
+ */
+export async function saveVlmResponse(recordingId, batchIndex, response) {
+    if (!DEBUG_ENABLED)
+        return;
+    const filePath = path.join(DEBUG_DIR, recordingId, 'vlm-responses', `batch-${String(batchIndex).padStart(3, '0')}-response.json`);
+    // Create parent directories if they don't exist
+    await mkdir(dirname(filePath), { recursive: true });
+    await writeFile(filePath, JSON.stringify(response, null, 2), 'utf-8');
+}
+/**
+ * Copy sampled frames to debug directory with batch naming.
+ * @deprecated
+ */
+export async function copyFramesForDebug(recordingId, batchIndex, frames) {
+    if (!DEBUG_ENABLED)
+        return;
+    const batchFramesDir = path.join(DEBUG_DIR, recordingId, 'frames', `batch-${String(batchIndex).padStart(3, '0')}`);
+    await mkdir(batchFramesDir, { recursive: true });
+    for (const frame of frames) {
+        const destFileName = `frame-${String(frame.index).padStart(3, '0')}-t${frame.timestamp.toFixed(1)}.jpg`;
+        const destPath = path.join(batchFramesDir, destFileName);
+        try {
+            await copyFile(frame.imagePath, destPath);
+        }
+        catch (error) {
+            console.warn(`[Debug] Failed to copy frame ${frame.imagePath}:`, error.message);
+        }
+    }
+}