@siftd/connect-agent 0.2.29 → 0.2.31

package/dist/core/assets.js

@@ -0,0 +1,192 @@
+/**
+ * Asset System - Fast-path preview architecture
+ *
+ * Assets are files produced by workers with metadata for instant preview.
+ * The preview-worker generates thumbnails/metadata WITHOUT using the LLM.
+ */
+import { createHash } from 'crypto';
+import { readFileSync, writeFileSync, existsSync, mkdirSync, statSync } from 'fs';
+import { basename, extname, join } from 'path';
+// ============================================================================
+// UTILITY FUNCTIONS
+// ============================================================================
+/**
+ * Generate SHA256 hash of file contents
+ */
+export function hashFile(filePath) {
+    const content = readFileSync(filePath);
+    return createHash('sha256').update(content).digest('hex');
+}
+/**
+ * Get MIME type from file extension
+ */
+export function getMimeType(filePath) {
+    const ext = extname(filePath).toLowerCase().slice(1);
+    const mimeMap = {
+        // Images
+        png: 'image/png',
+        jpg: 'image/jpeg',
+        jpeg: 'image/jpeg',
+        gif: 'image/gif',
+        svg: 'image/svg+xml',
+        webp: 'image/webp',
+        ico: 'image/x-icon',
+        // Documents
+        pdf: 'application/pdf',
+        doc: 'application/msword',
+        docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        // Code/Text
+        js: 'text/javascript',
+        ts: 'text/typescript',
+        jsx: 'text/javascript',
+        tsx: 'text/typescript',
+        json: 'application/json',
+        html: 'text/html',
+        css: 'text/css',
+        md: 'text/markdown',
+        txt: 'text/plain',
+        csv: 'text/csv',
+        xml: 'text/xml',
+        yaml: 'text/yaml',
+        yml: 'text/yaml',
+        sh: 'text/x-shellscript',
+        py: 'text/x-python',
+        // Data
+        sqlite: 'application/x-sqlite3',
+        db: 'application/x-sqlite3',
+        // Video/Audio
+        mp4: 'video/mp4',
+        webm: 'video/webm',
+        mp3: 'audio/mpeg',
+        wav: 'audio/wav',
+    };
+    return mimeMap[ext] || 'application/octet-stream';
+}
+/**
+ * Generate unique asset ID
+ */
+export function generateAssetId() {
+    const timestamp = Date.now().toString(36);
+    const random = Math.random().toString(36).substring(2, 8);
+    return `asset_${timestamp}_${random}`;
+}
+/**
+ * Create asset manifest for a file
+ */
+export function createAssetManifest(filePath, options = {}) {
+    const stats = statSync(filePath);
+    return {
+        id: generateAssetId(),
+        path: filePath,
+        name: basename(filePath),
+        mime: getMimeType(filePath),
+        sha256: hashFile(filePath),
+        size: stats.size,
+        createdAt: Date.now(),
+        groupId: options.groupId,
+        label: options.label,
+        kind: 'file',
+    };
+}
+/**
+ * Write asset manifest alongside the file
+ */
+export function writeAssetManifest(manifest) {
+    const manifestPath = manifest.path + '.asset.json';
+    writeFileSync(manifestPath, JSON.stringify(manifest, null, 2));
+}
+/**
+ * Read asset manifest for a file
+ */
+export function readAssetManifest(filePath) {
+    const manifestPath = filePath + '.asset.json';
+    if (!existsSync(manifestPath))
+        return null;
+    try {
+        return JSON.parse(readFileSync(manifestPath, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Get preview cache directory
+ */
+export function getPreviewCacheDir() {
+    const home = process.env.HOME || '/tmp';
+    const cacheDir = join(home, '.connect-hub', 'preview-cache');
+    if (!existsSync(cacheDir)) {
+        mkdirSync(cacheDir, { recursive: true });
+    }
+    return cacheDir;
+}
+/**
+ * Generate cache key for preview
+ */
+export function getPreviewCacheKey(sha256, params) {
+    const paramsStr = params ? JSON.stringify(params) : '';
+    return createHash('sha256').update(sha256 + paramsStr).digest('hex').substring(0, 16);
+}
+/**
+ * Check if preview exists in cache
+ */
+export function getPreviewFromCache(cacheKey) {
+    const cachePath = join(getPreviewCacheDir(), `${cacheKey}.json`);
+    if (!existsSync(cachePath))
+        return null;
+    try {
+        return JSON.parse(readFileSync(cachePath, 'utf-8'));
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Save preview to cache
+ */
+export function savePreviewToCache(preview) {
+    const cachePath = join(getPreviewCacheDir(), `${preview.cacheKey}.json`);
+    writeFileSync(cachePath, JSON.stringify(preview, null, 2));
+}
+/**
+ * Create a ViewSpec for comparing two assets
+ */
+export function createCompareView(beforeAssetId, afterAssetId, title = 'Comparison') {
+    return {
+        id: `view_${Date.now().toString(36)}`,
+        title,
+        layout: 'compare',
+        before: { type: 'asset', assetId: beforeAssetId, label: 'Before' },
+        after: { type: 'asset', assetId: afterAssetId, label: 'After' },
+        createdAt: Date.now(),
+    };
+}
+/**
+ * Create a ViewSpec for a grid of assets
+ */
+export function createGridView(assetIds, title = 'Gallery') {
+    return {
+        id: `view_${Date.now().toString(36)}`,
+        title,
+        layout: 'grid',
+        items: assetIds.map(id => ({ type: 'asset', assetId: id })),
+        createdAt: Date.now(),
+    };
+}
+/**
+ * Create a ViewSpec for a report with markdown and assets
+ */
+export function createReportView(title, sections) {
+    return {
+        id: `view_${Date.now().toString(36)}`,
+        title,
+        layout: 'stack',
+        items: sections.map(section => {
+            if (section.markdown) {
+                return { type: 'markdown', content: section.markdown, label: section.label };
+            }
+            return { type: 'asset', assetId: section.assetId, label: section.label };
+        }),
+        createdAt: Date.now(),
+    };
+}

package/dist/core/preview-worker.d.ts

@@ -0,0 +1,98 @@
+/**
+ * Preview Worker - Fast-path local preview generation
+ *
+ * Watches ~/Lia-Hub/shared/outputs/ and generates:
+ * - Thumbnails (for images)
+ * - Metadata extraction (dimensions, page count, text snippets)
+ * - Preview JSON for instant gallery display
+ *
+ * NO LLM involvement - pure local, deterministic processing.
+ * Targets sub-second response for most assets.
+ */
+import { AssetManifest, AssetPreview } from './assets.js';
+export interface PreviewWorkerOptions {
+    /** Directories to watch */
+    watchDirs?: string[];
+    /** Max concurrent preview jobs */
+    concurrency?: number;
+    /** Callback when asset is discovered */
+    onAsset?: (manifest: AssetManifest) => void;
+    /** Callback when preview is ready */
+    onPreview?: (preview: AssetPreview) => void;
+    /** Enable verbose logging */
+    verbose?: boolean;
+}
+export declare class PreviewWorker {
+    private watchers;
+    private queue;
+    private processing;
+    private concurrency;
+    private options;
+    private running;
+    constructor(options?: PreviewWorkerOptions);
+    /**
+     * Start watching directories
+     */
+    start(): void;
+    /**
+     * Stop watching
+     */
+    stop(): void;
+    /**
+     * Scan directory for existing files
+     */
+    private scanDirectory;
+    /**
+     * Add file to processing queue
+     */
+    private enqueue;
+    /**
+     * Process queue with concurrency limit
+     */
+    private processQueue;
+    /**
+     * Process a single file - generate manifest and preview
+     */
+    private processFile;
+    /**
+     * Generate preview for an asset
+     */
+    private generatePreview;
+    /**
+     * Get image dimensions using sips (macOS) or file command
+     */
+    private getImageInfo;
+    /**
+     * Generate thumbnail as base64 data URL
+     * Uses sips on macOS for fast resizing
+     */
+    private generateImageThumbnail;
+    /**
+     * Get PDF info using pdfinfo or mdls
+     */
+    private getPdfInfo;
+    /**
+     * Get text snippet from file
+     */
+    private getTextSnippet;
+    /**
+     * Extract group ID from file path
+     * e.g., ~/Lia-Hub/shared/outputs/run_abc123/file.png -> run_abc123
+     */
+    private extractGroupId;
+    private log;
+}
+export interface AssetRegistry {
+    assets: Map<string, AssetManifest>;
+    previews: Map<string, AssetPreview>;
+    byGroup: Map<string, Set<string>>;
+}
+export declare function getAssetRegistry(): AssetRegistry;
+export declare function registerAsset(manifest: AssetManifest): void;
+export declare function registerPreview(preview: AssetPreview): void;
+export declare function getAsset(assetId: string): AssetManifest | undefined;
+export declare function getPreview(assetId: string): AssetPreview | undefined;
+export declare function getAssetsByGroup(groupId: string): AssetManifest[];
+export declare function getAllAssets(): AssetManifest[];
+export declare function startPreviewWorker(options?: PreviewWorkerOptions): PreviewWorker;
+export declare function stopPreviewWorker(): void;

package/dist/core/preview-worker.js

@@ -0,0 +1,395 @@
+/**
+ * Preview Worker - Fast-path local preview generation
+ *
+ * Watches ~/Lia-Hub/shared/outputs/ and generates:
+ * - Thumbnails (for images)
+ * - Metadata extraction (dimensions, page count, text snippets)
+ * - Preview JSON for instant gallery display
+ *
+ * NO LLM involvement - pure local, deterministic processing.
+ * Targets sub-second response for most assets.
+ */
+import { watch } from 'fs';
+import { readFileSync, existsSync, readdirSync, statSync, mkdirSync } from 'fs';
+import { join, basename } from 'path';
+import { execSync } from 'child_process';
+import { createAssetManifest, writeAssetManifest, readAssetManifest, getPreviewCacheDir, getPreviewCacheKey, getPreviewFromCache, savePreviewToCache, } from './assets.js';
+import { getSharedOutputPath } from './hub.js';
+export class PreviewWorker {
+    watchers = [];
+    queue = [];
+    processing = new Set();
+    concurrency;
+    options;
+    running = false;
+    constructor(options = {}) {
+        this.options = options;
+        this.concurrency = options.concurrency || 4;
+    }
+    /**
+     * Start watching directories
+     */
+    start() {
+        if (this.running)
+            return;
+        this.running = true;
+        const dirs = this.options.watchDirs || [getSharedOutputPath()];
+        for (const dir of dirs) {
+            if (!existsSync(dir)) {
+                mkdirSync(dir, { recursive: true });
+            }
+            this.log(`Watching: ${dir}`);
+            // Initial scan
+            this.scanDirectory(dir);
+            // Watch for changes
+            const watcher = watch(dir, { recursive: true }, (event, filename) => {
+                if (!filename)
+                    return;
+                const filePath = join(dir, filename);
+                // Skip manifest files and hidden files
+                if (filename.endsWith('.asset.json') || filename.startsWith('.'))
+                    return;
+                // Skip directories
+                try {
+                    if (statSync(filePath).isDirectory())
+                        return;
+                }
+                catch {
+                    return; // File may have been deleted
+                }
+                this.enqueue(filePath);
+            });
+            this.watchers.push(watcher);
+        }
+        // Start processing queue
+        this.processQueue();
+    }
+    /**
+     * Stop watching
+     */
+    stop() {
+        this.running = false;
+        for (const watcher of this.watchers) {
+            watcher.close();
+        }
+        this.watchers = [];
+    }
+    /**
+     * Scan directory for existing files
+     */
+    scanDirectory(dir) {
+        try {
+            const entries = readdirSync(dir, { withFileTypes: true });
+            for (const entry of entries) {
+                const fullPath = join(dir, entry.name);
+                if (entry.isDirectory()) {
+                    this.scanDirectory(fullPath);
+                }
+                else if (!entry.name.endsWith('.asset.json') && !entry.name.startsWith('.')) {
+                    this.enqueue(fullPath);
+                }
+            }
+        }
+        catch (error) {
+            this.log(`Scan error: ${error}`);
+        }
+    }
+    /**
+     * Add file to processing queue
+     */
+    enqueue(filePath) {
+        if (this.queue.includes(filePath) || this.processing.has(filePath)) {
+            return;
+        }
+        this.queue.push(filePath);
+        this.processQueue();
+    }
+    /**
+     * Process queue with concurrency limit
+     */
+    async processQueue() {
+        while (this.running && this.queue.length > 0 && this.processing.size < this.concurrency) {
+            const filePath = this.queue.shift();
+            if (!filePath)
+                continue;
+            this.processing.add(filePath);
+            this.processFile(filePath)
+                .catch(err => this.log(`Error processing ${filePath}: ${err}`))
+                .finally(() => {
+                this.processing.delete(filePath);
+                this.processQueue();
+            });
+        }
+    }
+    /**
+     * Process a single file - generate manifest and preview
+     */
+    async processFile(filePath) {
+        // Skip if file no longer exists
+        if (!existsSync(filePath))
+            return;
+        const startTime = Date.now();
+        // Check for existing manifest
+        let manifest = readAssetManifest(filePath);
+        if (!manifest) {
+            // Create new manifest
+            manifest = createAssetManifest(filePath, {
+                groupId: this.extractGroupId(filePath),
+            });
+            writeAssetManifest(manifest);
+            this.log(`Created manifest: ${manifest.name} (${manifest.id})`);
+            this.options.onAsset?.(manifest);
+        }
+        // Check preview cache
+        const cacheKey = getPreviewCacheKey(manifest.sha256);
+        let preview = getPreviewFromCache(cacheKey);
+        if (!preview) {
+            // Generate preview
+            preview = await this.generatePreview(manifest, cacheKey);
+            savePreviewToCache(preview);
+            const elapsed = Date.now() - startTime;
+            this.log(`Generated preview: ${manifest.name} (${elapsed}ms)`);
+        }
+        this.options.onPreview?.(preview);
+    }
+    /**
+     * Generate preview for an asset
+     */
+    async generatePreview(manifest, cacheKey) {
+        const preview = {
+            assetId: manifest.id,
+            generatedAt: Date.now(),
+            cacheKey,
+        };
+        const mime = manifest.mime;
+        // Image preview
+        if (mime.startsWith('image/')) {
+            const imageInfo = this.getImageInfo(manifest.path);
+            if (imageInfo) {
+                preview.dimensions = imageInfo.dimensions;
+                preview.thumbnailData = this.generateImageThumbnail(manifest.path);
+            }
+        }
+        // PDF preview
+        else if (mime === 'application/pdf') {
+            const pdfInfo = this.getPdfInfo(manifest.path);
+            if (pdfInfo) {
+                preview.pageCount = pdfInfo.pageCount;
+                preview.textSnippet = pdfInfo.textSnippet;
+            }
+        }
+        // Text/code preview
+        else if (mime.startsWith('text/') || mime === 'application/json') {
+            preview.textSnippet = this.getTextSnippet(manifest.path);
+        }
+        return preview;
+    }
+    /**
+     * Get image dimensions using sips (macOS) or file command
+     */
+    getImageInfo(filePath) {
+        try {
+            // Try sips on macOS
+            if (process.platform === 'darwin') {
+                const output = execSync(`sips -g pixelWidth -g pixelHeight "${filePath}" 2>/dev/null`, { encoding: 'utf-8', timeout: 5000 });
+                const widthMatch = output.match(/pixelWidth:\s*(\d+)/);
+                const heightMatch = output.match(/pixelHeight:\s*(\d+)/);
+                if (widthMatch && heightMatch) {
+                    return {
+                        dimensions: {
+                            width: parseInt(widthMatch[1], 10),
+                            height: parseInt(heightMatch[1], 10),
+                        },
+                    };
+                }
+            }
+            // Try file command (cross-platform)
+            const output = execSync(`file "${filePath}"`, { encoding: 'utf-8', timeout: 5000 });
+            const match = output.match(/(\d+)\s*x\s*(\d+)/);
+            if (match) {
+                return {
+                    dimensions: {
+                        width: parseInt(match[1], 10),
+                        height: parseInt(match[2], 10),
+                    },
+                };
+            }
+        }
+        catch {
+            // Ignore errors
+        }
+        return null;
+    }
+    /**
+     * Generate thumbnail as base64 data URL
+     * Uses sips on macOS for fast resizing
+     */
+    generateImageThumbnail(filePath, maxSize = 200) {
+        try {
+            if (process.platform !== 'darwin')
+                return undefined;
+            const cacheDir = getPreviewCacheDir();
+            const thumbName = `thumb_${basename(filePath)}.jpg`;
+            const thumbPath = join(cacheDir, thumbName);
+            // Use sips to resize
+            execSync(`sips -Z ${maxSize} -s format jpeg "${filePath}" --out "${thumbPath}" 2>/dev/null`, { timeout: 10000 });
+            // Read as base64
+            const data = readFileSync(thumbPath);
+            return `data:image/jpeg;base64,${data.toString('base64')}`;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Get PDF info using pdfinfo or mdls
+     */
+    getPdfInfo(filePath) {
+        try {
+            let pageCount;
+            // Try pdfinfo
+            try {
+                const output = execSync(`pdfinfo "${filePath}" 2>/dev/null`, {
+                    encoding: 'utf-8',
+                    timeout: 5000,
+                });
+                const match = output.match(/Pages:\s*(\d+)/);
+                if (match) {
+                    pageCount = parseInt(match[1], 10);
+                }
+            }
+            catch {
+                // Try mdls on macOS
+                if (process.platform === 'darwin') {
+                    const output = execSync(`mdls -name kMDItemNumberOfPages "${filePath}" 2>/dev/null`, {
+                        encoding: 'utf-8',
+                        timeout: 5000,
+                    });
+                    const match = output.match(/kMDItemNumberOfPages\s*=\s*(\d+)/);
+                    if (match) {
+                        pageCount = parseInt(match[1], 10);
+                    }
+                }
+            }
+            // Try to extract text snippet
+            let textSnippet;
+            try {
+                const output = execSync(`pdftotext -l 1 -q "${filePath}" - 2>/dev/null | head -c 500`, {
+                    encoding: 'utf-8',
+                    timeout: 10000,
+                });
+                if (output.trim()) {
+                    textSnippet = output.trim().substring(0, 500);
+                }
+            }
+            catch {
+                // pdftotext not available
+            }
+            return { pageCount, textSnippet };
+        }
+        catch {
+            return null;
+        }
+    }
+    /**
+     * Get text snippet from file
+     */
+    getTextSnippet(filePath, maxChars = 500) {
+        try {
+            const content = readFileSync(filePath, 'utf-8');
+            return content.substring(0, maxChars);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * Extract group ID from file path
+     * e.g., ~/Lia-Hub/shared/outputs/run_abc123/file.png -> run_abc123
+     */
+    extractGroupId(filePath) {
+        const outputPath = getSharedOutputPath();
+        if (filePath.startsWith(outputPath)) {
+            const relative = filePath.substring(outputPath.length + 1);
+            const parts = relative.split('/');
+            if (parts.length > 1) {
+                return parts[0]; // First directory is the group
+            }
+        }
+        return undefined;
+    }
+    log(message) {
+        if (this.options.verbose) {
+            console.log(`[PREVIEW] ${message}`);
+        }
+    }
+}
+let registry = null;
+export function getAssetRegistry() {
+    if (!registry) {
+        registry = {
+            assets: new Map(),
+            previews: new Map(),
+            byGroup: new Map(),
+        };
+    }
+    return registry;
+}
+export function registerAsset(manifest) {
+    const reg = getAssetRegistry();
+    reg.assets.set(manifest.id, manifest);
+    if (manifest.groupId) {
+        if (!reg.byGroup.has(manifest.groupId)) {
+            reg.byGroup.set(manifest.groupId, new Set());
+        }
+        reg.byGroup.get(manifest.groupId).add(manifest.id);
+    }
+}
+export function registerPreview(preview) {
+    const reg = getAssetRegistry();
+    reg.previews.set(preview.assetId, preview);
+}
+export function getAsset(assetId) {
+    return getAssetRegistry().assets.get(assetId);
+}
+export function getPreview(assetId) {
+    return getAssetRegistry().previews.get(assetId);
+}
+export function getAssetsByGroup(groupId) {
+    const reg = getAssetRegistry();
+    const ids = reg.byGroup.get(groupId);
+    if (!ids)
+        return [];
+    return Array.from(ids).map(id => reg.assets.get(id)).filter(Boolean);
+}
+export function getAllAssets() {
+    return Array.from(getAssetRegistry().assets.values());
+}
+// ============================================================================
+// SINGLETON WORKER
+// ============================================================================
+let previewWorker = null;
+export function startPreviewWorker(options = {}) {
+    if (previewWorker) {
+        return previewWorker;
+    }
+    previewWorker = new PreviewWorker({
+        ...options,
+        onAsset: (manifest) => {
+            registerAsset(manifest);
+            options.onAsset?.(manifest);
+        },
+        onPreview: (preview) => {
+            registerPreview(preview);
+            options.onPreview?.(preview);
+        },
+    });
+    previewWorker.start();
+    return previewWorker;
+}
+export function stopPreviewWorker() {
+    if (previewWorker) {
+        previewWorker.stop();
+        previewWorker = null;
+    }
+}