@nathanvale/chatline 0.0.1

package/dist/enrich/audio-transcription.js

@@ -0,0 +1,370 @@
+/**
+ * Audio Transcription Module (ENRICH--T02)
+ *
+ * Implements audio transcription with structured output:
+ * - AC01: Structured prompt requesting timestamps and speaker identification
+ * - AC02: Extract speaker labels (Speaker 1, Speaker 2, etc.)
+ * - AC03: Generate short description (1-2 sentences)
+ * - AC04: Store under media.enrichment with kind='transcription'
+ * - AC05: Handle long audio files (>10min) with streaming/chunking
+ *
+ * Architecture:
+ * - transcribeAudioChunk: Transcribe single chunk with Gemini API
+ * - handleLongAudio: Split large files and process chunks
+ * - transcribeAudio: Call Gemini Audio API with structured prompt
+ * - analyzeAudio: Main entry point, handles single message enrichment
+ * - analyzeAudios: Batch processing wrapper
+ *
+ * Error Handling:
+ * - Non-fatal errors are logged and original message is returned
+ * - Transcription failures don't block enrichment pipeline
+ * - Pipeline never crashes on enrichment errors
+ */
+import { access, stat } from 'node:fs/promises';
+import { GoogleGenerativeAI } from '@google/generative-ai';
+import { createLogger } from '#utils/logger';
+const logger = createLogger('enrich:audio-transcription');
+/**
+ * Structured prompt for Gemini Audio API
+ * Requests transcription with speaker identification, timestamps, and summary
+ */
+const GEMINI_AUDIO_PROMPT = `You are an expert at transcribing audio. Please transcribe the audio and provide:
+
+1. Full Transcription:
+   Format with speaker labels as "Speaker 1: [text]", "Speaker 2: [text]", etc.
+   Keep the exact words spoken, preserving natural speech patterns.
+
+2. Timestamps:
+   Format as MM:SS - Speaker N: [brief content]
+   Include timestamp for each speaker change or major topic shift.
+
+3. Short Description:
+   Provide a 1-2 sentence summary of the audio content and main topics.
+
+Format your response exactly as:
+
+Transcription:
+[full transcription with Speaker labels here]
+
+Timestamps:
+[timestamps here]
+
+Short Description: [1-2 sentence summary here]`;
+/**
+ * Extract audio file duration in seconds (rough estimate from file size)
+ * Audio bitrate typically 128kbps for M4A/AAC
+ */
+function estimateAudioDuration(fileSizeBytes) {
+    const bitRate = 128 * 1024; // 128 kbps in bytes per second
+    return Math.ceil(fileSizeBytes / bitRate);
+}
+/**
+ * AC05: Split long audio into chunks for processing
+ * Returns chunk info needed for streaming API
+ */
+function getAudioChunks(durationSeconds, maxChunkDuration = 600) {
+    const chunks = [];
+    for (let i = 0; i * maxChunkDuration < durationSeconds; i++) {
+        chunks.push({
+            index: i,
+            startSec: i * maxChunkDuration,
+            endSec: Math.min((i + 1) * maxChunkDuration, durationSeconds),
+        });
+    }
+    return chunks;
+}
+/**
+ * AC01: Call Gemini Audio API with structured prompt for single chunk
+ * AC02, AC03: Parse response into transcription, speakers, and description
+ * AC04: Create enrichment with provenance
+ */
+export async function transcribeAudioChunk(audioPath, chunkIndex, config) {
+    const apiKey = config.geminiApiKey;
+    const modelName = config.geminiModel || 'gemini-1.5-pro';
+    if (!apiKey) {
+        throw new Error('GEMINI_API_KEY is required for audio transcription');
+    }
+    try {
+        // AC01: Create Gemini client and call with structured prompt
+        const genAI = new GoogleGenerativeAI(apiKey);
+        const model = genAI.getGenerativeModel({ model: modelName });
+        // Read and encode the actual audio file
+        const { readFile } = await import('node:fs/promises');
+        const audioBuffer = await readFile(audioPath);
+        const audioBase64 = audioBuffer.toString('base64');
+        // Determine MIME type from file extension
+        const ext = audioPath.toLowerCase().split('.').pop() || 'm4a';
+        const mimeTypeMap = {
+            m4a: 'audio/mp4',
+            mp3: 'audio/mpeg',
+            wav: 'audio/wav',
+            aac: 'audio/aac',
+            ogg: 'audio/ogg',
+            flac: 'audio/flac',
+        };
+        const mimeType = mimeTypeMap[ext] || 'audio/mp4';
+        const response = await model.generateContent([
+            {
+                inlineData: {
+                    mimeType,
+                    data: audioBase64,
+                },
+            },
+            GEMINI_AUDIO_PROMPT,
+        ]);
+        const responseText = response.response.text();
+        logger.debug(`Gemini response received (chunk ${chunkIndex}): ${responseText.substring(0, 200)}...`);
+        // AC02: Parse speaker labels from response
+        const speakerMatches = responseText.match(/Speaker \d+/g) ?? [];
+        const speakers = Array.from(new Set(speakerMatches)); // Unique speakers in order
+        // AC03: Extract short description
+        const shortDescriptionMatch = responseText.match(/Short Description:\\s*(.+?)(?=\n|$)/is);
+        const shortDescription = shortDescriptionMatch?.[1]?.trim() || 'Audio transcription available';
+        // Extract full transcription section
+        const transcriptionMatch = responseText.match(/Transcription:\s*([\s\S]+?)(?=\n\nTimestamps:|$)/i);
+        const transcription = transcriptionMatch?.[1]?.trim() || responseText;
+        // Extract timestamps section
+        const timestampsMatch = responseText.match(/Timestamps:\s*([\s\S]+?)(?=\n\nShort Description:|$)/i);
+        const timestampsText = timestampsMatch?.[1]?.trim() || '';
+        // Parse individual timestamps
+        const timestamps = timestampsText
+            .split('\n')
+            .filter((line) => line.trim())
+            .map((line) => {
+            const match = line.match(/(\d{2}:\d{2})\s*-\s*Speaker (\d+):\s*(.+)/);
+            return {
+                time: match?.[1] || '00:00',
+                speaker: `Speaker ${match?.[2] || '1'}`,
+                content: match?.[3] || line,
+            };
+        });
+        return {
+            transcription,
+            speakers,
+            timestamps,
+            shortDescription,
+        };
+    }
+    catch (error) {
+        logger.error(`Gemini API error for ${audioPath} (chunk ${chunkIndex})`, {
+            error,
+        });
+        throw error;
+    }
+}
+/**
+ * AC05: Handle long audio files by splitting and processing chunks
+ * Merges results from all chunks into single transcription
+ */
+export async function handleLongAudio(audioPath, durationSeconds, config) {
+    const maxChunkDuration = (config.maxAudioChunkDuration || 10) * 60; // Convert to seconds
+    if (durationSeconds <= maxChunkDuration) {
+        // Single chunk - call directly
+        return transcribeAudioChunk(audioPath, 0, config);
+    }
+    // AC05: Split into chunks
+    const chunks = getAudioChunks(durationSeconds, maxChunkDuration);
+    logger.info(`Processing ${chunks.length} audio chunks for ${audioPath}`, {
+        duration: durationSeconds,
+        chunkDuration: maxChunkDuration,
+    });
+    const chunkResults = [];
+    for (const chunk of chunks) {
+        try {
+            const result = await transcribeAudioChunk(audioPath, chunk.index, config);
+            chunkResults.push(result);
+            // AC05: Respect rate limiting between chunks
+            if (chunk.index < chunks.length - 1 && config.rateLimitDelay) {
+                await new Promise((resolve) => setTimeout(resolve, config.rateLimitDelay));
+            }
+        }
+        catch (err) {
+            logger.warn(`Failed to transcribe chunk ${chunk.index}, continuing with others`, {
+                error: err instanceof Error ? err.message : String(err),
+            });
+            // Continue with next chunk even if this one fails
+        }
+    }
+    if (chunkResults.length === 0) {
+        throw new Error(`Failed to transcribe any chunks for ${audioPath}`);
+    }
+    // AC05: Merge all chunk transcriptions
+    const mergedTranscription = chunkResults
+        .map((r) => r.transcription)
+        .join('\n\n');
+    const allSpeakers = [...new Set(chunkResults.flatMap((r) => r.speakers))];
+    const mergedTimestamps = chunkResults.flatMap((r) => r.timestamps);
+    // AC03: Generate merged short description (use last chunk's description as primary)
+    const shortDescription = chunkResults[chunkResults.length - 1]?.shortDescription ||
+        'Audio transcription available';
+    return {
+        transcription: mergedTranscription,
+        speakers: allSpeakers,
+        timestamps: mergedTimestamps,
+        shortDescription,
+    };
+}
+/**
+ * AC01-AC05: Main transcription orchestrator
+ * Handles chunk detection, API calls, and response parsing
+ */
+export async function transcribeAudio(audioPath, config) {
+    try {
+        // Estimate audio duration from file size
+        const fileStats = await stat(audioPath);
+        const durationSeconds = estimateAudioDuration(fileStats.size);
+        logger.info(`Transcribing audio: ${audioPath}`, {
+            fileSizeKB: Math.round(fileStats.size / 1024),
+            estimatedDuration: Math.round(durationSeconds / 60),
+        });
+        // AC05: Handle long audio with chunking if needed
+        const transcriptionData = await handleLongAudio(audioPath, durationSeconds, config);
+        // AC04: Create enrichment entry with full provenance
+        const modelName = config.geminiModel || 'gemini-1.5-pro';
+        const version = new Date().toISOString().split('T')[0] || 'unknown';
+        const enrichment = {
+            kind: 'transcription',
+            provider: 'gemini',
+            model: modelName,
+            version, // YYYY-MM-DD
+            createdAt: new Date().toISOString(),
+            transcription: transcriptionData.transcription,
+            speakers: transcriptionData.speakers,
+            timestamps: transcriptionData.timestamps,
+            shortDescription: transcriptionData.shortDescription,
+        };
+        logger.info(`Audio transcription complete for ${audioPath}`, {
+            kind: enrichment.kind,
+            speakerCount: enrichment.speakers?.length,
+            duration: Math.round(durationSeconds / 60),
+        });
+        return enrichment;
+    }
+    catch (error) {
+        logger.error(`Transcription error for ${audioPath}`, { error });
+        throw error;
+    }
+}
+/**
+ * Main entry point - analyze audio media message and enrich it
+ * Handles all ACs (AC01-AC05) through helper functions
+ *
+ * Responsibilities:
+ * 1. Check if media is audio type (skip non-audio)
+ * 2. Check if path is available
+ * 3. Call transcription with chunking support (AC05)
+ * 4. Parse response and extract data (AC01-AC03)
+ * 5. Add enrichment with provenance (AC04)
+ */
+export async function analyzeAudio(message, config) {
+    // Skip if not enabled
+    if (!config.enableAudioTranscription) {
+        logger.debug('Audio transcription disabled in config');
+        return message;
+    }
+    // Skip if not a media message
+    if (message.messageKind !== 'media' || !message.media) {
+        return message;
+    }
+    // Skip if media is not audio
+    if (message.media.mediaKind !== 'audio') {
+        logger.debug('Skipping non-audio media', {
+            mediaKind: message.media.mediaKind,
+        });
+        return message;
+    }
+    // Skip if path is missing
+    if (!message.media.path) {
+        logger.warn('Skipping audio with missing path', {
+            filename: message.media.filename,
+        });
+        return message;
+    }
+    // Check if audio file exists
+    try {
+        await access(message.media.path);
+    }
+    catch {
+        logger.warn('Audio file not found at path', { path: message.media.path });
+        return message;
+    }
+    try {
+        // AC01-AC05: Transcribe audio (handles chunking, API calls, parsing)
+        const enrichment = await transcribeAudio(message.media.path, config);
+        // Check idempotency: don't re-transcribe if already done
+        const existingTranscription = message.media.enrichment?.find((e) => e.kind === 'transcription' &&
+            e.provider === (config.geminiModel ? 'gemini' : 'gemini'));
+        if (existingTranscription) {
+            logger.debug('Transcription already exists, skipping re-analysis', {
+                model: existingTranscription.model,
+                guid: message.guid,
+            });
+            return message;
+        }
+        // Update message with enrichment
+        const updatedMedia = {
+            ...message.media,
+            enrichment: [...(message.media.enrichment || []), enrichment],
+        };
+        logger.info('Audio enriched', {
+            filename: message.media.filename,
+            guid: message.guid,
+        });
+        return {
+            ...message,
+            media: updatedMedia,
+        };
+    }
+    catch (error) {
+        logger.error('Error analyzing audio', {
+            filename: message.media?.filename,
+            guid: message.guid,
+            error: error instanceof Error ? error.message : String(error),
+        });
+        // Don't crash pipeline - return original message
+        return message;
+    }
+}
+/**
+ * Batch analyze multiple messages
+ * Useful for enrichment stage that processes arrays of messages
+ * Each message is processed independently; errors don't stop the batch
+ */
+export async function analyzeAudios(messages, config) {
+    const results = [];
+    let successCount = 0;
+    let skipCount = 0;
+    let errorCount = 0;
+    for (const message of messages) {
+        try {
+            const analyzed = await analyzeAudio(message, config);
+            // Track if enrichment was added
+            if (analyzed.media?.enrichment &&
+                analyzed.media.enrichment.length >
+                    (message.media?.enrichment?.length || 0)) {
+                successCount++;
+            }
+            else {
+                skipCount++;
+            }
+            results.push(analyzed);
+        }
+        catch (err) {
+            errorCount++;
+            logger.error('Failed to analyze message', {
+                guid: message.guid,
+                error: err instanceof Error ? err.message : String(err),
+            });
+            // Keep original message if analysis fails
+            results.push(message);
+        }
+    }
+    logger.info('Batch audio transcription complete', {
+        successCount,
+        skipCount,
+        errorCount,
+        total: messages.length,
+    });
+    return results;
+}
+//# sourceMappingURL=audio-transcription.js.map

package/dist/enrich/audio-transcription.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"audio-transcription.js","sourceRoot":"","sources":["../../src/enrich/audio-transcription.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAEH,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,kBAAkB,CAAA;AAE/C,OAAO,EAAE,kBAAkB,EAAE,MAAM,uBAAuB,CAAA;AAI1D,OAAO,EAAE,YAAY,EAAE,MAAM,eAAe,CAAA;AAkB5C,MAAM,MAAM,GAAG,YAAY,CAAC,4BAA4B,CAAC,CAAA;AAEzD;;;GAGG;AACH,MAAM,mBAAmB,GAAG;;;;;;;;;;;;;;;;;;;;;+CAqBmB,CAAA;AAE/C;;;GAGG;AACH,SAAS,qBAAqB,CAAC,aAAqB;IACnD,MAAM,OAAO,GAAG,GAAG,GAAG,IAAI,CAAA,CAAC,+BAA+B;IAC1D,OAAO,IAAI,CAAC,IAAI,CAAC,aAAa,GAAG,OAAO,CAAC,CAAA;AAC1C,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CACtB,eAAuB,EACvB,gBAAgB,GAAG,GAAG;IAEtB,MAAM,MAAM,GAA+D,EAAE,CAAA;IAE7E,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,gBAAgB,GAAG,eAAe,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7D,MAAM,CAAC,IAAI,CAAC;YACX,KAAK,EAAE,CAAC;YACR,QAAQ,EAAE,CAAC,GAAG,gBAAgB;YAC9B,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,gBAAgB,EAAE,eAAe,CAAC;SAC7D,CAAC,CAAA;IACH,CAAC;IAED,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,oBAAoB,CACzC,SAAiB,EACjB,UAAkB,EAClB,MAAyC;IAEzC,MAAM,MAAM,GAAG,MAAM,CAAC,YAAY,CAAA;IAClC,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,IAAI,gBAAgB,CAAA;IAExD,IAAI,CAAC,MAAM,EAAE,CAAC;QACb,MAAM,IAAI,KAAK,CAAC,oDAAoD,CAAC,CAAA;IACtE,CAAC;IAED,IAAI,CAAC;QACJ,6DAA6D;QAC7D,MAAM,KAAK,GAAG,IAAI,kBAAkB,CAAC,MAAM,CAAC,CAAA;QAC5C,MAAM,KAAK,GAAG,KAAK,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAA;QAE5D,wCAAwC;QACxC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QACrD,MAAM,WAAW,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,CAAA;QAC7C,MAAM,WAAW,GAAG,WAAW,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAA;QAElD,0CAA0C;QAC1C,MAAM,GAAG,GAAG,SAAS,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,KAAK,CAAA;QAC7D,MAAM,WAAW,GAA2B;YAC3C,GAAG,EAAE,WAAW;YAChB,GAAG,EAAE,YAAY;YACjB,GAAG,EAAE,WAAW;YAChB,GAAG,EAAE,WAAW;YAChB,GAAG,EAAE,WAAW;YAChB,IAAI,EAAE,YAAY;SAClB,CAAA;QACD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,WAAW,CAAA;QAEhD,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,eAAe,CAAC;YAC5C;gBACC,UAAU,EAAE;oBACX,QAAQ;oBACR,IAAI,EAAE,WAAW;iBACjB;aACD;YACD,mBAAmB;SACnB,CAAC,CAAA;QAEF,MAAM,YAAY,GAAG,QAAQ,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAA;QAC7C,MAAM,CAAC,KAAK,CACX,mCAAmC,UAAU,MAAM,YAAY,CAAC,SAAS,CAAC,CAAC,EAAE,GAAG,CAAC,KAAK,CACtF,CAAA;QAED,2CAA2C;QAC3C,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,EAAE,CAAA;QAC/D,MAAM,QAAQ,GAAa,KAAK,CAAC,IAAI,CAAC,IAAI,GAAG,CAAS,cAAc,CAAC,CAAC,CAAA,CAAC,2BAA2B;QAElG,kCAAkC;QAClC,MAAM,qBAAqB,GAAG,YAAY,CAAC,KAAK,CAC/C,uCAAuC,CACvC,CAAA;QACD,MAAM,gBAAgB,GACrB,qBAAqB,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,+BAA+B,CAAA;QAEtE,qCAAqC;QACrC,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAC5C,mDAAmD,CACnD,CAAA;QACD,MAAM,aAAa,GAAG,kBAAkB,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,YAAY,CAAA;QAErE,6BAA6B;QAC7B,MAAM,eAAe,GAAG,YAAY,CAAC,KAAK,CACzC,uDAAuD,CACvD,CAAA;QACD,MAAM,cAAc,GAAG,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;QAEzD,8BAA8B;QAC9B,MAAM,UAAU,GAAG,cAAc;aAC/B,KAAK,CAAC,IAAI,CAAC;aACX,MAAM,CAAC,CAAC,IAAY,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC;aACrC,GAAG,CAAC,CAAC,IAAY,EAAE,EAAE;YACrB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,2CAA2C,CAAC,CAAA;YACrE,OAAO;gBACN,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,OAAO;gBAC3B,OAAO,EAAE,WAAW,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,EAAE;gBACvC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI;aAC3B,CAAA;QACF,CAAC,CAAC,CAAA;QAEH,OAAO;YACN,aAAa;YACb,QAAQ;YACR,UAAU;YACV,gBAAgB;SAChB,CAAA;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,wBAAwB,SAAS,WAAW,UAAU,GAAG,EAAE;YACvE,KAAK;SACL,CAAC,CAAA;QACF,MAAM,KAAK,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,SAAiB,EACjB,eAAuB,EACvB,MAAyC;IAEzC,MAAM,gBAAgB,GAAG,CAAC,MAAM,CAAC,qBAAqB,IAAI,EAAE,CAAC,GAAG,EAAE,CAAA,CAAC,qBAAqB;IAExF,IAAI,eAAe,IAAI,gBAAgB,EAAE,CAAC;QACzC,+BAA+B;QAC/B,OAAO,oBAAoB,CAAC,SAAS,EAAE,CAAC,EAAE,MAAM,CAAC,CAAA;IAClD,CAAC;IAED,0BAA0B;IAC1B,MAAM,MAAM,GAAG,cAAc,CAAC,eAAe,EAAE,gBAAgB,CAAC,CAAA;IAChE,MAAM,CAAC,IAAI,CAAC,cAAc,MAAM,CAAC,MAAM,qBAAqB,SAAS,EAAE,EAAE;QACxE,QAAQ,EAAE,eAAe;QACzB,aAAa,EAAE,gBAAgB;KAC/B,CAAC,CAAA;IAEF,MAAM,YAAY,GAAwB,EAAE,CAAA;IAE5C,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,IAAI,CAAC;YACJ,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,SAAS,EAAE,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;YACzE,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;YAEzB,6CAA6C;YAC7C,IAAI,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,MAAM,CAAC,cAAc,EAAE,CAAC;gBAC9D,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC7B,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,cAAc,CAAC,CAC1C,CAAA;YACF,CAAC;QACF,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,MAAM,CAAC,IAAI,CACV,8BAA8B,KAAK,CAAC,KAAK,0BAA0B,EACnE;gBACC,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACvD,CACD,CAAA;YACD,kDAAkD;QACnD,CAAC;IACF,CAAC;IAED,IAAI,YAAY,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,uCAAuC,SAAS,EAAE,CAAC,CAAA;IACpE,CAAC;IAED,uCAAuC;IACvC,MAAM,mBAAmB,GAAG,YAAY;SACtC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC;SAC3B,IAAI,CAAC,MAAM,CAAC,CAAA;IACd,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAA;IACzE,MAAM,gBAAgB,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,CAAC,CAAA;IAElE,oFAAoF;IACpF,MAAM,gBAAgB,GACrB,YAAY,CAAC,YAAY,CAAC,MAAM,GAAG,CAAC,CAAC,EAAE,gBAAgB;QACvD,+BAA+B,CAAA;IAEhC,OAAO;QACN,aAAa,EAAE,mBAAmB;QAClC,QAAQ,EAAE,WAAW;QACrB,UAAU,EAAE,gBAAgB;QAC5B,gBAAgB;KAChB,CAAA;AACF,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,SAAiB,EACjB,MAAyC;IAEzC,IAAI,CAAC;QACJ,yCAAyC;QACzC,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,CAAA;QACvC,MAAM,eAAe,GAAG,qBAAqB,CAAC,SAAS,CAAC,IAAI,CAAC,CAAA;QAE7D,MAAM,CAAC,IAAI,CAAC,uBAAuB,SAAS,EAAE,EAAE;YAC/C,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,IAAI,GAAG,IAAI,CAAC;YAC7C,iBAAiB,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC;SACnD,CAAC,CAAA;QAEF,kDAAkD;QAClD,MAAM,iBAAiB,GAAG,MAAM,eAAe,CAC9C,SAAS,EACT,eAAe,EACf,MAAM,CACN,CAAA;QAED,qDAAqD;QACrD,MAAM,SAAS,GAAG,MAAM,CAAC,WAAW,IAAI,gBAAgB,CAAA;QACxD,MAAM,OAAO,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,SAAS,CAAA;QACnE,MAAM,UAAU,GAAoB;YACnC,IAAI,EAAE,eAAe;YACrB,QAAQ,EAAE,QAAQ;YAClB,KAAK,EAAE,SAAS;YAChB,OAAO,EAAE,aAAa;YACtB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,aAAa,EAAE,iBAAiB,CAAC,aAAa;YAC9C,QAAQ,EAAE,iBAAiB,CAAC,QAAQ;YACpC,UAAU,EAAE,iBAAiB,CAAC,UAAU;YACxC,gBAAgB,EAAE,iBAAiB,CAAC,gBAAgB;SACpD,CAAA;QAED,MAAM,CAAC,IAAI,CAAC,oCAAoC,SAAS,EAAE,EAAE;YAC5D,IAAI,EAAE,UAAU,CAAC,IAAI;YACrB,YAAY,EAAE,UAAU,CAAC,QAAQ,EAAE,MAAM;YACzC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,EAAE,CAAC;SAC1C,CAAC,CAAA;QAEF,OAAO,UAAU,CAAA;IAClB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,2BAA2B,SAAS,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAA;QAC/D,MAAM,KAAK,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CACjC,OAAgB,EAChB,MAAyC;IAEzC,sBAAsB;IACtB,IAAI,CAAC,MAAM,CAAC,wBAAwB,EAAE,CAAC;QACtC,MAAM,CAAC,KAAK,CAAC,wCAAwC,CAAC,CAAA;QACtD,OAAO,OAAO,CAAA;IACf,CAAC;IAED,8BAA8B;IAC9B,IAAI,OAAO,CAAC,WAAW,KAAK,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACvD,OAAO,OAAO,CAAA;IACf,CAAC;IAED,6BAA6B;IAC7B,IAAI,OAAO,CAAC,KAAK,CAAC,SAAS,KAAK,OAAO,EAAE,CAAC;QACzC,MAAM,CAAC,KAAK,CAAC,0BAA0B,EAAE;YACxC,SAAS,EAAE,OAAO,CAAC,KAAK,CAAC,SAAS;SAClC,CAAC,CAAA;QACF,OAAO,OAAO,CAAA;IACf,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC;QACzB,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;YAC/C,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ;SAChC,CAAC,CAAA;QACF,OAAO,OAAO,CAAA;IACf,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC;QACJ,MAAM,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;IACjC,CAAC;IAAC,MAAM,CAAC;QACR,MAAM,CAAC,IAAI,CAAC,8BAA8B,EAAE,EAAE,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,CAAC,CAAA;QACzE,OAAO,OAAO,CAAA;IACf,CAAC;IAED,IAAI,CAAC;QACJ,qEAAqE;QACrE,MAAM,UAAU,GAAG,MAAM,eAAe,CAAC,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAA;QAEpE,yDAAyD;QACzD,MAAM,qBAAqB,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,IAAI,CAC3D,CAAC,CAAC,EAAE,EAAE,CACL,CAAC,CAAC,IAAI,KAAK,eAAe;YAC1B,CAAC,CAAC,QAAQ,KAAK,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAC1D,CAAA;QAED,IAAI,qBAAqB,EAAE,CAAC;YAC3B,MAAM,CAAC,KAAK,CAAC,oDAAoD,EAAE;gBAClE,KAAK,EAAE,qBAAqB,CAAC,KAAK;gBAClC,IAAI,EAAE,OAAO,CAAC,IAAI;aAClB,CAAC,CAAA;YACF,OAAO,OAAO,CAAA;QACf,CAAC;QAED,iCAAiC;QACjC,MAAM,YAAY,GAAc;YAC/B,GAAG,OAAO,CAAC,KAAK;YAChB,UAAU,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAC,EAAE,UAAU,CAAC;SAC7D,CAAA;QAED,MAAM,CAAC,IAAI,CAAC,gBAAgB,EAAE;YAC7B,QAAQ,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ;YAChC,IAAI,EAAE,OAAO,CAAC,IAAI;SAClB,CAAC,CAAA;QAEF,OAAO;YACN,GAAG,OAAO;YACV,KAAK,EAAE,YAAY;SACnB,CAAA;IACF,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,MAAM,CAAC,KAAK,CAAC,uBAAuB,EAAE;YACrC,QAAQ,EAAE,OAAO,CAAC,KAAK,EAAE,QAAQ;YACjC,IAAI,EAAE,OAAO,CAAC,IAAI;YAClB,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;SAC7D,CAAC,CAAA;QACF,iDAAiD;QACjD,OAAO,OAAO,CAAA;IACf,CAAC;AACF,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAClC,QAAmB,EACnB,MAAyC;IAEzC,MAAM,OAAO,GAAc,EAAE,CAAA;IAC7B,IAAI,YAAY,GAAG,CAAC,CAAA;IACpB,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,UAAU,GAAG,CAAC,CAAA;IAElB,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAChC,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,OAAO,EAAE,MAAM,CAAC,CAAA;YACpD,gCAAgC;YAChC,IACC,QAAQ,CAAC,KAAK,EAAE,UAAU;gBAC1B,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM;oBAC/B,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,MAAM,IAAI,CAAC,CAAC,EACxC,CAAC;gBACF,YAAY,EAAE,CAAA;YACf,CAAC;iBAAM,CAAC;gBACP,SAAS,EAAE,CAAA;YACZ,CAAC;YACD,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACvB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACd,UAAU,EAAE,CAAA;YACZ,MAAM,CAAC,KAAK,CAAC,2BAA2B,EAAE;gBACzC,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,KAAK,EAAE,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC;aACvD,CAAC,CAAA;YACF,0CAA0C;YAC1C,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,CAAA;QACtB,CAAC;IACF,CAAC;IAED,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;QACjD,YAAY;QACZ,SAAS;QACT,UAAU;QACV,KAAK,EAAE,QAAQ,CAAC,MAAM;KACtB,CAAC,CAAA;IACF,OAAO,OAAO,CAAA;AACf,CAAC"}

package/dist/enrich/checkpoint.d.ts

@@ -0,0 +1,137 @@
+/**
+ * Checkpoint and Resume Module (ENRICH--T06)
+ *
+ * Implements resumable enrichment with:
+ * - AC01: Checkpoint writes after N items (configurable, default 100)
+ * - AC02: Full checkpoint schema with stats and failed items
+ * - AC03: Atomic writes using temp file + rename pattern
+ * - AC04: Resume within ≤1 item of last checkpoint
+ * - AC05: Config consistency verification with hash comparison
+ *
+ * Architecture:
+ * - createCheckpoint: Create new checkpoint with schema
+ * - shouldWriteCheckpoint: Determine if checkpoint should be written
+ * - getResumeIndex: Calculate resume position from checkpoint
+ * - verifyConfigHash: Validate config hasn't changed
+ * - getCheckpointPath: Generate deterministic checkpoint file path
+ * - loadCheckpoint: Load checkpoint from disk
+ * - saveCheckpoint: Write checkpoint atomically
+ */
+export type FailedItem = {
+    index: number;
+    guid: string;
+    kind: string;
+    error: string;
+};
+export type CheckpointStats = {
+    processedCount: number;
+    failedCount: number;
+    enrichmentsByKind: Record<string, number>;
+};
+export type EnrichCheckpoint = {
+    version: string;
+    configHash: string;
+    lastProcessedIndex: number;
+    totalProcessed: number;
+    totalFailed: number;
+    stats: CheckpointStats;
+    failedItems: FailedItem[];
+    createdAt: string;
+};
+export type CheckpointInput = {
+    lastProcessedIndex: number;
+    totalProcessed: number;
+    totalFailed: number;
+    stats: CheckpointStats;
+    failedItems: FailedItem[];
+    configHash: string;
+};
+/**
+ * AC01: Determine if checkpoint should be written after N items
+ *
+ * @param itemIndex - Current item index (0-based)
+ * @param checkpointInterval - Checkpoint interval (default 100)
+ * @returns true if checkpoint should be written
+ */
+export declare function shouldWriteCheckpoint(itemIndex: number, checkpointInterval?: number): boolean;
+/**
+ * AC02: Create checkpoint with full schema
+ *
+ * @param input - Checkpoint input data
+ * @returns EnrichCheckpoint with all required fields
+ */
+export declare function createCheckpoint(input: CheckpointInput): EnrichCheckpoint;
+/**
+ * AC03: Generate deterministic checkpoint file path
+ *
+ * @param checkpointDir - Directory for checkpoints
+ * @param configHash - Config hash for uniqueness
+ * @returns Path to checkpoint file
+ */
+export declare function getCheckpointPath(checkpointDir: string, configHash: string): string;
+/**
+ * AC03: Save checkpoint atomically using temp file + rename
+ *
+ * @param checkpoint - Checkpoint to save
+ * @param checkpointPath - Path to save checkpoint
+ */
+export declare function saveCheckpoint(checkpoint: EnrichCheckpoint, checkpointPath: string): Promise<void>;
+/**
+ * AC03: Load checkpoint from disk
+ *
+ * @param checkpointPath - Path to checkpoint file
+ * @returns Loaded checkpoint or null if not found
+ */
+export declare function loadCheckpoint(checkpointPath: string): Promise<EnrichCheckpoint | null>;
+/**
+ * AC04: Calculate resume index from checkpoint
+ *
+ * Resume at lastProcessedIndex + 1 to ensure we don't re-process
+ * the last item that was in the previous checkpoint.
+ *
+ * @param checkpoint - Checkpoint to resume from
+ * @returns Resume index (within ≤1 item of last checkpoint)
+ */
+export declare function getResumeIndex(checkpoint: EnrichCheckpoint): number;
+/**
+ * AC05: Compute config hash for consistency checking
+ *
+ * @param config - Configuration object
+ * @returns SHA-256 hash of config
+ */
+export declare function computeConfigHash(config: Record<string, unknown>): string;
+/**
+ * AC05: Verify config hasn't changed by comparing hashes
+ *
+ * @param checkpointHash - Hash from checkpoint
+ * @param currentHash - Hash of current config
+ * @returns true if hashes match (config unchanged)
+ */
+export declare function verifyConfigHash(checkpointHash: string, currentHash: string): boolean;
+export type CheckpointState = {
+    isResuming: boolean;
+    lastCheckpointIndex: number;
+    configHash: string;
+    failedItemsInCheckpoint: FailedItem[];
+};
+/**
+ * Initialize checkpoint state for enrichment run
+ *
+ * @param checkpoint - Loaded checkpoint or null
+ * @param currentConfigHash - Hash of current config
+ * @returns Checkpoint state or error
+ */
+export declare function initializeCheckpointState(checkpoint: EnrichCheckpoint | null, currentConfigHash: string): CheckpointState | Error;
+/**
+ * Create new checkpoint for saving after processing batch
+ *
+ * @param lastProcessedIndex - Index of last processed item
+ * @param totalProcessed - Total items processed so far
+ * @param totalFailed - Total failed items so far
+ * @param batchStats - Stats for this batch
+ * @param failedItems - Failed items in this batch
+ * @param configHash - Hash of current config
+ * @returns Checkpoint ready to save
+ */
+export declare function prepareCheckpoint(lastProcessedIndex: number, totalProcessed: number, totalFailed: number, batchStats: CheckpointStats, failedItems: FailedItem[], configHash: string): EnrichCheckpoint;
+//# sourceMappingURL=checkpoint.d.ts.map

package/dist/enrich/checkpoint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"checkpoint.d.ts","sourceRoot":"","sources":["../../src/enrich/checkpoint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAUH,MAAM,MAAM,UAAU,GAAG;IACxB,KAAK,EAAE,MAAM,CAAA;IACb,IAAI,EAAE,MAAM,CAAA;IACZ,IAAI,EAAE,MAAM,CAAA;IACZ,KAAK,EAAE,MAAM,CAAA;CACb,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC7B,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,iBAAiB,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;CACzC,CAAA;AAED,MAAM,MAAM,gBAAgB,GAAG;IAC9B,OAAO,EAAE,MAAM,CAAA;IACf,UAAU,EAAE,MAAM,CAAA;IAClB,kBAAkB,EAAE,MAAM,CAAA;IAC1B,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,eAAe,CAAA;IACtB,WAAW,EAAE,UAAU,EAAE,CAAA;IACzB,SAAS,EAAE,MAAM,CAAA;CACjB,CAAA;AAED,MAAM,MAAM,eAAe,GAAG;IAC7B,kBAAkB,EAAE,MAAM,CAAA;IAC1B,cAAc,EAAE,MAAM,CAAA;IACtB,WAAW,EAAE,MAAM,CAAA;IACnB,KAAK,EAAE,eAAe,CAAA;IACtB,WAAW,EAAE,UAAU,EAAE,CAAA;IACzB,UAAU,EAAE,MAAM,CAAA;CAClB,CAAA;AAMD;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACpC,SAAS,EAAE,MAAM,EACjB,kBAAkB,SAAM,GACtB,OAAO,CAGT;AAMD;;;;;GAKG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,eAAe,GAAG,gBAAgB,CAWzE;AAMD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAChC,aAAa,EAAE,MAAM,EACrB,UAAU,EAAE,MAAM,GAChB,MAAM,CAER;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,UAAU,EAAE,gBAAgB,EAC5B,cAAc,EAAE,MAAM,GACpB,OAAO,CAAC,IAAI,CAAC,CAqBf;AAED;;;;;GAKG;AACH,wBAAsB,cAAc,CACnC,cAAc,EAAE,MAAM,GACpB,OAAO,CAAC,gBAAgB,GAAG,IAAI,CAAC,CASlC;AAMD;;;;;;;;GAQG;AACH,wBAAgB,cAAc,CAAC,UAAU,EAAE,gBAAgB,GAAG,MAAM,CAGnE;AAMD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAGzE;AAED;;;;;;GAMG;AACH,wBAAgB,gBAAgB,CAC/B,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,MAAM,GACjB,OAAO,CAET;AAMD,MAAM,MAAM,eAAe,GAAG;IAC7B,UAAU,EAAE,OAAO,CAAA;IACnB,mBAAmB,EAAE,MAAM,CAAA;IAC3B,UAAU,EAAE,MAAM,CAAA;IAClB,uBAAuB,EAAE,UAAU,EAAE,CAAA;CACrC,CAAA;AAED;;;;;;GAMG;AACH,wBAAgB,yBAAyB,CACxC,UAAU,EAAE,gBAAgB,GAAG,IAAI,EACnC,iBAAiB,EAAE,MAAM,GACvB,eAAe,GAAG,KAAK,CAyBzB;AAED;;;;;;;;;;GAUG;AACH,wBAAgB,iBAAiB,CAChC,kBAAkB,EAAE,MAAM,EAC1B,cAAc,EAAE,MAAM,EACtB,WAAW,EAAE,MAAM,EACnB,UAAU,EAAE,eAAe,EAC3B,WAAW,EAAE,UAAU,EAAE,EACzB,UAAU,EAAE,MAAM,GAChB,gBAAgB,CASlB"}