@thunderkiller/video-clipper 1.5.6 → 1.6.0

package/dist/utils/mongoCacheBackend.js

@@ -0,0 +1,176 @@
+import { createHash } from 'node:crypto';
+import { z } from 'zod';
+import { log } from './logger.js';
+import { TranscriptLineSchema, ChunkEvaluationSchema, AudioEventSchema, SegmentRefinementSchema, } from '../types/index.js';
+// ---------------------------------------------------------------------------
+// Internal helpers
+// ---------------------------------------------------------------------------
+function hashContent(input) {
+    return createHash('sha256').update(input).digest('hex');
+}
+/**
+ * Serializes audio events into a stable string for cache keying.
+ * Events are sorted by time so the key is order-independent.
+ */
+function audioEventsKey(events) {
+    if (events.length === 0)
+        return '';
+    const sorted = [...events].sort((a, b) => a.time - b.time);
+    return JSON.stringify(sorted);
+}
+// ---------------------------------------------------------------------------
+// MongoCacheBackend
+// ---------------------------------------------------------------------------
+/**
+ * MongoDB-backed cache backend.
+ *
+ * Each cache category maps to one collection. Documents have the shape:
+ *   { cacheKey: string, data: <payload>, createdAt: Date }
+ *
+ * Indexes created on first use (idempotent):
+ *   - Unique index on `cacheKey` for O(1) lookups
+ *   - TTL index on `createdAt` when `ttlSeconds > 0`
+ *
+ * All reads validate the stored payload through the same zod schemas used by
+ * the file backend, giving identical runtime safety guarantees.
+ *
+ * Pass `ttlSeconds = 0` (or omit) to disable TTL expiration.
+ */
+export class MongoCacheBackend {
+    client;
+    db;
+    ttlSeconds;
+    /** Track which collections already have their indexes created this session. */
+    indexedCollections = new Set();
+    constructor(client, databaseName, ttlSeconds = 0) {
+        this.client = client;
+        this.db = client.db(databaseName);
+        this.ttlSeconds = ttlSeconds;
+    }
+    // ---------------------------------------------------------------------------
+    // Index management
+    // ---------------------------------------------------------------------------
+    /**
+     * Ensures indexes exist for a collection. Called lazily on first access per
+     * collection name. `createIndex` is idempotent so concurrent calls are safe.
+     */
+    async ensureIndexes(collectionName) {
+        if (this.indexedCollections.has(collectionName))
+            return;
+        try {
+            const col = this.db.collection(collectionName);
+            // Unique index for O(1) cache key lookups
+            await col.createIndex({ cacheKey: 1 }, { unique: true });
+            // TTL index — only when a positive TTL is configured
+            if (this.ttlSeconds > 0) {
+                await col.createIndex({ createdAt: 1 }, { expireAfterSeconds: this.ttlSeconds });
+            }
+            this.indexedCollections.add(collectionName);
+        }
+        catch (err) {
+            // Non-fatal — pipeline should not crash if indexes can't be created
+            log.warn(`[mongo-cache] Failed to ensure indexes on "${collectionName}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // Low-level read / write helpers
+    // ---------------------------------------------------------------------------
+    async readDoc(collectionName, cacheKey, schema) {
+        try {
+            await this.ensureIndexes(collectionName);
+            const col = this.db.collection(collectionName);
+            const doc = await col.findOne({ cacheKey });
+            if (!doc)
+                return null;
+            const parsed = schema.safeParse(doc.data);
+            if (!parsed.success) {
+                log.warn(`[mongo-cache] Corrupt entry in "${collectionName}" (key=${cacheKey}) — ignoring`);
+                return null;
+            }
+            return parsed.data;
+        }
+        catch (err) {
+            log.warn(`[mongo-cache] Read failed in "${collectionName}": ${err instanceof Error ? err.message : String(err)}`);
+            return null;
+        }
+    }
+    async writeDoc(collectionName, cacheKey, data) {
+        try {
+            await this.ensureIndexes(collectionName);
+            const col = this.db.collection(collectionName);
+            await col.updateOne({ cacheKey }, { $set: { cacheKey, data, createdAt: new Date() } }, { upsert: true });
+        }
+        catch (err) {
+            log.warn(`[mongo-cache] Write failed in "${collectionName}": ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+    // ---------------------------------------------------------------------------
+    // Transcript
+    // ---------------------------------------------------------------------------
+    static TRANSCRIPTS = 'transcripts';
+    async readTranscript(videoId) {
+        return this.readDoc(MongoCacheBackend.TRANSCRIPTS, hashContent(videoId), z.array(TranscriptLineSchema));
+    }
+    async writeTranscript(videoId, lines) {
+        await this.writeDoc(MongoCacheBackend.TRANSCRIPTS, hashContent(videoId), lines);
+    }
+    // ---------------------------------------------------------------------------
+    // LLM chunk results
+    // ---------------------------------------------------------------------------
+    static CHUNKS = 'chunkEvaluations';
+    chunkKey(chunk, chunkAudioEvents = []) {
+        const audioKey = audioEventsKey(chunkAudioEvents);
+        return hashContent(`${chunk.start}|${chunk.end}|${chunk.text}|${audioKey}`);
+    }
+    async readChunk(chunk, chunkAudioEvents = []) {
+        return this.readDoc(MongoCacheBackend.CHUNKS, this.chunkKey(chunk, chunkAudioEvents), ChunkEvaluationSchema);
+    }
+    async writeChunk(chunk, evaluation, chunkAudioEvents = []) {
+        // Only cache successful evaluations — mirrors the file backend behaviour
+        if (evaluation.status !== 'success')
+            return;
+        await this.writeDoc(MongoCacheBackend.CHUNKS, this.chunkKey(chunk, chunkAudioEvents), evaluation);
+    }
+    // ---------------------------------------------------------------------------
+    // Segment refinement
+    // ---------------------------------------------------------------------------
+    static SEGMENTS = 'segmentRefinements';
+    async readSegmentRefinement(start, end, reason) {
+        return this.readDoc(MongoCacheBackend.SEGMENTS, hashContent(`${start}|${end}|${reason}`), SegmentRefinementSchema);
+    }
+    async writeSegmentRefinement(start, end, reason, refined) {
+        await this.writeDoc(MongoCacheBackend.SEGMENTS, hashContent(`${start}|${end}|${reason}`), refined);
+    }
+    // ---------------------------------------------------------------------------
+    // Audio events (whole-video)
+    // ---------------------------------------------------------------------------
+    static AUDIO_EVENTS = 'audioEvents';
+    async readAudioEvents(videoId, gameProfile, provider) {
+        return this.readDoc(MongoCacheBackend.AUDIO_EVENTS, hashContent(`${videoId}|${gameProfile}|${provider}`), z.array(AudioEventSchema));
+    }
+    async writeAudioEvents(videoId, gameProfile, provider, events) {
+        await this.writeDoc(MongoCacheBackend.AUDIO_EVENTS, hashContent(`${videoId}|${gameProfile}|${provider}`), events);
+    }
+    // ---------------------------------------------------------------------------
+    // Audio events (per-chunk)
+    // ---------------------------------------------------------------------------
+    static AUDIO_CHUNKS = 'audioChunks';
+    async readAudioChunk(videoId, gameProfile, provider, windowStart, windowEnd) {
+        return this.readDoc(MongoCacheBackend.AUDIO_CHUNKS, hashContent(`${videoId}|${gameProfile}|${provider}|${windowStart}|${windowEnd}`), z.array(AudioEventSchema));
+    }
+    async writeAudioChunk(videoId, gameProfile, provider, windowStart, windowEnd, events) {
+        await this.writeDoc(MongoCacheBackend.AUDIO_CHUNKS, hashContent(`${videoId}|${gameProfile}|${provider}|${windowStart}|${windowEnd}`), events);
+    }
+    // ---------------------------------------------------------------------------
+    // Lifecycle
+    // ---------------------------------------------------------------------------
+    async close() {
+        try {
+            await this.client.close();
+        }
+        catch (err) {
+            log.warn(`[mongo-cache] Failed to close MongoDB connection: ${err instanceof Error ? err.message : String(err)}`);
+        }
+    }
+}
+//# sourceMappingURL=mongoCacheBackend.js.map

package/dist/utils/mongoCacheBackend.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"mongoCacheBackend.js","sourceRoot":"","sources":["../../src/utils/mongoCacheBackend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAuB3B,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,MAAoB;IAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3D,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,iBAAiB;IAOT;IANF,EAAE,CAAC;IACH,UAAU,CAAS;IACpC,+EAA+E;IAC9D,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAExD,YACmB,MAAmB,EACpC,YAAoB,EACpB,UAAU,GAAG,CAAC;QAFG,WAAM,GAAN,MAAM,CAAa;QAIpC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,8EAA8E;IAC9E,mBAAmB;IACnB,8EAA8E;IAE9E;;;OAGG;IACK,KAAK,CAAC,aAAa,CAAC,cAAsB;QAChD,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,cAAc,CAAC;YAAE,OAAO;QACxD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAC/C,0CAA0C;YAC1C,MAAM,GAAG,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,qDAAqD;YACrD,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,GAAG,CAAC,WAAW,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,oEAAoE;YACpE,GAAG,CAAC,IAAI,CACN,8CAA8C,cAAc,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACrH,CAAC;QACJ,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,iCAAiC;IACjC,8EAA8E;IAEtE,KAAK,CAAC,OAAO,CACnB,cAAsB,EACtB,QAAgB,EAChB,MAAoB;QAEpB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;YACzC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAyB,cAAc,CAAC,CAAC;YACvE,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC5C,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAC;YAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,GAAG,CAAC,IAAI,CAAC,mCAAmC,cAAc,UAAU,QAAQ,cAAc,CAAC,CAAC;gBAC5F,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,MAAM,CAAC,IAAI,CAAC;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CACN,iCAAiC,cAAc,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACxG,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,cAAsB,EAAE,QAAgB,EAAE,IAAa;QAC5E,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;YACzC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAC/C,MAAM,GAAG,CAAC,SAAS,CACjB,EAAE,QAAQ,EAAE,EACZ,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,EAAE,EACnD,EAAE,MAAM,EAAE,IAAI,EAAE,CACjB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CACN,kCAAkC,cAAc,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACzG,CAAC;QACJ,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,aAAa;IACb,8EAA8E;IAEtE,MAAM,CAAU,WAAW,GAAG,aAAa,CAAC;IAEpD,KAAK,CAAC,cAAc,CAAC,OAAe;QAClC,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,WAAW,EAC7B,WAAW,CAAC,OAAO,CAAC,EACpB,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,EAAE,KAAuB;QAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;IAClF,CAAC;IAED,8EAA8E;IAC9E,oBAAoB;IACpB,8EAA8E;IAEtE,MAAM,CAAU,MAAM,GAAG,kBAAkB,CAAC;IAE5C,QAAQ,CAAC,KAAe,EAAE,mBAAiC,EAAE;QACnE,MAAM,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAClD,OAAO,WAAW,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAe,EACf,mBAAiC,EAAE;QAEnC,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,MAAM,EACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC,EACtC,qBAAqB,CACtB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,UAA2B,EAC3B,mBAAiC,EAAE;QAEnC,yEAAyE;QACzE,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO;QAC5C,MAAM,IAAI,CAAC,QAAQ,CACjB,iBAAiB,CAAC,MAAM,EACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC,EACtC,UAAU,CACX,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,qBAAqB;IACrB,8EAA8E;IAEtE,MAAM,CAAU,QAAQ,GAAG,oBAAoB,CAAC;IAExD,KAAK,CAAC,qBAAqB,CACzB,KAAa,EACb,GAAW,EACX,MAAc;QAEd,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,QAAQ,EAC1B,WAAW,CAAC,GAAG,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC,EACxC,uBAAuB,CACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB,CAC1B,KAAa,EACb,GAAW,EACX,MAAc,EACd,OAA0B;QAE1B,MAAM,IAAI,CAAC,QAAQ,CACjB,iBAAiB,CAAC,QAAQ,EAC1B,WAAW,CAAC,GAAG,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC,EACxC,OAAO,CACR,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,6BAA6B;IAC7B,8EAA8E;IAEtE,MAAM,CAAU,YAAY,GAAG,aAAa,CAAC;IAErD,KAAK,CAAC,eAAe,CACnB,OAAe,EACf,WAAmB,EACnB,QAAgB;QAEhB,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,YAAY,EAC9B,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC,EACpD,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,MAAoB;QAEpB,MAAM,IAAI,CAAC,QAAQ,CACjB,iBAAiB,CAAC,YAAY,EAC9B,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC,EACpD,MAAM,CACP,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,2BAA2B;IAC3B,8EAA8E;IAEtE,MAAM,CAAU,YAAY,GAAG,aAAa,CAAC;IAErD,KAAK,CAAC,cAAc,CAClB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,WAAmB,EACnB,SAAiB;QAEjB,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,YAAY,EAC9B,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC,EAChF,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,WAAmB,EACnB,SAAiB,EACjB,MAAoB;QAEpB,MAAM,IAAI,CAAC,QAAQ,CACjB,iBAAiB,CAAC,YAAY,EAC9B,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC,EAChF,MAAM,CACP,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,YAAY;IACZ,8EAA8E;IAE9E,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CACN,qDAAqD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACxG,CAAC;QACJ,CAAC;IACH,CAAC"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thunderkiller/video-clipper",
-  "version": "1.5.6",
+  "version": "1.6.0",
   "description": "CLI that analyzes YouTube transcripts with an LLM to find interesting moments and cut clips",
   "license": "ISC",
   "author": "",
@@ -73,6 +73,7 @@
     "dotenv": "^16.4.7",
     "execa": "^9.5.2",
     "fluent-ffmpeg": "^2.1.3",
+    "mongodb": "^7.1.0",
     "p-limit": "^7.3.0",
     "zod": "^4.3.6"
   },