@thunderkiller/video-clipper 1.5.5 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +69 -349
- package/dist/config/env.d.ts +4 -0
- package/dist/config/env.d.ts.map +1 -1
- package/dist/config/index.d.ts +0 -1
- package/dist/config/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/pipeline/dumper.d.ts.map +1 -0
- package/dist/{utils → pipeline}/dumper.js +1 -1
- package/dist/pipeline/dumper.js.map +1 -0
- package/dist/pipeline/runner.d.ts.map +1 -1
- package/dist/pipeline/runner.js +80 -73
- package/dist/pipeline/runner.js.map +1 -1
- package/dist/pipeline/stages/audioProcessor.d.ts +0 -1
- package/dist/pipeline/stages/audioProcessor.d.ts.map +1 -1
- package/dist/pipeline/stages/audioProcessor.js +1 -1
- package/dist/pipeline/stages/audioProcessor.js.map +1 -1
- package/dist/pipeline/stages/clipExporter.d.ts +0 -1
- package/dist/pipeline/stages/clipExporter.d.ts.map +1 -1
- package/dist/pipeline/stages/clipExporter.js.map +1 -1
- package/dist/pipeline/stages/segmentAnalyzer.d.ts +1 -2
- package/dist/pipeline/stages/segmentAnalyzer.d.ts.map +1 -1
- package/dist/pipeline/stages/segmentAnalyzer.js +2 -2
- package/dist/pipeline/stages/segmentAnalyzer.js.map +1 -1
- package/dist/pipeline/stages/segmentSelector.d.ts +0 -1
- package/dist/pipeline/stages/segmentSelector.d.ts.map +1 -1
- package/dist/pipeline/stages/segmentSelector.js +1 -2
- package/dist/pipeline/stages/segmentSelector.js.map +1 -1
- package/dist/pipeline/stages/videoResolver.d.ts +0 -1
- package/dist/pipeline/stages/videoResolver.d.ts.map +1 -1
- package/dist/pipeline/stages/videoResolver.js.map +1 -1
- package/dist/services/audioAnalyzers/index.d.ts +0 -4
- package/dist/services/audioAnalyzers/index.d.ts.map +1 -1
- package/dist/services/audioAnalyzers/index.js +0 -3
- package/dist/services/audioAnalyzers/index.js.map +1 -1
- package/dist/services/audioAnalyzers/whisper.d.ts +0 -1
- package/dist/services/audioAnalyzers/whisper.d.ts.map +1 -1
- package/dist/services/audioAnalyzers/whisper.js +1 -21
- package/dist/services/audioAnalyzers/whisper.js.map +1 -1
- package/dist/services/audioAnalyzers/yamnet.js +1 -1
- package/dist/services/audioAnalyzers/yamnet.js.map +1 -1
- package/dist/services/audioDownloader/index.d.ts.map +1 -1
- package/dist/services/audioDownloader/index.js +3 -14
- package/dist/services/audioDownloader/index.js.map +1 -1
- package/dist/services/audioDownloader/sliceAudio.d.ts.map +1 -0
- package/dist/{utils → services/audioDownloader}/sliceAudio.js +1 -1
- package/dist/services/audioDownloader/sliceAudio.js.map +1 -0
- package/dist/services/clipGenerator/index.js +0 -16
- package/dist/services/clipGenerator/index.js.map +1 -1
- package/dist/services/clipRefiner/index.d.ts +2 -1
- package/dist/services/clipRefiner/index.d.ts.map +1 -1
- package/dist/services/clipRefiner/index.js +3 -5
- package/dist/services/clipRefiner/index.js.map +1 -1
- package/dist/services/llmAnalyzer/LLMAnalyzer.d.ts +2 -2
- package/dist/services/llmAnalyzer/LLMAnalyzer.d.ts.map +1 -1
- package/dist/services/llmAnalyzer/LLMAnalyzer.js +2 -2
- package/dist/services/llmAnalyzer/LLMAnalyzer.js.map +1 -1
- package/dist/services/llmAnalyzer/index.d.ts +3 -2
- package/dist/services/llmAnalyzer/index.d.ts.map +1 -1
- package/dist/services/llmAnalyzer/index.js +5 -7
- package/dist/services/llmAnalyzer/index.js.map +1 -1
- package/dist/services/segmentRanker/index.d.ts +2 -1
- package/dist/services/segmentRanker/index.d.ts.map +1 -1
- package/dist/services/segmentRanker/index.js +36 -0
- package/dist/services/segmentRanker/index.js.map +1 -1
- package/dist/services/transcriptAnalyzers/index.d.ts +0 -4
- package/dist/services/transcriptAnalyzers/index.d.ts.map +1 -1
- package/dist/services/transcriptAnalyzers/index.js +0 -3
- package/dist/services/transcriptAnalyzers/index.js.map +1 -1
- package/dist/services/transcriptAnalyzers/whisper.js +1 -1
- package/dist/services/transcriptAnalyzers/whisper.js.map +1 -1
- package/dist/services/transcriptAnalyzers/ytdlp.d.ts +11 -0
- package/dist/services/transcriptAnalyzers/ytdlp.d.ts.map +1 -1
- package/dist/services/transcriptAnalyzers/ytdlp.js +120 -1
- package/dist/services/transcriptAnalyzers/ytdlp.js.map +1 -1
- package/dist/services/transcriptDetector/index.d.ts +2 -2
- package/dist/services/transcriptDetector/index.d.ts.map +1 -1
- package/dist/services/transcriptDetector/index.js.map +1 -1
- package/dist/services/videoDownloader/index.d.ts.map +1 -1
- package/dist/services/videoDownloader/index.js +4 -19
- package/dist/services/videoDownloader/index.js.map +1 -1
- package/dist/types/analyzer.d.ts +2 -1
- package/dist/types/analyzer.d.ts.map +1 -1
- package/dist/types/config.d.ts +7 -3
- package/dist/types/config.d.ts.map +1 -1
- package/dist/types/config.js +12 -0
- package/dist/types/config.js.map +1 -1
- package/dist/types/downloader.d.ts.map +1 -1
- package/dist/types/pipeline.d.ts +3 -1
- package/dist/types/pipeline.d.ts.map +1 -1
- package/dist/utils/cache.d.ts +14 -18
- package/dist/utils/cache.d.ts.map +1 -1
- package/dist/utils/cache.js +32 -112
- package/dist/utils/cache.js.map +1 -1
- package/dist/utils/cacheBackend.d.ts +40 -0
- package/dist/utils/cacheBackend.d.ts.map +1 -0
- package/dist/utils/cacheBackend.js +28 -0
- package/dist/utils/cacheBackend.js.map +1 -0
- package/dist/utils/cacheFactory.d.ts +14 -0
- package/dist/utils/cacheFactory.d.ts.map +1 -0
- package/dist/utils/cacheFactory.js +31 -0
- package/dist/utils/cacheFactory.js.map +1 -0
- package/dist/utils/chunker.d.ts +0 -1
- package/dist/utils/chunker.d.ts.map +1 -1
- package/dist/utils/chunker.js.map +1 -1
- package/dist/utils/fileCacheBackend.d.ts +33 -0
- package/dist/utils/fileCacheBackend.d.ts.map +1 -0
- package/dist/utils/fileCacheBackend.js +123 -0
- package/dist/utils/fileCacheBackend.js.map +1 -0
- package/dist/utils/format.d.ts +6 -0
- package/dist/utils/format.d.ts.map +1 -1
- package/dist/utils/format.js +20 -0
- package/dist/utils/format.js.map +1 -1
- package/dist/utils/logger.d.ts +1 -0
- package/dist/utils/logger.d.ts.map +1 -1
- package/dist/utils/logger.js +10 -0
- package/dist/utils/logger.js.map +1 -1
- package/dist/utils/mongoCacheBackend.d.ts +51 -0
- package/dist/utils/mongoCacheBackend.d.ts.map +1 -0
- package/dist/utils/mongoCacheBackend.js +176 -0
- package/dist/utils/mongoCacheBackend.js.map +1 -0
- package/dist/utils/pythonBin.d.ts +2 -0
- package/dist/utils/pythonBin.d.ts.map +1 -0
- package/dist/utils/pythonBin.js +24 -0
- package/dist/utils/pythonBin.js.map +1 -0
- package/package.json +2 -1
- package/dist/services/signalMerger/index.d.ts +0 -5
- package/dist/services/signalMerger/index.d.ts.map +0 -1
- package/dist/services/signalMerger/index.js +0 -37
- package/dist/services/signalMerger/index.js.map +0 -1
- package/dist/services/transcriptFetcher/index.d.ts +0 -26
- package/dist/services/transcriptFetcher/index.d.ts.map +0 -1
- package/dist/services/transcriptFetcher/index.js +0 -121
- package/dist/services/transcriptFetcher/index.js.map +0 -1
- package/dist/utils/dumper.d.ts.map +0 -1
- package/dist/utils/dumper.js.map +0 -1
- package/dist/utils/redactConfig.d.ts +0 -7
- package/dist/utils/redactConfig.d.ts.map +0 -1
- package/dist/utils/redactConfig.js +0 -21
- package/dist/utils/redactConfig.js.map +0 -1
- package/dist/utils/sliceAudio.d.ts.map +0 -1
- package/dist/utils/sliceAudio.js.map +0 -1
- /package/dist/{utils → pipeline}/dumper.d.ts +0 -0
- /package/dist/{utils → services/audioDownloader}/sliceAudio.d.ts +0 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { TranscriptLine, LLMChunk, ChunkEvaluation, AudioEvent, SegmentRefinement } from '../types/index.js';
|
|
2
|
+
import type { CacheBackend } from './cacheBackend.js';
|
|
3
|
+
/**
|
|
4
|
+
* Disk-backed cache backend. Each cached item is stored as a pretty-printed
|
|
5
|
+
* JSON file under `cacheDir`, named by a SHA-256 hash of the cache key.
|
|
6
|
+
*
|
|
7
|
+
* Directory layout:
|
|
8
|
+
* <cacheDir>/transcript/<hash>.json
|
|
9
|
+
* <cacheDir>/chunks/<hash>.json
|
|
10
|
+
* <cacheDir>/segments/<hash>.json
|
|
11
|
+
* <cacheDir>/audio/<hash>.json
|
|
12
|
+
*/
|
|
13
|
+
export declare class FileCacheBackend implements CacheBackend {
|
|
14
|
+
private readonly cacheDir;
|
|
15
|
+
constructor(cacheDir: string);
|
|
16
|
+
private transcriptPath;
|
|
17
|
+
readTranscript(videoId: string): Promise<TranscriptLine[] | null>;
|
|
18
|
+
writeTranscript(videoId: string, lines: TranscriptLine[]): Promise<void>;
|
|
19
|
+
private chunkPath;
|
|
20
|
+
readChunk(chunk: LLMChunk, chunkAudioEvents?: AudioEvent[]): Promise<ChunkEvaluation | null>;
|
|
21
|
+
writeChunk(chunk: LLMChunk, evaluation: ChunkEvaluation, chunkAudioEvents?: AudioEvent[]): Promise<void>;
|
|
22
|
+
private segmentRefinementPath;
|
|
23
|
+
readSegmentRefinement(start: number, end: number, reason: string): Promise<SegmentRefinement | null>;
|
|
24
|
+
writeSegmentRefinement(start: number, end: number, reason: string, refined: SegmentRefinement): Promise<void>;
|
|
25
|
+
private audioEventPath;
|
|
26
|
+
readAudioEvents(videoId: string, gameProfile: string, provider: string): Promise<AudioEvent[] | null>;
|
|
27
|
+
writeAudioEvents(videoId: string, gameProfile: string, provider: string, events: AudioEvent[]): Promise<void>;
|
|
28
|
+
private audioChunkPath;
|
|
29
|
+
readAudioChunk(videoId: string, gameProfile: string, provider: string, windowStart: number, windowEnd: number): Promise<AudioEvent[] | null>;
|
|
30
|
+
writeAudioChunk(videoId: string, gameProfile: string, provider: string, windowStart: number, windowEnd: number, events: AudioEvent[]): Promise<void>;
|
|
31
|
+
close(): Promise<void>;
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=fileCacheBackend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fileCacheBackend.d.ts","sourceRoot":"","sources":["../../src/utils/fileCacheBackend.ts"],"names":[],"mappings":"AAWA,OAAO,KAAK,EACV,cAAc,EACd,QAAQ,EACR,eAAe,EACf,UAAU,EACV,iBAAiB,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAkDtD;;;;;;;;;GASG;AACH,qBAAa,gBAAiB,YAAW,YAAY;IACvC,OAAO,CAAC,QAAQ,CAAC,QAAQ;gBAAR,QAAQ,EAAE,MAAM;IAI7C,OAAO,CAAC,cAAc;IAIhB,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC;IAIjE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAM9E,OAAO,CAAC,SAAS;IASX,SAAS,CACb,KAAK,EAAE,QAAQ,EACf,gBAAgB,GAAE,UAAU,EAAO,GAClC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAI5B,UAAU,CACd,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,eAAe,EAC3B,gBAAgB,GAAE,UAAU,EAAO,GAClC,OAAO,CAAC,IAAI,CAAC;IAOhB,OAAO,CAAC,qBAAqB;IAIvB,qBAAqB,CACzB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAI9B,sBAAsB,CAC1B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,IAAI,CAAC;IAMhB,OAAO,CAAC,cAAc;IAQhB,eAAe,CACnB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC;IAOzB,gBAAgB,CACpB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,UAAU,EAAE,GACnB,OAAO,CAAC,IAAI,CAAC;IAMhB,OAAO,CAAC,cAAc;IAchB,cAAc,CAClB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC;IAOzB,eAAe,CACnB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,UAAU,EAAE,GACnB,OAAO,CAAC,IAAI,CAAC;IASV,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAG7B"}
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { promises as fs } from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
import { z } from 'zod';
|
|
5
|
+
import { log } from './logger.js';
|
|
6
|
+
import { TranscriptLineSchema, ChunkEvaluationSchema, AudioEventSchema, SegmentRefinementSchema, } from '../types/index.js';
|
|
7
|
+
// ---------------------------------------------------------------------------
|
|
8
|
+
// Internal helpers
|
|
9
|
+
// ---------------------------------------------------------------------------
|
|
10
|
+
/**
|
|
11
|
+
* Serializes audio events into a stable string for cache keying.
|
|
12
|
+
* Events are sorted by time so the key is order-independent.
|
|
13
|
+
*/
|
|
14
|
+
function audioEventsKey(events) {
|
|
15
|
+
if (events.length === 0)
|
|
16
|
+
return '';
|
|
17
|
+
const sorted = [...events].sort((a, b) => a.time - b.time);
|
|
18
|
+
return JSON.stringify(sorted);
|
|
19
|
+
}
|
|
20
|
+
function hashContent(input) {
|
|
21
|
+
return createHash('sha256').update(input).digest('hex');
|
|
22
|
+
}
|
|
23
|
+
async function readCacheFile(filePath, schema) {
|
|
24
|
+
try {
|
|
25
|
+
const raw = await fs.readFile(filePath, 'utf-8');
|
|
26
|
+
const parsed = schema.safeParse(JSON.parse(raw));
|
|
27
|
+
if (!parsed.success) {
|
|
28
|
+
log.warn(`[cache] Corrupt entry at ${filePath} — ignoring`);
|
|
29
|
+
return null;
|
|
30
|
+
}
|
|
31
|
+
return parsed.data;
|
|
32
|
+
}
|
|
33
|
+
catch {
|
|
34
|
+
// File not found or unreadable — normal cache miss, stay silent
|
|
35
|
+
return null;
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
async function writeCacheFile(filePath, data) {
|
|
39
|
+
try {
|
|
40
|
+
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
|
41
|
+
await fs.writeFile(filePath, JSON.stringify(data, null, 2), 'utf-8');
|
|
42
|
+
}
|
|
43
|
+
catch (err) {
|
|
44
|
+
log.warn(`[cache] Failed to write ${filePath}: ${err instanceof Error ? err.message : String(err)}`);
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
// ---------------------------------------------------------------------------
|
|
48
|
+
// FileCacheBackend
|
|
49
|
+
// ---------------------------------------------------------------------------
|
|
50
|
+
/**
|
|
51
|
+
* Disk-backed cache backend. Each cached item is stored as a pretty-printed
|
|
52
|
+
* JSON file under `cacheDir`, named by a SHA-256 hash of the cache key.
|
|
53
|
+
*
|
|
54
|
+
* Directory layout:
|
|
55
|
+
* <cacheDir>/transcript/<hash>.json
|
|
56
|
+
* <cacheDir>/chunks/<hash>.json
|
|
57
|
+
* <cacheDir>/segments/<hash>.json
|
|
58
|
+
* <cacheDir>/audio/<hash>.json
|
|
59
|
+
*/
|
|
60
|
+
export class FileCacheBackend {
|
|
61
|
+
cacheDir;
|
|
62
|
+
constructor(cacheDir) {
|
|
63
|
+
this.cacheDir = cacheDir;
|
|
64
|
+
}
|
|
65
|
+
// ---- Transcript -----------------------------------------------------------
|
|
66
|
+
transcriptPath(videoId) {
|
|
67
|
+
return path.join(this.cacheDir, 'transcript', `${hashContent(videoId)}.json`);
|
|
68
|
+
}
|
|
69
|
+
async readTranscript(videoId) {
|
|
70
|
+
return readCacheFile(this.transcriptPath(videoId), z.array(TranscriptLineSchema));
|
|
71
|
+
}
|
|
72
|
+
async writeTranscript(videoId, lines) {
|
|
73
|
+
await writeCacheFile(this.transcriptPath(videoId), lines);
|
|
74
|
+
}
|
|
75
|
+
// ---- LLM chunk results ----------------------------------------------------
|
|
76
|
+
chunkPath(chunk, chunkAudioEvents = []) {
|
|
77
|
+
const audioKey = audioEventsKey(chunkAudioEvents);
|
|
78
|
+
return path.join(this.cacheDir, 'chunks', `${hashContent(`${chunk.start}|${chunk.end}|${chunk.text}|${audioKey}`)}.json`);
|
|
79
|
+
}
|
|
80
|
+
async readChunk(chunk, chunkAudioEvents = []) {
|
|
81
|
+
return readCacheFile(this.chunkPath(chunk, chunkAudioEvents), ChunkEvaluationSchema);
|
|
82
|
+
}
|
|
83
|
+
async writeChunk(chunk, evaluation, chunkAudioEvents = []) {
|
|
84
|
+
if (evaluation.status !== 'success')
|
|
85
|
+
return;
|
|
86
|
+
await writeCacheFile(this.chunkPath(chunk, chunkAudioEvents), evaluation);
|
|
87
|
+
}
|
|
88
|
+
// ---- Segment refinement ---------------------------------------------------
|
|
89
|
+
segmentRefinementPath(start, end, reason) {
|
|
90
|
+
return path.join(this.cacheDir, 'segments', `${hashContent(`${start}|${end}|${reason}`)}.json`);
|
|
91
|
+
}
|
|
92
|
+
async readSegmentRefinement(start, end, reason) {
|
|
93
|
+
return readCacheFile(this.segmentRefinementPath(start, end, reason), SegmentRefinementSchema);
|
|
94
|
+
}
|
|
95
|
+
async writeSegmentRefinement(start, end, reason, refined) {
|
|
96
|
+
await writeCacheFile(this.segmentRefinementPath(start, end, reason), refined);
|
|
97
|
+
}
|
|
98
|
+
// ---- Audio events (whole-video) -------------------------------------------
|
|
99
|
+
audioEventPath(videoId, gameProfile, provider) {
|
|
100
|
+
return path.join(this.cacheDir, 'audio', `${hashContent(`${videoId}|${gameProfile}|${provider}`)}.json`);
|
|
101
|
+
}
|
|
102
|
+
async readAudioEvents(videoId, gameProfile, provider) {
|
|
103
|
+
return readCacheFile(this.audioEventPath(videoId, gameProfile, provider), z.array(AudioEventSchema));
|
|
104
|
+
}
|
|
105
|
+
async writeAudioEvents(videoId, gameProfile, provider, events) {
|
|
106
|
+
await writeCacheFile(this.audioEventPath(videoId, gameProfile, provider), events);
|
|
107
|
+
}
|
|
108
|
+
// ---- Audio events (per-chunk) ---------------------------------------------
|
|
109
|
+
audioChunkPath(videoId, gameProfile, provider, windowStart, windowEnd) {
|
|
110
|
+
return path.join(this.cacheDir, 'audio', `${hashContent(`${videoId}|${gameProfile}|${provider}|${windowStart}|${windowEnd}`)}.json`);
|
|
111
|
+
}
|
|
112
|
+
async readAudioChunk(videoId, gameProfile, provider, windowStart, windowEnd) {
|
|
113
|
+
return readCacheFile(this.audioChunkPath(videoId, gameProfile, provider, windowStart, windowEnd), z.array(AudioEventSchema));
|
|
114
|
+
}
|
|
115
|
+
async writeAudioChunk(videoId, gameProfile, provider, windowStart, windowEnd, events) {
|
|
116
|
+
await writeCacheFile(this.audioChunkPath(videoId, gameProfile, provider, windowStart, windowEnd), events);
|
|
117
|
+
}
|
|
118
|
+
// ---- Lifecycle ------------------------------------------------------------
|
|
119
|
+
async close() {
|
|
120
|
+
// No-op — file handles are not kept open between operations
|
|
121
|
+
}
|
|
122
|
+
}
|
|
123
|
+
//# sourceMappingURL=fileCacheBackend.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fileCacheBackend.js","sourceRoot":"","sources":["../../src/utils/fileCacheBackend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,QAAQ,IAAI,EAAE,EAAE,MAAM,IAAI,CAAC;AACpC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAU3B,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;GAGG;AACH,SAAS,cAAc,CAAC,MAAoB;IAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3D,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,KAAK,UAAU,aAAa,CAAI,QAAgB,EAAE,MAAoB;IACpE,IAAI,CAAC;QACH,MAAM,GAAG,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;YACpB,GAAG,CAAC,IAAI,CAAC,4BAA4B,QAAQ,aAAa,CAAC,CAAC;YAC5D,OAAO,IAAI,CAAC;QACd,CAAC;QACD,OAAO,MAAM,CAAC,IAAI,CAAC;IACrB,CAAC;IAAC,MAAM,CAAC;QACP,gEAAgE;QAChE,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,QAAgB,EAAE,IAAa;IAC3D,IAAI,CAAC;QACH,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC5D,MAAM,EAAE,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;IACvE,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,GAAG,CAAC,IAAI,CACN,2BAA2B,QAAQ,KAAK,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAC3F,CAAC;IACJ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E;;;;;;;;;GASG;AACH,MAAM,OAAO,gBAAgB;IACE;IAA7B,YAA6B,QAAgB;QAAhB,aAAQ,GAAR,QAAQ,CAAQ;IAAG,CAAC;IAEjD,8EAA8E;IAEtE,cAAc,CAAC,OAAe;QACpC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,WAAW,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAChF,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,OAAe;QAClC,OAAO,aAAa,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC,CAAC;IACpF,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,EAAE,KAAuB;QAC5D,MAAM,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;IAC5D,CAAC;IAED,8EAA8E;IAEtE,SAAS,CAAC,KAAe,EAAE,mBAAiC,EAAE;QACpE,MAAM,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAClD,OAAO,IAAI,CAAC,IAAI,CACd,IAAI,CAAC,QAAQ,EACb,QAAQ,EACR,GAAG,WAAW,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,OAAO,CAC/E,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAe,EACf,mBAAiC,EAAE;QAEnC,OAAO,aAAa,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,gBAAgB,CAAC,EAAE,qBAAqB,CAAC,CAAC;IACvF,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,UAA2B,EAC3B,mBAAiC,EAAE;QAEnC,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO;QAC5C,MAAM,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,gBAAgB,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5E,CAAC;IAED,8EAA8E;IAEtE,qBAAqB,CAAC,KAAa,EAAE,GAAW,EAAE,MAAc;QACtE,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,EAAE,UAAU,EAAE,GAAG,WAAW,CAAC,GAAG,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC,OAAO,CAAC,CAAC;IAClG,CAAC;IAED,KAAK,CAAC,qBAAqB,CACzB,KAAa,EACb,GAAW,EACX,MAAc;QAEd,OAAO,aAAa,CAAC,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,uBAAuB,CAAC,CAAC;IAChG,CAAC;IAED,KAAK,CAAC,sBAAsB,CAC1B,KAAa,EACb,GAAW,EACX,MAAc,EACd,OAA0B;QAE1B,MAAM,cAAc,CAAC,IAAI,CAAC,qBAAqB,CAAC,KAAK,EAAE,GAAG,EAAE,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;IAChF,CAAC;IAED,8EAA8E;IAEtE,cAAc,CAAC,OAAe,EAAE,WAAmB,EAAE,QAAgB;QAC3E,OAAO,IAAI,CAAC,IAAI,CACd,IAAI,CAAC,QAAQ,EACb,OAAO,EACP,GAAG,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC,OAAO,CAC/D,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,OAAe,EACf,WAAmB,EACnB,QAAgB;QAEhB,OAAO,aAAa,CAClB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,EACnD,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,MAAoB;QAEpB,MAAM,cAAc,CAAC,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,CAAC,EAAE,MAAM,CAAC,CAAC;IACpF,CAAC;IAED,8EAA8E;IAEtE,cAAc,CACpB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,WAAmB,EACnB,SAAiB;QAEjB,OAAO,IAAI,CAAC,IAAI,CACd,IAAI,CAAC,QAAQ,EACb,OAAO,EACP,GAAG,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC,OAAO,CAC3F,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,cAAc,CAClB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,WAAmB,EACnB,SAAiB;QAEjB,OAAO,aAAa,CAClB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,EAC3E,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,WAAmB,EACnB,SAAiB,EACjB,MAAoB;QAEpB,MAAM,cAAc,CAClB,IAAI,CAAC,cAAc,CAAC,OAAO,EAAE,WAAW,EAAE,QAAQ,EAAE,WAAW,EAAE,SAAS,CAAC,EAC3E,MAAM,CACP,CAAC;IACJ,CAAC;IAED,8EAA8E;IAE9E,KAAK,CAAC,KAAK;QACT,4DAA4D;IAC9D,CAAC;CACF"}
|
package/dist/utils/format.d.ts
CHANGED
|
@@ -1,3 +1,9 @@
|
|
|
1
|
+
import type { Config } from '../types/config.js';
|
|
2
|
+
/**
|
|
3
|
+
* Formats the resolved config as a single-line key=value string,
|
|
4
|
+
* omitting all API key fields and any undefined optional values.
|
|
5
|
+
*/
|
|
6
|
+
export declare function formatConfig(cfg: Config): string;
|
|
1
7
|
/**
|
|
2
8
|
* Converts a duration in seconds to HH:MM:SS string.
|
|
3
9
|
* e.g. 3723 → "01:02:03"
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/utils/format.ts"],"names":[],"mappings":"AAAA;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAKrD"}
|
|
1
|
+
{"version":3,"file":"format.d.ts","sourceRoot":"","sources":["../../src/utils/format.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,oBAAoB,CAAC;AAYjD;;;GAGG;AACH,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,MAAM,CAMhD;AAED;;;GAGG;AACH,wBAAgB,aAAa,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAKrD"}
|
package/dist/utils/format.js
CHANGED
|
@@ -1,3 +1,23 @@
|
|
|
1
|
+
const SENSITIVE_KEYS = new Set([
|
|
2
|
+
'OPENAI_API_KEY',
|
|
3
|
+
'ANTHROPIC_API_KEY',
|
|
4
|
+
'GOOGLE_GENERATIVE_AI_API_KEY',
|
|
5
|
+
'XAI_API_KEY',
|
|
6
|
+
'MISTRAL_API_KEY',
|
|
7
|
+
'GROQ_API_KEY',
|
|
8
|
+
'ZAI_API_KEY',
|
|
9
|
+
]);
|
|
10
|
+
/**
|
|
11
|
+
* Formats the resolved config as a single-line key=value string,
|
|
12
|
+
* omitting all API key fields and any undefined optional values.
|
|
13
|
+
*/
|
|
14
|
+
export function formatConfig(cfg) {
|
|
15
|
+
return Object.entries(cfg)
|
|
16
|
+
.filter(([k]) => !SENSITIVE_KEYS.has(k))
|
|
17
|
+
.filter(([, v]) => v !== undefined)
|
|
18
|
+
.map(([k, v]) => `${k}=${String(v)}`)
|
|
19
|
+
.join(' ');
|
|
20
|
+
}
|
|
1
21
|
/**
|
|
2
22
|
* Converts a duration in seconds to HH:MM:SS string.
|
|
3
23
|
* e.g. 3723 → "01:02:03"
|
package/dist/utils/format.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/utils/format.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"format.js","sourceRoot":"","sources":["../../src/utils/format.ts"],"names":[],"mappings":"AAEA,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC;IAC7B,gBAAgB;IAChB,mBAAmB;IACnB,8BAA8B;IAC9B,aAAa;IACb,iBAAiB;IACjB,cAAc;IACd,aAAa;CACd,CAAC,CAAC;AAEH;;;GAGG;AACH,MAAM,UAAU,YAAY,CAAC,GAAW;IACtC,OAAQ,MAAM,CAAC,OAAO,CAAC,GAAG,CAAyB;SAChD,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,cAAc,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SACvC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC;SAClC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC;SACpC,IAAI,CAAC,GAAG,CAAC,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,OAAe;IAC3C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,CAAC;IAC5C,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,EAAE,CAAC,CAAC;IACnC,OAAO,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;AACpE,CAAC"}
|
package/dist/utils/logger.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,GAAG;gBACF,MAAM,KAAG,IAAI;gBAGb,MAAM,KAAG,IAAI;iBAGZ,MAAM,KAAG,IAAI;
|
|
1
|
+
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAMA,eAAO,MAAM,GAAG;gBACF,MAAM,KAAG,IAAI;gBAGb,MAAM,KAAG,IAAI;iBAGZ,MAAM,KAAG,IAAI;qBAGT,MAAM,GAAG,MAAM,KAAG,IAAI;CAWxC,CAAC"}
|
package/dist/utils/logger.js
CHANGED
|
@@ -13,5 +13,15 @@ export const log = {
|
|
|
13
13
|
error: (msg) => {
|
|
14
14
|
console.error(`${LEVELS.error} ${msg}`);
|
|
15
15
|
},
|
|
16
|
+
progress: (data) => {
|
|
17
|
+
const text = String(data);
|
|
18
|
+
const lines = text.split('\n').filter((line) => line.trim());
|
|
19
|
+
for (const line of lines) {
|
|
20
|
+
const progressMatch = line.match(/\[download\]\s+(\d+\.?\d*%)/);
|
|
21
|
+
if (progressMatch) {
|
|
22
|
+
process.stdout.write(`\r${progressMatch[0]}`);
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
},
|
|
16
26
|
};
|
|
17
27
|
//# sourceMappingURL=logger.js.map
|
package/dist/utils/logger.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,GAAG;IACb,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,KAAK,EAAE,SAAS;CACR,CAAC;AAEX,MAAM,CAAC,MAAM,GAAG,GAAG;IACjB,IAAI,EAAE,CAAC,GAAW,EAAQ,EAAE;QAC1B,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,EAAE,CAAC,GAAW,EAAQ,EAAE;QAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,KAAK,EAAE,CAAC,GAAW,EAAQ,EAAE;QAC3B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;CACF,CAAC"}
|
|
1
|
+
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/utils/logger.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,GAAG;IACb,IAAI,EAAE,QAAQ;IACd,IAAI,EAAE,QAAQ;IACd,KAAK,EAAE,SAAS;CACR,CAAC;AAEX,MAAM,CAAC,MAAM,GAAG,GAAG;IACjB,IAAI,EAAE,CAAC,GAAW,EAAQ,EAAE;QAC1B,OAAO,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;IACvC,CAAC;IACD,IAAI,EAAE,CAAC,GAAW,EAAQ,EAAE;QAC1B,OAAO,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;IACxC,CAAC;IACD,KAAK,EAAE,CAAC,GAAW,EAAQ,EAAE;QAC3B,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC,KAAK,IAAI,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;IACD,QAAQ,EAAE,CAAC,IAAqB,EAAQ,EAAE;QACxC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,CAAC;QAC1B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC;QAE7D,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,aAAa,GAAG,IAAI,CAAC,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAChE,IAAI,aAAa,EAAE,CAAC;gBAClB,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,aAAa,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;YAChD,CAAC;QACH,CAAC;IACH,CAAC;CACF,CAAC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { MongoClient } from 'mongodb';
|
|
2
|
+
import type { TranscriptLine, LLMChunk, ChunkEvaluation, AudioEvent, SegmentRefinement } from '../types/index.js';
|
|
3
|
+
import type { CacheBackend } from './cacheBackend.js';
|
|
4
|
+
/**
|
|
5
|
+
* MongoDB-backed cache backend.
|
|
6
|
+
*
|
|
7
|
+
* Each cache category maps to one collection. Documents have the shape:
|
|
8
|
+
* { cacheKey: string, data: <payload>, createdAt: Date }
|
|
9
|
+
*
|
|
10
|
+
* Indexes created on first use (idempotent):
|
|
11
|
+
* - Unique index on `cacheKey` for O(1) lookups
|
|
12
|
+
* - TTL index on `createdAt` when `ttlSeconds > 0`
|
|
13
|
+
*
|
|
14
|
+
* All reads validate the stored payload through the same zod schemas used by
|
|
15
|
+
* the file backend, giving identical runtime safety guarantees.
|
|
16
|
+
*
|
|
17
|
+
* Pass `ttlSeconds = 0` (or omit) to disable TTL expiration.
|
|
18
|
+
*/
|
|
19
|
+
export declare class MongoCacheBackend implements CacheBackend {
|
|
20
|
+
private readonly client;
|
|
21
|
+
private readonly db;
|
|
22
|
+
private readonly ttlSeconds;
|
|
23
|
+
/** Track which collections already have their indexes created this session. */
|
|
24
|
+
private readonly indexedCollections;
|
|
25
|
+
constructor(client: MongoClient, databaseName: string, ttlSeconds?: number);
|
|
26
|
+
/**
|
|
27
|
+
* Ensures indexes exist for a collection. Called lazily on first access per
|
|
28
|
+
* collection name. `createIndex` is idempotent so concurrent calls are safe.
|
|
29
|
+
*/
|
|
30
|
+
private ensureIndexes;
|
|
31
|
+
private readDoc;
|
|
32
|
+
private writeDoc;
|
|
33
|
+
private static readonly TRANSCRIPTS;
|
|
34
|
+
readTranscript(videoId: string): Promise<TranscriptLine[] | null>;
|
|
35
|
+
writeTranscript(videoId: string, lines: TranscriptLine[]): Promise<void>;
|
|
36
|
+
private static readonly CHUNKS;
|
|
37
|
+
private chunkKey;
|
|
38
|
+
readChunk(chunk: LLMChunk, chunkAudioEvents?: AudioEvent[]): Promise<ChunkEvaluation | null>;
|
|
39
|
+
writeChunk(chunk: LLMChunk, evaluation: ChunkEvaluation, chunkAudioEvents?: AudioEvent[]): Promise<void>;
|
|
40
|
+
private static readonly SEGMENTS;
|
|
41
|
+
readSegmentRefinement(start: number, end: number, reason: string): Promise<SegmentRefinement | null>;
|
|
42
|
+
writeSegmentRefinement(start: number, end: number, reason: string, refined: SegmentRefinement): Promise<void>;
|
|
43
|
+
private static readonly AUDIO_EVENTS;
|
|
44
|
+
readAudioEvents(videoId: string, gameProfile: string, provider: string): Promise<AudioEvent[] | null>;
|
|
45
|
+
writeAudioEvents(videoId: string, gameProfile: string, provider: string, events: AudioEvent[]): Promise<void>;
|
|
46
|
+
private static readonly AUDIO_CHUNKS;
|
|
47
|
+
readAudioChunk(videoId: string, gameProfile: string, provider: string, windowStart: number, windowEnd: number): Promise<AudioEvent[] | null>;
|
|
48
|
+
writeAudioChunk(videoId: string, gameProfile: string, provider: string, windowStart: number, windowEnd: number, events: AudioEvent[]): Promise<void>;
|
|
49
|
+
close(): Promise<void>;
|
|
50
|
+
}
|
|
51
|
+
//# sourceMappingURL=mongoCacheBackend.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mongoCacheBackend.d.ts","sourceRoot":"","sources":["../../src/utils/mongoCacheBackend.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAS3C,OAAO,KAAK,EACV,cAAc,EACd,QAAQ,EACR,eAAe,EACf,UAAU,EACV,iBAAiB,EAClB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAqCtD;;;;;;;;;;;;;;GAcG;AACH,qBAAa,iBAAkB,YAAW,YAAY;IAOlD,OAAO,CAAC,QAAQ,CAAC,MAAM;IANzB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;IACpB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;IACpC,+EAA+E;IAC/E,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAqB;gBAGrC,MAAM,EAAE,WAAW,EACpC,YAAY,EAAE,MAAM,EACpB,UAAU,SAAI;IAUhB;;;OAGG;YACW,aAAa;YAuBb,OAAO;YAyBP,QAAQ;IAoBtB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,WAAW,CAAiB;IAE9C,cAAc,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,GAAG,IAAI,CAAC;IAQjE,eAAe,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,cAAc,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAQ9E,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAsB;IAEpD,OAAO,CAAC,QAAQ;IAKV,SAAS,CACb,KAAK,EAAE,QAAQ,EACf,gBAAgB,GAAE,UAAU,EAAO,GAClC,OAAO,CAAC,eAAe,GAAG,IAAI,CAAC;IAQ5B,UAAU,CACd,KAAK,EAAE,QAAQ,EACf,UAAU,EAAE,eAAe,EAC3B,gBAAgB,GAAE,UAAU,EAAO,GAClC,OAAO,CAAC,IAAI,CAAC;IAchB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IAElD,qBAAqB,CACzB,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,GACb,OAAO,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAQ9B,sBAAsB,CAC1B,KAAK,EAAE,MAAM,EACb,GAAG,EAAE,MAAM,EACX,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,iBAAiB,GACzB,OAAO,CAAC,IAAI,CAAC;IAYhB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAiB;IAE/C,eAAe,CACnB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,GACf,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC;IAQzB,gBAAgB,CACpB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,UAAU,EAAE,GACnB,OAAO,CAAC,IAAI,CAAC;IAYhB,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,YAAY,CAAiB;IAE/C,cAAc,CAClB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,GAChB,OAAO,CAAC,UAAU,EAAE,GAAG,IAAI,CAAC;IAQzB,eAAe,CACnB,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,MAAM,EACnB,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,UAAU,EAAE,GACnB,OAAO,CAAC,IAAI,CAAC;IAYV,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;CAS7B"}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { log } from './logger.js';
|
|
4
|
+
import { TranscriptLineSchema, ChunkEvaluationSchema, AudioEventSchema, SegmentRefinementSchema, } from '../types/index.js';
|
|
5
|
+
// ---------------------------------------------------------------------------
|
|
6
|
+
// Internal helpers
|
|
7
|
+
// ---------------------------------------------------------------------------
|
|
8
|
+
function hashContent(input) {
|
|
9
|
+
return createHash('sha256').update(input).digest('hex');
|
|
10
|
+
}
|
|
11
|
+
/**
|
|
12
|
+
* Serializes audio events into a stable string for cache keying.
|
|
13
|
+
* Events are sorted by time so the key is order-independent.
|
|
14
|
+
*/
|
|
15
|
+
function audioEventsKey(events) {
|
|
16
|
+
if (events.length === 0)
|
|
17
|
+
return '';
|
|
18
|
+
const sorted = [...events].sort((a, b) => a.time - b.time);
|
|
19
|
+
return JSON.stringify(sorted);
|
|
20
|
+
}
|
|
21
|
+
// ---------------------------------------------------------------------------
|
|
22
|
+
// MongoCacheBackend
|
|
23
|
+
// ---------------------------------------------------------------------------
|
|
24
|
+
/**
|
|
25
|
+
* MongoDB-backed cache backend.
|
|
26
|
+
*
|
|
27
|
+
* Each cache category maps to one collection. Documents have the shape:
|
|
28
|
+
* { cacheKey: string, data: <payload>, createdAt: Date }
|
|
29
|
+
*
|
|
30
|
+
* Indexes created on first use (idempotent):
|
|
31
|
+
* - Unique index on `cacheKey` for O(1) lookups
|
|
32
|
+
* - TTL index on `createdAt` when `ttlSeconds > 0`
|
|
33
|
+
*
|
|
34
|
+
* All reads validate the stored payload through the same zod schemas used by
|
|
35
|
+
* the file backend, giving identical runtime safety guarantees.
|
|
36
|
+
*
|
|
37
|
+
* Pass `ttlSeconds = 0` (or omit) to disable TTL expiration.
|
|
38
|
+
*/
|
|
39
|
+
export class MongoCacheBackend {
|
|
40
|
+
client;
|
|
41
|
+
db;
|
|
42
|
+
ttlSeconds;
|
|
43
|
+
/** Track which collections already have their indexes created this session. */
|
|
44
|
+
indexedCollections = new Set();
|
|
45
|
+
constructor(client, databaseName, ttlSeconds = 0) {
|
|
46
|
+
this.client = client;
|
|
47
|
+
this.db = client.db(databaseName);
|
|
48
|
+
this.ttlSeconds = ttlSeconds;
|
|
49
|
+
}
|
|
50
|
+
// ---------------------------------------------------------------------------
|
|
51
|
+
// Index management
|
|
52
|
+
// ---------------------------------------------------------------------------
|
|
53
|
+
/**
|
|
54
|
+
* Ensures indexes exist for a collection. Called lazily on first access per
|
|
55
|
+
* collection name. `createIndex` is idempotent so concurrent calls are safe.
|
|
56
|
+
*/
|
|
57
|
+
async ensureIndexes(collectionName) {
|
|
58
|
+
if (this.indexedCollections.has(collectionName))
|
|
59
|
+
return;
|
|
60
|
+
try {
|
|
61
|
+
const col = this.db.collection(collectionName);
|
|
62
|
+
// Unique index for O(1) cache key lookups
|
|
63
|
+
await col.createIndex({ cacheKey: 1 }, { unique: true });
|
|
64
|
+
// TTL index — only when a positive TTL is configured
|
|
65
|
+
if (this.ttlSeconds > 0) {
|
|
66
|
+
await col.createIndex({ createdAt: 1 }, { expireAfterSeconds: this.ttlSeconds });
|
|
67
|
+
}
|
|
68
|
+
this.indexedCollections.add(collectionName);
|
|
69
|
+
}
|
|
70
|
+
catch (err) {
|
|
71
|
+
// Non-fatal — pipeline should not crash if indexes can't be created
|
|
72
|
+
log.warn(`[mongo-cache] Failed to ensure indexes on "${collectionName}": ${err instanceof Error ? err.message : String(err)}`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
// ---------------------------------------------------------------------------
|
|
76
|
+
// Low-level read / write helpers
|
|
77
|
+
// ---------------------------------------------------------------------------
|
|
78
|
+
async readDoc(collectionName, cacheKey, schema) {
|
|
79
|
+
try {
|
|
80
|
+
await this.ensureIndexes(collectionName);
|
|
81
|
+
const col = this.db.collection(collectionName);
|
|
82
|
+
const doc = await col.findOne({ cacheKey });
|
|
83
|
+
if (!doc)
|
|
84
|
+
return null;
|
|
85
|
+
const parsed = schema.safeParse(doc.data);
|
|
86
|
+
if (!parsed.success) {
|
|
87
|
+
log.warn(`[mongo-cache] Corrupt entry in "${collectionName}" (key=${cacheKey}) — ignoring`);
|
|
88
|
+
return null;
|
|
89
|
+
}
|
|
90
|
+
return parsed.data;
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
log.warn(`[mongo-cache] Read failed in "${collectionName}": ${err instanceof Error ? err.message : String(err)}`);
|
|
94
|
+
return null;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
async writeDoc(collectionName, cacheKey, data) {
|
|
98
|
+
try {
|
|
99
|
+
await this.ensureIndexes(collectionName);
|
|
100
|
+
const col = this.db.collection(collectionName);
|
|
101
|
+
await col.updateOne({ cacheKey }, { $set: { cacheKey, data, createdAt: new Date() } }, { upsert: true });
|
|
102
|
+
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
log.warn(`[mongo-cache] Write failed in "${collectionName}": ${err instanceof Error ? err.message : String(err)}`);
|
|
105
|
+
}
|
|
106
|
+
}
|
|
107
|
+
// ---------------------------------------------------------------------------
|
|
108
|
+
// Transcript
|
|
109
|
+
// ---------------------------------------------------------------------------
|
|
110
|
+
static TRANSCRIPTS = 'transcripts';
|
|
111
|
+
async readTranscript(videoId) {
|
|
112
|
+
return this.readDoc(MongoCacheBackend.TRANSCRIPTS, hashContent(videoId), z.array(TranscriptLineSchema));
|
|
113
|
+
}
|
|
114
|
+
async writeTranscript(videoId, lines) {
|
|
115
|
+
await this.writeDoc(MongoCacheBackend.TRANSCRIPTS, hashContent(videoId), lines);
|
|
116
|
+
}
|
|
117
|
+
// ---------------------------------------------------------------------------
|
|
118
|
+
// LLM chunk results
|
|
119
|
+
// ---------------------------------------------------------------------------
|
|
120
|
+
static CHUNKS = 'chunkEvaluations';
|
|
121
|
+
chunkKey(chunk, chunkAudioEvents = []) {
|
|
122
|
+
const audioKey = audioEventsKey(chunkAudioEvents);
|
|
123
|
+
return hashContent(`${chunk.start}|${chunk.end}|${chunk.text}|${audioKey}`);
|
|
124
|
+
}
|
|
125
|
+
async readChunk(chunk, chunkAudioEvents = []) {
|
|
126
|
+
return this.readDoc(MongoCacheBackend.CHUNKS, this.chunkKey(chunk, chunkAudioEvents), ChunkEvaluationSchema);
|
|
127
|
+
}
|
|
128
|
+
async writeChunk(chunk, evaluation, chunkAudioEvents = []) {
|
|
129
|
+
// Only cache successful evaluations — mirrors the file backend behaviour
|
|
130
|
+
if (evaluation.status !== 'success')
|
|
131
|
+
return;
|
|
132
|
+
await this.writeDoc(MongoCacheBackend.CHUNKS, this.chunkKey(chunk, chunkAudioEvents), evaluation);
|
|
133
|
+
}
|
|
134
|
+
// ---------------------------------------------------------------------------
|
|
135
|
+
// Segment refinement
|
|
136
|
+
// ---------------------------------------------------------------------------
|
|
137
|
+
static SEGMENTS = 'segmentRefinements';
|
|
138
|
+
async readSegmentRefinement(start, end, reason) {
|
|
139
|
+
return this.readDoc(MongoCacheBackend.SEGMENTS, hashContent(`${start}|${end}|${reason}`), SegmentRefinementSchema);
|
|
140
|
+
}
|
|
141
|
+
async writeSegmentRefinement(start, end, reason, refined) {
|
|
142
|
+
await this.writeDoc(MongoCacheBackend.SEGMENTS, hashContent(`${start}|${end}|${reason}`), refined);
|
|
143
|
+
}
|
|
144
|
+
// ---------------------------------------------------------------------------
|
|
145
|
+
// Audio events (whole-video)
|
|
146
|
+
// ---------------------------------------------------------------------------
|
|
147
|
+
static AUDIO_EVENTS = 'audioEvents';
|
|
148
|
+
async readAudioEvents(videoId, gameProfile, provider) {
|
|
149
|
+
return this.readDoc(MongoCacheBackend.AUDIO_EVENTS, hashContent(`${videoId}|${gameProfile}|${provider}`), z.array(AudioEventSchema));
|
|
150
|
+
}
|
|
151
|
+
async writeAudioEvents(videoId, gameProfile, provider, events) {
|
|
152
|
+
await this.writeDoc(MongoCacheBackend.AUDIO_EVENTS, hashContent(`${videoId}|${gameProfile}|${provider}`), events);
|
|
153
|
+
}
|
|
154
|
+
// ---------------------------------------------------------------------------
|
|
155
|
+
// Audio events (per-chunk)
|
|
156
|
+
// ---------------------------------------------------------------------------
|
|
157
|
+
static AUDIO_CHUNKS = 'audioChunks';
|
|
158
|
+
async readAudioChunk(videoId, gameProfile, provider, windowStart, windowEnd) {
|
|
159
|
+
return this.readDoc(MongoCacheBackend.AUDIO_CHUNKS, hashContent(`${videoId}|${gameProfile}|${provider}|${windowStart}|${windowEnd}`), z.array(AudioEventSchema));
|
|
160
|
+
}
|
|
161
|
+
async writeAudioChunk(videoId, gameProfile, provider, windowStart, windowEnd, events) {
|
|
162
|
+
await this.writeDoc(MongoCacheBackend.AUDIO_CHUNKS, hashContent(`${videoId}|${gameProfile}|${provider}|${windowStart}|${windowEnd}`), events);
|
|
163
|
+
}
|
|
164
|
+
// ---------------------------------------------------------------------------
|
|
165
|
+
// Lifecycle
|
|
166
|
+
// ---------------------------------------------------------------------------
|
|
167
|
+
async close() {
|
|
168
|
+
try {
|
|
169
|
+
await this.client.close();
|
|
170
|
+
}
|
|
171
|
+
catch (err) {
|
|
172
|
+
log.warn(`[mongo-cache] Failed to close MongoDB connection: ${err instanceof Error ? err.message : String(err)}`);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
}
|
|
176
|
+
//# sourceMappingURL=mongoCacheBackend.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"mongoCacheBackend.js","sourceRoot":"","sources":["../../src/utils/mongoCacheBackend.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAClC,OAAO,EACL,oBAAoB,EACpB,qBAAqB,EACrB,gBAAgB,EAChB,uBAAuB,GACxB,MAAM,mBAAmB,CAAC;AAuB3B,8EAA8E;AAC9E,mBAAmB;AACnB,8EAA8E;AAE9E,SAAS,WAAW,CAAC,KAAa;IAChC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,MAAoB;IAC1C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACnC,MAAM,MAAM,GAAG,CAAC,GAAG,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3D,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;;;;;;;;;;;;;GAcG;AACH,MAAM,OAAO,iBAAiB;IAOT;IANF,EAAE,CAAC;IACH,UAAU,CAAS;IACpC,+EAA+E;IAC9D,kBAAkB,GAAG,IAAI,GAAG,EAAU,CAAC;IAExD,YACmB,MAAmB,EACpC,YAAoB,EACpB,UAAU,GAAG,CAAC;QAFG,WAAM,GAAN,MAAM,CAAa;QAIpC,IAAI,CAAC,EAAE,GAAG,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,CAAC;QAClC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,8EAA8E;IAC9E,mBAAmB;IACnB,8EAA8E;IAE9E;;;OAGG;IACK,KAAK,CAAC,aAAa,CAAC,cAAsB;QAChD,IAAI,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,cAAc,CAAC;YAAE,OAAO;QACxD,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAC/C,0CAA0C;YAC1C,MAAM,GAAG,CAAC,WAAW,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;YACzD,qDAAqD;YACrD,IAAI,IAAI,CAAC,UAAU,GAAG,CAAC,EAAE,CAAC;gBACxB,MAAM,GAAG,CAAC,WAAW,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,EAAE,kBAAkB,EAAE,IAAI,CAAC,UAAU,EAAE,CAAC,CAAC;YACnF,CAAC;YACD,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;QAC9C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,oEAAoE;YACpE,GAAG,CAAC,IAAI,CACN,8CAA8C,cAAc,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACrH,CAAC;QACJ,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,iCAAiC;IACjC,8EAA8E;IAEtE,KAAK,CAAC,OAAO,CACnB,cAAsB,EACtB,QAAgB,EAChB,MAAoB;QAEpB,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;YACzC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAyB,cAAc,CAAC,CAAC;YACvE,MAAM,GAAG,GAAG,MAAM,GAAG,CAAC,OAAO,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;YAC5C,IAAI,CAAC,GAAG;gBAAE,OAAO,IAAI,CAAC;YAEtB,MAAM,MAAM,GAAG,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YAC1C,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;gBACpB,GAAG,CAAC,IAAI,CAAC,mCAAmC,cAAc,UAAU,QAAQ,cAAc,CAAC,CAAC;gBAC5F,OAAO,IAAI,CAAC;YACd,CAAC;YACD,OAAO,MAAM,CAAC,IAAI,CAAC;QACrB,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CACN,iCAAiC,cAAc,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACxG,CAAC;YACF,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,QAAQ,CAAC,cAAsB,EAAE,QAAgB,EAAE,IAAa;QAC5E,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,CAAC;YACzC,MAAM,GAAG,GAAG,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,cAAc,CAAC,CAAC;YAC/C,MAAM,GAAG,CAAC,SAAS,CACjB,EAAE,QAAQ,EAAE,EACZ,EAAE,IAAI,EAAE,EAAE,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,IAAI,EAAE,EAAE,EAAE,EACnD,EAAE,MAAM,EAAE,IAAI,EAAE,CACjB,CAAC;QACJ,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CACN,kCAAkC,cAAc,MAAM,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACzG,CAAC;QACJ,CAAC;IACH,CAAC;IAED,8EAA8E;IAC9E,aAAa;IACb,8EAA8E;IAEtE,MAAM,CAAU,WAAW,GAAG,aAAa,CAAC;IAEpD,KAAK,CAAC,cAAc,CAAC,OAAe;QAClC,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,WAAW,EAC7B,WAAW,CAAC,OAAO,CAAC,EACpB,CAAC,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAC9B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CAAC,OAAe,EAAE,KAAuB;QAC5D,MAAM,IAAI,CAAC,QAAQ,CAAC,iBAAiB,CAAC,WAAW,EAAE,WAAW,CAAC,OAAO,CAAC,EAAE,KAAK,CAAC,CAAC;IAClF,CAAC;IAED,8EAA8E;IAC9E,oBAAoB;IACpB,8EAA8E;IAEtE,MAAM,CAAU,MAAM,GAAG,kBAAkB,CAAC;IAE5C,QAAQ,CAAC,KAAe,EAAE,mBAAiC,EAAE;QACnE,MAAM,QAAQ,GAAG,cAAc,CAAC,gBAAgB,CAAC,CAAC;QAClD,OAAO,WAAW,CAAC,GAAG,KAAK,CAAC,KAAK,IAAI,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,IAAI,IAAI,QAAQ,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,CAAC,SAAS,CACb,KAAe,EACf,mBAAiC,EAAE;QAEnC,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,MAAM,EACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC,EACtC,qBAAqB,CACtB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,UAAU,CACd,KAAe,EACf,UAA2B,EAC3B,mBAAiC,EAAE;QAEnC,yEAAyE;QACzE,IAAI,UAAU,CAAC,MAAM,KAAK,SAAS;YAAE,OAAO;QAC5C,MAAM,IAAI,CAAC,QAAQ,CACjB,iBAAiB,CAAC,MAAM,EACxB,IAAI,CAAC,QAAQ,CAAC,KAAK,EAAE,gBAAgB,CAAC,EACtC,UAAU,CACX,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,qBAAqB;IACrB,8EAA8E;IAEtE,MAAM,CAAU,QAAQ,GAAG,oBAAoB,CAAC;IAExD,KAAK,CAAC,qBAAqB,CACzB,KAAa,EACb,GAAW,EACX,MAAc;QAEd,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,QAAQ,EAC1B,WAAW,CAAC,GAAG,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC,EACxC,uBAAuB,CACxB,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,sBAAsB,CAC1B,KAAa,EACb,GAAW,EACX,MAAc,EACd,OAA0B;QAE1B,MAAM,IAAI,CAAC,QAAQ,CACjB,iBAAiB,CAAC,QAAQ,EAC1B,WAAW,CAAC,GAAG,KAAK,IAAI,GAAG,IAAI,MAAM,EAAE,CAAC,EACxC,OAAO,CACR,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,6BAA6B;IAC7B,8EAA8E;IAEtE,MAAM,CAAU,YAAY,GAAG,aAAa,CAAC;IAErD,KAAK,CAAC,eAAe,CACnB,OAAe,EACf,WAAmB,EACnB,QAAgB;QAEhB,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,YAAY,EAC9B,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC,EACpD,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,gBAAgB,CACpB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,MAAoB;QAEpB,MAAM,IAAI,CAAC,QAAQ,CACjB,iBAAiB,CAAC,YAAY,EAC9B,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,EAAE,CAAC,EACpD,MAAM,CACP,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,2BAA2B;IAC3B,8EAA8E;IAEtE,MAAM,CAAU,YAAY,GAAG,aAAa,CAAC;IAErD,KAAK,CAAC,cAAc,CAClB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,WAAmB,EACnB,SAAiB;QAEjB,OAAO,IAAI,CAAC,OAAO,CACjB,iBAAiB,CAAC,YAAY,EAC9B,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC,EAChF,CAAC,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAC1B,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,eAAe,CACnB,OAAe,EACf,WAAmB,EACnB,QAAgB,EAChB,WAAmB,EACnB,SAAiB,EACjB,MAAoB;QAEpB,MAAM,IAAI,CAAC,QAAQ,CACjB,iBAAiB,CAAC,YAAY,EAC9B,WAAW,CAAC,GAAG,OAAO,IAAI,WAAW,IAAI,QAAQ,IAAI,WAAW,IAAI,SAAS,EAAE,CAAC,EAChF,MAAM,CACP,CAAC;IACJ,CAAC;IAED,8EAA8E;IAC9E,YAAY;IACZ,8EAA8E;IAE9E,KAAK,CAAC,KAAK;QACT,IAAI,CAAC;YACH,MAAM,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QAC5B,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,GAAG,CAAC,IAAI,CACN,qDAAqD,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CACxG,CAAC;QACJ,CAAC;IACH,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pythonBin.d.ts","sourceRoot":"","sources":["../../src/utils/pythonBin.ts"],"names":[],"mappings":"AAUA,wBAAsB,YAAY,IAAI,OAAO,CAAC,MAAM,CAAC,CAgBpD"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { execa } from 'execa';
|
|
2
|
+
import { log } from './logger.js';
|
|
3
|
+
/**
|
|
4
|
+
* Resolves the Python interpreter binary, caching the result after the first
|
|
5
|
+
* successful lookup. Shared by both Python-based analyzers (Whisper, YAMNet)
|
|
6
|
+
* and the Whisper transcript analyzer.
|
|
7
|
+
*/
|
|
8
|
+
let _pythonBin = null;
|
|
9
|
+
export async function getPythonBin() {
|
|
10
|
+
if (_pythonBin)
|
|
11
|
+
return _pythonBin;
|
|
12
|
+
for (const bin of ['python3', 'python']) {
|
|
13
|
+
try {
|
|
14
|
+
await execa(bin, ['--version']);
|
|
15
|
+
_pythonBin = bin;
|
|
16
|
+
return bin;
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
log.warn(`${bin} not found, trying next binary...`);
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
throw new Error('No Python interpreter found (tried python3, python). Install Python 3 to use YAMNet or Whisper.');
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=pythonBin.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pythonBin.js","sourceRoot":"","sources":["../../src/utils/pythonBin.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,OAAO,CAAC;AAC9B,OAAO,EAAE,GAAG,EAAE,MAAM,aAAa,CAAC;AAElC;;;;GAIG;AACH,IAAI,UAAU,GAAkB,IAAI,CAAC;AAErC,MAAM,CAAC,KAAK,UAAU,YAAY;IAChC,IAAI,UAAU;QAAE,OAAO,UAAU,CAAC;IAElC,KAAK,MAAM,GAAG,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC;QACxC,IAAI,CAAC;YACH,MAAM,KAAK,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,CAAC,CAAC;YAChC,UAAU,GAAG,GAAG,CAAC;YACjB,OAAO,GAAG,CAAC;QACb,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,CAAC,IAAI,CAAC,GAAG,GAAG,mCAAmC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,MAAM,IAAI,KAAK,CACb,iGAAiG,CAClG,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@thunderkiller/video-clipper",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.6.0",
|
|
4
4
|
"description": "CLI that analyzes YouTube transcripts with an LLM to find interesting moments and cut clips",
|
|
5
5
|
"license": "ISC",
|
|
6
6
|
"author": "",
|
|
@@ -73,6 +73,7 @@
|
|
|
73
73
|
"dotenv": "^16.4.7",
|
|
74
74
|
"execa": "^9.5.2",
|
|
75
75
|
"fluent-ffmpeg": "^2.1.3",
|
|
76
|
+
"mongodb": "^7.1.0",
|
|
76
77
|
"p-limit": "^7.3.0",
|
|
77
78
|
"zod": "^4.3.6"
|
|
78
79
|
},
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import type { MergedCandidate } from '../../types/index.js';
|
|
2
|
-
import type { ChunkEvaluation } from '../../types/index.js';
|
|
3
|
-
import type { AudioEvent } from '../../types/index.js';
|
|
4
|
-
export declare function mergeSignals(llmSegments: ChunkEvaluation[], audioEvents: AudioEvent[], boostWindow?: number, scoreBoost?: number, preRoll?: number, postRoll?: number): MergedCandidate[];
|
|
5
|
-
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/services/signalMerger/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAGvD,wBAAgB,YAAY,CAC1B,WAAW,EAAE,eAAe,EAAE,EAC9B,WAAW,EAAE,UAAU,EAAE,EACzB,WAAW,CAAC,EAAE,MAAM,EACpB,UAAU,CAAC,EAAE,MAAM,EACnB,OAAO,CAAC,EAAE,MAAM,EAChB,QAAQ,CAAC,EAAE,MAAM,GAChB,eAAe,EAAE,CAwCnB"}
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import { config } from '../../config/index.js';
|
|
2
|
-
export function mergeSignals(llmSegments, audioEvents, boostWindow, scoreBoost, preRoll, postRoll) {
|
|
3
|
-
const candidates = [];
|
|
4
|
-
const windowSec = boostWindow ?? config.AUDIO_LLM_BOOST_WINDOW;
|
|
5
|
-
const boost = scoreBoost ?? config.AUDIO_LLM_SCORE_BOOST;
|
|
6
|
-
const pre = preRoll ?? config.AUDIO_CLIP_PRE_ROLL;
|
|
7
|
-
const post = postRoll ?? config.AUDIO_CLIP_POST_ROLL;
|
|
8
|
-
const successfulSegments = llmSegments.filter((s) => s.status === 'success');
|
|
9
|
-
for (const seg of successfulSegments) {
|
|
10
|
-
if (!seg.interesting)
|
|
11
|
-
continue;
|
|
12
|
-
const nearby = audioEvents.filter((e) => Math.abs(e.time - seg.clip_start) < windowSec);
|
|
13
|
-
candidates.push({
|
|
14
|
-
start: seg.clip_start,
|
|
15
|
-
end: seg.clip_end,
|
|
16
|
-
score: Math.min(10, seg.score + (nearby.length > 0 ? boost : 0)),
|
|
17
|
-
source: nearby.length > 0 ? 'both' : 'transcript',
|
|
18
|
-
reason: seg.reason,
|
|
19
|
-
audio_event: nearby.length > 0 ? nearby[0].event : undefined,
|
|
20
|
-
});
|
|
21
|
-
}
|
|
22
|
-
for (const evt of audioEvents) {
|
|
23
|
-
const hasLLM = successfulSegments.some((s) => Math.abs(s.clip_start - evt.time) < windowSec);
|
|
24
|
-
if (!hasLLM) {
|
|
25
|
-
candidates.push({
|
|
26
|
-
start: Math.max(0, evt.time - pre),
|
|
27
|
-
end: evt.time + post,
|
|
28
|
-
score: Math.round(evt.confidence * 10),
|
|
29
|
-
source: 'audio',
|
|
30
|
-
reason: `Audio event: ${evt.event} (${(evt.confidence * 100).toFixed(0)}% confidence)`,
|
|
31
|
-
audio_event: evt.event,
|
|
32
|
-
});
|
|
33
|
-
}
|
|
34
|
-
}
|
|
35
|
-
return candidates;
|
|
36
|
-
}
|
|
37
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/services/signalMerger/index.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAE/C,MAAM,UAAU,YAAY,CAC1B,WAA8B,EAC9B,WAAyB,EACzB,WAAoB,EACpB,UAAmB,EACnB,OAAgB,EAChB,QAAiB;IAEjB,MAAM,UAAU,GAAsB,EAAE,CAAC;IACzC,MAAM,SAAS,GAAG,WAAW,IAAI,MAAM,CAAC,sBAAsB,CAAC;IAC/D,MAAM,KAAK,GAAG,UAAU,IAAI,MAAM,CAAC,qBAAqB,CAAC;IACzD,MAAM,GAAG,GAAG,OAAO,IAAI,MAAM,CAAC,mBAAmB,CAAC;IAClD,MAAM,IAAI,GAAG,QAAQ,IAAI,MAAM,CAAC,oBAAoB,CAAC;IAErD,MAAM,kBAAkB,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC;IAE7E,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACrC,IAAI,CAAC,GAAG,CAAC,WAAW;YAAE,SAAS;QAE/B,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,GAAG,GAAG,CAAC,UAAU,CAAC,GAAG,SAAS,CAAC,CAAC;QAExF,UAAU,CAAC,IAAI,CAAC;YACd,KAAK,EAAE,GAAG,CAAC,UAAU;YACrB,GAAG,EAAE,GAAG,CAAC,QAAQ;YACjB,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,KAAK,GAAG,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;YAChE,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,YAAY;YACjD,MAAM,EAAE,GAAG,CAAC,MAAM;YAClB,WAAW,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,SAAS;SAC7D,CAAC,CAAC;IACL,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,kBAAkB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,UAAU,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,SAAS,CAAC,CAAC;QAE7F,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,UAAU,CAAC,IAAI,CAAC;gBACd,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,GAAG,CAAC,IAAI,GAAG,GAAG,CAAC;gBAClC,GAAG,EAAE,GAAG,CAAC,IAAI,GAAG,IAAI;gBACpB,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,EAAE,CAAC;gBACtC,MAAM,EAAE,OAAO;gBACf,MAAM,EAAE,gBAAgB,GAAG,CAAC,KAAK,KAAK,CAAC,GAAG,CAAC,UAAU,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe;gBACtF,WAAW,EAAE,GAAG,CAAC,KAAK;aACvB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IAED,OAAO,UAAU,CAAC;AACpB,CAAC"}
|