escribano 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +297 -0
- package/dist/0_types.js +279 -0
- package/dist/actions/classify-session.js +77 -0
- package/dist/actions/create-contexts.js +44 -0
- package/dist/actions/create-topic-blocks.js +68 -0
- package/dist/actions/extract-metadata.js +24 -0
- package/dist/actions/generate-artifact-v3.js +296 -0
- package/dist/actions/generate-artifact.js +61 -0
- package/dist/actions/generate-summary-v3.js +260 -0
- package/dist/actions/outline-index.js +204 -0
- package/dist/actions/process-recording-v2.js +494 -0
- package/dist/actions/process-recording-v3.js +412 -0
- package/dist/actions/process-session.js +183 -0
- package/dist/actions/publish-summary-v3.js +303 -0
- package/dist/actions/sync-to-outline.js +196 -0
- package/dist/adapters/audio.silero.adapter.js +69 -0
- package/dist/adapters/cap.adapter.js +94 -0
- package/dist/adapters/capture.cap.adapter.js +107 -0
- package/dist/adapters/capture.filesystem.adapter.js +124 -0
- package/dist/adapters/embedding.ollama.adapter.js +141 -0
- package/dist/adapters/intelligence.adapter.js +202 -0
- package/dist/adapters/intelligence.mlx.adapter.js +395 -0
- package/dist/adapters/intelligence.ollama.adapter.js +741 -0
- package/dist/adapters/publishing.outline.adapter.js +75 -0
- package/dist/adapters/storage.adapter.js +81 -0
- package/dist/adapters/storage.fs.adapter.js +83 -0
- package/dist/adapters/transcription.whisper.adapter.js +206 -0
- package/dist/adapters/video.ffmpeg.adapter.js +405 -0
- package/dist/adapters/whisper.adapter.js +168 -0
- package/dist/batch-context.js +329 -0
- package/dist/db/helpers.js +50 -0
- package/dist/db/index.js +95 -0
- package/dist/db/migrate.js +80 -0
- package/dist/db/repositories/artifact.sqlite.js +77 -0
- package/dist/db/repositories/cluster.sqlite.js +92 -0
- package/dist/db/repositories/context.sqlite.js +75 -0
- package/dist/db/repositories/index.js +10 -0
- package/dist/db/repositories/observation.sqlite.js +70 -0
- package/dist/db/repositories/recording.sqlite.js +56 -0
- package/dist/db/repositories/subject.sqlite.js +64 -0
- package/dist/db/repositories/topic-block.sqlite.js +45 -0
- package/dist/db/types.js +4 -0
- package/dist/domain/classification.js +60 -0
- package/dist/domain/context.js +97 -0
- package/dist/domain/index.js +2 -0
- package/dist/domain/observation.js +17 -0
- package/dist/domain/recording.js +41 -0
- package/dist/domain/segment.js +93 -0
- package/dist/domain/session.js +93 -0
- package/dist/domain/time-range.js +38 -0
- package/dist/domain/transcript.js +79 -0
- package/dist/index.js +173 -0
- package/dist/pipeline/context.js +162 -0
- package/dist/pipeline/events.js +2 -0
- package/dist/prerequisites.js +226 -0
- package/dist/scripts/rebuild-index.js +53 -0
- package/dist/scripts/seed-fixtures.js +290 -0
- package/dist/services/activity-segmentation.js +333 -0
- package/dist/services/activity-segmentation.test.js +191 -0
- package/dist/services/app-normalization.js +212 -0
- package/dist/services/cluster-merge.js +69 -0
- package/dist/services/clustering.js +237 -0
- package/dist/services/debug.js +58 -0
- package/dist/services/frame-sampling.js +318 -0
- package/dist/services/signal-extraction.js +106 -0
- package/dist/services/subject-grouping.js +342 -0
- package/dist/services/temporal-alignment.js +99 -0
- package/dist/services/vlm-enrichment.js +84 -0
- package/dist/services/vlm-service.js +130 -0
- package/dist/stats/index.js +3 -0
- package/dist/stats/observer.js +65 -0
- package/dist/stats/repository.js +36 -0
- package/dist/stats/resource-tracker.js +86 -0
- package/dist/stats/types.js +1 -0
- package/dist/test-classification-prompts.js +181 -0
- package/dist/tests/cap.adapter.test.js +75 -0
- package/dist/tests/capture.cap.adapter.test.js +69 -0
- package/dist/tests/classify-session.test.js +140 -0
- package/dist/tests/db/repositories.test.js +243 -0
- package/dist/tests/domain/time-range.test.js +31 -0
- package/dist/tests/integration.test.js +84 -0
- package/dist/tests/intelligence.adapter.test.js +102 -0
- package/dist/tests/intelligence.ollama.adapter.test.js +178 -0
- package/dist/tests/process-v2.test.js +90 -0
- package/dist/tests/services/clustering.test.js +112 -0
- package/dist/tests/services/frame-sampling.test.js +152 -0
- package/dist/tests/utils/ocr.test.js +76 -0
- package/dist/tests/utils/parallel.test.js +57 -0
- package/dist/tests/visual-observer.test.js +175 -0
- package/dist/utils/id-normalization.js +15 -0
- package/dist/utils/index.js +9 -0
- package/dist/utils/model-detector.js +154 -0
- package/dist/utils/ocr.js +80 -0
- package/dist/utils/parallel.js +32 -0
- package/migrations/001_initial.sql +109 -0
- package/migrations/002_clusters.sql +41 -0
- package/migrations/003_observations_vlm_fields.sql +14 -0
- package/migrations/004_observations_unique.sql +18 -0
- package/migrations/005_processing_stats.sql +29 -0
- package/migrations/006_vlm_raw_response.sql +6 -0
- package/migrations/007_subjects.sql +23 -0
- package/migrations/008_artifacts_recording.sql +6 -0
- package/migrations/009_artifact_subjects.sql +10 -0
- package/package.json +82 -0
- package/prompts/action-items.md +55 -0
- package/prompts/blog-draft.md +54 -0
- package/prompts/blog-research.md +87 -0
- package/prompts/card.md +54 -0
- package/prompts/classify-segment.md +38 -0
- package/prompts/classify.md +37 -0
- package/prompts/code-snippets.md +163 -0
- package/prompts/extract-metadata.md +149 -0
- package/prompts/notes.md +83 -0
- package/prompts/runbook.md +123 -0
- package/prompts/standup.md +50 -0
- package/prompts/step-by-step.md +125 -0
- package/prompts/subject-grouping.md +31 -0
- package/prompts/summary-v3.md +89 -0
- package/prompts/summary.md +77 -0
- package/prompts/topic-classifier.md +24 -0
- package/prompts/topic-extract.md +13 -0
- package/prompts/vlm-batch.md +21 -0
- package/prompts/vlm-single.md +19 -0
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Process Recording V2 Tests
|
|
3
|
+
*/
|
|
4
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
5
|
+
import { processRecordingV2 } from '../actions/process-recording-v2.js';
|
|
6
|
+
describe('processRecordingV2', () => {
|
|
7
|
+
it('should call extractFramesAtInterval instead of extractFramesAtTimestamps during visual processing', async () => {
|
|
8
|
+
// Mock Recording in DB
|
|
9
|
+
const mockDbRecording = {
|
|
10
|
+
id: 'rec-123',
|
|
11
|
+
status: 'raw',
|
|
12
|
+
processing_step: 'idle',
|
|
13
|
+
video_path: '/tmp/test.mp4',
|
|
14
|
+
audio_mic_path: null,
|
|
15
|
+
audio_system_path: null,
|
|
16
|
+
captured_at: new Date(),
|
|
17
|
+
duration: 60,
|
|
18
|
+
};
|
|
19
|
+
const mockRepos = {
|
|
20
|
+
recordings: {
|
|
21
|
+
findById: vi.fn().mockReturnValue(mockDbRecording),
|
|
22
|
+
updateStatus: vi.fn(),
|
|
23
|
+
},
|
|
24
|
+
observations: {
|
|
25
|
+
saveBatch: vi.fn(),
|
|
26
|
+
save: vi.fn(),
|
|
27
|
+
findByRecording: vi.fn().mockReturnValue([]),
|
|
28
|
+
findByRecordingAndType: vi.fn().mockReturnValue([]),
|
|
29
|
+
updateEmbedding: vi.fn(),
|
|
30
|
+
deleteByRecording: vi.fn(),
|
|
31
|
+
},
|
|
32
|
+
clusters: {
|
|
33
|
+
deleteByRecording: vi.fn(),
|
|
34
|
+
save: vi.fn(),
|
|
35
|
+
linkObservationsBatch: vi.fn(),
|
|
36
|
+
findByRecording: vi.fn().mockReturnValue([]),
|
|
37
|
+
findByRecordingAndType: vi.fn().mockReturnValue([]),
|
|
38
|
+
getObservations: vi.fn().mockReturnValue([]),
|
|
39
|
+
updateClassification: vi.fn(),
|
|
40
|
+
getMergedAudioClusters: vi.fn().mockReturnValue([]),
|
|
41
|
+
saveMerge: vi.fn(),
|
|
42
|
+
},
|
|
43
|
+
contexts: {
|
|
44
|
+
findByTypeAndName: vi.fn(),
|
|
45
|
+
save: vi.fn(),
|
|
46
|
+
linkObservation: vi.fn(),
|
|
47
|
+
getLinksByRecording: vi.fn().mockReturnValue([]),
|
|
48
|
+
},
|
|
49
|
+
topicBlocks: {
|
|
50
|
+
deleteByRecording: vi.fn(),
|
|
51
|
+
save: vi.fn(),
|
|
52
|
+
},
|
|
53
|
+
};
|
|
54
|
+
const mockVideoService = {
|
|
55
|
+
extractFramesAtInterval: vi.fn().mockResolvedValue([
|
|
56
|
+
{ imagePath: '/tmp/f1.jpg', timestamp: 0 },
|
|
57
|
+
{ imagePath: '/tmp/f2.jpg', timestamp: 2 },
|
|
58
|
+
]),
|
|
59
|
+
extractFramesAtTimestamps: vi.fn(),
|
|
60
|
+
runVisualIndexing: vi.fn().mockResolvedValue({ frames: [] }),
|
|
61
|
+
};
|
|
62
|
+
const mockIntelligence = {
|
|
63
|
+
describeImages: vi.fn().mockResolvedValue({ descriptions: [] }),
|
|
64
|
+
embedText: vi.fn().mockResolvedValue([]),
|
|
65
|
+
extractTopics: vi.fn().mockResolvedValue([]),
|
|
66
|
+
};
|
|
67
|
+
const mockEmbedding = {
|
|
68
|
+
embedBatch: vi.fn().mockResolvedValue([]),
|
|
69
|
+
};
|
|
70
|
+
const mockPreprocessor = {
|
|
71
|
+
extractSpeechSegments: vi
|
|
72
|
+
.fn()
|
|
73
|
+
.mockResolvedValue({ segments: [], tempDir: '/tmp' }),
|
|
74
|
+
cleanup: vi.fn(),
|
|
75
|
+
};
|
|
76
|
+
const mockTranscription = {};
|
|
77
|
+
await processRecordingV2('rec-123', mockRepos, {
|
|
78
|
+
video: mockVideoService,
|
|
79
|
+
intelligence: mockIntelligence,
|
|
80
|
+
embedding: mockEmbedding,
|
|
81
|
+
preprocessor: mockPreprocessor,
|
|
82
|
+
transcription: mockTranscription,
|
|
83
|
+
});
|
|
84
|
+
// Verify correct function was called
|
|
85
|
+
expect(mockVideoService.extractFramesAtInterval).toHaveBeenCalled();
|
|
86
|
+
expect(mockVideoService.extractFramesAtTimestamps).not.toHaveBeenCalled();
|
|
87
|
+
// Verify interval-based extraction was used with correct threshold
|
|
88
|
+
expect(mockVideoService.extractFramesAtInterval).toHaveBeenCalledWith('/tmp/test.mp4', 0.3, expect.stringContaining('frames'));
|
|
89
|
+
});
|
|
90
|
+
});
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import { embeddingToBlob } from '../../db/helpers.js';
|
|
3
|
+
import { clusterObservations } from '../../services/clustering.js';
|
|
4
|
+
describe('clusterObservations', () => {
|
|
5
|
+
const mockEmbeddingService = {
|
|
6
|
+
embed: vi.fn(),
|
|
7
|
+
embedBatch: vi.fn(),
|
|
8
|
+
similarity: vi.fn((a, b) => {
|
|
9
|
+
// Simple mock: if first element is same, they are identical
|
|
10
|
+
return a[0] === b[0] ? 1 : 0;
|
|
11
|
+
}),
|
|
12
|
+
centroid: vi.fn((embeddings) => {
|
|
13
|
+
if (embeddings.length === 0)
|
|
14
|
+
return [];
|
|
15
|
+
const dim = embeddings[0].length;
|
|
16
|
+
const res = new Array(dim).fill(0);
|
|
17
|
+
for (const e of embeddings) {
|
|
18
|
+
for (let i = 0; i < dim; i++)
|
|
19
|
+
res[i] += e[i];
|
|
20
|
+
}
|
|
21
|
+
return res.map((v) => v / embeddings.length);
|
|
22
|
+
}),
|
|
23
|
+
};
|
|
24
|
+
const createObs = (id, timestamp, typeVal) => ({
|
|
25
|
+
id,
|
|
26
|
+
recording_id: 'rec1',
|
|
27
|
+
type: 'visual',
|
|
28
|
+
timestamp,
|
|
29
|
+
end_timestamp: timestamp + 1,
|
|
30
|
+
image_path: null,
|
|
31
|
+
ocr_text: `text ${typeVal}`,
|
|
32
|
+
vlm_description: null,
|
|
33
|
+
vlm_raw_response: null,
|
|
34
|
+
activity_type: null,
|
|
35
|
+
apps: null,
|
|
36
|
+
topics: null,
|
|
37
|
+
text: null,
|
|
38
|
+
audio_source: null,
|
|
39
|
+
audio_type: null,
|
|
40
|
+
embedding: embeddingToBlob([typeVal, 0, 0]),
|
|
41
|
+
created_at: new Date().toISOString(),
|
|
42
|
+
});
|
|
43
|
+
it('should cluster identical observations within time window', () => {
|
|
44
|
+
const obs = [
|
|
45
|
+
createObs('1', 10, 1),
|
|
46
|
+
createObs('2', 20, 1),
|
|
47
|
+
createObs('3', 30, 1),
|
|
48
|
+
];
|
|
49
|
+
const clusters = clusterObservations(obs, mockEmbeddingService, {
|
|
50
|
+
timeWindowSeconds: 60,
|
|
51
|
+
distanceThreshold: 0.1,
|
|
52
|
+
minClusterSize: 2,
|
|
53
|
+
});
|
|
54
|
+
expect(clusters).toHaveLength(1);
|
|
55
|
+
expect(clusters[0].observations).toHaveLength(3);
|
|
56
|
+
});
|
|
57
|
+
it('should not cluster observations outside time window', () => {
|
|
58
|
+
const obs = [
|
|
59
|
+
createObs('1', 10, 1),
|
|
60
|
+
createObs('2', 1000, 1), // Far away in time
|
|
61
|
+
];
|
|
62
|
+
const clusters = clusterObservations(obs, mockEmbeddingService, {
|
|
63
|
+
timeWindowSeconds: 60,
|
|
64
|
+
distanceThreshold: 0.1,
|
|
65
|
+
minClusterSize: 1, // Set to 1 to see two clusters
|
|
66
|
+
});
|
|
67
|
+
expect(clusters).toHaveLength(2);
|
|
68
|
+
});
|
|
69
|
+
it('should not cluster semantically different observations', () => {
|
|
70
|
+
const obs = [
|
|
71
|
+
createObs('1', 10, 1),
|
|
72
|
+
createObs('2', 20, 2), // Different embedding
|
|
73
|
+
];
|
|
74
|
+
const clusters = clusterObservations(obs, mockEmbeddingService, {
|
|
75
|
+
timeWindowSeconds: 60,
|
|
76
|
+
distanceThreshold: 0.1,
|
|
77
|
+
minClusterSize: 1,
|
|
78
|
+
});
|
|
79
|
+
expect(clusters).toHaveLength(2);
|
|
80
|
+
});
|
|
81
|
+
it('should absorb small clusters into nearest large cluster', () => {
|
|
82
|
+
// 3 similar obs (large cluster)
|
|
83
|
+
// 1 similar obs (small cluster, should be absorbed)
|
|
84
|
+
const obs = [
|
|
85
|
+
createObs('1', 10, 1),
|
|
86
|
+
createObs('2', 20, 1),
|
|
87
|
+
createObs('3', 30, 1),
|
|
88
|
+
createObs('4', 40, 1),
|
|
89
|
+
];
|
|
90
|
+
const clusters = clusterObservations(obs, mockEmbeddingService, {
|
|
91
|
+
timeWindowSeconds: 60,
|
|
92
|
+
distanceThreshold: 0.1,
|
|
93
|
+
minClusterSize: 3,
|
|
94
|
+
});
|
|
95
|
+
expect(clusters).toHaveLength(1);
|
|
96
|
+
expect(clusters[0].observations).toHaveLength(4);
|
|
97
|
+
});
|
|
98
|
+
it('should handle empty input', () => {
|
|
99
|
+
const clusters = clusterObservations([], mockEmbeddingService);
|
|
100
|
+
expect(clusters).toEqual([]);
|
|
101
|
+
});
|
|
102
|
+
it('should throw if an observation in validObs is missing embedding', () => {
|
|
103
|
+
// We mock bufferToEmbedding to fail if we want to test the throw,
|
|
104
|
+
// but the current code in clustering.ts uses a local bufferToEmbedding.
|
|
105
|
+
// However, if we pass an obs that PASSES the filter but then has no embedding
|
|
106
|
+
// (impossible due to filter), it would throw.
|
|
107
|
+
// Let's test the filter instead
|
|
108
|
+
const obsNoEmbed = [{ id: '1', embedding: null }];
|
|
109
|
+
const clusters = clusterObservations(obsNoEmbed, mockEmbeddingService);
|
|
110
|
+
expect(clusters).toHaveLength(0);
|
|
111
|
+
});
|
|
112
|
+
});
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { adaptiveSample, adaptiveSampleWithScenes, calculateAdaptiveBaseInterval, getSamplingStats, } from '../../services/frame-sampling.js';
|
|
3
|
+
describe('adaptiveSample', () => {
|
|
4
|
+
it('should return empty array for empty input', () => {
|
|
5
|
+
const result = adaptiveSample([]);
|
|
6
|
+
expect(result).toEqual([]);
|
|
7
|
+
});
|
|
8
|
+
it('should sample at base interval', () => {
|
|
9
|
+
// Create frames every 2 seconds for 60 seconds
|
|
10
|
+
const frames = [];
|
|
11
|
+
for (let t = 0; t <= 60; t += 2) {
|
|
12
|
+
frames.push({ imagePath: `frame_${t}.jpg`, timestamp: t });
|
|
13
|
+
}
|
|
14
|
+
const result = adaptiveSample(frames, { baseIntervalSeconds: 10 });
|
|
15
|
+
// Should get frames at 0, 10, 20, 30, 40, 50, 60 = 7 frames
|
|
16
|
+
expect(result.length).toBe(7);
|
|
17
|
+
expect(result.every((f) => f.reason === 'base')).toBe(true);
|
|
18
|
+
expect(result.map((f) => f.timestamp)).toEqual([0, 10, 20, 30, 40, 50, 60]);
|
|
19
|
+
});
|
|
20
|
+
it('should fill gaps larger than threshold', () => {
|
|
21
|
+
// Create frames with a gap
|
|
22
|
+
const frames = [
|
|
23
|
+
{ imagePath: 'frame_0.jpg', timestamp: 0 },
|
|
24
|
+
{ imagePath: 'frame_2.jpg', timestamp: 2 },
|
|
25
|
+
{ imagePath: 'frame_4.jpg', timestamp: 4 },
|
|
26
|
+
// Gap from 4 to 30 seconds
|
|
27
|
+
{ imagePath: 'frame_30.jpg', timestamp: 30 },
|
|
28
|
+
{ imagePath: 'frame_32.jpg', timestamp: 32 },
|
|
29
|
+
];
|
|
30
|
+
const result = adaptiveSample(frames, {
|
|
31
|
+
baseIntervalSeconds: 10,
|
|
32
|
+
gapThresholdSeconds: 15,
|
|
33
|
+
gapFillIntervalSeconds: 5,
|
|
34
|
+
});
|
|
35
|
+
// Base samples: 0, 30
|
|
36
|
+
// Gap detected: 30 - 0 = 30 > 15
|
|
37
|
+
// Gap fill should add samples between 0 and 30
|
|
38
|
+
expect(result.length).toBeGreaterThan(2);
|
|
39
|
+
expect(result.some((f) => f.reason === 'gap_fill')).toBe(true);
|
|
40
|
+
});
|
|
41
|
+
it('should not fill gaps smaller than threshold', () => {
|
|
42
|
+
const frames = [];
|
|
43
|
+
for (let t = 0; t <= 30; t += 2) {
|
|
44
|
+
frames.push({ imagePath: `frame_${t}.jpg`, timestamp: t });
|
|
45
|
+
}
|
|
46
|
+
const result = adaptiveSample(frames, {
|
|
47
|
+
baseIntervalSeconds: 10,
|
|
48
|
+
gapThresholdSeconds: 15,
|
|
49
|
+
});
|
|
50
|
+
// Gap between samples is 10s, which is < 15s threshold
|
|
51
|
+
expect(result.every((f) => f.reason === 'base')).toBe(true);
|
|
52
|
+
});
|
|
53
|
+
it('should handle unsorted input', () => {
|
|
54
|
+
const frames = [
|
|
55
|
+
{ imagePath: 'frame_20.jpg', timestamp: 20 },
|
|
56
|
+
{ imagePath: 'frame_0.jpg', timestamp: 0 },
|
|
57
|
+
{ imagePath: 'frame_10.jpg', timestamp: 10 },
|
|
58
|
+
];
|
|
59
|
+
const result = adaptiveSample(frames, { baseIntervalSeconds: 10 });
|
|
60
|
+
expect(result.map((f) => f.timestamp)).toEqual([0, 10, 20]);
|
|
61
|
+
});
|
|
62
|
+
});
|
|
63
|
+
describe('getSamplingStats', () => {
|
|
64
|
+
it('should calculate correct statistics', () => {
|
|
65
|
+
const original = Array.from({ length: 100 }, (_, i) => ({
|
|
66
|
+
imagePath: `frame_${i}.jpg`,
|
|
67
|
+
timestamp: i * 2,
|
|
68
|
+
}));
|
|
69
|
+
const sampled = adaptiveSample(original, { baseIntervalSeconds: 10 });
|
|
70
|
+
const stats = getSamplingStats(original, sampled);
|
|
71
|
+
expect(stats.originalCount).toBe(100);
|
|
72
|
+
expect(stats.sampledCount).toBeLessThan(100);
|
|
73
|
+
expect(stats.reductionPercent).toBeGreaterThan(0);
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
describe('calculateAdaptiveBaseInterval', () => {
|
|
77
|
+
it('should return config base for few scene changes (<= 20)', () => {
|
|
78
|
+
expect(calculateAdaptiveBaseInterval(0, 10)).toBe(10);
|
|
79
|
+
expect(calculateAdaptiveBaseInterval(5, 10)).toBe(10);
|
|
80
|
+
expect(calculateAdaptiveBaseInterval(20, 10)).toBe(10);
|
|
81
|
+
});
|
|
82
|
+
it('should return 20s for moderate scene changes (21-50)', () => {
|
|
83
|
+
expect(calculateAdaptiveBaseInterval(21, 10)).toBe(20);
|
|
84
|
+
expect(calculateAdaptiveBaseInterval(35, 10)).toBe(20);
|
|
85
|
+
expect(calculateAdaptiveBaseInterval(50, 10)).toBe(20);
|
|
86
|
+
});
|
|
87
|
+
it('should return 30s for high scene density (> 50)', () => {
|
|
88
|
+
expect(calculateAdaptiveBaseInterval(51, 10)).toBe(30);
|
|
89
|
+
expect(calculateAdaptiveBaseInterval(109, 10)).toBe(30);
|
|
90
|
+
expect(calculateAdaptiveBaseInterval(200, 10)).toBe(30);
|
|
91
|
+
});
|
|
92
|
+
it('should never go below configured base interval', () => {
|
|
93
|
+
// If user configured 40s, keep it even for low scene counts
|
|
94
|
+
expect(calculateAdaptiveBaseInterval(5, 40)).toBe(40);
|
|
95
|
+
expect(calculateAdaptiveBaseInterval(30, 40)).toBe(40);
|
|
96
|
+
expect(calculateAdaptiveBaseInterval(100, 40)).toBe(40);
|
|
97
|
+
});
|
|
98
|
+
});
|
|
99
|
+
describe('adaptiveSampleWithScenes', () => {
|
|
100
|
+
// Helper: create frames every 2s for N minutes
|
|
101
|
+
function makeFrames(durationMinutes) {
|
|
102
|
+
const frames = [];
|
|
103
|
+
const total = durationMinutes * 60;
|
|
104
|
+
for (let t = 0; t <= total; t += 2) {
|
|
105
|
+
frames.push({ imagePath: `frame_${t}.jpg`, timestamp: t });
|
|
106
|
+
}
|
|
107
|
+
return frames;
|
|
108
|
+
}
|
|
109
|
+
it('should return empty array for empty input', () => {
|
|
110
|
+
expect(adaptiveSampleWithScenes([], [])).toEqual([]);
|
|
111
|
+
});
|
|
112
|
+
it('should sample at base interval when no scene changes', () => {
|
|
113
|
+
const frames = makeFrames(1); // 1 minute
|
|
114
|
+
const result = adaptiveSampleWithScenes(frames, []);
|
|
115
|
+
// Should behave like regular sampling
|
|
116
|
+
expect(result.length).toBeGreaterThan(0);
|
|
117
|
+
expect(result.every((f) => f.reason === 'base')).toBe(true);
|
|
118
|
+
});
|
|
119
|
+
it('should include scene change frames', () => {
|
|
120
|
+
const frames = makeFrames(1);
|
|
121
|
+
const sceneChanges = [10, 20, 40];
|
|
122
|
+
const result = adaptiveSampleWithScenes(frames, sceneChanges);
|
|
123
|
+
const sceneFrames = result.filter((f) => f.reason === 'scene_change');
|
|
124
|
+
expect(sceneFrames.length).toBe(3);
|
|
125
|
+
});
|
|
126
|
+
it('should produce fewer total frames with high scene density', () => {
|
|
127
|
+
const frames = makeFrames(10); // 10 minutes, 300 frames
|
|
128
|
+
// Low density: 5 scene changes
|
|
129
|
+
const lowDensityResult = adaptiveSampleWithScenes(frames, [30, 100, 200, 400, 500], { baseIntervalSeconds: 10 });
|
|
130
|
+
// High density: 60 scene changes (every ~10s)
|
|
131
|
+
const highDensityScenes = Array.from({ length: 60 }, (_, i) => i * 10);
|
|
132
|
+
const highDensityResult = adaptiveSampleWithScenes(frames, highDensityScenes, { baseIntervalSeconds: 10 });
|
|
133
|
+
// High density should use 30s base interval, resulting in fewer
|
|
134
|
+
// base samples (but more scene samples). Total should still be
|
|
135
|
+
// manageable and not explode.
|
|
136
|
+
const highStats = getSamplingStats(frames, highDensityResult);
|
|
137
|
+
expect(highStats.sampledCount).toBeLessThan(frames.length);
|
|
138
|
+
expect(highStats.reductionPercent).toBeGreaterThan(50);
|
|
139
|
+
});
|
|
140
|
+
it('should match real-world scenario: 59 min, 109 scenes', () => {
|
|
141
|
+
const frames = makeFrames(59); // 1776 frames
|
|
142
|
+
// Simulate 109 scene changes spread across 59 minutes
|
|
143
|
+
const sceneChanges = Array.from({ length: 109 }, (_, i) => Math.round((i / 109) * 59 * 60));
|
|
144
|
+
const result = adaptiveSampleWithScenes(frames, sceneChanges);
|
|
145
|
+
const stats = getSamplingStats(frames, result);
|
|
146
|
+
// Target: under 250 frames total (was 1199 before adaptive interval + gap threshold fix)
|
|
147
|
+
expect(stats.sampledCount).toBeLessThan(250);
|
|
148
|
+
expect(stats.sceneChangeCount).toBe(109);
|
|
149
|
+
// Reduction should be significant
|
|
150
|
+
expect(stats.reductionPercent).toBeGreaterThan(85);
|
|
151
|
+
});
|
|
152
|
+
});
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { describe, expect, it } from 'vitest';
|
|
2
|
+
import { cleanOcrText, isOcrMeaningful } from '../../utils/ocr.js';
|
|
3
|
+
describe('OCR Cleanup Utilities', () => {
|
|
4
|
+
describe('cleanOcrText', () => {
|
|
5
|
+
it('preserves valid content and short commands', () => {
|
|
6
|
+
const input = 'git status\nnpm install\nThis is a valid sentence.';
|
|
7
|
+
const output = cleanOcrText(input);
|
|
8
|
+
expect(output).toContain('git status');
|
|
9
|
+
expect(output).toContain('npm install');
|
|
10
|
+
expect(output).toContain('This is a valid sentence.');
|
|
11
|
+
});
|
|
12
|
+
it('filters out system clock artifacts', () => {
|
|
13
|
+
const input = '© 15/01 Thu 12:12\nActual Content';
|
|
14
|
+
const output = cleanOcrText(input);
|
|
15
|
+
expect(output).not.toContain('© 15/01');
|
|
16
|
+
expect(output).toContain('Actual Content');
|
|
17
|
+
});
|
|
18
|
+
it('filters out window title artifacts', () => {
|
|
19
|
+
const input = '> Google Chrome\nSome webpage content';
|
|
20
|
+
const output = cleanOcrText(input);
|
|
21
|
+
expect(output).not.toContain('> Google Chrome');
|
|
22
|
+
expect(output).toContain('Some webpage content');
|
|
23
|
+
});
|
|
24
|
+
it('filters out repeated character garbage', () => {
|
|
25
|
+
const input = 'eee\nGPassssss\nValid text';
|
|
26
|
+
const output = cleanOcrText(input);
|
|
27
|
+
expect(output).not.toContain('eee');
|
|
28
|
+
expect(output).not.toContain('GPassssss');
|
|
29
|
+
expect(output).toContain('Valid text');
|
|
30
|
+
});
|
|
31
|
+
it('filters out common UI bookmarks and status lines', () => {
|
|
32
|
+
const input = 'All Bookmarks\nZoho Mail\nProject code here';
|
|
33
|
+
const output = cleanOcrText(input);
|
|
34
|
+
expect(output).not.toContain('All Bookmarks');
|
|
35
|
+
expect(output).not.toContain('Zoho Mail');
|
|
36
|
+
expect(output).toContain('Project code here');
|
|
37
|
+
});
|
|
38
|
+
it('filters out UI separators and indicators', () => {
|
|
39
|
+
const input = '| BSI - Bundesamt f.\nDetected Language\nEnglish\nActual information';
|
|
40
|
+
const output = cleanOcrText(input);
|
|
41
|
+
expect(output).not.toContain('BSI - Bundesamt');
|
|
42
|
+
expect(output).not.toContain('Detected Language');
|
|
43
|
+
expect(output).not.toContain('English');
|
|
44
|
+
expect(output).toContain('Actual information');
|
|
45
|
+
});
|
|
46
|
+
it('removes duplicate consecutive lines', () => {
|
|
47
|
+
const input = 'Line 1\nLine 1\nLine 2\nLine 1';
|
|
48
|
+
const output = cleanOcrText(input);
|
|
49
|
+
const lines = output.split('\n');
|
|
50
|
+
expect(lines).toHaveLength(3);
|
|
51
|
+
expect(lines[0]).toBe('Line 1');
|
|
52
|
+
expect(lines[1]).toBe('Line 2');
|
|
53
|
+
expect(lines[2]).toBe('Line 1');
|
|
54
|
+
});
|
|
55
|
+
it('filters out short garbage fragments while keeping short commands', () => {
|
|
56
|
+
// 'cid' was found in DB, should be filtered if it doesn't look like a word
|
|
57
|
+
// 'git' should be kept.
|
|
58
|
+
const input = 'cid\ngit\nnpm\nsh Twelv';
|
|
59
|
+
const output = cleanOcrText(input);
|
|
60
|
+
expect(output).not.toContain('cid');
|
|
61
|
+
expect(output).not.toContain('sh Twelv');
|
|
62
|
+
expect(output).toContain('git');
|
|
63
|
+
expect(output).toContain('npm');
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
describe('isOcrMeaningful', () => {
|
|
67
|
+
it('returns false for empty or very short text', () => {
|
|
68
|
+
expect(isOcrMeaningful('')).toBe(false);
|
|
69
|
+
expect(isOcrMeaningful('Too short')).toBe(false);
|
|
70
|
+
});
|
|
71
|
+
it('returns true for substantial content', () => {
|
|
72
|
+
const content = 'Line one of content\nLine two of content\nLine three of content';
|
|
73
|
+
expect(isOcrMeaningful(content)).toBe(true);
|
|
74
|
+
});
|
|
75
|
+
});
|
|
76
|
+
});
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
2
|
+
import { chunkArray, parallelMap } from '../../utils/parallel.js';
|
|
3
|
+
describe('parallelMap', () => {
|
|
4
|
+
it('should process items and return results in order', async () => {
|
|
5
|
+
const items = [1, 2, 3, 4, 5];
|
|
6
|
+
const fn = async (n) => n * 2;
|
|
7
|
+
const results = await parallelMap(items, fn, 2);
|
|
8
|
+
expect(results).toEqual([2, 4, 6, 8, 10]);
|
|
9
|
+
});
|
|
10
|
+
it('should respect concurrency limits', async () => {
|
|
11
|
+
const items = [100, 100, 100, 100];
|
|
12
|
+
let activeCount = 0;
|
|
13
|
+
let maxActive = 0;
|
|
14
|
+
const fn = async (ms) => {
|
|
15
|
+
activeCount++;
|
|
16
|
+
maxActive = Math.max(maxActive, activeCount);
|
|
17
|
+
await new Promise((resolve) => setTimeout(resolve, ms));
|
|
18
|
+
activeCount--;
|
|
19
|
+
return ms;
|
|
20
|
+
};
|
|
21
|
+
const concurrency = 2;
|
|
22
|
+
await parallelMap(items, fn, concurrency);
|
|
23
|
+
expect(maxActive).toBeLessThanOrEqual(concurrency);
|
|
24
|
+
});
|
|
25
|
+
it('should handle empty input array', async () => {
|
|
26
|
+
const results = await parallelMap([], async (x) => x, 2);
|
|
27
|
+
expect(results).toEqual([]);
|
|
28
|
+
});
|
|
29
|
+
it('should handle concurrency <= 0 by defaulting to 1', async () => {
|
|
30
|
+
const items = [1, 2, 3];
|
|
31
|
+
const fn = vi.fn(async (x) => x);
|
|
32
|
+
const results0 = await parallelMap(items, fn, 0);
|
|
33
|
+
expect(results0).toEqual([1, 2, 3]);
|
|
34
|
+
const resultsNeg = await parallelMap(items, fn, -5);
|
|
35
|
+
expect(resultsNeg).toEqual([1, 2, 3]);
|
|
36
|
+
});
|
|
37
|
+
it('should propagate errors and stop processing', async () => {
|
|
38
|
+
const items = [1, 2, 3, 4, 5];
|
|
39
|
+
const fn = async (n) => {
|
|
40
|
+
if (n === 3)
|
|
41
|
+
throw new Error('Failed');
|
|
42
|
+
return n;
|
|
43
|
+
};
|
|
44
|
+
await expect(parallelMap(items, fn, 2)).rejects.toThrow('Failed');
|
|
45
|
+
});
|
|
46
|
+
});
|
|
47
|
+
describe('chunkArray', () => {
|
|
48
|
+
it('should split array into chunks', () => {
|
|
49
|
+
const items = [1, 2, 3, 4, 5];
|
|
50
|
+
expect(chunkArray(items, 2)).toEqual([[1, 2], [3, 4], [5]]);
|
|
51
|
+
expect(chunkArray(items, 5)).toEqual([[1, 2, 3, 4, 5]]);
|
|
52
|
+
expect(chunkArray(items, 10)).toEqual([[1, 2, 3, 4, 5]]);
|
|
53
|
+
});
|
|
54
|
+
it('should handle empty array', () => {
|
|
55
|
+
expect(chunkArray([], 2)).toEqual([]);
|
|
56
|
+
});
|
|
57
|
+
});
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Visual Observer Integration Tests
|
|
3
|
+
*/
|
|
4
|
+
import { describe, expect, it, vi } from 'vitest';
|
|
5
|
+
import { processSession } from '../actions/process-session.js';
|
|
6
|
+
vi.mock('node:fs/promises', async (importOriginal) => {
|
|
7
|
+
const actual = await importOriginal();
|
|
8
|
+
return {
|
|
9
|
+
...actual,
|
|
10
|
+
writeFile: vi.fn().mockResolvedValue(undefined),
|
|
11
|
+
};
|
|
12
|
+
});
|
|
13
|
+
// Mock data
|
|
14
|
+
const mockRecording = {
|
|
15
|
+
id: 'test-recording-123',
|
|
16
|
+
source: {
|
|
17
|
+
type: 'cap',
|
|
18
|
+
originalPath: '/tmp/test.cap',
|
|
19
|
+
},
|
|
20
|
+
videoPath: '/tmp/test.mp4',
|
|
21
|
+
audioMicPath: '/tmp/mic.wav',
|
|
22
|
+
audioSystemPath: null,
|
|
23
|
+
duration: 60,
|
|
24
|
+
capturedAt: new Date(),
|
|
25
|
+
};
|
|
26
|
+
// Note: Transcript mock not used in current test setup
|
|
27
|
+
// const _mockTranscript: Transcript = {
|
|
28
|
+
// fullText: 'Hello world',
|
|
29
|
+
// segments: [{ id: '1', start: 0, end: 5, text: 'Hello world' }],
|
|
30
|
+
// language: 'en',
|
|
31
|
+
// duration: 60,
|
|
32
|
+
// };
|
|
33
|
+
const mockVisualIndex = {
|
|
34
|
+
frames: [
|
|
35
|
+
{
|
|
36
|
+
index: 0,
|
|
37
|
+
timestamp: 0,
|
|
38
|
+
imagePath: '/tmp/frame_0.jpg',
|
|
39
|
+
ocrText: 'def foo(): pass',
|
|
40
|
+
clusterId: 0,
|
|
41
|
+
changeScore: 0,
|
|
42
|
+
},
|
|
43
|
+
],
|
|
44
|
+
clusters: [
|
|
45
|
+
{
|
|
46
|
+
id: 0,
|
|
47
|
+
heuristicLabel: 'code-editor',
|
|
48
|
+
timeRange: [0, 60],
|
|
49
|
+
frameCount: 1,
|
|
50
|
+
representativeIdx: 0,
|
|
51
|
+
avgOcrCharacters: 15,
|
|
52
|
+
mediaIndicators: [],
|
|
53
|
+
},
|
|
54
|
+
],
|
|
55
|
+
processingTime: { ocrMs: 100, clipMs: 100, totalMs: 200 },
|
|
56
|
+
};
|
|
57
|
+
// Mock for describeImages() - returns array of parsed VLM results (not VisualDescriptions)
|
|
58
|
+
const mockVlmResults = [
|
|
59
|
+
{
|
|
60
|
+
index: 0,
|
|
61
|
+
timestamp: 0,
|
|
62
|
+
imagePath: '/tmp/frame_0.jpg',
|
|
63
|
+
activity: 'coding',
|
|
64
|
+
description: 'User is writing python code',
|
|
65
|
+
apps: ['VSCode'],
|
|
66
|
+
topics: ['python', 'programming'],
|
|
67
|
+
},
|
|
68
|
+
];
|
|
69
|
+
const mockStorageService = {
|
|
70
|
+
saveSession: vi.fn().mockResolvedValue(undefined),
|
|
71
|
+
loadSession: vi.fn().mockResolvedValue(null),
|
|
72
|
+
listSessions: vi.fn().mockResolvedValue([]),
|
|
73
|
+
saveArtifact: vi.fn().mockResolvedValue(undefined),
|
|
74
|
+
loadArtifacts: vi.fn().mockResolvedValue([]),
|
|
75
|
+
};
|
|
76
|
+
describe('Visual Observer Pipeline', () => {
|
|
77
|
+
it('should process session with visual intelligence', async () => {
|
|
78
|
+
// Mock services
|
|
79
|
+
const mockTranscriber = {
|
|
80
|
+
transcribe: vi.fn().mockResolvedValue({
|
|
81
|
+
fullText: '',
|
|
82
|
+
segments: [],
|
|
83
|
+
language: 'en',
|
|
84
|
+
duration: 0,
|
|
85
|
+
}),
|
|
86
|
+
transcribeSegment: vi.fn().mockResolvedValue(''),
|
|
87
|
+
};
|
|
88
|
+
const mockVideoService = {
|
|
89
|
+
extractFramesAtTimestamps: vi
|
|
90
|
+
.fn()
|
|
91
|
+
.mockResolvedValue(['/tmp/frame_0.jpg']),
|
|
92
|
+
extractFramesAtInterval: vi
|
|
93
|
+
.fn()
|
|
94
|
+
.mockResolvedValue([{ imagePath: '/tmp/frame_0.jpg', timestamp: 0 }]),
|
|
95
|
+
extractFramesAtTimestampsBatch: vi
|
|
96
|
+
.fn()
|
|
97
|
+
.mockResolvedValue([{ imagePath: '/tmp/frame_0.jpg', timestamp: 0 }]),
|
|
98
|
+
getMetadata: vi
|
|
99
|
+
.fn()
|
|
100
|
+
.mockResolvedValue({ duration: 60, width: 1920, height: 1080 }),
|
|
101
|
+
runVisualIndexing: vi.fn().mockResolvedValue(mockVisualIndex),
|
|
102
|
+
detectSceneChanges: vi.fn().mockResolvedValue([]),
|
|
103
|
+
};
|
|
104
|
+
const mockIntelligenceService = {
|
|
105
|
+
classify: vi.fn(),
|
|
106
|
+
classifySegment: vi.fn(),
|
|
107
|
+
extractMetadata: vi.fn(),
|
|
108
|
+
generate: vi.fn(),
|
|
109
|
+
describeImages: vi.fn().mockResolvedValue(mockVlmResults),
|
|
110
|
+
embedText: vi.fn(),
|
|
111
|
+
extractTopics: vi.fn(),
|
|
112
|
+
generateText: vi.fn().mockResolvedValue('Mock generated summary'),
|
|
113
|
+
};
|
|
114
|
+
const session = await processSession(mockRecording, mockTranscriber, mockVideoService, mockStorageService, mockIntelligenceService);
|
|
115
|
+
expect(session.id).toBe(mockRecording.id);
|
|
116
|
+
expect(session.visualLogs).toHaveLength(1);
|
|
117
|
+
expect(session.visualLogs[0].entries).toHaveLength(1);
|
|
118
|
+
const entry = session.visualLogs[0].entries[0];
|
|
119
|
+
expect(entry.description).toBe('User is writing python code');
|
|
120
|
+
expect(entry.heuristicLabel).toBe('code-editor');
|
|
121
|
+
expect(entry.ocrSummary).toContain('def foo()');
|
|
122
|
+
});
|
|
123
|
+
it('should skip VLM when discriminator rules say so', async () => {
|
|
124
|
+
// Case: Rich audio transcript overlaps with cluster
|
|
125
|
+
const richTranscript = {
|
|
126
|
+
fullText: 'I am writing a long explanation of this code for 60 seconds.',
|
|
127
|
+
segments: [
|
|
128
|
+
{
|
|
129
|
+
id: '1',
|
|
130
|
+
start: 0,
|
|
131
|
+
end: 60,
|
|
132
|
+
text: 'I am writing a long explanation of this code for 60 seconds.',
|
|
133
|
+
},
|
|
134
|
+
],
|
|
135
|
+
language: 'en',
|
|
136
|
+
duration: 60,
|
|
137
|
+
};
|
|
138
|
+
// Cluster with high OCR density and audio overlap
|
|
139
|
+
const denseIndex = {
|
|
140
|
+
...mockVisualIndex,
|
|
141
|
+
clusters: [
|
|
142
|
+
{
|
|
143
|
+
...mockVisualIndex.clusters[0],
|
|
144
|
+
avgOcrCharacters: 2000, // High density
|
|
145
|
+
},
|
|
146
|
+
],
|
|
147
|
+
};
|
|
148
|
+
const mockTranscriber = {
|
|
149
|
+
transcribe: vi.fn().mockResolvedValue(richTranscript),
|
|
150
|
+
transcribeSegment: vi.fn().mockResolvedValue(''),
|
|
151
|
+
};
|
|
152
|
+
const mockVideoService = {
|
|
153
|
+
// biome-ignore lint/suspicious/noExplicitAny: mock
|
|
154
|
+
...vi.fn(), // Other methods mocked as needed
|
|
155
|
+
extractFramesAtTimestamps: vi.fn(),
|
|
156
|
+
extractFramesAtInterval: vi
|
|
157
|
+
.fn()
|
|
158
|
+
.mockResolvedValue([{ imagePath: '/tmp/f.jpg', timestamp: 0 }]),
|
|
159
|
+
runVisualIndexing: vi.fn().mockResolvedValue(denseIndex),
|
|
160
|
+
};
|
|
161
|
+
const mockIntelligenceService = {
|
|
162
|
+
classify: vi.fn(),
|
|
163
|
+
classifySegment: vi.fn(),
|
|
164
|
+
extractMetadata: vi.fn(),
|
|
165
|
+
generate: vi.fn(),
|
|
166
|
+
describeImages: vi.fn(), // Should NOT be called
|
|
167
|
+
embedText: vi.fn(),
|
|
168
|
+
extractTopics: vi.fn(),
|
|
169
|
+
generateText: vi.fn().mockResolvedValue('Mock generated summary'),
|
|
170
|
+
};
|
|
171
|
+
const session = await processSession(mockRecording, mockTranscriber, mockVideoService, mockStorageService, mockIntelligenceService);
|
|
172
|
+
expect(mockIntelligenceService.describeImages).not.toHaveBeenCalled();
|
|
173
|
+
expect(session.visualLogs[0].entries[0].description).toBeUndefined();
|
|
174
|
+
});
|
|
175
|
+
});
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ID Normalization Utilities
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Normalizes a raw ID (e.g. from a folder name) by removing spaces and special characters.
|
|
6
|
+
*/
|
|
7
|
+
export function normalizeSessionId(rawId) {
|
|
8
|
+
return rawId
|
|
9
|
+
.replace(/\s+/g, '-') // Spaces → hyphens
|
|
10
|
+
.replace(/[()[\]{}]/g, '') // Remove brackets
|
|
11
|
+
.replace(/—/g, '-') // Em-dash → hyphen
|
|
12
|
+
.replace(/-+/g, '-') // Collapse multiple hyphens
|
|
13
|
+
.replace(/^-|-$/g, '') // Trim leading/trailing hyphens
|
|
14
|
+
.replace(/\.cap$/i, ''); // Remove .cap extension
|
|
15
|
+
}
|