escribano 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (124) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +297 -0
  3. package/dist/0_types.js +279 -0
  4. package/dist/actions/classify-session.js +77 -0
  5. package/dist/actions/create-contexts.js +44 -0
  6. package/dist/actions/create-topic-blocks.js +68 -0
  7. package/dist/actions/extract-metadata.js +24 -0
  8. package/dist/actions/generate-artifact-v3.js +296 -0
  9. package/dist/actions/generate-artifact.js +61 -0
  10. package/dist/actions/generate-summary-v3.js +260 -0
  11. package/dist/actions/outline-index.js +204 -0
  12. package/dist/actions/process-recording-v2.js +494 -0
  13. package/dist/actions/process-recording-v3.js +412 -0
  14. package/dist/actions/process-session.js +183 -0
  15. package/dist/actions/publish-summary-v3.js +303 -0
  16. package/dist/actions/sync-to-outline.js +196 -0
  17. package/dist/adapters/audio.silero.adapter.js +69 -0
  18. package/dist/adapters/cap.adapter.js +94 -0
  19. package/dist/adapters/capture.cap.adapter.js +107 -0
  20. package/dist/adapters/capture.filesystem.adapter.js +124 -0
  21. package/dist/adapters/embedding.ollama.adapter.js +141 -0
  22. package/dist/adapters/intelligence.adapter.js +202 -0
  23. package/dist/adapters/intelligence.mlx.adapter.js +395 -0
  24. package/dist/adapters/intelligence.ollama.adapter.js +741 -0
  25. package/dist/adapters/publishing.outline.adapter.js +75 -0
  26. package/dist/adapters/storage.adapter.js +81 -0
  27. package/dist/adapters/storage.fs.adapter.js +83 -0
  28. package/dist/adapters/transcription.whisper.adapter.js +206 -0
  29. package/dist/adapters/video.ffmpeg.adapter.js +405 -0
  30. package/dist/adapters/whisper.adapter.js +168 -0
  31. package/dist/batch-context.js +329 -0
  32. package/dist/db/helpers.js +50 -0
  33. package/dist/db/index.js +95 -0
  34. package/dist/db/migrate.js +80 -0
  35. package/dist/db/repositories/artifact.sqlite.js +77 -0
  36. package/dist/db/repositories/cluster.sqlite.js +92 -0
  37. package/dist/db/repositories/context.sqlite.js +75 -0
  38. package/dist/db/repositories/index.js +10 -0
  39. package/dist/db/repositories/observation.sqlite.js +70 -0
  40. package/dist/db/repositories/recording.sqlite.js +56 -0
  41. package/dist/db/repositories/subject.sqlite.js +64 -0
  42. package/dist/db/repositories/topic-block.sqlite.js +45 -0
  43. package/dist/db/types.js +4 -0
  44. package/dist/domain/classification.js +60 -0
  45. package/dist/domain/context.js +97 -0
  46. package/dist/domain/index.js +2 -0
  47. package/dist/domain/observation.js +17 -0
  48. package/dist/domain/recording.js +41 -0
  49. package/dist/domain/segment.js +93 -0
  50. package/dist/domain/session.js +93 -0
  51. package/dist/domain/time-range.js +38 -0
  52. package/dist/domain/transcript.js +79 -0
  53. package/dist/index.js +173 -0
  54. package/dist/pipeline/context.js +162 -0
  55. package/dist/pipeline/events.js +2 -0
  56. package/dist/prerequisites.js +226 -0
  57. package/dist/scripts/rebuild-index.js +53 -0
  58. package/dist/scripts/seed-fixtures.js +290 -0
  59. package/dist/services/activity-segmentation.js +333 -0
  60. package/dist/services/activity-segmentation.test.js +191 -0
  61. package/dist/services/app-normalization.js +212 -0
  62. package/dist/services/cluster-merge.js +69 -0
  63. package/dist/services/clustering.js +237 -0
  64. package/dist/services/debug.js +58 -0
  65. package/dist/services/frame-sampling.js +318 -0
  66. package/dist/services/signal-extraction.js +106 -0
  67. package/dist/services/subject-grouping.js +342 -0
  68. package/dist/services/temporal-alignment.js +99 -0
  69. package/dist/services/vlm-enrichment.js +84 -0
  70. package/dist/services/vlm-service.js +130 -0
  71. package/dist/stats/index.js +3 -0
  72. package/dist/stats/observer.js +65 -0
  73. package/dist/stats/repository.js +36 -0
  74. package/dist/stats/resource-tracker.js +86 -0
  75. package/dist/stats/types.js +1 -0
  76. package/dist/test-classification-prompts.js +181 -0
  77. package/dist/tests/cap.adapter.test.js +75 -0
  78. package/dist/tests/capture.cap.adapter.test.js +69 -0
  79. package/dist/tests/classify-session.test.js +140 -0
  80. package/dist/tests/db/repositories.test.js +243 -0
  81. package/dist/tests/domain/time-range.test.js +31 -0
  82. package/dist/tests/integration.test.js +84 -0
  83. package/dist/tests/intelligence.adapter.test.js +102 -0
  84. package/dist/tests/intelligence.ollama.adapter.test.js +178 -0
  85. package/dist/tests/process-v2.test.js +90 -0
  86. package/dist/tests/services/clustering.test.js +112 -0
  87. package/dist/tests/services/frame-sampling.test.js +152 -0
  88. package/dist/tests/utils/ocr.test.js +76 -0
  89. package/dist/tests/utils/parallel.test.js +57 -0
  90. package/dist/tests/visual-observer.test.js +175 -0
  91. package/dist/utils/id-normalization.js +15 -0
  92. package/dist/utils/index.js +9 -0
  93. package/dist/utils/model-detector.js +154 -0
  94. package/dist/utils/ocr.js +80 -0
  95. package/dist/utils/parallel.js +32 -0
  96. package/migrations/001_initial.sql +109 -0
  97. package/migrations/002_clusters.sql +41 -0
  98. package/migrations/003_observations_vlm_fields.sql +14 -0
  99. package/migrations/004_observations_unique.sql +18 -0
  100. package/migrations/005_processing_stats.sql +29 -0
  101. package/migrations/006_vlm_raw_response.sql +6 -0
  102. package/migrations/007_subjects.sql +23 -0
  103. package/migrations/008_artifacts_recording.sql +6 -0
  104. package/migrations/009_artifact_subjects.sql +10 -0
  105. package/package.json +82 -0
  106. package/prompts/action-items.md +55 -0
  107. package/prompts/blog-draft.md +54 -0
  108. package/prompts/blog-research.md +87 -0
  109. package/prompts/card.md +54 -0
  110. package/prompts/classify-segment.md +38 -0
  111. package/prompts/classify.md +37 -0
  112. package/prompts/code-snippets.md +163 -0
  113. package/prompts/extract-metadata.md +149 -0
  114. package/prompts/notes.md +83 -0
  115. package/prompts/runbook.md +123 -0
  116. package/prompts/standup.md +50 -0
  117. package/prompts/step-by-step.md +125 -0
  118. package/prompts/subject-grouping.md +31 -0
  119. package/prompts/summary-v3.md +89 -0
  120. package/prompts/summary.md +77 -0
  121. package/prompts/topic-classifier.md +24 -0
  122. package/prompts/topic-extract.md +13 -0
  123. package/prompts/vlm-batch.md +21 -0
  124. package/prompts/vlm-single.md +19 -0
@@ -0,0 +1,90 @@
1
+ /**
2
+ * Process Recording V2 Tests
3
+ */
4
+ import { describe, expect, it, vi } from 'vitest';
5
+ import { processRecordingV2 } from '../actions/process-recording-v2.js';
6
+ describe('processRecordingV2', () => {
7
+ it('should call extractFramesAtInterval instead of extractFramesAtTimestamps during visual processing', async () => {
8
+ // Mock Recording in DB
9
+ const mockDbRecording = {
10
+ id: 'rec-123',
11
+ status: 'raw',
12
+ processing_step: 'idle',
13
+ video_path: '/tmp/test.mp4',
14
+ audio_mic_path: null,
15
+ audio_system_path: null,
16
+ captured_at: new Date(),
17
+ duration: 60,
18
+ };
19
+ const mockRepos = {
20
+ recordings: {
21
+ findById: vi.fn().mockReturnValue(mockDbRecording),
22
+ updateStatus: vi.fn(),
23
+ },
24
+ observations: {
25
+ saveBatch: vi.fn(),
26
+ save: vi.fn(),
27
+ findByRecording: vi.fn().mockReturnValue([]),
28
+ findByRecordingAndType: vi.fn().mockReturnValue([]),
29
+ updateEmbedding: vi.fn(),
30
+ deleteByRecording: vi.fn(),
31
+ },
32
+ clusters: {
33
+ deleteByRecording: vi.fn(),
34
+ save: vi.fn(),
35
+ linkObservationsBatch: vi.fn(),
36
+ findByRecording: vi.fn().mockReturnValue([]),
37
+ findByRecordingAndType: vi.fn().mockReturnValue([]),
38
+ getObservations: vi.fn().mockReturnValue([]),
39
+ updateClassification: vi.fn(),
40
+ getMergedAudioClusters: vi.fn().mockReturnValue([]),
41
+ saveMerge: vi.fn(),
42
+ },
43
+ contexts: {
44
+ findByTypeAndName: vi.fn(),
45
+ save: vi.fn(),
46
+ linkObservation: vi.fn(),
47
+ getLinksByRecording: vi.fn().mockReturnValue([]),
48
+ },
49
+ topicBlocks: {
50
+ deleteByRecording: vi.fn(),
51
+ save: vi.fn(),
52
+ },
53
+ };
54
+ const mockVideoService = {
55
+ extractFramesAtInterval: vi.fn().mockResolvedValue([
56
+ { imagePath: '/tmp/f1.jpg', timestamp: 0 },
57
+ { imagePath: '/tmp/f2.jpg', timestamp: 2 },
58
+ ]),
59
+ extractFramesAtTimestamps: vi.fn(),
60
+ runVisualIndexing: vi.fn().mockResolvedValue({ frames: [] }),
61
+ };
62
+ const mockIntelligence = {
63
+ describeImages: vi.fn().mockResolvedValue({ descriptions: [] }),
64
+ embedText: vi.fn().mockResolvedValue([]),
65
+ extractTopics: vi.fn().mockResolvedValue([]),
66
+ };
67
+ const mockEmbedding = {
68
+ embedBatch: vi.fn().mockResolvedValue([]),
69
+ };
70
+ const mockPreprocessor = {
71
+ extractSpeechSegments: vi
72
+ .fn()
73
+ .mockResolvedValue({ segments: [], tempDir: '/tmp' }),
74
+ cleanup: vi.fn(),
75
+ };
76
+ const mockTranscription = {};
77
+ await processRecordingV2('rec-123', mockRepos, {
78
+ video: mockVideoService,
79
+ intelligence: mockIntelligence,
80
+ embedding: mockEmbedding,
81
+ preprocessor: mockPreprocessor,
82
+ transcription: mockTranscription,
83
+ });
84
+ // Verify correct function was called
85
+ expect(mockVideoService.extractFramesAtInterval).toHaveBeenCalled();
86
+ expect(mockVideoService.extractFramesAtTimestamps).not.toHaveBeenCalled();
87
+ // Verify interval-based extraction was used with correct threshold
88
+ expect(mockVideoService.extractFramesAtInterval).toHaveBeenCalledWith('/tmp/test.mp4', 0.3, expect.stringContaining('frames'));
89
+ });
90
+ });
@@ -0,0 +1,112 @@
1
+ import { describe, expect, it, vi } from 'vitest';
2
+ import { embeddingToBlob } from '../../db/helpers.js';
3
+ import { clusterObservations } from '../../services/clustering.js';
4
+ describe('clusterObservations', () => {
5
+ const mockEmbeddingService = {
6
+ embed: vi.fn(),
7
+ embedBatch: vi.fn(),
8
+ similarity: vi.fn((a, b) => {
9
+ // Simple mock: if first element is same, they are identical
10
+ return a[0] === b[0] ? 1 : 0;
11
+ }),
12
+ centroid: vi.fn((embeddings) => {
13
+ if (embeddings.length === 0)
14
+ return [];
15
+ const dim = embeddings[0].length;
16
+ const res = new Array(dim).fill(0);
17
+ for (const e of embeddings) {
18
+ for (let i = 0; i < dim; i++)
19
+ res[i] += e[i];
20
+ }
21
+ return res.map((v) => v / embeddings.length);
22
+ }),
23
+ };
24
+ const createObs = (id, timestamp, typeVal) => ({
25
+ id,
26
+ recording_id: 'rec1',
27
+ type: 'visual',
28
+ timestamp,
29
+ end_timestamp: timestamp + 1,
30
+ image_path: null,
31
+ ocr_text: `text ${typeVal}`,
32
+ vlm_description: null,
33
+ vlm_raw_response: null,
34
+ activity_type: null,
35
+ apps: null,
36
+ topics: null,
37
+ text: null,
38
+ audio_source: null,
39
+ audio_type: null,
40
+ embedding: embeddingToBlob([typeVal, 0, 0]),
41
+ created_at: new Date().toISOString(),
42
+ });
43
+ it('should cluster identical observations within time window', () => {
44
+ const obs = [
45
+ createObs('1', 10, 1),
46
+ createObs('2', 20, 1),
47
+ createObs('3', 30, 1),
48
+ ];
49
+ const clusters = clusterObservations(obs, mockEmbeddingService, {
50
+ timeWindowSeconds: 60,
51
+ distanceThreshold: 0.1,
52
+ minClusterSize: 2,
53
+ });
54
+ expect(clusters).toHaveLength(1);
55
+ expect(clusters[0].observations).toHaveLength(3);
56
+ });
57
+ it('should not cluster observations outside time window', () => {
58
+ const obs = [
59
+ createObs('1', 10, 1),
60
+ createObs('2', 1000, 1), // Far away in time
61
+ ];
62
+ const clusters = clusterObservations(obs, mockEmbeddingService, {
63
+ timeWindowSeconds: 60,
64
+ distanceThreshold: 0.1,
65
+ minClusterSize: 1, // Set to 1 to see two clusters
66
+ });
67
+ expect(clusters).toHaveLength(2);
68
+ });
69
+ it('should not cluster semantically different observations', () => {
70
+ const obs = [
71
+ createObs('1', 10, 1),
72
+ createObs('2', 20, 2), // Different embedding
73
+ ];
74
+ const clusters = clusterObservations(obs, mockEmbeddingService, {
75
+ timeWindowSeconds: 60,
76
+ distanceThreshold: 0.1,
77
+ minClusterSize: 1,
78
+ });
79
+ expect(clusters).toHaveLength(2);
80
+ });
81
+ it('should absorb small clusters into nearest large cluster', () => {
82
+ // 3 similar obs (large cluster)
83
+ // 1 similar obs (small cluster, should be absorbed)
84
+ const obs = [
85
+ createObs('1', 10, 1),
86
+ createObs('2', 20, 1),
87
+ createObs('3', 30, 1),
88
+ createObs('4', 40, 1),
89
+ ];
90
+ const clusters = clusterObservations(obs, mockEmbeddingService, {
91
+ timeWindowSeconds: 60,
92
+ distanceThreshold: 0.1,
93
+ minClusterSize: 3,
94
+ });
95
+ expect(clusters).toHaveLength(1);
96
+ expect(clusters[0].observations).toHaveLength(4);
97
+ });
98
+ it('should handle empty input', () => {
99
+ const clusters = clusterObservations([], mockEmbeddingService);
100
+ expect(clusters).toEqual([]);
101
+ });
102
+ it('should throw if an observation in validObs is missing embedding', () => {
103
+ // We mock bufferToEmbedding to fail if we want to test the throw,
104
+ // but the current code in clustering.ts uses a local bufferToEmbedding.
105
+ // However, if we pass an obs that PASSES the filter but then has no embedding
106
+ // (impossible due to filter), it would throw.
107
+ // Let's test the filter instead
108
+ const obsNoEmbed = [{ id: '1', embedding: null }];
109
+ const clusters = clusterObservations(obsNoEmbed, mockEmbeddingService);
110
+ expect(clusters).toHaveLength(0);
111
+ });
112
+ });
@@ -0,0 +1,152 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { adaptiveSample, adaptiveSampleWithScenes, calculateAdaptiveBaseInterval, getSamplingStats, } from '../../services/frame-sampling.js';
3
+ describe('adaptiveSample', () => {
4
+ it('should return empty array for empty input', () => {
5
+ const result = adaptiveSample([]);
6
+ expect(result).toEqual([]);
7
+ });
8
+ it('should sample at base interval', () => {
9
+ // Create frames every 2 seconds for 60 seconds
10
+ const frames = [];
11
+ for (let t = 0; t <= 60; t += 2) {
12
+ frames.push({ imagePath: `frame_${t}.jpg`, timestamp: t });
13
+ }
14
+ const result = adaptiveSample(frames, { baseIntervalSeconds: 10 });
15
+ // Should get frames at 0, 10, 20, 30, 40, 50, 60 = 7 frames
16
+ expect(result.length).toBe(7);
17
+ expect(result.every((f) => f.reason === 'base')).toBe(true);
18
+ expect(result.map((f) => f.timestamp)).toEqual([0, 10, 20, 30, 40, 50, 60]);
19
+ });
20
+ it('should fill gaps larger than threshold', () => {
21
+ // Create frames with a gap
22
+ const frames = [
23
+ { imagePath: 'frame_0.jpg', timestamp: 0 },
24
+ { imagePath: 'frame_2.jpg', timestamp: 2 },
25
+ { imagePath: 'frame_4.jpg', timestamp: 4 },
26
+ // Gap from 4 to 30 seconds
27
+ { imagePath: 'frame_30.jpg', timestamp: 30 },
28
+ { imagePath: 'frame_32.jpg', timestamp: 32 },
29
+ ];
30
+ const result = adaptiveSample(frames, {
31
+ baseIntervalSeconds: 10,
32
+ gapThresholdSeconds: 15,
33
+ gapFillIntervalSeconds: 5,
34
+ });
35
+ // Base samples: 0, 30
36
+ // Gap detected: 30 - 0 = 30 > 15
37
+ // Gap fill should add samples between 0 and 30
38
+ expect(result.length).toBeGreaterThan(2);
39
+ expect(result.some((f) => f.reason === 'gap_fill')).toBe(true);
40
+ });
41
+ it('should not fill gaps smaller than threshold', () => {
42
+ const frames = [];
43
+ for (let t = 0; t <= 30; t += 2) {
44
+ frames.push({ imagePath: `frame_${t}.jpg`, timestamp: t });
45
+ }
46
+ const result = adaptiveSample(frames, {
47
+ baseIntervalSeconds: 10,
48
+ gapThresholdSeconds: 15,
49
+ });
50
+ // Gap between samples is 10s, which is < 15s threshold
51
+ expect(result.every((f) => f.reason === 'base')).toBe(true);
52
+ });
53
+ it('should handle unsorted input', () => {
54
+ const frames = [
55
+ { imagePath: 'frame_20.jpg', timestamp: 20 },
56
+ { imagePath: 'frame_0.jpg', timestamp: 0 },
57
+ { imagePath: 'frame_10.jpg', timestamp: 10 },
58
+ ];
59
+ const result = adaptiveSample(frames, { baseIntervalSeconds: 10 });
60
+ expect(result.map((f) => f.timestamp)).toEqual([0, 10, 20]);
61
+ });
62
+ });
63
+ describe('getSamplingStats', () => {
64
+ it('should calculate correct statistics', () => {
65
+ const original = Array.from({ length: 100 }, (_, i) => ({
66
+ imagePath: `frame_${i}.jpg`,
67
+ timestamp: i * 2,
68
+ }));
69
+ const sampled = adaptiveSample(original, { baseIntervalSeconds: 10 });
70
+ const stats = getSamplingStats(original, sampled);
71
+ expect(stats.originalCount).toBe(100);
72
+ expect(stats.sampledCount).toBeLessThan(100);
73
+ expect(stats.reductionPercent).toBeGreaterThan(0);
74
+ });
75
+ });
76
+ describe('calculateAdaptiveBaseInterval', () => {
77
+ it('should return config base for few scene changes (<= 20)', () => {
78
+ expect(calculateAdaptiveBaseInterval(0, 10)).toBe(10);
79
+ expect(calculateAdaptiveBaseInterval(5, 10)).toBe(10);
80
+ expect(calculateAdaptiveBaseInterval(20, 10)).toBe(10);
81
+ });
82
+ it('should return 20s for moderate scene changes (21-50)', () => {
83
+ expect(calculateAdaptiveBaseInterval(21, 10)).toBe(20);
84
+ expect(calculateAdaptiveBaseInterval(35, 10)).toBe(20);
85
+ expect(calculateAdaptiveBaseInterval(50, 10)).toBe(20);
86
+ });
87
+ it('should return 30s for high scene density (> 50)', () => {
88
+ expect(calculateAdaptiveBaseInterval(51, 10)).toBe(30);
89
+ expect(calculateAdaptiveBaseInterval(109, 10)).toBe(30);
90
+ expect(calculateAdaptiveBaseInterval(200, 10)).toBe(30);
91
+ });
92
+ it('should never go below configured base interval', () => {
93
+ // If user configured 40s, keep it even for low scene counts
94
+ expect(calculateAdaptiveBaseInterval(5, 40)).toBe(40);
95
+ expect(calculateAdaptiveBaseInterval(30, 40)).toBe(40);
96
+ expect(calculateAdaptiveBaseInterval(100, 40)).toBe(40);
97
+ });
98
+ });
99
+ describe('adaptiveSampleWithScenes', () => {
100
+ // Helper: create frames every 2s for N minutes
101
+ function makeFrames(durationMinutes) {
102
+ const frames = [];
103
+ const total = durationMinutes * 60;
104
+ for (let t = 0; t <= total; t += 2) {
105
+ frames.push({ imagePath: `frame_${t}.jpg`, timestamp: t });
106
+ }
107
+ return frames;
108
+ }
109
+ it('should return empty array for empty input', () => {
110
+ expect(adaptiveSampleWithScenes([], [])).toEqual([]);
111
+ });
112
+ it('should sample at base interval when no scene changes', () => {
113
+ const frames = makeFrames(1); // 1 minute
114
+ const result = adaptiveSampleWithScenes(frames, []);
115
+ // Should behave like regular sampling
116
+ expect(result.length).toBeGreaterThan(0);
117
+ expect(result.every((f) => f.reason === 'base')).toBe(true);
118
+ });
119
+ it('should include scene change frames', () => {
120
+ const frames = makeFrames(1);
121
+ const sceneChanges = [10, 20, 40];
122
+ const result = adaptiveSampleWithScenes(frames, sceneChanges);
123
+ const sceneFrames = result.filter((f) => f.reason === 'scene_change');
124
+ expect(sceneFrames.length).toBe(3);
125
+ });
126
+ it('should produce fewer total frames with high scene density', () => {
127
+ const frames = makeFrames(10); // 10 minutes, 300 frames
128
+ // Low density: 5 scene changes
129
+ const lowDensityResult = adaptiveSampleWithScenes(frames, [30, 100, 200, 400, 500], { baseIntervalSeconds: 10 });
130
+ // High density: 60 scene changes (every ~10s)
131
+ const highDensityScenes = Array.from({ length: 60 }, (_, i) => i * 10);
132
+ const highDensityResult = adaptiveSampleWithScenes(frames, highDensityScenes, { baseIntervalSeconds: 10 });
133
+ // High density should use 30s base interval, resulting in fewer
134
+ // base samples (but more scene samples). Total should still be
135
+ // manageable and not explode.
136
+ const highStats = getSamplingStats(frames, highDensityResult);
137
+ expect(highStats.sampledCount).toBeLessThan(frames.length);
138
+ expect(highStats.reductionPercent).toBeGreaterThan(50);
139
+ });
140
+ it('should match real-world scenario: 59 min, 109 scenes', () => {
141
+ const frames = makeFrames(59); // 1776 frames
142
+ // Simulate 109 scene changes spread across 59 minutes
143
+ const sceneChanges = Array.from({ length: 109 }, (_, i) => Math.round((i / 109) * 59 * 60));
144
+ const result = adaptiveSampleWithScenes(frames, sceneChanges);
145
+ const stats = getSamplingStats(frames, result);
146
+ // Target: under 250 frames total (was 1199 before adaptive interval + gap threshold fix)
147
+ expect(stats.sampledCount).toBeLessThan(250);
148
+ expect(stats.sceneChangeCount).toBe(109);
149
+ // Reduction should be significant
150
+ expect(stats.reductionPercent).toBeGreaterThan(85);
151
+ });
152
+ });
@@ -0,0 +1,76 @@
1
+ import { describe, expect, it } from 'vitest';
2
+ import { cleanOcrText, isOcrMeaningful } from '../../utils/ocr.js';
3
+ describe('OCR Cleanup Utilities', () => {
4
+ describe('cleanOcrText', () => {
5
+ it('preserves valid content and short commands', () => {
6
+ const input = 'git status\nnpm install\nThis is a valid sentence.';
7
+ const output = cleanOcrText(input);
8
+ expect(output).toContain('git status');
9
+ expect(output).toContain('npm install');
10
+ expect(output).toContain('This is a valid sentence.');
11
+ });
12
+ it('filters out system clock artifacts', () => {
13
+ const input = '© 15/01 Thu 12:12\nActual Content';
14
+ const output = cleanOcrText(input);
15
+ expect(output).not.toContain('© 15/01');
16
+ expect(output).toContain('Actual Content');
17
+ });
18
+ it('filters out window title artifacts', () => {
19
+ const input = '> Google Chrome\nSome webpage content';
20
+ const output = cleanOcrText(input);
21
+ expect(output).not.toContain('> Google Chrome');
22
+ expect(output).toContain('Some webpage content');
23
+ });
24
+ it('filters out repeated character garbage', () => {
25
+ const input = 'eee\nGPassssss\nValid text';
26
+ const output = cleanOcrText(input);
27
+ expect(output).not.toContain('eee');
28
+ expect(output).not.toContain('GPassssss');
29
+ expect(output).toContain('Valid text');
30
+ });
31
+ it('filters out common UI bookmarks and status lines', () => {
32
+ const input = 'All Bookmarks\nZoho Mail\nProject code here';
33
+ const output = cleanOcrText(input);
34
+ expect(output).not.toContain('All Bookmarks');
35
+ expect(output).not.toContain('Zoho Mail');
36
+ expect(output).toContain('Project code here');
37
+ });
38
+ it('filters out UI separators and indicators', () => {
39
+ const input = '| BSI - Bundesamt f.\nDetected Language\nEnglish\nActual information';
40
+ const output = cleanOcrText(input);
41
+ expect(output).not.toContain('BSI - Bundesamt');
42
+ expect(output).not.toContain('Detected Language');
43
+ expect(output).not.toContain('English');
44
+ expect(output).toContain('Actual information');
45
+ });
46
+ it('removes duplicate consecutive lines', () => {
47
+ const input = 'Line 1\nLine 1\nLine 2\nLine 1';
48
+ const output = cleanOcrText(input);
49
+ const lines = output.split('\n');
50
+ expect(lines).toHaveLength(3);
51
+ expect(lines[0]).toBe('Line 1');
52
+ expect(lines[1]).toBe('Line 2');
53
+ expect(lines[2]).toBe('Line 1');
54
+ });
55
+ it('filters out short garbage fragments while keeping short commands', () => {
56
+ // 'cid' was found in DB, should be filtered if it doesn't look like a word
57
+ // 'git' should be kept.
58
+ const input = 'cid\ngit\nnpm\nsh Twelv';
59
+ const output = cleanOcrText(input);
60
+ expect(output).not.toContain('cid');
61
+ expect(output).not.toContain('sh Twelv');
62
+ expect(output).toContain('git');
63
+ expect(output).toContain('npm');
64
+ });
65
+ });
66
+ describe('isOcrMeaningful', () => {
67
+ it('returns false for empty or very short text', () => {
68
+ expect(isOcrMeaningful('')).toBe(false);
69
+ expect(isOcrMeaningful('Too short')).toBe(false);
70
+ });
71
+ it('returns true for substantial content', () => {
72
+ const content = 'Line one of content\nLine two of content\nLine three of content';
73
+ expect(isOcrMeaningful(content)).toBe(true);
74
+ });
75
+ });
76
+ });
@@ -0,0 +1,57 @@
1
+ import { describe, expect, it, vi } from 'vitest';
2
+ import { chunkArray, parallelMap } from '../../utils/parallel.js';
3
+ describe('parallelMap', () => {
4
+ it('should process items and return results in order', async () => {
5
+ const items = [1, 2, 3, 4, 5];
6
+ const fn = async (n) => n * 2;
7
+ const results = await parallelMap(items, fn, 2);
8
+ expect(results).toEqual([2, 4, 6, 8, 10]);
9
+ });
10
+ it('should respect concurrency limits', async () => {
11
+ const items = [100, 100, 100, 100];
12
+ let activeCount = 0;
13
+ let maxActive = 0;
14
+ const fn = async (ms) => {
15
+ activeCount++;
16
+ maxActive = Math.max(maxActive, activeCount);
17
+ await new Promise((resolve) => setTimeout(resolve, ms));
18
+ activeCount--;
19
+ return ms;
20
+ };
21
+ const concurrency = 2;
22
+ await parallelMap(items, fn, concurrency);
23
+ expect(maxActive).toBeLessThanOrEqual(concurrency);
24
+ });
25
+ it('should handle empty input array', async () => {
26
+ const results = await parallelMap([], async (x) => x, 2);
27
+ expect(results).toEqual([]);
28
+ });
29
+ it('should handle concurrency <= 0 by defaulting to 1', async () => {
30
+ const items = [1, 2, 3];
31
+ const fn = vi.fn(async (x) => x);
32
+ const results0 = await parallelMap(items, fn, 0);
33
+ expect(results0).toEqual([1, 2, 3]);
34
+ const resultsNeg = await parallelMap(items, fn, -5);
35
+ expect(resultsNeg).toEqual([1, 2, 3]);
36
+ });
37
+ it('should propagate errors and stop processing', async () => {
38
+ const items = [1, 2, 3, 4, 5];
39
+ const fn = async (n) => {
40
+ if (n === 3)
41
+ throw new Error('Failed');
42
+ return n;
43
+ };
44
+ await expect(parallelMap(items, fn, 2)).rejects.toThrow('Failed');
45
+ });
46
+ });
47
+ describe('chunkArray', () => {
48
+ it('should split array into chunks', () => {
49
+ const items = [1, 2, 3, 4, 5];
50
+ expect(chunkArray(items, 2)).toEqual([[1, 2], [3, 4], [5]]);
51
+ expect(chunkArray(items, 5)).toEqual([[1, 2, 3, 4, 5]]);
52
+ expect(chunkArray(items, 10)).toEqual([[1, 2, 3, 4, 5]]);
53
+ });
54
+ it('should handle empty array', () => {
55
+ expect(chunkArray([], 2)).toEqual([]);
56
+ });
57
+ });
@@ -0,0 +1,175 @@
1
+ /**
2
+ * Visual Observer Integration Tests
3
+ */
4
+ import { describe, expect, it, vi } from 'vitest';
5
+ import { processSession } from '../actions/process-session.js';
6
+ vi.mock('node:fs/promises', async (importOriginal) => {
7
+ const actual = await importOriginal();
8
+ return {
9
+ ...actual,
10
+ writeFile: vi.fn().mockResolvedValue(undefined),
11
+ };
12
+ });
13
+ // Mock data
14
+ const mockRecording = {
15
+ id: 'test-recording-123',
16
+ source: {
17
+ type: 'cap',
18
+ originalPath: '/tmp/test.cap',
19
+ },
20
+ videoPath: '/tmp/test.mp4',
21
+ audioMicPath: '/tmp/mic.wav',
22
+ audioSystemPath: null,
23
+ duration: 60,
24
+ capturedAt: new Date(),
25
+ };
26
+ // Note: Transcript mock not used in current test setup
27
+ // const _mockTranscript: Transcript = {
28
+ // fullText: 'Hello world',
29
+ // segments: [{ id: '1', start: 0, end: 5, text: 'Hello world' }],
30
+ // language: 'en',
31
+ // duration: 60,
32
+ // };
33
+ const mockVisualIndex = {
34
+ frames: [
35
+ {
36
+ index: 0,
37
+ timestamp: 0,
38
+ imagePath: '/tmp/frame_0.jpg',
39
+ ocrText: 'def foo(): pass',
40
+ clusterId: 0,
41
+ changeScore: 0,
42
+ },
43
+ ],
44
+ clusters: [
45
+ {
46
+ id: 0,
47
+ heuristicLabel: 'code-editor',
48
+ timeRange: [0, 60],
49
+ frameCount: 1,
50
+ representativeIdx: 0,
51
+ avgOcrCharacters: 15,
52
+ mediaIndicators: [],
53
+ },
54
+ ],
55
+ processingTime: { ocrMs: 100, clipMs: 100, totalMs: 200 },
56
+ };
57
+ // Mock for describeImages() - returns array of parsed VLM results (not VisualDescriptions)
58
+ const mockVlmResults = [
59
+ {
60
+ index: 0,
61
+ timestamp: 0,
62
+ imagePath: '/tmp/frame_0.jpg',
63
+ activity: 'coding',
64
+ description: 'User is writing python code',
65
+ apps: ['VSCode'],
66
+ topics: ['python', 'programming'],
67
+ },
68
+ ];
69
+ const mockStorageService = {
70
+ saveSession: vi.fn().mockResolvedValue(undefined),
71
+ loadSession: vi.fn().mockResolvedValue(null),
72
+ listSessions: vi.fn().mockResolvedValue([]),
73
+ saveArtifact: vi.fn().mockResolvedValue(undefined),
74
+ loadArtifacts: vi.fn().mockResolvedValue([]),
75
+ };
76
+ describe('Visual Observer Pipeline', () => {
77
+ it('should process session with visual intelligence', async () => {
78
+ // Mock services
79
+ const mockTranscriber = {
80
+ transcribe: vi.fn().mockResolvedValue({
81
+ fullText: '',
82
+ segments: [],
83
+ language: 'en',
84
+ duration: 0,
85
+ }),
86
+ transcribeSegment: vi.fn().mockResolvedValue(''),
87
+ };
88
+ const mockVideoService = {
89
+ extractFramesAtTimestamps: vi
90
+ .fn()
91
+ .mockResolvedValue(['/tmp/frame_0.jpg']),
92
+ extractFramesAtInterval: vi
93
+ .fn()
94
+ .mockResolvedValue([{ imagePath: '/tmp/frame_0.jpg', timestamp: 0 }]),
95
+ extractFramesAtTimestampsBatch: vi
96
+ .fn()
97
+ .mockResolvedValue([{ imagePath: '/tmp/frame_0.jpg', timestamp: 0 }]),
98
+ getMetadata: vi
99
+ .fn()
100
+ .mockResolvedValue({ duration: 60, width: 1920, height: 1080 }),
101
+ runVisualIndexing: vi.fn().mockResolvedValue(mockVisualIndex),
102
+ detectSceneChanges: vi.fn().mockResolvedValue([]),
103
+ };
104
+ const mockIntelligenceService = {
105
+ classify: vi.fn(),
106
+ classifySegment: vi.fn(),
107
+ extractMetadata: vi.fn(),
108
+ generate: vi.fn(),
109
+ describeImages: vi.fn().mockResolvedValue(mockVlmResults),
110
+ embedText: vi.fn(),
111
+ extractTopics: vi.fn(),
112
+ generateText: vi.fn().mockResolvedValue('Mock generated summary'),
113
+ };
114
+ const session = await processSession(mockRecording, mockTranscriber, mockVideoService, mockStorageService, mockIntelligenceService);
115
+ expect(session.id).toBe(mockRecording.id);
116
+ expect(session.visualLogs).toHaveLength(1);
117
+ expect(session.visualLogs[0].entries).toHaveLength(1);
118
+ const entry = session.visualLogs[0].entries[0];
119
+ expect(entry.description).toBe('User is writing python code');
120
+ expect(entry.heuristicLabel).toBe('code-editor');
121
+ expect(entry.ocrSummary).toContain('def foo()');
122
+ });
123
+ it('should skip VLM when discriminator rules say so', async () => {
124
+ // Case: Rich audio transcript overlaps with cluster
125
+ const richTranscript = {
126
+ fullText: 'I am writing a long explanation of this code for 60 seconds.',
127
+ segments: [
128
+ {
129
+ id: '1',
130
+ start: 0,
131
+ end: 60,
132
+ text: 'I am writing a long explanation of this code for 60 seconds.',
133
+ },
134
+ ],
135
+ language: 'en',
136
+ duration: 60,
137
+ };
138
+ // Cluster with high OCR density and audio overlap
139
+ const denseIndex = {
140
+ ...mockVisualIndex,
141
+ clusters: [
142
+ {
143
+ ...mockVisualIndex.clusters[0],
144
+ avgOcrCharacters: 2000, // High density
145
+ },
146
+ ],
147
+ };
148
+ const mockTranscriber = {
149
+ transcribe: vi.fn().mockResolvedValue(richTranscript),
150
+ transcribeSegment: vi.fn().mockResolvedValue(''),
151
+ };
152
+ const mockVideoService = {
153
+ // biome-ignore lint/suspicious/noExplicitAny: mock
154
+ ...vi.fn(), // Other methods mocked as needed
155
+ extractFramesAtTimestamps: vi.fn(),
156
+ extractFramesAtInterval: vi
157
+ .fn()
158
+ .mockResolvedValue([{ imagePath: '/tmp/f.jpg', timestamp: 0 }]),
159
+ runVisualIndexing: vi.fn().mockResolvedValue(denseIndex),
160
+ };
161
+ const mockIntelligenceService = {
162
+ classify: vi.fn(),
163
+ classifySegment: vi.fn(),
164
+ extractMetadata: vi.fn(),
165
+ generate: vi.fn(),
166
+ describeImages: vi.fn(), // Should NOT be called
167
+ embedText: vi.fn(),
168
+ extractTopics: vi.fn(),
169
+ generateText: vi.fn().mockResolvedValue('Mock generated summary'),
170
+ };
171
+ const session = await processSession(mockRecording, mockTranscriber, mockVideoService, mockStorageService, mockIntelligenceService);
172
+ expect(mockIntelligenceService.describeImages).not.toHaveBeenCalled();
173
+ expect(session.visualLogs[0].entries[0].description).toBeUndefined();
174
+ });
175
+ });
@@ -0,0 +1,15 @@
1
+ /**
2
+ * ID Normalization Utilities
3
+ */
4
+ /**
5
+ * Normalizes a raw ID (e.g. from a folder name) by removing spaces and special characters.
6
+ */
7
+ export function normalizeSessionId(rawId) {
8
+ return rawId
9
+ .replace(/\s+/g, '-') // Spaces → hyphens
10
+ .replace(/[()[\]{}]/g, '') // Remove brackets
11
+ .replace(/—/g, '-') // Em-dash → hyphen
12
+ .replace(/-+/g, '-') // Collapse multiple hyphens
13
+ .replace(/^-|-$/g, '') // Trim leading/trailing hyphens
14
+ .replace(/\.cap$/i, ''); // Remove .cap extension
15
+ }