@nathanvale/chatline 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +1 -0
- package/LICENSE +21 -0
- package/README.md +1535 -0
- package/dist/bin/index.js +5121 -0
- package/dist/cli/commands/clean.d.ts +17 -0
- package/dist/cli/commands/clean.d.ts.map +1 -0
- package/dist/cli/commands/clean.js +142 -0
- package/dist/cli/commands/clean.js.map +1 -0
- package/dist/cli/commands/doctor.d.ts +17 -0
- package/dist/cli/commands/doctor.d.ts.map +1 -0
- package/dist/cli/commands/doctor.js +202 -0
- package/dist/cli/commands/doctor.js.map +1 -0
- package/dist/cli/commands/enrich-ai.d.ts +17 -0
- package/dist/cli/commands/enrich-ai.d.ts.map +1 -0
- package/dist/cli/commands/enrich-ai.js +371 -0
- package/dist/cli/commands/enrich-ai.js.map +1 -0
- package/dist/cli/commands/index.d.ts +16 -0
- package/dist/cli/commands/index.d.ts.map +1 -0
- package/dist/cli/commands/index.js +16 -0
- package/dist/cli/commands/index.js.map +1 -0
- package/dist/cli/commands/ingest-csv.d.ts +17 -0
- package/dist/cli/commands/ingest-csv.d.ts.map +1 -0
- package/dist/cli/commands/ingest-csv.js +138 -0
- package/dist/cli/commands/ingest-csv.js.map +1 -0
- package/dist/cli/commands/ingest-db.d.ts +17 -0
- package/dist/cli/commands/ingest-db.d.ts.map +1 -0
- package/dist/cli/commands/ingest-db.js +159 -0
- package/dist/cli/commands/ingest-db.js.map +1 -0
- package/dist/cli/commands/init.d.ts +17 -0
- package/dist/cli/commands/init.d.ts.map +1 -0
- package/dist/cli/commands/init.js +110 -0
- package/dist/cli/commands/init.js.map +1 -0
- package/dist/cli/commands/normalize-link.d.ts +16 -0
- package/dist/cli/commands/normalize-link.d.ts.map +1 -0
- package/dist/cli/commands/normalize-link.js +144 -0
- package/dist/cli/commands/normalize-link.js.map +1 -0
- package/dist/cli/commands/render-markdown.d.ts +17 -0
- package/dist/cli/commands/render-markdown.d.ts.map +1 -0
- package/dist/cli/commands/render-markdown.js +218 -0
- package/dist/cli/commands/render-markdown.js.map +1 -0
- package/dist/cli/commands/stats.d.ts +17 -0
- package/dist/cli/commands/stats.d.ts.map +1 -0
- package/dist/cli/commands/stats.js +175 -0
- package/dist/cli/commands/stats.js.map +1 -0
- package/dist/cli/commands/validate.d.ts +17 -0
- package/dist/cli/commands/validate.d.ts.map +1 -0
- package/dist/cli/commands/validate.js +152 -0
- package/dist/cli/commands/validate.js.map +1 -0
- package/dist/cli/index.d.ts +13 -0
- package/dist/cli/index.d.ts.map +1 -0
- package/dist/cli/index.js +121 -0
- package/dist/cli/index.js.map +1 -0
- package/dist/cli/types.d.ts +93 -0
- package/dist/cli/types.d.ts.map +1 -0
- package/dist/cli/types.js +7 -0
- package/dist/cli/types.js.map +1 -0
- package/dist/cli/utils.d.ts +29 -0
- package/dist/cli/utils.d.ts.map +1 -0
- package/dist/cli/utils.js +53 -0
- package/dist/cli/utils.js.map +1 -0
- package/dist/cli.d.ts +9 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +1805 -0
- package/dist/config/generator.d.ts +90 -0
- package/dist/config/generator.d.ts.map +1 -0
- package/dist/config/generator.js +320 -0
- package/dist/config/generator.js.map +1 -0
- package/dist/config/loader.d.ts +107 -0
- package/dist/config/loader.d.ts.map +1 -0
- package/dist/config/loader.js +251 -0
- package/dist/config/loader.js.map +1 -0
- package/dist/config/schema.d.ts +107 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +169 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/enrich/audio-transcription.d.ts +77 -0
- package/dist/enrich/audio-transcription.d.ts.map +1 -0
- package/dist/enrich/audio-transcription.js +370 -0
- package/dist/enrich/audio-transcription.js.map +1 -0
- package/dist/enrich/checkpoint.d.ts +137 -0
- package/dist/enrich/checkpoint.d.ts.map +1 -0
- package/dist/enrich/checkpoint.js +205 -0
- package/dist/enrich/checkpoint.js.map +1 -0
- package/dist/enrich/idempotency.d.ts +90 -0
- package/dist/enrich/idempotency.d.ts.map +1 -0
- package/dist/enrich/idempotency.js +188 -0
- package/dist/enrich/idempotency.js.map +1 -0
- package/dist/enrich/image-analysis.d.ts +62 -0
- package/dist/enrich/image-analysis.d.ts.map +1 -0
- package/dist/enrich/image-analysis.js +264 -0
- package/dist/enrich/image-analysis.js.map +1 -0
- package/dist/enrich/index.d.ts +60 -0
- package/dist/enrich/index.d.ts.map +1 -0
- package/dist/enrich/index.js +74 -0
- package/dist/enrich/index.js.map +1 -0
- package/dist/enrich/link-enrichment.d.ts +37 -0
- package/dist/enrich/link-enrichment.d.ts.map +1 -0
- package/dist/enrich/link-enrichment.js +202 -0
- package/dist/enrich/link-enrichment.js.map +1 -0
- package/dist/enrich/pdf-video-handling.d.ts +49 -0
- package/dist/enrich/pdf-video-handling.d.ts.map +1 -0
- package/dist/enrich/pdf-video-handling.js +325 -0
- package/dist/enrich/pdf-video-handling.js.map +1 -0
- package/dist/enrich/progress-tracker.d.ts +120 -0
- package/dist/enrich/progress-tracker.d.ts.map +1 -0
- package/dist/enrich/progress-tracker.js +220 -0
- package/dist/enrich/progress-tracker.js.map +1 -0
- package/dist/enrich/providers/firecrawl.d.ts +18 -0
- package/dist/enrich/providers/firecrawl.d.ts.map +1 -0
- package/dist/enrich/providers/firecrawl.js +48 -0
- package/dist/enrich/providers/firecrawl.js.map +1 -0
- package/dist/enrich/providers/generic.d.ts +16 -0
- package/dist/enrich/providers/generic.d.ts.map +1 -0
- package/dist/enrich/providers/generic.js +36 -0
- package/dist/enrich/providers/generic.js.map +1 -0
- package/dist/enrich/providers/index.d.ts +14 -0
- package/dist/enrich/providers/index.d.ts.map +1 -0
- package/dist/enrich/providers/index.js +13 -0
- package/dist/enrich/providers/index.js.map +1 -0
- package/dist/enrich/providers/instagram.d.ts +16 -0
- package/dist/enrich/providers/instagram.d.ts.map +1 -0
- package/dist/enrich/providers/instagram.js +43 -0
- package/dist/enrich/providers/instagram.js.map +1 -0
- package/dist/enrich/providers/spotify.d.ts +16 -0
- package/dist/enrich/providers/spotify.d.ts.map +1 -0
- package/dist/enrich/providers/spotify.js +45 -0
- package/dist/enrich/providers/spotify.js.map +1 -0
- package/dist/enrich/providers/twitter.d.ts +16 -0
- package/dist/enrich/providers/twitter.d.ts.map +1 -0
- package/dist/enrich/providers/twitter.js +43 -0
- package/dist/enrich/providers/twitter.js.map +1 -0
- package/dist/enrich/providers/types.d.ts +47 -0
- package/dist/enrich/providers/types.d.ts.map +1 -0
- package/dist/enrich/providers/types.js +15 -0
- package/dist/enrich/providers/types.js.map +1 -0
- package/dist/enrich/providers/youtube.d.ts +16 -0
- package/dist/enrich/providers/youtube.d.ts.map +1 -0
- package/dist/enrich/providers/youtube.js +43 -0
- package/dist/enrich/providers/youtube.js.map +1 -0
- package/dist/enrich/rate-limiting.d.ts +118 -0
- package/dist/enrich/rate-limiting.d.ts.map +1 -0
- package/dist/enrich/rate-limiting.js +258 -0
- package/dist/enrich/rate-limiting.js.map +1 -0
- package/dist/index.d.ts +688 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +1729 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/dedup-merge.d.ts +82 -0
- package/dist/ingest/dedup-merge.d.ts.map +1 -0
- package/dist/ingest/dedup-merge.js +262 -0
- package/dist/ingest/dedup-merge.js.map +1 -0
- package/dist/ingest/ingest-csv.d.ts +62 -0
- package/dist/ingest/ingest-csv.d.ts.map +1 -0
- package/dist/ingest/ingest-csv.js +300 -0
- package/dist/ingest/ingest-csv.js.map +1 -0
- package/dist/ingest/ingest-db.d.ts +64 -0
- package/dist/ingest/ingest-db.d.ts.map +1 -0
- package/dist/ingest/ingest-db.js +172 -0
- package/dist/ingest/ingest-db.js.map +1 -0
- package/dist/ingest/link-replies-and-tapbacks.d.ts +53 -0
- package/dist/ingest/link-replies-and-tapbacks.d.ts.map +1 -0
- package/dist/ingest/link-replies-and-tapbacks.js +381 -0
- package/dist/ingest/link-replies-and-tapbacks.js.map +1 -0
- package/dist/normalize/date-converters.d.ts +45 -0
- package/dist/normalize/date-converters.d.ts.map +1 -0
- package/dist/normalize/date-converters.js +166 -0
- package/dist/normalize/date-converters.js.map +1 -0
- package/dist/normalize/path-validator.d.ts +65 -0
- package/dist/normalize/path-validator.d.ts.map +1 -0
- package/dist/normalize/path-validator.js +221 -0
- package/dist/normalize/path-validator.js.map +1 -0
- package/dist/normalize/validate-normalized.d.ts +45 -0
- package/dist/normalize/validate-normalized.d.ts.map +1 -0
- package/dist/normalize/validate-normalized.js +144 -0
- package/dist/normalize/validate-normalized.js.map +1 -0
- package/dist/render/embeds-blockquotes.d.ts +84 -0
- package/dist/render/embeds-blockquotes.d.ts.map +1 -0
- package/dist/render/embeds-blockquotes.js +204 -0
- package/dist/render/embeds-blockquotes.js.map +1 -0
- package/dist/render/grouping.d.ts +78 -0
- package/dist/render/grouping.d.ts.map +1 -0
- package/dist/render/grouping.js +134 -0
- package/dist/render/grouping.js.map +1 -0
- package/dist/render/index.d.ts +47 -0
- package/dist/render/index.d.ts.map +1 -0
- package/dist/render/index.js +245 -0
- package/dist/render/index.js.map +1 -0
- package/dist/render/reply-rendering.d.ts +88 -0
- package/dist/render/reply-rendering.d.ts.map +1 -0
- package/dist/render/reply-rendering.js +196 -0
- package/dist/render/reply-rendering.js.map +1 -0
- package/dist/schema/message.d.ts +125 -0
- package/dist/schema/message.d.ts.map +1 -0
- package/dist/schema/message.js +331 -0
- package/dist/schema/message.js.map +1 -0
- package/dist/utils/delta-detection.d.ts +107 -0
- package/dist/utils/delta-detection.d.ts.map +1 -0
- package/dist/utils/delta-detection.js +199 -0
- package/dist/utils/delta-detection.js.map +1 -0
- package/dist/utils/enrichment-merge.d.ts +135 -0
- package/dist/utils/enrichment-merge.d.ts.map +1 -0
- package/dist/utils/enrichment-merge.js +280 -0
- package/dist/utils/enrichment-merge.js.map +1 -0
- package/dist/utils/human.d.ts +15 -0
- package/dist/utils/human.d.ts.map +1 -0
- package/dist/utils/human.js +27 -0
- package/dist/utils/human.js.map +1 -0
- package/dist/utils/incremental-state.d.ts +133 -0
- package/dist/utils/incremental-state.d.ts.map +1 -0
- package/dist/utils/incremental-state.js +237 -0
- package/dist/utils/incremental-state.js.map +1 -0
- package/dist/utils/logger.d.ts +40 -0
- package/dist/utils/logger.d.ts.map +1 -0
- package/dist/utils/logger.js +176 -0
- package/dist/utils/logger.js.map +1 -0
- package/package.json +165 -0
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Checkpoint and Resume Module (ENRICH--T06)
|
|
3
|
+
*
|
|
4
|
+
* Implements resumable enrichment with:
|
|
5
|
+
* - AC01: Checkpoint writes after N items (configurable, default 100)
|
|
6
|
+
* - AC02: Full checkpoint schema with stats and failed items
|
|
7
|
+
* - AC03: Atomic writes using temp file + rename pattern
|
|
8
|
+
* - AC04: Resume within ≤1 item of last checkpoint
|
|
9
|
+
* - AC05: Config consistency verification with hash comparison
|
|
10
|
+
*
|
|
11
|
+
* Architecture:
|
|
12
|
+
* - createCheckpoint: Create new checkpoint with schema
|
|
13
|
+
* - shouldWriteCheckpoint: Determine if checkpoint should be written
|
|
14
|
+
* - getResumeIndex: Calculate resume position from checkpoint
|
|
15
|
+
* - verifyConfigHash: Validate config hasn't changed
|
|
16
|
+
* - getCheckpointPath: Generate deterministic checkpoint file path
|
|
17
|
+
* - loadCheckpoint: Load checkpoint from disk
|
|
18
|
+
* - saveCheckpoint: Write checkpoint atomically
|
|
19
|
+
*/
|
|
20
|
+
import crypto from 'node:crypto';
|
|
21
|
+
import { access, readFile, writeFile } from 'node:fs/promises';
|
|
22
|
+
import path from 'node:path';
|
|
23
|
+
// ============================================================================
|
|
24
|
+
// AC01: Write checkpoint after N items
|
|
25
|
+
// ============================================================================
|
|
26
|
+
/**
|
|
27
|
+
* AC01: Determine if checkpoint should be written after N items
|
|
28
|
+
*
|
|
29
|
+
* @param itemIndex - Current item index (0-based)
|
|
30
|
+
* @param checkpointInterval - Checkpoint interval (default 100)
|
|
31
|
+
* @returns true if checkpoint should be written
|
|
32
|
+
*/
|
|
33
|
+
export function shouldWriteCheckpoint(itemIndex, checkpointInterval = 100) {
|
|
34
|
+
// Write checkpoint at multiples of interval (100, 200, 300, etc.)
|
|
35
|
+
return itemIndex > 0 && itemIndex % checkpointInterval === 0;
|
|
36
|
+
}
|
|
37
|
+
// ============================================================================
|
|
38
|
+
// AC02: Checkpoint structure with stats and failed items
|
|
39
|
+
// ============================================================================
|
|
40
|
+
/**
|
|
41
|
+
* AC02: Create checkpoint with full schema
|
|
42
|
+
*
|
|
43
|
+
* @param input - Checkpoint input data
|
|
44
|
+
* @returns EnrichCheckpoint with all required fields
|
|
45
|
+
*/
|
|
46
|
+
export function createCheckpoint(input) {
|
|
47
|
+
return {
|
|
48
|
+
version: '1.0',
|
|
49
|
+
configHash: input.configHash,
|
|
50
|
+
lastProcessedIndex: input.lastProcessedIndex,
|
|
51
|
+
totalProcessed: input.totalProcessed,
|
|
52
|
+
totalFailed: input.totalFailed,
|
|
53
|
+
stats: input.stats,
|
|
54
|
+
failedItems: input.failedItems,
|
|
55
|
+
createdAt: new Date().toISOString(),
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
// ============================================================================
|
|
59
|
+
// AC03: Atomic checkpoint writes
|
|
60
|
+
// ============================================================================
|
|
61
|
+
/**
|
|
62
|
+
* AC03: Generate deterministic checkpoint file path
|
|
63
|
+
*
|
|
64
|
+
* @param checkpointDir - Directory for checkpoints
|
|
65
|
+
* @param configHash - Config hash for uniqueness
|
|
66
|
+
* @returns Path to checkpoint file
|
|
67
|
+
*/
|
|
68
|
+
export function getCheckpointPath(checkpointDir, configHash) {
|
|
69
|
+
return path.join(checkpointDir, `checkpoint-${configHash}.json`);
|
|
70
|
+
}
|
|
71
|
+
/**
|
|
72
|
+
* AC03: Save checkpoint atomically using temp file + rename
|
|
73
|
+
*
|
|
74
|
+
* @param checkpoint - Checkpoint to save
|
|
75
|
+
* @param checkpointPath - Path to save checkpoint
|
|
76
|
+
*/
|
|
77
|
+
export async function saveCheckpoint(checkpoint, checkpointPath) {
|
|
78
|
+
const tempPath = `${checkpointPath}.tmp`;
|
|
79
|
+
try {
|
|
80
|
+
// Write to temp file
|
|
81
|
+
await writeFile(tempPath, JSON.stringify(checkpoint, null, 2), 'utf-8');
|
|
82
|
+
// Atomic rename (replaces original if exists)
|
|
83
|
+
// In Node.js, fs.rename is atomic on most filesystems
|
|
84
|
+
const fs = await import('node:fs/promises');
|
|
85
|
+
await fs.rename(tempPath, checkpointPath);
|
|
86
|
+
}
|
|
87
|
+
catch (error) {
|
|
88
|
+
// Clean up temp file on error
|
|
89
|
+
try {
|
|
90
|
+
const fs = await import('node:fs/promises');
|
|
91
|
+
await fs.unlink(tempPath);
|
|
92
|
+
}
|
|
93
|
+
catch {
|
|
94
|
+
// Ignore cleanup errors
|
|
95
|
+
}
|
|
96
|
+
throw error;
|
|
97
|
+
}
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* AC03: Load checkpoint from disk
|
|
101
|
+
*
|
|
102
|
+
* @param checkpointPath - Path to checkpoint file
|
|
103
|
+
* @returns Loaded checkpoint or null if not found
|
|
104
|
+
*/
|
|
105
|
+
export async function loadCheckpoint(checkpointPath) {
|
|
106
|
+
try {
|
|
107
|
+
await access(checkpointPath);
|
|
108
|
+
const content = await readFile(checkpointPath, 'utf-8');
|
|
109
|
+
return JSON.parse(content);
|
|
110
|
+
}
|
|
111
|
+
catch {
|
|
112
|
+
// File doesn't exist or is invalid
|
|
113
|
+
return null;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
// ============================================================================
|
|
117
|
+
// AC04: Resume within ≤1 item
|
|
118
|
+
// ============================================================================
|
|
119
|
+
/**
|
|
120
|
+
* AC04: Calculate resume index from checkpoint
|
|
121
|
+
*
|
|
122
|
+
* Resume at lastProcessedIndex + 1 to ensure we don't re-process
|
|
123
|
+
* the last item that was in the previous checkpoint.
|
|
124
|
+
*
|
|
125
|
+
* @param checkpoint - Checkpoint to resume from
|
|
126
|
+
* @returns Resume index (within ≤1 item of last checkpoint)
|
|
127
|
+
*/
|
|
128
|
+
export function getResumeIndex(checkpoint) {
|
|
129
|
+
// Resume at next item after last processed
|
|
130
|
+
return checkpoint.lastProcessedIndex + 1;
|
|
131
|
+
}
|
|
132
|
+
// ============================================================================
|
|
133
|
+
// AC05: Config consistency verification
|
|
134
|
+
// ============================================================================
|
|
135
|
+
/**
|
|
136
|
+
* AC05: Compute config hash for consistency checking
|
|
137
|
+
*
|
|
138
|
+
* @param config - Configuration object
|
|
139
|
+
* @returns SHA-256 hash of config
|
|
140
|
+
*/
|
|
141
|
+
export function computeConfigHash(config) {
|
|
142
|
+
const configStr = JSON.stringify(config);
|
|
143
|
+
return crypto.createHash('sha256').update(configStr).digest('hex');
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* AC05: Verify config hasn't changed by comparing hashes
|
|
147
|
+
*
|
|
148
|
+
* @param checkpointHash - Hash from checkpoint
|
|
149
|
+
* @param currentHash - Hash of current config
|
|
150
|
+
* @returns true if hashes match (config unchanged)
|
|
151
|
+
*/
|
|
152
|
+
export function verifyConfigHash(checkpointHash, currentHash) {
|
|
153
|
+
return checkpointHash === currentHash;
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Initialize checkpoint state for enrichment run
|
|
157
|
+
*
|
|
158
|
+
* @param checkpoint - Loaded checkpoint or null
|
|
159
|
+
* @param currentConfigHash - Hash of current config
|
|
160
|
+
* @returns Checkpoint state or error
|
|
161
|
+
*/
|
|
162
|
+
export function initializeCheckpointState(checkpoint, currentConfigHash) {
|
|
163
|
+
if (!checkpoint) {
|
|
164
|
+
// No checkpoint, starting fresh
|
|
165
|
+
return {
|
|
166
|
+
isResuming: false,
|
|
167
|
+
lastCheckpointIndex: -1,
|
|
168
|
+
configHash: currentConfigHash,
|
|
169
|
+
failedItemsInCheckpoint: [],
|
|
170
|
+
};
|
|
171
|
+
}
|
|
172
|
+
// Verify config matches (AC05)
|
|
173
|
+
if (!verifyConfigHash(checkpoint.configHash, currentConfigHash)) {
|
|
174
|
+
return new Error(`Config mismatch: checkpoint was created with config ${checkpoint.configHash.substring(0, 8)}, but current config is ${currentConfigHash.substring(0, 8)}. Cannot resume with different configuration.`);
|
|
175
|
+
}
|
|
176
|
+
// Initialize resume state
|
|
177
|
+
return {
|
|
178
|
+
isResuming: true,
|
|
179
|
+
lastCheckpointIndex: getResumeIndex(checkpoint) - 1,
|
|
180
|
+
configHash: checkpoint.configHash,
|
|
181
|
+
failedItemsInCheckpoint: checkpoint.failedItems,
|
|
182
|
+
};
|
|
183
|
+
}
|
|
184
|
+
/**
|
|
185
|
+
* Create new checkpoint for saving after processing batch
|
|
186
|
+
*
|
|
187
|
+
* @param lastProcessedIndex - Index of last processed item
|
|
188
|
+
* @param totalProcessed - Total items processed so far
|
|
189
|
+
* @param totalFailed - Total failed items so far
|
|
190
|
+
* @param batchStats - Stats for this batch
|
|
191
|
+
* @param failedItems - Failed items in this batch
|
|
192
|
+
* @param configHash - Hash of current config
|
|
193
|
+
* @returns Checkpoint ready to save
|
|
194
|
+
*/
|
|
195
|
+
export function prepareCheckpoint(lastProcessedIndex, totalProcessed, totalFailed, batchStats, failedItems, configHash) {
|
|
196
|
+
return createCheckpoint({
|
|
197
|
+
lastProcessedIndex,
|
|
198
|
+
totalProcessed,
|
|
199
|
+
totalFailed,
|
|
200
|
+
stats: batchStats,
|
|
201
|
+
failedItems,
|
|
202
|
+
configHash,
|
|
203
|
+
});
|
|
204
|
+
}
|
|
205
|
+
//# sourceMappingURL=checkpoint.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"checkpoint.js","sourceRoot":"","sources":["../../src/enrich/checkpoint.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;GAkBG;AAEH,OAAO,MAAM,MAAM,aAAa,CAAA;AAChC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAA;AAC9D,OAAO,IAAI,MAAM,WAAW,CAAA;AAuC5B,+EAA+E;AAC/E,uCAAuC;AACvC,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACpC,SAAiB,EACjB,kBAAkB,GAAG,GAAG;IAExB,kEAAkE;IAClE,OAAO,SAAS,GAAG,CAAC,IAAI,SAAS,GAAG,kBAAkB,KAAK,CAAC,CAAA;AAC7D,CAAC;AAED,+EAA+E;AAC/E,yDAAyD;AACzD,+EAA+E;AAE/E;;;;;GAKG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAsB;IACtD,OAAO;QACN,OAAO,EAAE,KAAK;QACd,UAAU,EAAE,KAAK,CAAC,UAAU;QAC5B,kBAAkB,EAAE,KAAK,CAAC,kBAAkB;QAC5C,cAAc,EAAE,KAAK,CAAC,cAAc;QACpC,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,KAAK,EAAE,KAAK,CAAC,KAAK;QAClB,WAAW,EAAE,KAAK,CAAC,WAAW;QAC9B,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACnC,CAAA;AACF,CAAC;AAED,+EAA+E;AAC/E,iCAAiC;AACjC,+EAA+E;AAE/E;;;;;;GAMG;AACH,MAAM,UAAU,iBAAiB,CAChC,aAAqB,EACrB,UAAkB;IAElB,OAAO,IAAI,CAAC,IAAI,CAAC,aAAa,EAAE,cAAc,UAAU,OAAO,CAAC,CAAA;AACjE,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,UAA4B,EAC5B,cAAsB;IAEtB,MAAM,QAAQ,GAAG,GAAG,cAAc,MAAM,CAAA;IAExC,IAAI,CAAC;QACJ,qBAAqB;QACrB,MAAM,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;QAEvE,8CAA8C;QAC9C,sDAAsD;QACtD,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;QAC3C,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,cAAc,CAAC,CAAA;IAC1C,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAChB,8BAA8B;QAC9B,IAAI,CAAC;YACJ,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAA;YAC3C,MAAM,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAA;QAC1B,CAAC;QAAC,MAAM,CAAC;YACR,wBAAwB;QACzB,CAAC;QACD,MAAM,KAAK,CAAA;IACZ,CAAC;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,cAAsB;IAEtB,IAAI,CAAC;QACJ,MAAM,MAAM,CAAC,cAAc,CAAC,CAAA;QAC5B,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,cAAc,EAAE,OAAO,CAAC,CAAA;QACvD,OAAO,IAAI,CAAC,KAAK,CAAC,OAAO,CAAqB,CAAA;IAC/C,CAAC;IAAC,MAAM,CAAC;QACR,mCAAmC;QACnC,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,MAAM,UAAU,cAAc,CAAC,UAA4B;IAC1D,2CAA2C;IAC3C,OAAO,UAAU,CAAC,kBAAkB,GAAG,CAAC,CAAA;AACzC,CAAC;AAED,+EAA+E;AAC/E,wCAAwC;AACxC,+EAA+E;AAE/E;;;;;GAKG;AACH,MAAM,UAAU,iBAAiB,CAAC,MAA+B;IAChE,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAA;IACxC,OAAO,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AACnE,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,gBAAgB,CAC/B,cAAsB,EACtB,WAAmB;IAEnB,OAAO,cAAc,KAAK,WAAW,CAAA;AACtC,CAAC;AAaD;;;;;;GAMG;AACH,MAAM,UAAU,yBAAyB,CACxC,UAAmC,EACnC,iBAAyB;IAEzB,IAAI,CAAC,UAAU,EAAE,CAAC;QACjB,gCAAgC;QAChC,OAAO;YACN,UAAU,EAAE,KAAK;YACjB,mBAAmB,EAAE,CAAC,CAAC;YACvB,UAAU,EAAE,iBAAiB;YAC7B,uBAAuB,EAAE,EAAE;SAC3B,CAAA;IACF,CAAC;IAED,+BAA+B;IAC/B,IAAI,CAAC,gBAAgB,CAAC,UAAU,CAAC,UAAU,EAAE,iBAAiB,CAAC,EAAE,CAAC;QACjE,OAAO,IAAI,KAAK,CACf,uDAAuD,UAAU,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,2BAA2B,iBAAiB,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC,CAAC,+CAA+C,CACvM,CAAA;IACF,CAAC;IAED,0BAA0B;IAC1B,OAAO;QACN,UAAU,EAAE,IAAI;QAChB,mBAAmB,EAAE,cAAc,CAAC,UAAU,CAAC,GAAG,CAAC;QACnD,UAAU,EAAE,UAAU,CAAC,UAAU;QACjC,uBAAuB,EAAE,UAAU,CAAC,WAAW;KAC/C,CAAA;AACF,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,UAAU,iBAAiB,CAChC,kBAA0B,EAC1B,cAAsB,EACtB,WAAmB,EACnB,UAA2B,EAC3B,WAAyB,EACzB,UAAkB;IAElB,OAAO,gBAAgB,CAAC;QACvB,kBAAkB;QAClB,cAAc;QACd,WAAW;QACX,KAAK,EAAE,UAAU;QACjB,WAAW;QACX,UAAU;KACV,CAAC,CAAA;AACH,CAAC"}
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Enrichment Idempotency Module (ENRICH--T05)
|
|
3
|
+
*
|
|
4
|
+
* Implements idempotent enrichment operations to prevent duplicate entries:
|
|
5
|
+
* - AC01: Skip enrichment if media.enrichment already contains entry with matching kind
|
|
6
|
+
* - AC02: Deduplicate enrichment array by kind before adding new entries
|
|
7
|
+
* - AC03: Re-running enrich-ai does not create duplicate entries (verified with tests)
|
|
8
|
+
* - AC04: Support --force-refresh flag to override idempotency and re-enrich
|
|
9
|
+
*
|
|
10
|
+
* Key Design:
|
|
11
|
+
* - Idempotency is keyed by enrichment.kind
|
|
12
|
+
* - Deduplication keeps the latest enrichment by createdAt timestamp
|
|
13
|
+
* - Force-refresh bypasses idempotency checks
|
|
14
|
+
*/
|
|
15
|
+
import type { MediaEnrichment, Message } from '#schema/message';
|
|
16
|
+
type IdempotencyOptions = {
|
|
17
|
+
forceRefresh?: boolean;
|
|
18
|
+
};
|
|
19
|
+
/**
|
|
20
|
+
* AC01: Check if enrichment with matching kind already exists
|
|
21
|
+
*
|
|
22
|
+
* @param message - Message to check
|
|
23
|
+
* @param kind - Enrichment kind to check for
|
|
24
|
+
* @returns true if enrichment with matching kind exists, false otherwise
|
|
25
|
+
*/
|
|
26
|
+
export declare function shouldSkipEnrichment(message: Message, kind: MediaEnrichment['kind']): boolean;
|
|
27
|
+
/**
|
|
28
|
+
* AC02: Deduplicate enrichment array by kind
|
|
29
|
+
*
|
|
30
|
+
* Removes duplicate enrichments with the same kind, keeping the most recent
|
|
31
|
+
* (determined by createdAt timestamp).
|
|
32
|
+
*
|
|
33
|
+
* @param enrichments - Array of enrichments to deduplicate
|
|
34
|
+
* @returns Deduplicated array with latest enrichment per kind
|
|
35
|
+
*/
|
|
36
|
+
export declare function deduplicateEnrichmentByKind(enrichments: MediaEnrichment[]): MediaEnrichment[];
|
|
37
|
+
/**
|
|
38
|
+
* AC03/AC04: Add enrichment idempotently
|
|
39
|
+
*
|
|
40
|
+
* Adds new enrichment to message while ensuring no duplicates are created.
|
|
41
|
+
* - If forceRefresh=false (default): skips if enrichment with same kind exists
|
|
42
|
+
* - If forceRefresh=true: replaces existing enrichment with same kind
|
|
43
|
+
*
|
|
44
|
+
* Also deduplicates the enrichment array before returning.
|
|
45
|
+
*
|
|
46
|
+
* @param message - Message to enrich
|
|
47
|
+
* @param enrichment - New enrichment to add
|
|
48
|
+
* @param options - Idempotency options (forceRefresh flag)
|
|
49
|
+
* @returns Updated message with enrichment added (or original if skipped)
|
|
50
|
+
*/
|
|
51
|
+
export declare function addEnrichmentIdempotent(message: Message, enrichment: MediaEnrichment, options?: IdempotencyOptions): Message;
|
|
52
|
+
/**
|
|
53
|
+
* Batch add enrichments idempotently
|
|
54
|
+
*
|
|
55
|
+
* Applies idempotent enrichment to multiple messages.
|
|
56
|
+
*
|
|
57
|
+
* @param messages - Messages to enrich
|
|
58
|
+
* @param enrichments - Map of message GUID to enrichment to add
|
|
59
|
+
* @param options - Idempotency options
|
|
60
|
+
* @returns Updated messages
|
|
61
|
+
*/
|
|
62
|
+
export declare function addEnrichmentsIdempotent(messages: Message[], enrichments: Map<string, MediaEnrichment>, options?: IdempotencyOptions): Message[];
|
|
63
|
+
/**
|
|
64
|
+
* Check if a message has all required enrichments
|
|
65
|
+
*
|
|
66
|
+
* Useful for checking if enrichment stage is complete.
|
|
67
|
+
*
|
|
68
|
+
* @param message - Message to check
|
|
69
|
+
* @param requiredKinds - Required enrichment kinds
|
|
70
|
+
* @returns true if message has all required enrichment kinds
|
|
71
|
+
*/
|
|
72
|
+
export declare function hasAllEnrichments(message: Message, requiredKinds: MediaEnrichment['kind'][]): boolean;
|
|
73
|
+
/**
|
|
74
|
+
* Get enrichment by kind
|
|
75
|
+
*
|
|
76
|
+
* @param message - Message to search
|
|
77
|
+
* @param kind - Enrichment kind to find
|
|
78
|
+
* @returns Enrichment if found, undefined otherwise
|
|
79
|
+
*/
|
|
80
|
+
export declare function getEnrichmentByKind(message: Message, kind: MediaEnrichment['kind']): MediaEnrichment | undefined;
|
|
81
|
+
/**
|
|
82
|
+
* Clear enrichments of a specific kind from a message
|
|
83
|
+
*
|
|
84
|
+
* @param message - Message to update
|
|
85
|
+
* @param kind - Enrichment kind to remove
|
|
86
|
+
* @returns Updated message without enrichment of specified kind
|
|
87
|
+
*/
|
|
88
|
+
export declare function clearEnrichmentByKind(message: Message, kind: MediaEnrichment['kind']): Message;
|
|
89
|
+
export {};
|
|
90
|
+
//# sourceMappingURL=idempotency.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"idempotency.d.ts","sourceRoot":"","sources":["../../src/enrich/idempotency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAE/D,KAAK,kBAAkB,GAAG;IACzB,YAAY,CAAC,EAAE,OAAO,CAAA;CACtB,CAAA;AAED;;;;;;GAMG;AACH,wBAAgB,oBAAoB,CACnC,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,GAC3B,OAAO,CAMT;AAED;;;;;;;;GAQG;AACH,wBAAgB,2BAA2B,CAC1C,WAAW,EAAE,eAAe,EAAE,GAC5B,eAAe,EAAE,CAsBnB;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,uBAAuB,CACtC,OAAO,EAAE,OAAO,EAChB,UAAU,EAAE,eAAe,EAC3B,OAAO,GAAE,kBAAuB,GAC9B,OAAO,CA8CT;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CACvC,QAAQ,EAAE,OAAO,EAAE,EACnB,WAAW,EAAE,GAAG,CAAC,MAAM,EAAE,eAAe,CAAC,EACzC,OAAO,GAAE,kBAAuB,GAC9B,OAAO,EAAE,CAQX;AAED;;;;;;;;GAQG;AACH,wBAAgB,iBAAiB,CAChC,OAAO,EAAE,OAAO,EAChB,aAAa,EAAE,eAAe,CAAC,MAAM,CAAC,EAAE,GACtC,OAAO,CAOT;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAClC,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,GAC3B,eAAe,GAAG,SAAS,CAM7B;AAED;;;;;;GAMG;AACH,wBAAgB,qBAAqB,CACpC,OAAO,EAAE,OAAO,EAChB,IAAI,EAAE,eAAe,CAAC,MAAM,CAAC,GAC3B,OAAO,CAwBT"}
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Enrichment Idempotency Module (ENRICH--T05)
|
|
3
|
+
*
|
|
4
|
+
* Implements idempotent enrichment operations to prevent duplicate entries:
|
|
5
|
+
* - AC01: Skip enrichment if media.enrichment already contains entry with matching kind
|
|
6
|
+
* - AC02: Deduplicate enrichment array by kind before adding new entries
|
|
7
|
+
* - AC03: Re-running enrich-ai does not create duplicate entries (verified with tests)
|
|
8
|
+
* - AC04: Support --force-refresh flag to override idempotency and re-enrich
|
|
9
|
+
*
|
|
10
|
+
* Key Design:
|
|
11
|
+
* - Idempotency is keyed by enrichment.kind
|
|
12
|
+
* - Deduplication keeps the latest enrichment by createdAt timestamp
|
|
13
|
+
* - Force-refresh bypasses idempotency checks
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* AC01: Check if enrichment with matching kind already exists
|
|
17
|
+
*
|
|
18
|
+
* @param message - Message to check
|
|
19
|
+
* @param kind - Enrichment kind to check for
|
|
20
|
+
* @returns true if enrichment with matching kind exists, false otherwise
|
|
21
|
+
*/
|
|
22
|
+
export function shouldSkipEnrichment(message, kind) {
|
|
23
|
+
if (!message.media?.enrichment) {
|
|
24
|
+
return false;
|
|
25
|
+
}
|
|
26
|
+
return message.media.enrichment.some((enrichment) => enrichment.kind === kind);
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* AC02: Deduplicate enrichment array by kind
|
|
30
|
+
*
|
|
31
|
+
* Removes duplicate enrichments with the same kind, keeping the most recent
|
|
32
|
+
* (determined by createdAt timestamp).
|
|
33
|
+
*
|
|
34
|
+
* @param enrichments - Array of enrichments to deduplicate
|
|
35
|
+
* @returns Deduplicated array with latest enrichment per kind
|
|
36
|
+
*/
|
|
37
|
+
export function deduplicateEnrichmentByKind(enrichments) {
|
|
38
|
+
const kindMap = new Map();
|
|
39
|
+
for (const enrichment of enrichments) {
|
|
40
|
+
const existing = kindMap.get(enrichment.kind);
|
|
41
|
+
if (!existing) {
|
|
42
|
+
// First enrichment of this kind
|
|
43
|
+
kindMap.set(enrichment.kind, enrichment);
|
|
44
|
+
}
|
|
45
|
+
else {
|
|
46
|
+
// Compare timestamps - keep the most recent
|
|
47
|
+
const existingTime = new Date(existing.createdAt).getTime();
|
|
48
|
+
const newTime = new Date(enrichment.createdAt).getTime();
|
|
49
|
+
if (newTime > existingTime) {
|
|
50
|
+
kindMap.set(enrichment.kind, enrichment);
|
|
51
|
+
}
|
|
52
|
+
// If existing is more recent, keep it (no update)
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
return Array.from(kindMap.values());
|
|
56
|
+
}
|
|
57
|
+
/**
|
|
58
|
+
* AC03/AC04: Add enrichment idempotently
|
|
59
|
+
*
|
|
60
|
+
* Adds new enrichment to message while ensuring no duplicates are created.
|
|
61
|
+
* - If forceRefresh=false (default): skips if enrichment with same kind exists
|
|
62
|
+
* - If forceRefresh=true: replaces existing enrichment with same kind
|
|
63
|
+
*
|
|
64
|
+
* Also deduplicates the enrichment array before returning.
|
|
65
|
+
*
|
|
66
|
+
* @param message - Message to enrich
|
|
67
|
+
* @param enrichment - New enrichment to add
|
|
68
|
+
* @param options - Idempotency options (forceRefresh flag)
|
|
69
|
+
* @returns Updated message with enrichment added (or original if skipped)
|
|
70
|
+
*/
|
|
71
|
+
export function addEnrichmentIdempotent(message, enrichment, options = {}) {
|
|
72
|
+
const { forceRefresh = false } = options;
|
|
73
|
+
// Early return if not a media message or media is null
|
|
74
|
+
if (message.messageKind !== 'media' || !message.media) {
|
|
75
|
+
return message;
|
|
76
|
+
}
|
|
77
|
+
// Initialize enrichment array if missing
|
|
78
|
+
const currentEnrichments = message.media.enrichment || [];
|
|
79
|
+
// Check if enrichment with same kind already exists
|
|
80
|
+
const existingIndex = currentEnrichments.findIndex((e) => e.kind === enrichment.kind);
|
|
81
|
+
let updatedEnrichments;
|
|
82
|
+
if (existingIndex >= 0) {
|
|
83
|
+
// Enrichment with same kind already exists
|
|
84
|
+
if (forceRefresh) {
|
|
85
|
+
// AC04: Replace existing enrichment (force-refresh mode)
|
|
86
|
+
updatedEnrichments = [
|
|
87
|
+
...currentEnrichments.slice(0, existingIndex),
|
|
88
|
+
enrichment,
|
|
89
|
+
...currentEnrichments.slice(existingIndex + 1),
|
|
90
|
+
];
|
|
91
|
+
}
|
|
92
|
+
else {
|
|
93
|
+
// AC01: Skip adding (default idempotent behavior)
|
|
94
|
+
updatedEnrichments = currentEnrichments;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
else {
|
|
98
|
+
// New enrichment kind, add it
|
|
99
|
+
updatedEnrichments = [...currentEnrichments, enrichment];
|
|
100
|
+
}
|
|
101
|
+
// AC02: Deduplicate before returning
|
|
102
|
+
const deduped = deduplicateEnrichmentByKind(updatedEnrichments);
|
|
103
|
+
return {
|
|
104
|
+
...message,
|
|
105
|
+
media: {
|
|
106
|
+
...message.media,
|
|
107
|
+
enrichment: deduped,
|
|
108
|
+
},
|
|
109
|
+
};
|
|
110
|
+
}
|
|
111
|
+
/**
|
|
112
|
+
* Batch add enrichments idempotently
|
|
113
|
+
*
|
|
114
|
+
* Applies idempotent enrichment to multiple messages.
|
|
115
|
+
*
|
|
116
|
+
* @param messages - Messages to enrich
|
|
117
|
+
* @param enrichments - Map of message GUID to enrichment to add
|
|
118
|
+
* @param options - Idempotency options
|
|
119
|
+
* @returns Updated messages
|
|
120
|
+
*/
|
|
121
|
+
export function addEnrichmentsIdempotent(messages, enrichments, options = {}) {
|
|
122
|
+
return messages.map((message) => {
|
|
123
|
+
const enrichment = enrichments.get(message.guid);
|
|
124
|
+
if (!enrichment) {
|
|
125
|
+
return message;
|
|
126
|
+
}
|
|
127
|
+
return addEnrichmentIdempotent(message, enrichment, options);
|
|
128
|
+
});
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Check if a message has all required enrichments
|
|
132
|
+
*
|
|
133
|
+
* Useful for checking if enrichment stage is complete.
|
|
134
|
+
*
|
|
135
|
+
* @param message - Message to check
|
|
136
|
+
* @param requiredKinds - Required enrichment kinds
|
|
137
|
+
* @returns true if message has all required enrichment kinds
|
|
138
|
+
*/
|
|
139
|
+
export function hasAllEnrichments(message, requiredKinds) {
|
|
140
|
+
if (!message.media?.enrichment) {
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
const enrichedKinds = new Set(message.media.enrichment.map((e) => e.kind));
|
|
144
|
+
return requiredKinds.every((kind) => enrichedKinds.has(kind));
|
|
145
|
+
}
|
|
146
|
+
/**
|
|
147
|
+
* Get enrichment by kind
|
|
148
|
+
*
|
|
149
|
+
* @param message - Message to search
|
|
150
|
+
* @param kind - Enrichment kind to find
|
|
151
|
+
* @returns Enrichment if found, undefined otherwise
|
|
152
|
+
*/
|
|
153
|
+
export function getEnrichmentByKind(message, kind) {
|
|
154
|
+
if (!message.media?.enrichment) {
|
|
155
|
+
return undefined;
|
|
156
|
+
}
|
|
157
|
+
return message.media.enrichment.find((e) => e.kind === kind);
|
|
158
|
+
}
|
|
159
|
+
/**
|
|
160
|
+
* Clear enrichments of a specific kind from a message
|
|
161
|
+
*
|
|
162
|
+
* @param message - Message to update
|
|
163
|
+
* @param kind - Enrichment kind to remove
|
|
164
|
+
* @returns Updated message without enrichment of specified kind
|
|
165
|
+
*/
|
|
166
|
+
export function clearEnrichmentByKind(message, kind) {
|
|
167
|
+
if (!message.media?.enrichment) {
|
|
168
|
+
return message;
|
|
169
|
+
}
|
|
170
|
+
const filtered = message.media.enrichment.filter((e) => e.kind !== kind);
|
|
171
|
+
// If no enrichments remain, omit the enrichment field entirely
|
|
172
|
+
if (filtered.length === 0) {
|
|
173
|
+
const { enrichment: _enrichment, ...mediaWithoutEnrichment } = message.media;
|
|
174
|
+
return {
|
|
175
|
+
...message,
|
|
176
|
+
media: mediaWithoutEnrichment,
|
|
177
|
+
};
|
|
178
|
+
}
|
|
179
|
+
// If enrichments remain, update with filtered array
|
|
180
|
+
return {
|
|
181
|
+
...message,
|
|
182
|
+
media: {
|
|
183
|
+
...message.media,
|
|
184
|
+
enrichment: filtered,
|
|
185
|
+
},
|
|
186
|
+
};
|
|
187
|
+
}
|
|
188
|
+
//# sourceMappingURL=idempotency.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"idempotency.js","sourceRoot":"","sources":["../../src/enrich/idempotency.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAQH;;;;;;GAMG;AACH,MAAM,UAAU,oBAAoB,CACnC,OAAgB,EAChB,IAA6B;IAE7B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAA;IACb,CAAC;IAED,OAAO,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC/E,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,2BAA2B,CAC1C,WAA8B;IAE9B,MAAM,OAAO,GAAG,IAAI,GAAG,EAA4C,CAAA;IAEnE,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACtC,MAAM,QAAQ,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,CAAC,CAAA;QAE7C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,gCAAgC;YAChC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;QACzC,CAAC;aAAM,CAAC;YACP,4CAA4C;YAC5C,MAAM,YAAY,GAAG,IAAI,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAA;YAC3D,MAAM,OAAO,GAAG,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,CAAC,OAAO,EAAE,CAAA;YAExD,IAAI,OAAO,GAAG,YAAY,EAAE,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,UAAU,CAAC,CAAA;YACzC,CAAC;YACD,kDAAkD;QACnD,CAAC;IACF,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAA;AACpC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,uBAAuB,CACtC,OAAgB,EAChB,UAA2B,EAC3B,UAA8B,EAAE;IAEhC,MAAM,EAAE,YAAY,GAAG,KAAK,EAAE,GAAG,OAAO,CAAA;IAExC,uDAAuD;IACvD,IAAI,OAAO,CAAC,WAAW,KAAK,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;QACvD,OAAO,OAAO,CAAA;IACf,CAAC;IAED,yCAAyC;IACzC,MAAM,kBAAkB,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,IAAI,EAAE,CAAA;IAEzD,oDAAoD;IACpD,MAAM,aAAa,GAAG,kBAAkB,CAAC,SAAS,CACjD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,UAAU,CAAC,IAAI,CACjC,CAAA;IAED,IAAI,kBAAqC,CAAA;IAEzC,IAAI,aAAa,IAAI,CAAC,EAAE,CAAC;QACxB,2CAA2C;QAC3C,IAAI,YAAY,EAAE,CAAC;YAClB,yDAAyD;YACzD,kBAAkB,GAAG;gBACpB,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC,EAAE,aAAa,CAAC;gBAC7C,UAAU;gBACV,GAAG,kBAAkB,CAAC,KAAK,CAAC,aAAa,GAAG,CAAC,CAAC;aAC9C,CAAA;QACF,CAAC;aAAM,CAAC;YACP,kDAAkD;YAClD,kBAAkB,GAAG,kBAAkB,CAAA;QACxC,CAAC;IACF,CAAC;SAAM,CAAC;QACP,8BAA8B;QAC9B,kBAAkB,GAAG,CAAC,GAAG,kBAAkB,EAAE,UAAU,CAAC,CAAA;IACzD,CAAC;IAED,qCAAqC;IACrC,MAAM,OAAO,GAAG,2BAA2B,CAAC,kBAAkB,CAAC,CAAA;IAE/D,OAAO;QACN,GAAG,OAAO;QACV,KAAK,EAAE;YACN,GAAG,OAAO,CAAC,KAAK;YAChB,UAAU,EAAE,OAAO;SACnB;KACD,CAAA;AACF,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,wBAAwB,CACvC,QAAmB,EACnB,WAAyC,EACzC,UAA8B,EAAE;IAEhC,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE;QAC/B,MAAM,UAAU,GAAG,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAA;QAChD,IAAI,CAAC,UAAU,EAAE,CAAC;YACjB,OAAO,OAAO,CAAA;QACf,CAAC;QACD,OAAO,uBAAuB,CAAC,OAAO,EAAE,UAAU,EAAE,OAAO,CAAC,CAAA;IAC7D,CAAC,CAAC,CAAA;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,iBAAiB,CAChC,OAAgB,EAChB,aAAwC;IAExC,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC;QAChC,OAAO,KAAK,CAAA;IACb,CAAC;IAED,MAAM,aAAa,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAA;IAC1E,OAAO,aAAa,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAA;AAC9D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAClC,OAAgB,EAChB,IAA6B;IAE7B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC;QAChC,OAAO,SAAS,CAAA;IACjB,CAAC;IAED,OAAO,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;AAC7D,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,qBAAqB,CACpC,OAAgB,EAChB,IAA6B;IAE7B,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,UAAU,EAAE,CAAC;QAChC,OAAO,OAAO,CAAA;IACf,CAAC;IAED,MAAM,QAAQ,GAAG,OAAO,CAAC,KAAK,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAA;IAExE,+DAA+D;IAC/D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3B,MAAM,EAAE,UAAU,EAAE,WAAW,EAAE,GAAG,sBAAsB,EAAE,GAAG,OAAO,CAAC,KAAK,CAAA;QAC5E,OAAO;YACN,GAAG,OAAO;YACV,KAAK,EAAE,sBAAsB;SAC7B,CAAA;IACF,CAAC;IAED,oDAAoD;IACpD,OAAO;QACN,GAAG,OAAO;QACV,KAAK,EAAE;YACN,GAAG,OAAO,CAAC,KAAK;YAChB,UAAU,EAAE,QAAQ;SACpB;KACD,CAAA;AACF,CAAC"}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Image Analysis Module (ENRICH--T01)
|
|
3
|
+
*
|
|
4
|
+
* Implements image analysis with preview generation:
|
|
5
|
+
* - AC01: HEIC → JPG conversion with ≥90% quality
|
|
6
|
+
* - AC02: TIFF → JPG conversion
|
|
7
|
+
* - AC03: Preview caching by filename (generate once, skip if exists)
|
|
8
|
+
* - AC04: Gemini Vision API with structured prompt
|
|
9
|
+
* - AC05: Parse response into enrichment array with kind='image_analysis'
|
|
10
|
+
* - AC06: Store provenance (provider, model, version, timestamp)
|
|
11
|
+
*
|
|
12
|
+
* Architecture:
|
|
13
|
+
* - convertToJpgPreview: Handles format conversion with caching
|
|
14
|
+
* - analyzeImageWithGemini: Calls Gemini Vision API with structured prompt
|
|
15
|
+
* - analyzeImage: Main entry point, handles single message enrichment
|
|
16
|
+
* - analyzeImages: Batch processing wrapper
|
|
17
|
+
*
|
|
18
|
+
* Error Handling:
|
|
19
|
+
* - Non-fatal errors are logged and original message is returned
|
|
20
|
+
* - Preview generation failures don't block Gemini analysis
|
|
21
|
+
* - Pipeline never crashes on enrichment errors
|
|
22
|
+
*/
|
|
23
|
+
import type { MediaEnrichment, Message } from '#schema/message';
|
|
24
|
+
type ImageAnalysisConfig = {
|
|
25
|
+
enableVisionAnalysis: boolean;
|
|
26
|
+
geminiApiKey: string;
|
|
27
|
+
geminiModel?: string;
|
|
28
|
+
imageCacheDir: string;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* AC03: Convert image to JPG preview
|
|
32
|
+
* - Input: path to HEIC, TIFF, or other format
|
|
33
|
+
* - Output: path to cached JPG preview
|
|
34
|
+
* - Behavior: Generate once, cache by filename, skip if exists
|
|
35
|
+
*/
|
|
36
|
+
export declare function convertToJpgPreview(inputPath: string, cacheDir: string, quality?: number): Promise<string>;
|
|
37
|
+
/**
|
|
38
|
+
* AC04: Call Gemini Vision with structured prompt
|
|
39
|
+
* AC05: Parse response into enrichment array
|
|
40
|
+
* AC06: Store provenance
|
|
41
|
+
*/
|
|
42
|
+
export declare function analyzeImageWithGemini(imagePath: string, config: Partial<ImageAnalysisConfig>): Promise<MediaEnrichment>;
|
|
43
|
+
/**
|
|
44
|
+
* Main entry point - analyze image media message and enrich it
|
|
45
|
+
* Handles all ACs (AC01-AC06) through helper functions
|
|
46
|
+
*
|
|
47
|
+
* Responsibilities:
|
|
48
|
+
* 1. Check if media is image type (skip non-images)
|
|
49
|
+
* 2. Convert HEIC/TIFF to JPG preview (AC01-AC03)
|
|
50
|
+
* 3. Call Gemini Vision API (AC04)
|
|
51
|
+
* 4. Parse response (AC05)
|
|
52
|
+
* 5. Add enrichment with provenance (AC06)
|
|
53
|
+
*/
|
|
54
|
+
export declare function analyzeImage(message: Message, config: Partial<ImageAnalysisConfig>): Promise<Message>;
|
|
55
|
+
/**
|
|
56
|
+
* Batch analyze multiple messages
|
|
57
|
+
* Useful for enrichment stage that processes arrays of messages
|
|
58
|
+
* Each message is processed independently; errors don't stop the batch
|
|
59
|
+
*/
|
|
60
|
+
export declare function analyzeImages(messages: Message[], config: Partial<ImageAnalysisConfig>): Promise<Message[]>;
|
|
61
|
+
export {};
|
|
62
|
+
//# sourceMappingURL=image-analysis.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"image-analysis.d.ts","sourceRoot":"","sources":["../../src/enrich/image-analysis.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;GAqBG;AAQH,OAAO,KAAK,EAAE,eAAe,EAAa,OAAO,EAAE,MAAM,iBAAiB,CAAA;AAI1E,KAAK,mBAAmB,GAAG;IAC1B,oBAAoB,EAAE,OAAO,CAAA;IAC7B,YAAY,EAAE,MAAM,CAAA;IACpB,WAAW,CAAC,EAAE,MAAM,CAAA;IACpB,aAAa,EAAE,MAAM,CAAA;CACrB,CAAA;AAwBD;;;;;GAKG;AACH,wBAAsB,mBAAmB,CACxC,SAAS,EAAE,MAAM,EACjB,QAAQ,EAAE,MAAM,EAChB,OAAO,SAAK,GACV,OAAO,CAAC,MAAM,CAAC,CAmCjB;AAED;;;;GAIG;AACH,wBAAsB,sBAAsB,CAC3C,SAAS,EAAE,MAAM,EACjB,MAAM,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAClC,OAAO,CAAC,eAAe,CAAC,CAwE1B;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,YAAY,CACjC,OAAO,EAAE,OAAO,EAChB,MAAM,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAClC,OAAO,CAAC,OAAO,CAAC,CA6ElB;AAED;;;;GAIG;AACH,wBAAsB,aAAa,CAClC,QAAQ,EAAE,OAAO,EAAE,EACnB,MAAM,EAAE,OAAO,CAAC,mBAAmB,CAAC,GAClC,OAAO,CAAC,OAAO,EAAE,CAAC,CAsCpB"}
|