@nathanvale/chatline 0.0.1 → 0.0.2-next.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/dist/bin/index.js +1 -1
- package/dist/index.js +1 -1
- package/package.json +1 -1
- package/dist/cli/commands/clean.d.ts +0 -17
- package/dist/cli/commands/clean.d.ts.map +0 -1
- package/dist/cli/commands/clean.js +0 -142
- package/dist/cli/commands/clean.js.map +0 -1
- package/dist/cli/commands/doctor.d.ts +0 -17
- package/dist/cli/commands/doctor.d.ts.map +0 -1
- package/dist/cli/commands/doctor.js +0 -202
- package/dist/cli/commands/doctor.js.map +0 -1
- package/dist/cli/commands/enrich-ai.d.ts +0 -17
- package/dist/cli/commands/enrich-ai.d.ts.map +0 -1
- package/dist/cli/commands/enrich-ai.js +0 -371
- package/dist/cli/commands/enrich-ai.js.map +0 -1
- package/dist/cli/commands/index.d.ts +0 -16
- package/dist/cli/commands/index.d.ts.map +0 -1
- package/dist/cli/commands/index.js +0 -16
- package/dist/cli/commands/index.js.map +0 -1
- package/dist/cli/commands/ingest-csv.d.ts +0 -17
- package/dist/cli/commands/ingest-csv.d.ts.map +0 -1
- package/dist/cli/commands/ingest-csv.js +0 -138
- package/dist/cli/commands/ingest-csv.js.map +0 -1
- package/dist/cli/commands/ingest-db.d.ts +0 -17
- package/dist/cli/commands/ingest-db.d.ts.map +0 -1
- package/dist/cli/commands/ingest-db.js +0 -159
- package/dist/cli/commands/ingest-db.js.map +0 -1
- package/dist/cli/commands/init.d.ts +0 -17
- package/dist/cli/commands/init.d.ts.map +0 -1
- package/dist/cli/commands/init.js +0 -110
- package/dist/cli/commands/init.js.map +0 -1
- package/dist/cli/commands/normalize-link.d.ts +0 -16
- package/dist/cli/commands/normalize-link.d.ts.map +0 -1
- package/dist/cli/commands/normalize-link.js +0 -144
- package/dist/cli/commands/normalize-link.js.map +0 -1
- package/dist/cli/commands/render-markdown.d.ts +0 -17
- package/dist/cli/commands/render-markdown.d.ts.map +0 -1
- package/dist/cli/commands/render-markdown.js +0 -218
- package/dist/cli/commands/render-markdown.js.map +0 -1
- package/dist/cli/commands/stats.d.ts +0 -17
- package/dist/cli/commands/stats.d.ts.map +0 -1
- package/dist/cli/commands/stats.js +0 -175
- package/dist/cli/commands/stats.js.map +0 -1
- package/dist/cli/commands/validate.d.ts +0 -17
- package/dist/cli/commands/validate.d.ts.map +0 -1
- package/dist/cli/commands/validate.js +0 -152
- package/dist/cli/commands/validate.js.map +0 -1
- package/dist/cli/index.d.ts +0 -13
- package/dist/cli/index.d.ts.map +0 -1
- package/dist/cli/index.js +0 -121
- package/dist/cli/index.js.map +0 -1
- package/dist/cli/types.d.ts +0 -93
- package/dist/cli/types.d.ts.map +0 -1
- package/dist/cli/types.js +0 -7
- package/dist/cli/types.js.map +0 -1
- package/dist/cli/utils.d.ts +0 -29
- package/dist/cli/utils.d.ts.map +0 -1
- package/dist/cli/utils.js +0 -53
- package/dist/cli/utils.js.map +0 -1
- package/dist/cli.d.ts +0 -9
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js +0 -1805
- package/dist/config/generator.d.ts +0 -90
- package/dist/config/generator.d.ts.map +0 -1
- package/dist/config/generator.js +0 -320
- package/dist/config/generator.js.map +0 -1
- package/dist/config/loader.d.ts +0 -107
- package/dist/config/loader.d.ts.map +0 -1
- package/dist/config/loader.js +0 -251
- package/dist/config/loader.js.map +0 -1
- package/dist/config/schema.d.ts +0 -107
- package/dist/config/schema.d.ts.map +0 -1
- package/dist/config/schema.js +0 -169
- package/dist/config/schema.js.map +0 -1
- package/dist/enrich/audio-transcription.d.ts +0 -77
- package/dist/enrich/audio-transcription.d.ts.map +0 -1
- package/dist/enrich/audio-transcription.js +0 -370
- package/dist/enrich/audio-transcription.js.map +0 -1
- package/dist/enrich/checkpoint.d.ts +0 -137
- package/dist/enrich/checkpoint.d.ts.map +0 -1
- package/dist/enrich/checkpoint.js +0 -205
- package/dist/enrich/checkpoint.js.map +0 -1
- package/dist/enrich/idempotency.d.ts +0 -90
- package/dist/enrich/idempotency.d.ts.map +0 -1
- package/dist/enrich/idempotency.js +0 -188
- package/dist/enrich/idempotency.js.map +0 -1
- package/dist/enrich/image-analysis.d.ts +0 -62
- package/dist/enrich/image-analysis.d.ts.map +0 -1
- package/dist/enrich/image-analysis.js +0 -264
- package/dist/enrich/image-analysis.js.map +0 -1
- package/dist/enrich/index.d.ts +0 -60
- package/dist/enrich/index.d.ts.map +0 -1
- package/dist/enrich/index.js +0 -74
- package/dist/enrich/index.js.map +0 -1
- package/dist/enrich/link-enrichment.d.ts +0 -37
- package/dist/enrich/link-enrichment.d.ts.map +0 -1
- package/dist/enrich/link-enrichment.js +0 -202
- package/dist/enrich/link-enrichment.js.map +0 -1
- package/dist/enrich/pdf-video-handling.d.ts +0 -49
- package/dist/enrich/pdf-video-handling.d.ts.map +0 -1
- package/dist/enrich/pdf-video-handling.js +0 -325
- package/dist/enrich/pdf-video-handling.js.map +0 -1
- package/dist/enrich/progress-tracker.d.ts +0 -120
- package/dist/enrich/progress-tracker.d.ts.map +0 -1
- package/dist/enrich/progress-tracker.js +0 -220
- package/dist/enrich/progress-tracker.js.map +0 -1
- package/dist/enrich/providers/firecrawl.d.ts +0 -18
- package/dist/enrich/providers/firecrawl.d.ts.map +0 -1
- package/dist/enrich/providers/firecrawl.js +0 -48
- package/dist/enrich/providers/firecrawl.js.map +0 -1
- package/dist/enrich/providers/generic.d.ts +0 -16
- package/dist/enrich/providers/generic.d.ts.map +0 -1
- package/dist/enrich/providers/generic.js +0 -36
- package/dist/enrich/providers/generic.js.map +0 -1
- package/dist/enrich/providers/index.d.ts +0 -14
- package/dist/enrich/providers/index.d.ts.map +0 -1
- package/dist/enrich/providers/index.js +0 -13
- package/dist/enrich/providers/index.js.map +0 -1
- package/dist/enrich/providers/instagram.d.ts +0 -16
- package/dist/enrich/providers/instagram.d.ts.map +0 -1
- package/dist/enrich/providers/instagram.js +0 -43
- package/dist/enrich/providers/instagram.js.map +0 -1
- package/dist/enrich/providers/spotify.d.ts +0 -16
- package/dist/enrich/providers/spotify.d.ts.map +0 -1
- package/dist/enrich/providers/spotify.js +0 -45
- package/dist/enrich/providers/spotify.js.map +0 -1
- package/dist/enrich/providers/twitter.d.ts +0 -16
- package/dist/enrich/providers/twitter.d.ts.map +0 -1
- package/dist/enrich/providers/twitter.js +0 -43
- package/dist/enrich/providers/twitter.js.map +0 -1
- package/dist/enrich/providers/types.d.ts +0 -47
- package/dist/enrich/providers/types.d.ts.map +0 -1
- package/dist/enrich/providers/types.js +0 -15
- package/dist/enrich/providers/types.js.map +0 -1
- package/dist/enrich/providers/youtube.d.ts +0 -16
- package/dist/enrich/providers/youtube.d.ts.map +0 -1
- package/dist/enrich/providers/youtube.js +0 -43
- package/dist/enrich/providers/youtube.js.map +0 -1
- package/dist/enrich/rate-limiting.d.ts +0 -118
- package/dist/enrich/rate-limiting.d.ts.map +0 -1
- package/dist/enrich/rate-limiting.js +0 -258
- package/dist/enrich/rate-limiting.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/ingest/dedup-merge.d.ts +0 -82
- package/dist/ingest/dedup-merge.d.ts.map +0 -1
- package/dist/ingest/dedup-merge.js +0 -262
- package/dist/ingest/dedup-merge.js.map +0 -1
- package/dist/ingest/ingest-csv.d.ts +0 -62
- package/dist/ingest/ingest-csv.d.ts.map +0 -1
- package/dist/ingest/ingest-csv.js +0 -300
- package/dist/ingest/ingest-csv.js.map +0 -1
- package/dist/ingest/ingest-db.d.ts +0 -64
- package/dist/ingest/ingest-db.d.ts.map +0 -1
- package/dist/ingest/ingest-db.js +0 -172
- package/dist/ingest/ingest-db.js.map +0 -1
- package/dist/ingest/link-replies-and-tapbacks.d.ts +0 -53
- package/dist/ingest/link-replies-and-tapbacks.d.ts.map +0 -1
- package/dist/ingest/link-replies-and-tapbacks.js +0 -381
- package/dist/ingest/link-replies-and-tapbacks.js.map +0 -1
- package/dist/normalize/date-converters.d.ts +0 -45
- package/dist/normalize/date-converters.d.ts.map +0 -1
- package/dist/normalize/date-converters.js +0 -166
- package/dist/normalize/date-converters.js.map +0 -1
- package/dist/normalize/path-validator.d.ts +0 -65
- package/dist/normalize/path-validator.d.ts.map +0 -1
- package/dist/normalize/path-validator.js +0 -221
- package/dist/normalize/path-validator.js.map +0 -1
- package/dist/normalize/validate-normalized.d.ts +0 -45
- package/dist/normalize/validate-normalized.d.ts.map +0 -1
- package/dist/normalize/validate-normalized.js +0 -144
- package/dist/normalize/validate-normalized.js.map +0 -1
- package/dist/render/embeds-blockquotes.d.ts +0 -84
- package/dist/render/embeds-blockquotes.d.ts.map +0 -1
- package/dist/render/embeds-blockquotes.js +0 -204
- package/dist/render/embeds-blockquotes.js.map +0 -1
- package/dist/render/grouping.d.ts +0 -78
- package/dist/render/grouping.d.ts.map +0 -1
- package/dist/render/grouping.js +0 -134
- package/dist/render/grouping.js.map +0 -1
- package/dist/render/index.d.ts +0 -47
- package/dist/render/index.d.ts.map +0 -1
- package/dist/render/index.js +0 -245
- package/dist/render/index.js.map +0 -1
- package/dist/render/reply-rendering.d.ts +0 -88
- package/dist/render/reply-rendering.d.ts.map +0 -1
- package/dist/render/reply-rendering.js +0 -196
- package/dist/render/reply-rendering.js.map +0 -1
- package/dist/schema/message.d.ts +0 -125
- package/dist/schema/message.d.ts.map +0 -1
- package/dist/schema/message.js +0 -331
- package/dist/schema/message.js.map +0 -1
- package/dist/utils/delta-detection.d.ts +0 -107
- package/dist/utils/delta-detection.d.ts.map +0 -1
- package/dist/utils/delta-detection.js +0 -199
- package/dist/utils/delta-detection.js.map +0 -1
- package/dist/utils/enrichment-merge.d.ts +0 -135
- package/dist/utils/enrichment-merge.d.ts.map +0 -1
- package/dist/utils/enrichment-merge.js +0 -280
- package/dist/utils/enrichment-merge.js.map +0 -1
- package/dist/utils/human.d.ts +0 -15
- package/dist/utils/human.d.ts.map +0 -1
- package/dist/utils/human.js +0 -27
- package/dist/utils/human.js.map +0 -1
- package/dist/utils/incremental-state.d.ts +0 -133
- package/dist/utils/incremental-state.d.ts.map +0 -1
- package/dist/utils/incremental-state.js +0 -237
- package/dist/utils/incremental-state.js.map +0 -1
- package/dist/utils/logger.d.ts +0 -40
- package/dist/utils/logger.d.ts.map +0 -1
- package/dist/utils/logger.js +0 -176
- package/dist/utils/logger.js.map +0 -1
|
@@ -1,371 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Enrich AI Command
|
|
3
|
-
*
|
|
4
|
-
* Add AI-powered enrichment to media messages.
|
|
5
|
-
* CLI-T03-AC02, AC03, AC04, AC05: Enrich with checkpoints, incremental, rate limiting
|
|
6
|
-
*/
|
|
7
|
-
import { humanError, humanInfo, humanWarn } from '#utils/human';
|
|
8
|
-
import { applyLogLevel, cliLogger, logEvent } from '../utils.js';
|
|
9
|
-
/**
|
|
10
|
-
* Execute the enrich-ai command logic
|
|
11
|
-
*/
|
|
12
|
-
export async function executeEnrichAI(options, globalOptions) {
|
|
13
|
-
const { input, output, checkpointDir, resume, incremental, stateFile: userProvidedStateFile, resetState, forceRefresh: _forceRefresh, rateLimitMs, maxRetries, checkpointInterval, enableVision, enableAudio, enableLinks, } = options;
|
|
14
|
-
const { verbose, quiet } = globalOptions;
|
|
15
|
-
applyLogLevel(verbose, quiet);
|
|
16
|
-
logEvent('enrich-start', {
|
|
17
|
-
command: 'enrich',
|
|
18
|
-
phase: 'start',
|
|
19
|
-
options: {
|
|
20
|
-
input,
|
|
21
|
-
output,
|
|
22
|
-
checkpointDir,
|
|
23
|
-
resume,
|
|
24
|
-
incremental,
|
|
25
|
-
stateFile: userProvidedStateFile,
|
|
26
|
-
resetState,
|
|
27
|
-
rateLimitMs,
|
|
28
|
-
maxRetries,
|
|
29
|
-
checkpointInterval,
|
|
30
|
-
enableVision,
|
|
31
|
-
enableAudio,
|
|
32
|
-
enableLinks,
|
|
33
|
-
},
|
|
34
|
-
});
|
|
35
|
-
// Validate inputs
|
|
36
|
-
const fs = await import('node:fs');
|
|
37
|
-
if (!fs.existsSync(input)) {
|
|
38
|
-
humanError(`❌ Input file not found: ${input}`);
|
|
39
|
-
process.exit(1);
|
|
40
|
-
}
|
|
41
|
-
// Parse rate limit and retry options
|
|
42
|
-
const rateLimitDelay = Number.parseInt(rateLimitMs, 10);
|
|
43
|
-
const maxRetriesNum = Number.parseInt(maxRetries, 10);
|
|
44
|
-
const checkpointIntervalNum = Number.parseInt(checkpointInterval, 10);
|
|
45
|
-
if (Number.isNaN(rateLimitDelay) || rateLimitDelay < 0) {
|
|
46
|
-
humanError('❌ --rate-limit must be a non-negative number (milliseconds)');
|
|
47
|
-
process.exit(1);
|
|
48
|
-
}
|
|
49
|
-
if (Number.isNaN(maxRetriesNum) || maxRetriesNum < 0) {
|
|
50
|
-
humanError('❌ --max-retries must be a non-negative number');
|
|
51
|
-
process.exit(1);
|
|
52
|
-
}
|
|
53
|
-
if (Number.isNaN(checkpointIntervalNum) || checkpointIntervalNum < 1) {
|
|
54
|
-
humanError('❌ --checkpoint-interval must be a positive number');
|
|
55
|
-
process.exit(1);
|
|
56
|
-
}
|
|
57
|
-
if (verbose) {
|
|
58
|
-
cliLogger.info('Enrich config', {
|
|
59
|
-
input,
|
|
60
|
-
output,
|
|
61
|
-
checkpointDir,
|
|
62
|
-
rateLimitDelay,
|
|
63
|
-
maxRetries: maxRetriesNum,
|
|
64
|
-
checkpointInterval: checkpointIntervalNum,
|
|
65
|
-
enableVision,
|
|
66
|
-
enableAudio,
|
|
67
|
-
enableLinks,
|
|
68
|
-
incremental,
|
|
69
|
-
});
|
|
70
|
-
}
|
|
71
|
-
// Create checkpoint directory if needed
|
|
72
|
-
if (!fs.existsSync(checkpointDir)) {
|
|
73
|
-
await import('node:fs/promises').then((fsp) => fsp.mkdir(checkpointDir, { recursive: true }));
|
|
74
|
-
}
|
|
75
|
-
// Load normalized messages
|
|
76
|
-
const content = fs.readFileSync(input, 'utf-8');
|
|
77
|
-
const data = JSON.parse(content);
|
|
78
|
-
const messages = Array.isArray(data) ? data : data.messages || [];
|
|
79
|
-
humanInfo(`✓ Loaded ${messages.length.toLocaleString()} messages`);
|
|
80
|
-
// Import enrichment modules
|
|
81
|
-
const { loadCheckpoint, computeConfigHash, saveCheckpoint, createCheckpoint, } = await import('../../enrich/checkpoint.js');
|
|
82
|
-
// Import actual enrichment functions
|
|
83
|
-
const { analyzeImage } = await import('../../enrich/image-analysis.js');
|
|
84
|
-
const { analyzeAudio } = await import('../../enrich/audio-transcription.js');
|
|
85
|
-
const { enrichLinkContext } = await import('../../enrich/link-enrichment.js');
|
|
86
|
-
const { createRateLimiter } = await import('../../enrich/rate-limiting.js');
|
|
87
|
-
// Create rate limiter with circuit breaker
|
|
88
|
-
const rateLimiter = createRateLimiter({
|
|
89
|
-
rateLimitDelay,
|
|
90
|
-
maxRetries: maxRetriesNum,
|
|
91
|
-
circuitBreakerThreshold: 5,
|
|
92
|
-
circuitBreakerResetMs: 60000,
|
|
93
|
-
});
|
|
94
|
-
// Compute config hash for checkpoint verification (AC05: Config consistency)
|
|
95
|
-
const enrichConfig = {
|
|
96
|
-
enableVisionAnalysis: enableVision,
|
|
97
|
-
enableLinkAnalysis: enableLinks,
|
|
98
|
-
enableAudioTranscription: enableAudio,
|
|
99
|
-
rateLimitDelay,
|
|
100
|
-
maxRetries: maxRetriesNum,
|
|
101
|
-
};
|
|
102
|
-
const configHash = computeConfigHash(enrichConfig);
|
|
103
|
-
const checkpointPath = `${checkpointDir}/enrich-checkpoint-${configHash}.json`;
|
|
104
|
-
// INCREMENTAL--T04-AC02/AC03: Handle incremental state file
|
|
105
|
-
const stateFilePath = userProvidedStateFile || '.imessage-state.json';
|
|
106
|
-
const stateFileExists = fs.existsSync(stateFilePath);
|
|
107
|
-
// INCREMENTAL--T04-AC04: Handle --reset-state flag
|
|
108
|
-
if (resetState && stateFileExists) {
|
|
109
|
-
fs.unlinkSync(stateFilePath);
|
|
110
|
-
if (verbose) {
|
|
111
|
-
humanInfo(`🗑️ Reset incremental state file: ${stateFilePath}`);
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
// Import incremental state module for AC02, AC03, AC04, AC05
|
|
115
|
-
const { loadIncrementalState, detectNewMessages } = await import('../../utils/incremental-state.js');
|
|
116
|
-
// INCREMENTAL--T04-AC02: Auto-detect state file and load previous state
|
|
117
|
-
let previousState = null;
|
|
118
|
-
let newMessageGuids = [];
|
|
119
|
-
let newMessageCount = messages.length;
|
|
120
|
-
if (incremental && stateFileExists && !resetState) {
|
|
121
|
-
previousState = await loadIncrementalState(stateFilePath);
|
|
122
|
-
if (previousState) {
|
|
123
|
-
// Detect new messages using GUID comparison
|
|
124
|
-
const currentGuids = new Set(messages
|
|
125
|
-
.map((m) => m.guid)
|
|
126
|
-
.filter((g) => Boolean(g)));
|
|
127
|
-
newMessageGuids = detectNewMessages(currentGuids, previousState);
|
|
128
|
-
newMessageCount = newMessageGuids.length;
|
|
129
|
-
if (verbose) {
|
|
130
|
-
humanInfo(`♻️ Incremental mode: detected ${newMessageCount.toLocaleString()} new messages`);
|
|
131
|
-
humanInfo(` Total messages: ${messages.length.toLocaleString()}`);
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
}
|
|
135
|
-
else if (incremental && !stateFileExists && !resetState) {
|
|
136
|
-
if (verbose) {
|
|
137
|
-
humanInfo(`♻️ Incremental mode enabled but no state file found: ${stateFilePath}`);
|
|
138
|
-
humanInfo(` Enriching all ${messages.length.toLocaleString()} messages`);
|
|
139
|
-
}
|
|
140
|
-
}
|
|
141
|
-
// AC05: Load checkpoint and verify config hash
|
|
142
|
-
let startIndex = 0;
|
|
143
|
-
if (resume) {
|
|
144
|
-
const checkpoint = await loadCheckpoint(checkpointPath);
|
|
145
|
-
if (checkpoint) {
|
|
146
|
-
if (checkpoint.configHash !== configHash) {
|
|
147
|
-
humanError('❌ Config has changed since last checkpoint');
|
|
148
|
-
humanError('Use --force-refresh to re-enrich or delete checkpoint file');
|
|
149
|
-
process.exit(1);
|
|
150
|
-
}
|
|
151
|
-
startIndex = checkpoint.lastProcessedIndex + 1;
|
|
152
|
-
cliLogger.info('Resuming from checkpoint', {
|
|
153
|
-
startIndex,
|
|
154
|
-
alreadyProcessed: checkpoint.totalProcessed,
|
|
155
|
-
failedItems: checkpoint.totalFailed,
|
|
156
|
-
});
|
|
157
|
-
}
|
|
158
|
-
else if (resume) {
|
|
159
|
-
humanWarn('⚠️ No checkpoint found, starting from beginning');
|
|
160
|
-
}
|
|
161
|
-
}
|
|
162
|
-
// AC02: Enrich messages with checkpoint support
|
|
163
|
-
const enrichedMessages = [];
|
|
164
|
-
let totalProcessed = 0;
|
|
165
|
-
let totalFailed = 0;
|
|
166
|
-
const failedItems = [];
|
|
167
|
-
// INCREMENTAL--T04-AC05: Show progress with new message count
|
|
168
|
-
const progressMsg = incremental && newMessageCount < messages.length
|
|
169
|
-
? `Enriching ${newMessageCount.toLocaleString()} new messages (${messages.length.toLocaleString()} total)`
|
|
170
|
-
: `Processing ${messages.length.toLocaleString()} messages`;
|
|
171
|
-
humanInfo(`\n🚀 Starting enrichment: ${progressMsg}`);
|
|
172
|
-
// Build enrichment configs
|
|
173
|
-
const geminiApiKey = process.env.GEMINI_API_KEY || '';
|
|
174
|
-
const firecrawlApiKey = process.env.FIRECRAWL_API_KEY;
|
|
175
|
-
const imageConfig = {
|
|
176
|
-
enableVisionAnalysis: enableVision ?? true,
|
|
177
|
-
geminiApiKey,
|
|
178
|
-
geminiModel: 'gemini-1.5-pro',
|
|
179
|
-
imageCacheDir: '/tmp/image-cache',
|
|
180
|
-
};
|
|
181
|
-
const audioConfig = {
|
|
182
|
-
enableAudioTranscription: enableAudio ?? true,
|
|
183
|
-
geminiApiKey,
|
|
184
|
-
geminiModel: 'gemini-1.5-pro',
|
|
185
|
-
rateLimitDelay,
|
|
186
|
-
maxRetries: maxRetriesNum,
|
|
187
|
-
};
|
|
188
|
-
const linkConfig = {
|
|
189
|
-
enableLinkAnalysis: enableLinks ?? true,
|
|
190
|
-
...(firecrawlApiKey ? { firecrawlApiKey } : {}),
|
|
191
|
-
rateLimitDelay,
|
|
192
|
-
maxRetries: maxRetriesNum,
|
|
193
|
-
};
|
|
194
|
-
// Build set of new message GUIDs for incremental filtering
|
|
195
|
-
const newGuidSet = new Set(newMessageGuids);
|
|
196
|
-
for (let i = startIndex; i < messages.length; i++) {
|
|
197
|
-
const message = messages[i];
|
|
198
|
-
try {
|
|
199
|
-
// INCREMENTAL--T04: Skip already-enriched messages in incremental mode
|
|
200
|
-
const shouldEnrich = !incremental || !previousState || newGuidSet.has(message.guid || '');
|
|
201
|
-
let enrichedMessage = message;
|
|
202
|
-
if (shouldEnrich) {
|
|
203
|
-
// Check circuit breaker before making API calls
|
|
204
|
-
if (rateLimiter.isCircuitOpen()) {
|
|
205
|
-
// Circuit is open - skip enrichment but don't fail
|
|
206
|
-
if (verbose) {
|
|
207
|
-
humanWarn(`⚠️ Circuit breaker open - skipping enrichment for message ${i}`);
|
|
208
|
-
}
|
|
209
|
-
}
|
|
210
|
-
else {
|
|
211
|
-
// Apply rate limiting delay between API calls
|
|
212
|
-
const rateLimitDelayMs = rateLimiter.shouldRateLimit();
|
|
213
|
-
if (rateLimitDelayMs > 0) {
|
|
214
|
-
await new Promise((resolve) => setTimeout(resolve, rateLimitDelayMs));
|
|
215
|
-
}
|
|
216
|
-
rateLimiter.recordCall();
|
|
217
|
-
// Enrich based on message type and config
|
|
218
|
-
if (enableVision &&
|
|
219
|
-
message.messageKind === 'media' &&
|
|
220
|
-
message.media?.mediaKind === 'image') {
|
|
221
|
-
enrichedMessage = await analyzeImage(enrichedMessage, imageConfig);
|
|
222
|
-
rateLimiter.recordSuccess();
|
|
223
|
-
}
|
|
224
|
-
else if (enableAudio &&
|
|
225
|
-
message.messageKind === 'media' &&
|
|
226
|
-
message.media?.mediaKind === 'audio') {
|
|
227
|
-
enrichedMessage = await analyzeAudio(enrichedMessage, audioConfig);
|
|
228
|
-
rateLimiter.recordSuccess();
|
|
229
|
-
}
|
|
230
|
-
else if (enableLinks &&
|
|
231
|
-
message.messageKind === 'text' &&
|
|
232
|
-
message.text) {
|
|
233
|
-
enrichedMessage = await enrichLinkContext(enrichedMessage, linkConfig);
|
|
234
|
-
rateLimiter.recordSuccess();
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
}
|
|
238
|
-
enrichedMessages.push(enrichedMessage);
|
|
239
|
-
totalProcessed++;
|
|
240
|
-
// AC01: Write checkpoint at intervals
|
|
241
|
-
if ((i + 1) % checkpointIntervalNum === 0) {
|
|
242
|
-
const checkpoint = createCheckpoint({
|
|
243
|
-
lastProcessedIndex: i,
|
|
244
|
-
totalProcessed,
|
|
245
|
-
totalFailed,
|
|
246
|
-
stats: {
|
|
247
|
-
processedCount: totalProcessed,
|
|
248
|
-
failedCount: totalFailed,
|
|
249
|
-
enrichmentsByKind: {},
|
|
250
|
-
},
|
|
251
|
-
failedItems,
|
|
252
|
-
configHash,
|
|
253
|
-
});
|
|
254
|
-
await saveCheckpoint(checkpoint, checkpointPath);
|
|
255
|
-
if (verbose) {
|
|
256
|
-
cliLogger.info('Checkpoint written', { index: i + 1 });
|
|
257
|
-
}
|
|
258
|
-
}
|
|
259
|
-
}
|
|
260
|
-
catch (error) {
|
|
261
|
-
totalFailed++;
|
|
262
|
-
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
263
|
-
failedItems.push({
|
|
264
|
-
index: i,
|
|
265
|
-
guid: message.guid || 'unknown',
|
|
266
|
-
kind: message.messageKind || 'unknown',
|
|
267
|
-
error: errorMessage,
|
|
268
|
-
});
|
|
269
|
-
if (verbose) {
|
|
270
|
-
humanWarn(`⚠️ Failed to enrich message ${i}: ${errorMessage}`);
|
|
271
|
-
logEvent('enrich-item-failed', {
|
|
272
|
-
command: 'enrich',
|
|
273
|
-
phase: 'warning',
|
|
274
|
-
context: {
|
|
275
|
-
index: i,
|
|
276
|
-
guid: message.guid || 'unknown',
|
|
277
|
-
kind: message.messageKind || 'unknown',
|
|
278
|
-
},
|
|
279
|
-
error: {
|
|
280
|
-
type: error instanceof Error ? error.name : 'Unknown',
|
|
281
|
-
message: errorMessage,
|
|
282
|
-
...(error instanceof Error && error.stack
|
|
283
|
-
? { stack: error.stack }
|
|
284
|
-
: {}),
|
|
285
|
-
},
|
|
286
|
-
});
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
}
|
|
290
|
-
// Write final checkpoint
|
|
291
|
-
const finalCheckpoint = createCheckpoint({
|
|
292
|
-
lastProcessedIndex: messages.length - 1,
|
|
293
|
-
totalProcessed,
|
|
294
|
-
totalFailed,
|
|
295
|
-
stats: {
|
|
296
|
-
processedCount: totalProcessed,
|
|
297
|
-
failedCount: totalFailed,
|
|
298
|
-
enrichmentsByKind: {},
|
|
299
|
-
},
|
|
300
|
-
failedItems,
|
|
301
|
-
configHash,
|
|
302
|
-
});
|
|
303
|
-
await saveCheckpoint(finalCheckpoint, checkpointPath);
|
|
304
|
-
// Write output
|
|
305
|
-
const { createExportEnvelope } = await import('../../ingest/ingest-csv.js');
|
|
306
|
-
const envelope = createExportEnvelope(enrichedMessages);
|
|
307
|
-
envelope.source = 'merged';
|
|
308
|
-
fs.writeFileSync(output, JSON.stringify(envelope, null, 2), 'utf-8');
|
|
309
|
-
humanInfo('\n✅ Enrichment complete');
|
|
310
|
-
humanInfo(`✓ Processed: ${totalProcessed.toLocaleString()} messages`);
|
|
311
|
-
if (totalFailed > 0) {
|
|
312
|
-
humanInfo(`⚠️ Failed: ${totalFailed.toLocaleString()} messages`);
|
|
313
|
-
}
|
|
314
|
-
humanInfo(`✓ Wrote to ${output}`);
|
|
315
|
-
logEvent('enrich-summary', {
|
|
316
|
-
command: 'enrich',
|
|
317
|
-
phase: 'summary',
|
|
318
|
-
metrics: { processed: totalProcessed, failed: totalFailed },
|
|
319
|
-
options: { output, checkpointInterval: checkpointIntervalNum },
|
|
320
|
-
context: { checkpointPath },
|
|
321
|
-
exitCode: 0,
|
|
322
|
-
});
|
|
323
|
-
}
|
|
324
|
-
/**
|
|
325
|
-
* Register the enrich-ai command with Commander
|
|
326
|
-
*/
|
|
327
|
-
export function registerEnrichAICommand(program, getGlobalOptions) {
|
|
328
|
-
program
|
|
329
|
-
.command('enrich-ai')
|
|
330
|
-
.description('Add AI-powered enrichment to media messages')
|
|
331
|
-
.requiredOption('-i, --input <path>', 'input normalized JSON file')
|
|
332
|
-
.option('-o, --output <path>', 'output JSON file path', './messages.enriched.json')
|
|
333
|
-
.option('-c, --checkpoint-dir <path>', 'checkpoint directory', './.checkpoints')
|
|
334
|
-
.option('--resume', 'resume from last checkpoint', false)
|
|
335
|
-
.option('--incremental', 'only enrich messages new since last enrichment run', false)
|
|
336
|
-
.option('--state-file <path>', 'path to incremental state file (auto-detects .imessage-state.json by default)')
|
|
337
|
-
.option('--reset-state', 'clear incremental state and enrich all messages', false)
|
|
338
|
-
.option('--force-refresh', 'force re-enrichment even if already done', false)
|
|
339
|
-
.option('--rate-limit <ms>', 'delay between API calls (milliseconds)', '1000')
|
|
340
|
-
.option('--max-retries <n>', 'max retries on API errors', '3')
|
|
341
|
-
.option('--checkpoint-interval <n>', 'write checkpoint every N items', '100')
|
|
342
|
-
.option('--enable-vision', 'enable image analysis with Gemini Vision', true)
|
|
343
|
-
.option('--enable-audio', 'enable audio transcription with Gemini Audio', true)
|
|
344
|
-
.option('--enable-links', 'enable link enrichment with Firecrawl', true)
|
|
345
|
-
.action(async (options) => {
|
|
346
|
-
try {
|
|
347
|
-
await executeEnrichAI(options, getGlobalOptions());
|
|
348
|
-
process.exit(0);
|
|
349
|
-
}
|
|
350
|
-
catch (error) {
|
|
351
|
-
humanError('❌ Failed to enrich:', error instanceof Error ? error.message : String(error));
|
|
352
|
-
if (getGlobalOptions().verbose && error instanceof Error) {
|
|
353
|
-
humanError(error.stack);
|
|
354
|
-
}
|
|
355
|
-
logEvent('enrich-error', {
|
|
356
|
-
command: 'enrich',
|
|
357
|
-
phase: 'error',
|
|
358
|
-
error: {
|
|
359
|
-
type: error instanceof Error ? error.name : 'Unknown',
|
|
360
|
-
message: error instanceof Error ? error.message : String(error),
|
|
361
|
-
...(error instanceof Error && error.stack
|
|
362
|
-
? { stack: error.stack }
|
|
363
|
-
: {}),
|
|
364
|
-
},
|
|
365
|
-
exitCode: 2,
|
|
366
|
-
});
|
|
367
|
-
process.exit(2);
|
|
368
|
-
}
|
|
369
|
-
});
|
|
370
|
-
}
|
|
371
|
-
//# sourceMappingURL=enrich-ai.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"enrich-ai.js","sourceRoot":"","sources":["../../../src/cli/commands/enrich-ai.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AAG/D,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEhE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CACpC,OAAwB,EACxB,aAA4B;IAE5B,MAAM,EACL,KAAK,EACL,MAAM,EACN,aAAa,EACb,MAAM,EACN,WAAW,EACX,SAAS,EAAE,qBAAqB,EAChC,UAAU,EACV,YAAY,EAAE,aAAa,EAC3B,WAAW,EACX,UAAU,EACV,kBAAkB,EAClB,YAAY,EACZ,WAAW,EACX,WAAW,GACX,GAAG,OAAO,CAAA;IACX,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,aAAa,CAAA;IAExC,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IAE7B,QAAQ,CAAC,cAAc,EAAE;QACxB,OAAO,EAAE,QAAQ;QACjB,KAAK,EAAE,OAAO;QACd,OAAO,EAAE;YACR,KAAK;YACL,MAAM;YACN,aAAa;YACb,MAAM;YACN,WAAW;YACX,SAAS,EAAE,qBAAqB;YAChC,UAAU;YACV,WAAW;YACX,UAAU;YACV,kBAAkB;YAClB,YAAY;YACZ,WAAW;YACX,WAAW;SACX;KACD,CAAC,CAAA;IAEF,kBAAkB;IAClB,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAA;IAClC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,UAAU,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAA;QAC9C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IAED,qCAAqC;IACrC,MAAM,cAAc,GAAG,MAAM,CAAC,QAAQ,CAAC,WAAqB,EAAE,EAAE,CAAC,CAAA;IACjE,MAAM,aAAa,GAAG,MAAM,CAAC,QAAQ,CAAC,UAAoB,EAAE,EAAE,CAAC,CAAA;IAC/D,MAAM,qBAAqB,GAAG,MAAM,CAAC,QAAQ,CAC5C,kBAA4B,EAC5B,EAAE,CACF,CAAA;IAED,IAAI,MAAM,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,cAAc,GAAG,CAAC,EAAE,CAAC;QACxD,UAAU,CAAC,6DAA6D,CAAC,CAAA;QACzE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,aAAa,GAAG,CAAC,EAAE,CAAC;QACtD,UAAU,CAAC,+CAA+C,CAAC,CAAA;QAC3D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IACD,IAAI,MAAM,CAAC,KAAK,CAAC,qBAAqB,CAAC,IAAI,qBAAqB,GAAG,CAAC,EAAE,CAAC;QACtE,UAAU,CAAC,mDAAmD,CAAC,CAAA;QAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IAED,IAAI,OAAO,EAAE,CAAC;QACb,SAAS,CAAC,IAAI,CAAC,eAAe,EAAE;YAC/B,KAAK;YACL,MAAM;YACN,aAAa;YACb,cAAc;YACd,UAAU,EAAE,aAAa;YACzB,kBAAkB,EAAE,qBAAqB;YACzC,YAAY;YACZ,WAAW;YACX,WAAW;YACX,WAAW;SACX,CAAC,CAAA;IACH,CAAC;IAED,wCAAwC;IACxC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,EAAE,CAAC;QACnC,MAAM,MAAM,CAAC,kBAAkB,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAC7C,GAAG,CAAC,KAAK,CAAC,aAAa,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAC7C,CAAA;IACF,CAAC;IAED,2BAA2B;IAC3B,MAAM,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAA;IAC/C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAA;IAChC,MAAM,QAAQ,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAA;IAEjE,SAAS,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;IAElE,4BAA4B;IAC5B,MAAM,EACL,cAAc,EACd,iBAAiB,EACjB,cAAc,EACd,gBAAgB,GAChB,GAAG,MAAM,MAAM,CAAC,4BAA4B,CAAC,CAAA;IAE9C,qCAAqC;IACrC,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,gCAAgC,CAAC,CAAA;IACvE,MAAM,EAAE,YAAY,EAAE,GAAG,MAAM,MAAM,CAAC,qCAAqC,CAAC,CAAA;IAC5E,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAA;IAC7E,MAAM,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAAC,+BAA+B,CAAC,CAAA;IAE3E,2CAA2C;IAC3C,MAAM,WAAW,GAAG,iBAAiB,CAAC;QACrC,cAAc;QACd,UAAU,EAAE,aAAa;QACzB,uBAAuB,EAAE,CAAC;QAC1B,qBAAqB,EAAE,KAAK;KAC5B,CAAC,CAAA;IAEF,6EAA6E;IAC7E,MAAM,YAAY,GAAG;QACpB,oBAAoB,EAAE,YAAY;QAClC,kBAAkB,EAAE,WAAW;QAC/B,wBAAwB,EAAE,WAAW;QACrC,cAAc;QACd,UAAU,EAAE,aAAa;KACzB,CAAA;IACD,MAAM,UAAU,GAAG,iBAAiB,CAAC,YAAY,CAAC,CAAA;IAClD,MAAM,cAAc,GAAG,GAAG,aAAa,sBAAsB,UAAU,OAAO,CAAA;IAE9E,4DAA4D;IAC5D,MAAM,aAAa,GAAG,qBAAqB,IAAI,sBAAsB,CAAA;IACrE,MAAM,eAAe,GAAG,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAA;IAEpD,mDAAmD;IACnD,IAAI,UAAU,IAAI,eAAe,EAAE,CAAC;QACnC,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAA;QAC5B,IAAI,OAAO,EAAE,CAAC;YACb,SAAS,CAAC,sCAAsC,aAAa,EAAE,CAAC,CAAA;QACjE,CAAC;IACF,CAAC;IAED,6DAA6D;IAC7D,MAAM,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,GAAG,MAAM,MAAM,CAC/D,kCAAkC,CAClC,CAAA;IAED,wEAAwE;IACxE,IAAI,aAAa,GAChB,IAAI,CAAA;IACL,IAAI,eAAe,GAAa,EAAE,CAAA;IAClC,IAAI,eAAe,GAAG,QAAQ,CAAC,MAAM,CAAA;IAErC,IAAI,WAAW,IAAI,eAAe,IAAI,CAAC,UAAU,EAAE,CAAC;QACnD,aAAa,GAAG,MAAM,oBAAoB,CAAC,aAAa,CAAC,CAAA;QACzD,IAAI,aAAa,EAAE,CAAC;YACnB,4CAA4C;YAC5C,MAAM,YAAY,GAAG,IAAI,GAAG,CAC1B,QAAsB;iBACrB,GAAG,CAAC,CAAC,CAAU,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;iBAC3B,MAAM,CAAC,CAAC,CAAqB,EAAe,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAC5D,CAAA;YACD,eAAe,GAAG,iBAAiB,CAAC,YAAY,EAAE,aAAa,CAAC,CAAA;YAChE,eAAe,GAAG,eAAe,CAAC,MAAM,CAAA;YACxC,IAAI,OAAO,EAAE,CAAC;gBACb,SAAS,CACR,kCAAkC,eAAe,CAAC,cAAc,EAAE,eAAe,CACjF,CAAA;gBACD,SAAS,CAAC,sBAAsB,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,EAAE,CAAC,CAAA;YACpE,CAAC;QACF,CAAC;IACF,CAAC;SAAM,IAAI,WAAW,IAAI,CAAC,eAAe,IAAI,CAAC,UAAU,EAAE,CAAC;QAC3D,IAAI,OAAO,EAAE,CAAC;YACb,SAAS,CACR,yDAAyD,aAAa,EAAE,CACxE,CAAA;YACD,SAAS,CAAC,oBAAoB,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;QAC3E,CAAC;IACF,CAAC;IAED,+CAA+C;IAC/C,IAAI,UAAU,GAAG,CAAC,CAAA;IAClB,IAAI,MAAM,EAAE,CAAC;QACZ,MAAM,UAAU,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,CAAA;QACvD,IAAI,UAAU,EAAE,CAAC;YAChB,IAAI,UAAU,CAAC,UAAU,KAAK,UAAU,EAAE,CAAC;gBAC1C,UAAU,CAAC,4CAA4C,CAAC,CAAA;gBACxD,UAAU,CAAC,4DAA4D,CAAC,CAAA;gBACxE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAChB,CAAC;YACD,UAAU,GAAG,UAAU,CAAC,kBAAkB,GAAG,CAAC,CAAA;YAC9C,SAAS,CAAC,IAAI,CAAC,0BAA0B,EAAE;gBAC1C,UAAU;gBACV,gBAAgB,EAAE,UAAU,CAAC,cAAc;gBAC3C,WAAW,EAAE,UAAU,CAAC,WAAW;aACnC,CAAC,CAAA;QACH,CAAC;aAAM,IAAI,MAAM,EAAE,CAAC;YACnB,SAAS,CAAC,kDAAkD,CAAC,CAAA;QAC9D,CAAC;IACF,CAAC;IAED,gDAAgD;IAChD,MAAM,gBAAgB,GAAc,EAAE,CAAA;IACtC,IAAI,cAAc,GAAG,CAAC,CAAA;IACtB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,WAAW,GAKZ,EAAE,CAAA;IAEP,8DAA8D;IAC9D,MAAM,WAAW,GAChB,WAAW,IAAI,eAAe,GAAG,QAAQ,CAAC,MAAM;QAC/C,CAAC,CAAC,aAAa,eAAe,CAAC,cAAc,EAAE,kBAAkB,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,SAAS;QAC1G,CAAC,CAAC,cAAc,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,WAAW,CAAA;IAC7D,SAAS,CAAC,6BAA6B,WAAW,EAAE,CAAC,CAAA;IAErD,2BAA2B;IAC3B,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,cAAc,IAAI,EAAE,CAAA;IACrD,MAAM,eAAe,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAA;IAErD,MAAM,WAAW,GAAG;QACnB,oBAAoB,EAAE,YAAY,IAAI,IAAI;QAC1C,YAAY;QACZ,WAAW,EAAE,gBAAgB;QAC7B,aAAa,EAAE,kBAAkB;KACjC,CAAA;IAED,MAAM,WAAW,GAAG;QACnB,wBAAwB,EAAE,WAAW,IAAI,IAAI;QAC7C,YAAY;QACZ,WAAW,EAAE,gBAAgB;QAC7B,cAAc;QACd,UAAU,EAAE,aAAa;KACzB,CAAA;IAED,MAAM,UAAU,GAAG;QAClB,kBAAkB,EAAE,WAAW,IAAI,IAAI;QACvC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QAC/C,cAAc;QACd,UAAU,EAAE,aAAa;KACzB,CAAA;IAED,2DAA2D;IAC3D,MAAM,UAAU,GAAG,IAAI,GAAG,CAAC,eAAe,CAAC,CAAA;IAE3C,KAAK,IAAI,CAAC,GAAG,UAAU,EAAE,CAAC,GAAG,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACnD,MAAM,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAA;QAE3B,IAAI,CAAC;YACJ,uEAAuE;YACvE,MAAM,YAAY,GACjB,CAAC,WAAW,IAAI,CAAC,aAAa,IAAI,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,IAAI,EAAE,CAAC,CAAA;YAErE,IAAI,eAAe,GAAG,OAAO,CAAA;YAE7B,IAAI,YAAY,EAAE,CAAC;gBAClB,gDAAgD;gBAChD,IAAI,WAAW,CAAC,aAAa,EAAE,EAAE,CAAC;oBACjC,mDAAmD;oBACnD,IAAI,OAAO,EAAE,CAAC;wBACb,SAAS,CACR,8DAA8D,CAAC,EAAE,CACjE,CAAA;oBACF,CAAC;gBACF,CAAC;qBAAM,CAAC;oBACP,8CAA8C;oBAC9C,MAAM,gBAAgB,GAAG,WAAW,CAAC,eAAe,EAAE,CAAA;oBACtD,IAAI,gBAAgB,GAAG,CAAC,EAAE,CAAC;wBAC1B,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAC7B,UAAU,CAAC,OAAO,EAAE,gBAAgB,CAAC,CACrC,CAAA;oBACF,CAAC;oBACD,WAAW,CAAC,UAAU,EAAE,CAAA;oBAExB,0CAA0C;oBAC1C,IACC,YAAY;wBACZ,OAAO,CAAC,WAAW,KAAK,OAAO;wBAC/B,OAAO,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,EACnC,CAAC;wBACF,eAAe,GAAG,MAAM,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAA;wBAClE,WAAW,CAAC,aAAa,EAAE,CAAA;oBAC5B,CAAC;yBAAM,IACN,WAAW;wBACX,OAAO,CAAC,WAAW,KAAK,OAAO;wBAC/B,OAAO,CAAC,KAAK,EAAE,SAAS,KAAK,OAAO,EACnC,CAAC;wBACF,eAAe,GAAG,MAAM,YAAY,CAAC,eAAe,EAAE,WAAW,CAAC,CAAA;wBAClE,WAAW,CAAC,aAAa,EAAE,CAAA;oBAC5B,CAAC;yBAAM,IACN,WAAW;wBACX,OAAO,CAAC,WAAW,KAAK,MAAM;wBAC9B,OAAO,CAAC,IAAI,EACX,CAAC;wBACF,eAAe,GAAG,MAAM,iBAAiB,CACxC,eAAe,EACf,UAAU,CACV,CAAA;wBACD,WAAW,CAAC,aAAa,EAAE,CAAA;oBAC5B,CAAC;gBACF,CAAC;YACF,CAAC;YAED,gBAAgB,CAAC,IAAI,CAAC,eAAe,CAAC,CAAA;YACtC,cAAc,EAAE,CAAA;YAEhB,sCAAsC;YACtC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,qBAAqB,KAAK,CAAC,EAAE,CAAC;gBAC3C,MAAM,UAAU,GAAG,gBAAgB,CAAC;oBACnC,kBAAkB,EAAE,CAAC;oBACrB,cAAc;oBACd,WAAW;oBACX,KAAK,EAAE;wBACN,cAAc,EAAE,cAAc;wBAC9B,WAAW,EAAE,WAAW;wBACxB,iBAAiB,EAAE,EAAE;qBACrB;oBACD,WAAW;oBACX,UAAU;iBACV,CAAC,CAAA;gBACF,MAAM,cAAc,CAAC,UAAU,EAAE,cAAc,CAAC,CAAA;gBAChD,IAAI,OAAO,EAAE,CAAC;oBACb,SAAS,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAA;gBACvD,CAAC;YACF,CAAC;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,WAAW,EAAE,CAAA;YACb,MAAM,YAAY,GACjB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;YACvD,WAAW,CAAC,IAAI,CAAC;gBAChB,KAAK,EAAE,CAAC;gBACR,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,SAAS;gBAC/B,IAAI,EAAE,OAAO,CAAC,WAAW,IAAI,SAAS;gBACtC,KAAK,EAAE,YAAY;aACnB,CAAC,CAAA;YACF,IAAI,OAAO,EAAE,CAAC;gBACb,SAAS,CAAC,gCAAgC,CAAC,KAAK,YAAY,EAAE,CAAC,CAAA;gBAC/D,QAAQ,CAAC,oBAAoB,EAAE;oBAC9B,OAAO,EAAE,QAAQ;oBACjB,KAAK,EAAE,SAAS;oBAChB,OAAO,EAAE;wBACR,KAAK,EAAE,CAAC;wBACR,IAAI,EAAE,OAAO,CAAC,IAAI,IAAI,SAAS;wBAC/B,IAAI,EAAE,OAAO,CAAC,WAAW,IAAI,SAAS;qBACtC;oBACD,KAAK,EAAE;wBACN,IAAI,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;wBACrD,OAAO,EAAE,YAAY;wBACrB,GAAG,CAAC,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,KAAK;4BACxC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;4BACxB,CAAC,CAAC,EAAE,CAAC;qBACN;iBACD,CAAC,CAAA;YACH,CAAC;QACF,CAAC;IACF,CAAC;IAED,yBAAyB;IACzB,MAAM,eAAe,GAAG,gBAAgB,CAAC;QACxC,kBAAkB,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC;QACvC,cAAc;QACd,WAAW;QACX,KAAK,EAAE;YACN,cAAc,EAAE,cAAc;YAC9B,WAAW,EAAE,WAAW;YACxB,iBAAiB,EAAE,EAAE;SACrB;QACD,WAAW;QACX,UAAU;KACV,CAAC,CAAA;IACF,MAAM,cAAc,CAAC,eAAe,EAAE,cAAc,CAAC,CAAA;IAErD,eAAe;IACf,MAAM,EAAE,oBAAoB,EAAE,GAAG,MAAM,MAAM,CAAC,4BAA4B,CAAC,CAAA;IAC3E,MAAM,QAAQ,GAAG,oBAAoB,CAAC,gBAAgB,CAAC,CAAA;IACvD,QAAQ,CAAC,MAAM,GAAG,QAAQ,CAAA;IAC1B,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IAEpE,SAAS,CAAC,yBAAyB,CAAC,CAAA;IACpC,SAAS,CAAC,gBAAgB,cAAc,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;IACrE,IAAI,WAAW,GAAG,CAAC,EAAE,CAAC;QACrB,SAAS,CAAC,eAAe,WAAW,CAAC,cAAc,EAAE,WAAW,CAAC,CAAA;IAClE,CAAC;IACD,SAAS,CAAC,cAAc,MAAM,EAAE,CAAC,CAAA;IAEjC,QAAQ,CAAC,gBAAgB,EAAE;QAC1B,OAAO,EAAE,QAAQ;QACjB,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,EAAE,WAAW,EAAE;QAC3D,OAAO,EAAE,EAAE,MAAM,EAAE,kBAAkB,EAAE,qBAAqB,EAAE;QAC9D,OAAO,EAAE,EAAE,cAAc,EAAE;QAC3B,QAAQ,EAAE,CAAC;KACX,CAAC,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACtC,OAAgB,EAChB,gBAAqC;IAErC,OAAO;SACL,OAAO,CAAC,WAAW,CAAC;SACpB,WAAW,CAAC,6CAA6C,CAAC;SAC1D,cAAc,CAAC,oBAAoB,EAAE,4BAA4B,CAAC;SAClE,MAAM,CACN,qBAAqB,EACrB,uBAAuB,EACvB,0BAA0B,CAC1B;SACA,MAAM,CACN,6BAA6B,EAC7B,sBAAsB,EACtB,gBAAgB,CAChB;SACA,MAAM,CAAC,UAAU,EAAE,6BAA6B,EAAE,KAAK,CAAC;SACxD,MAAM,CACN,eAAe,EACf,oDAAoD,EACpD,KAAK,CACL;SACA,MAAM,CACN,qBAAqB,EACrB,+EAA+E,CAC/E;SACA,MAAM,CACN,eAAe,EACf,iDAAiD,EACjD,KAAK,CACL;SACA,MAAM,CACN,iBAAiB,EACjB,0CAA0C,EAC1C,KAAK,CACL;SACA,MAAM,CACN,mBAAmB,EACnB,wCAAwC,EACxC,MAAM,CACN;SACA,MAAM,CAAC,mBAAmB,EAAE,2BAA2B,EAAE,GAAG,CAAC;SAC7D,MAAM,CACN,2BAA2B,EAC3B,gCAAgC,EAChC,KAAK,CACL;SACA,MAAM,CAAC,iBAAiB,EAAE,0CAA0C,EAAE,IAAI,CAAC;SAC3E,MAAM,CACN,gBAAgB,EAChB,8CAA8C,EAC9C,IAAI,CACJ;SACA,MAAM,CAAC,gBAAgB,EAAE,uCAAuC,EAAE,IAAI,CAAC;SACvE,MAAM,CAAC,KAAK,EAAE,OAAwB,EAAE,EAAE;QAC1C,IAAI,CAAC;YACJ,MAAM,eAAe,CAAC,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAA;YAClD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,UAAU,CACT,qBAAqB,EACrB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACtD,CAAA;YACD,IAAI,gBAAgB,EAAE,CAAC,OAAO,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;gBAC1D,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACxB,CAAC;YACD,QAAQ,CAAC,cAAc,EAAE;gBACxB,OAAO,EAAE,QAAQ;gBACjB,KAAK,EAAE,OAAO;gBACd,KAAK,EAAE;oBACN,IAAI,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;oBACrD,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC/D,GAAG,CAAC,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,KAAK;wBACxC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;wBACxB,CAAC,CAAC,EAAE,CAAC;iBACN;gBACD,QAAQ,EAAE,CAAC;aACX,CAAC,CAAA;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAChB,CAAC;IACF,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* CLI Commands Index
|
|
3
|
-
*
|
|
4
|
-
* Barrel export for all CLI command modules.
|
|
5
|
-
*/
|
|
6
|
-
export { executeClean, registerCleanCommand } from './clean.js';
|
|
7
|
-
export { executeDoctor, registerDoctorCommand } from './doctor.js';
|
|
8
|
-
export { executeEnrichAI, registerEnrichAICommand } from './enrich-ai.js';
|
|
9
|
-
export { executeIngestCSV, registerIngestCSVCommand } from './ingest-csv.js';
|
|
10
|
-
export { executeIngestDB, registerIngestDBCommand } from './ingest-db.js';
|
|
11
|
-
export { executeInit, registerInitCommand } from './init.js';
|
|
12
|
-
export { executeNormalizeLink, registerNormalizeLinkCommand, } from './normalize-link.js';
|
|
13
|
-
export { executeRenderMarkdown, registerRenderMarkdownCommand, } from './render-markdown.js';
|
|
14
|
-
export { executeStats, registerStatsCommand } from './stats.js';
|
|
15
|
-
export { executeValidate, registerValidateCommand } from './validate.js';
|
|
16
|
-
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAA;AAClE,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AACzE,OAAO,EAAE,gBAAgB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAA;AAC5E,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AACzE,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAA;AAC5D,OAAO,EACN,oBAAoB,EACpB,4BAA4B,GAC5B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,qBAAqB,EACrB,6BAA6B,GAC7B,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA"}
|
|
@@ -1,16 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* CLI Commands Index
|
|
3
|
-
*
|
|
4
|
-
* Barrel export for all CLI command modules.
|
|
5
|
-
*/
|
|
6
|
-
export { executeClean, registerCleanCommand } from './clean.js';
|
|
7
|
-
export { executeDoctor, registerDoctorCommand } from './doctor.js';
|
|
8
|
-
export { executeEnrichAI, registerEnrichAICommand } from './enrich-ai.js';
|
|
9
|
-
export { executeIngestCSV, registerIngestCSVCommand } from './ingest-csv.js';
|
|
10
|
-
export { executeIngestDB, registerIngestDBCommand } from './ingest-db.js';
|
|
11
|
-
export { executeInit, registerInitCommand } from './init.js';
|
|
12
|
-
export { executeNormalizeLink, registerNormalizeLinkCommand, } from './normalize-link.js';
|
|
13
|
-
export { executeRenderMarkdown, registerRenderMarkdownCommand, } from './render-markdown.js';
|
|
14
|
-
export { executeStats, registerStatsCommand } from './stats.js';
|
|
15
|
-
export { executeValidate, registerValidateCommand } from './validate.js';
|
|
16
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/cli/commands/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EAAE,aAAa,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAA;AAClE,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AACzE,OAAO,EAAE,gBAAgB,EAAE,wBAAwB,EAAE,MAAM,iBAAiB,CAAA;AAC5E,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,gBAAgB,CAAA;AACzE,OAAO,EAAE,WAAW,EAAE,mBAAmB,EAAE,MAAM,WAAW,CAAA;AAC5D,OAAO,EACN,oBAAoB,EACpB,4BAA4B,GAC5B,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,qBAAqB,EACrB,6BAA6B,GAC7B,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,YAAY,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAA;AAC/D,OAAO,EAAE,eAAe,EAAE,uBAAuB,EAAE,MAAM,eAAe,CAAA"}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Ingest CSV Command
|
|
3
|
-
*
|
|
4
|
-
* Import messages from iMazing CSV export.
|
|
5
|
-
* CLI-T02-AC01: ingest-csv command with all options from usage guide
|
|
6
|
-
*/
|
|
7
|
-
import type { Command } from 'commander';
|
|
8
|
-
import type { GlobalOptions, IngestCSVOptions } from '../types.js';
|
|
9
|
-
/**
|
|
10
|
-
* Execute the ingest-csv command logic
|
|
11
|
-
*/
|
|
12
|
-
export declare function executeIngestCSV(options: IngestCSVOptions, globalOptions: GlobalOptions): Promise<void>;
|
|
13
|
-
/**
|
|
14
|
-
* Register the ingest-csv command with Commander
|
|
15
|
-
*/
|
|
16
|
-
export declare function registerIngestCSVCommand(program: Command, getGlobalOptions: () => GlobalOptions): void;
|
|
17
|
-
//# sourceMappingURL=ingest-csv.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ingest-csv.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAGxC,OAAO,KAAK,EAAE,aAAa,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAA;AAGlE;;GAEG;AACH,wBAAsB,gBAAgB,CACrC,OAAO,EAAE,gBAAgB,EACzB,aAAa,EAAE,aAAa,GAC1B,OAAO,CAAC,IAAI,CAAC,CA4Gf;AAED;;GAEG;AACH,wBAAgB,wBAAwB,CACvC,OAAO,EAAE,OAAO,EAChB,gBAAgB,EAAE,MAAM,aAAa,GACnC,IAAI,CAsCN"}
|
|
@@ -1,138 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Ingest CSV Command
|
|
3
|
-
*
|
|
4
|
-
* Import messages from iMazing CSV export.
|
|
5
|
-
* CLI-T02-AC01: ingest-csv command with all options from usage guide
|
|
6
|
-
*/
|
|
7
|
-
import { humanError, humanInfo } from '#utils/human';
|
|
8
|
-
import { setCorrelationId } from '#utils/logger';
|
|
9
|
-
import { applyLogLevel, cliLogger, logEvent } from '../utils.js';
|
|
10
|
-
/**
|
|
11
|
-
* Execute the ingest-csv command logic
|
|
12
|
-
*/
|
|
13
|
-
export async function executeIngestCSV(options, globalOptions) {
|
|
14
|
-
const { input, output, attachments } = options;
|
|
15
|
-
const { verbose, quiet } = globalOptions;
|
|
16
|
-
applyLogLevel(verbose, quiet);
|
|
17
|
-
setCorrelationId(`ingest-csv:${Date.now().toString(36)}`);
|
|
18
|
-
logEvent('ingest-start', {
|
|
19
|
-
command: 'ingest-csv',
|
|
20
|
-
phase: 'start',
|
|
21
|
-
options: { input, output, attachmentsCount: attachments?.length },
|
|
22
|
-
});
|
|
23
|
-
// CLI-T02-AC04: Input file validation with clear error messages
|
|
24
|
-
const fs = await import('node:fs');
|
|
25
|
-
if (!fs.existsSync(input)) {
|
|
26
|
-
humanError(`❌ Input CSV file not found: ${input}`);
|
|
27
|
-
humanError('\nPlease check:');
|
|
28
|
-
humanError(' • File path is correct');
|
|
29
|
-
humanError(' • File exists and is readable');
|
|
30
|
-
process.exit(1);
|
|
31
|
-
}
|
|
32
|
-
// CLI-T02-AC03: Attachment root validation (check directories exist)
|
|
33
|
-
const attachmentRoots = [];
|
|
34
|
-
if (attachments && attachments.length > 0) {
|
|
35
|
-
for (const dir of attachments) {
|
|
36
|
-
if (!fs.existsSync(dir)) {
|
|
37
|
-
humanError(`❌ Attachment directory not found: ${dir}`);
|
|
38
|
-
process.exit(1);
|
|
39
|
-
}
|
|
40
|
-
if (!fs.statSync(dir).isDirectory()) {
|
|
41
|
-
humanError(`❌ Not a directory: ${dir}`);
|
|
42
|
-
process.exit(1);
|
|
43
|
-
}
|
|
44
|
-
attachmentRoots.push(dir);
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
else {
|
|
48
|
-
// Default: ~/Library/Messages/Attachments
|
|
49
|
-
const os = await import('node:os');
|
|
50
|
-
const path = await import('node:path');
|
|
51
|
-
const defaultRoot = path.join(os.homedir(), 'Library', 'Messages', 'Attachments');
|
|
52
|
-
if (fs.existsSync(defaultRoot)) {
|
|
53
|
-
attachmentRoots.push(defaultRoot);
|
|
54
|
-
if (verbose) {
|
|
55
|
-
cliLogger.info('Using default attachment root', { defaultRoot });
|
|
56
|
-
}
|
|
57
|
-
}
|
|
58
|
-
}
|
|
59
|
-
// CLI-T02-AC01: ingest-csv command with all options from usage guide
|
|
60
|
-
const { ingestCSV, createExportEnvelope, validateMessages } = await import('../../ingest/ingest-csv.js');
|
|
61
|
-
if (verbose) {
|
|
62
|
-
cliLogger.info('Reading CSV ingest', { input, attachmentRoots });
|
|
63
|
-
}
|
|
64
|
-
const messages = ingestCSV(input, { attachmentRoots });
|
|
65
|
-
// CLI-T02-AC05: Progress output: ✓ Parsed 2,847 messages from CSV
|
|
66
|
-
humanInfo(`✓ Parsed ${messages.length.toLocaleString()} messages from CSV`);
|
|
67
|
-
// Validate messages before writing
|
|
68
|
-
const validation = validateMessages(messages);
|
|
69
|
-
if (!validation.valid) {
|
|
70
|
-
humanError(`❌ ${validation.errors.length} messages failed validation`);
|
|
71
|
-
if (verbose) {
|
|
72
|
-
validation.errors.slice(0, 5).forEach((err) => {
|
|
73
|
-
humanError(` Message ${err.index}:`, err.issues);
|
|
74
|
-
});
|
|
75
|
-
}
|
|
76
|
-
process.exit(1);
|
|
77
|
-
}
|
|
78
|
-
// Write export envelope
|
|
79
|
-
const envelope = createExportEnvelope(messages);
|
|
80
|
-
fs.writeFileSync(output, JSON.stringify(envelope, null, 2), 'utf-8');
|
|
81
|
-
humanInfo(`✓ Wrote ${messages.length.toLocaleString()} messages to ${output}`);
|
|
82
|
-
humanInfo('\n📊 Summary:');
|
|
83
|
-
const textCount = messages.filter((m) => m.messageKind === 'text').length;
|
|
84
|
-
const mediaCount = messages.filter((m) => m.messageKind === 'media').length;
|
|
85
|
-
const notifCount = messages.filter((m) => m.messageKind === 'notification').length;
|
|
86
|
-
humanInfo(` Text: ${textCount}`);
|
|
87
|
-
humanInfo(` Media: ${mediaCount}`);
|
|
88
|
-
humanInfo(` Notifications: ${notifCount}`);
|
|
89
|
-
logEvent('ingest-summary', {
|
|
90
|
-
command: 'ingest-csv',
|
|
91
|
-
phase: 'summary',
|
|
92
|
-
metrics: {
|
|
93
|
-
total: messages.length,
|
|
94
|
-
text: textCount,
|
|
95
|
-
media: mediaCount,
|
|
96
|
-
notifications: notifCount,
|
|
97
|
-
},
|
|
98
|
-
options: { output },
|
|
99
|
-
exitCode: 0,
|
|
100
|
-
});
|
|
101
|
-
}
|
|
102
|
-
/**
|
|
103
|
-
* Register the ingest-csv command with Commander
|
|
104
|
-
*/
|
|
105
|
-
export function registerIngestCSVCommand(program, getGlobalOptions) {
|
|
106
|
-
program
|
|
107
|
-
.command('ingest-csv')
|
|
108
|
-
.description('Import messages from iMazing CSV export')
|
|
109
|
-
.requiredOption('-i, --input <path>', 'path to CSV file')
|
|
110
|
-
.option('-o, --output <path>', 'output JSON file path', './messages.csv.ingested.json')
|
|
111
|
-
.option('-a, --attachments <dir...>', 'attachment root directories')
|
|
112
|
-
.action(async (options) => {
|
|
113
|
-
try {
|
|
114
|
-
await executeIngestCSV(options, getGlobalOptions());
|
|
115
|
-
process.exit(0);
|
|
116
|
-
}
|
|
117
|
-
catch (error) {
|
|
118
|
-
humanError('❌ Failed to ingest CSV:', error instanceof Error ? error.message : String(error));
|
|
119
|
-
if (getGlobalOptions().verbose && error instanceof Error) {
|
|
120
|
-
humanError(error.stack);
|
|
121
|
-
}
|
|
122
|
-
logEvent('ingest-error', {
|
|
123
|
-
command: 'ingest-csv',
|
|
124
|
-
phase: 'error',
|
|
125
|
-
error: {
|
|
126
|
-
type: error instanceof Error ? error.name : 'Unknown',
|
|
127
|
-
message: error instanceof Error ? error.message : String(error),
|
|
128
|
-
...(error instanceof Error && error.stack
|
|
129
|
-
? { stack: error.stack }
|
|
130
|
-
: {}),
|
|
131
|
-
},
|
|
132
|
-
exitCode: 2,
|
|
133
|
-
});
|
|
134
|
-
process.exit(2);
|
|
135
|
-
}
|
|
136
|
-
});
|
|
137
|
-
}
|
|
138
|
-
//# sourceMappingURL=ingest-csv.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ingest-csv.js","sourceRoot":"","sources":["../../../src/cli/commands/ingest-csv.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,cAAc,CAAA;AACpD,OAAO,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAA;AAEhD,OAAO,EAAE,aAAa,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAA;AAEhE;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACrC,OAAyB,EACzB,aAA4B;IAE5B,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAA;IAC9C,MAAM,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,aAAa,CAAA;IAExC,aAAa,CAAC,OAAO,EAAE,KAAK,CAAC,CAAA;IAC7B,gBAAgB,CAAC,cAAc,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAA;IAEzD,QAAQ,CAAC,cAAc,EAAE;QACxB,OAAO,EAAE,YAAY;QACrB,KAAK,EAAE,OAAO;QACd,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,gBAAgB,EAAE,WAAW,EAAE,MAAM,EAAE;KACjE,CAAC,CAAA;IAEF,gEAAgE;IAChE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAA;IAClC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,UAAU,CAAC,+BAA+B,KAAK,EAAE,CAAC,CAAA;QAClD,UAAU,CAAC,iBAAiB,CAAC,CAAA;QAC7B,UAAU,CAAC,0BAA0B,CAAC,CAAA;QACtC,UAAU,CAAC,iCAAiC,CAAC,CAAA;QAC7C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IAED,qEAAqE;IACrE,MAAM,eAAe,GAAa,EAAE,CAAA;IACpC,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAC3C,KAAK,MAAM,GAAG,IAAI,WAAW,EAAE,CAAC;YAC/B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACzB,UAAU,CAAC,qCAAqC,GAAG,EAAE,CAAC,CAAA;gBACtD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAChB,CAAC;YACD,IAAI,CAAC,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC;gBACrC,UAAU,CAAC,sBAAsB,GAAG,EAAE,CAAC,CAAA;gBACvC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;YAChB,CAAC;YACD,eAAe,CAAC,IAAI,CAAC,GAAG,CAAC,CAAA;QAC1B,CAAC;IACF,CAAC;SAAM,CAAC;QACP,0CAA0C;QAC1C,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,CAAA;QAClC,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,WAAW,CAAC,CAAA;QACtC,MAAM,WAAW,GAAG,IAAI,CAAC,IAAI,CAC5B,EAAE,CAAC,OAAO,EAAE,EACZ,SAAS,EACT,UAAU,EACV,aAAa,CACb,CAAA;QACD,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;YAChC,eAAe,CAAC,IAAI,CAAC,WAAW,CAAC,CAAA;YACjC,IAAI,OAAO,EAAE,CAAC;gBACb,SAAS,CAAC,IAAI,CAAC,+BAA+B,EAAE,EAAE,WAAW,EAAE,CAAC,CAAA;YACjE,CAAC;QACF,CAAC;IACF,CAAC;IAED,qEAAqE;IACrE,MAAM,EAAE,SAAS,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,GAAG,MAAM,MAAM,CACzE,4BAA4B,CAC5B,CAAA;IAED,IAAI,OAAO,EAAE,CAAC;QACb,SAAS,CAAC,IAAI,CAAC,oBAAoB,EAAE,EAAE,KAAK,EAAE,eAAe,EAAE,CAAC,CAAA;IACjE,CAAC;IAED,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,EAAE,EAAE,eAAe,EAAE,CAAC,CAAA;IAEtD,kEAAkE;IAClE,SAAS,CAAC,YAAY,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,oBAAoB,CAAC,CAAA;IAE3E,mCAAmC;IACnC,MAAM,UAAU,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAA;IAC7C,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;QACvB,UAAU,CAAC,KAAK,UAAU,CAAC,MAAM,CAAC,MAAM,6BAA6B,CAAC,CAAA;QACtE,IAAI,OAAO,EAAE,CAAC;YACb,UAAU,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC7C,UAAU,CAAC,aAAa,GAAG,CAAC,KAAK,GAAG,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;YAClD,CAAC,CAAC,CAAA;QACH,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChB,CAAC;IAED,wBAAwB;IACxB,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAA;IAC/C,EAAE,CAAC,aAAa,CAAC,MAAM,EAAE,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IAEpE,SAAS,CAAC,WAAW,QAAQ,CAAC,MAAM,CAAC,cAAc,EAAE,gBAAgB,MAAM,EAAE,CAAC,CAAA;IAC9E,SAAS,CAAC,eAAe,CAAC,CAAA;IAC1B,MAAM,SAAS,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,MAAM,CAAC,CAAC,MAAM,CAAA;IACzE,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,OAAO,CAAC,CAAC,MAAM,CAAA;IAC3E,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CACjC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,KAAK,cAAc,CACvC,CAAC,MAAM,CAAA;IACR,SAAS,CAAC,WAAW,SAAS,EAAE,CAAC,CAAA;IACjC,SAAS,CAAC,YAAY,UAAU,EAAE,CAAC,CAAA;IACnC,SAAS,CAAC,oBAAoB,UAAU,EAAE,CAAC,CAAA;IAE3C,QAAQ,CAAC,gBAAgB,EAAE;QAC1B,OAAO,EAAE,YAAY;QACrB,KAAK,EAAE,SAAS;QAChB,OAAO,EAAE;YACR,KAAK,EAAE,QAAQ,CAAC,MAAM;YACtB,IAAI,EAAE,SAAS;YACf,KAAK,EAAE,UAAU;YACjB,aAAa,EAAE,UAAU;SACzB;QACD,OAAO,EAAE,EAAE,MAAM,EAAE;QACnB,QAAQ,EAAE,CAAC;KACX,CAAC,CAAA;AACH,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,wBAAwB,CACvC,OAAgB,EAChB,gBAAqC;IAErC,OAAO;SACL,OAAO,CAAC,YAAY,CAAC;SACrB,WAAW,CAAC,yCAAyC,CAAC;SACtD,cAAc,CAAC,oBAAoB,EAAE,kBAAkB,CAAC;SACxD,MAAM,CACN,qBAAqB,EACrB,uBAAuB,EACvB,8BAA8B,CAC9B;SACA,MAAM,CAAC,4BAA4B,EAAE,6BAA6B,CAAC;SACnE,MAAM,CAAC,KAAK,EAAE,OAAyB,EAAE,EAAE;QAC3C,IAAI,CAAC;YACJ,MAAM,gBAAgB,CAAC,OAAO,EAAE,gBAAgB,EAAE,CAAC,CAAA;YACnD,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,UAAU,CACT,yBAAyB,EACzB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CACtD,CAAA;YACD,IAAI,gBAAgB,EAAE,CAAC,OAAO,IAAI,KAAK,YAAY,KAAK,EAAE,CAAC;gBAC1D,UAAU,CAAC,KAAK,CAAC,KAAK,CAAC,CAAA;YACxB,CAAC;YACD,QAAQ,CAAC,cAAc,EAAE;gBACxB,OAAO,EAAE,YAAY;gBACrB,KAAK,EAAE,OAAO;gBACd,KAAK,EAAE;oBACN,IAAI,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS;oBACrD,OAAO,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC/D,GAAG,CAAC,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,KAAK;wBACxC,CAAC,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,KAAK,EAAE;wBACxB,CAAC,CAAC,EAAE,CAAC;iBACN;gBACD,QAAQ,EAAE,CAAC;aACX,CAAC,CAAA;YACF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAChB,CAAC;IACF,CAAC,CAAC,CAAA;AACJ,CAAC"}
|
|
@@ -1,17 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Ingest DB Command
|
|
3
|
-
*
|
|
4
|
-
* Import messages from macOS Messages.app database export (JSON).
|
|
5
|
-
* CLI-T02-AC02: ingest-db command with database path and contact filtering
|
|
6
|
-
*/
|
|
7
|
-
import type { Command } from 'commander';
|
|
8
|
-
import type { GlobalOptions, IngestDBOptions } from '../types.js';
|
|
9
|
-
/**
|
|
10
|
-
* Execute the ingest-db command logic
|
|
11
|
-
*/
|
|
12
|
-
export declare function executeIngestDB(options: IngestDBOptions, globalOptions: GlobalOptions): Promise<void>;
|
|
13
|
-
/**
|
|
14
|
-
* Register the ingest-db command with Commander
|
|
15
|
-
*/
|
|
16
|
-
export declare function registerIngestDBCommand(program: Command, getGlobalOptions: () => GlobalOptions): void;
|
|
17
|
-
//# sourceMappingURL=ingest-db.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ingest-db.d.ts","sourceRoot":"","sources":["../../../src/cli/commands/ingest-db.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAIxC,OAAO,KAAK,EAAE,aAAa,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAGjE;;GAEG;AACH,wBAAsB,eAAe,CACpC,OAAO,EAAE,eAAe,EACxB,aAAa,EAAE,aAAa,GAC1B,OAAO,CAAC,IAAI,CAAC,CA2If;AAED;;GAEG;AACH,wBAAgB,uBAAuB,CACtC,OAAO,EAAE,OAAO,EAChB,gBAAgB,EAAE,MAAM,aAAa,GACnC,IAAI,CA0CN"}
|