@arela/uploader 0.2.13 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.template +66 -0
- package/README.md +263 -62
- package/docs/API_ENDPOINTS_FOR_DETECTION.md +647 -0
- package/docs/QUICK_REFERENCE_API_DETECTION.md +264 -0
- package/docs/REFACTORING_SUMMARY_DETECT_PEDIMENTOS.md +200 -0
- package/package.json +3 -2
- package/scripts/cleanup-ds-store.js +109 -0
- package/scripts/cleanup-system-files.js +69 -0
- package/scripts/tests/phase-7-features.test.js +415 -0
- package/scripts/tests/signal-handling.test.js +275 -0
- package/scripts/tests/smart-watch-integration.test.js +554 -0
- package/scripts/tests/watch-service-integration.test.js +584 -0
- package/src/commands/UploadCommand.js +31 -4
- package/src/commands/WatchCommand.js +1342 -0
- package/src/config/config.js +270 -2
- package/src/document-type-shared.js +2 -0
- package/src/document-types/support-document.js +200 -0
- package/src/file-detection.js +9 -1
- package/src/index.js +163 -4
- package/src/services/AdvancedFilterService.js +505 -0
- package/src/services/AutoProcessingService.js +749 -0
- package/src/services/BenchmarkingService.js +381 -0
- package/src/services/DatabaseService.js +1019 -539
- package/src/services/ErrorMonitor.js +275 -0
- package/src/services/LoggingService.js +419 -1
- package/src/services/MonitoringService.js +401 -0
- package/src/services/PerformanceOptimizer.js +511 -0
- package/src/services/ReportingService.js +511 -0
- package/src/services/SignalHandler.js +255 -0
- package/src/services/SmartWatchDatabaseService.js +527 -0
- package/src/services/WatchService.js +783 -0
- package/src/services/upload/ApiUploadService.js +447 -3
- package/src/services/upload/MultiApiUploadService.js +233 -0
- package/src/services/upload/SupabaseUploadService.js +12 -5
- package/src/services/upload/UploadServiceFactory.js +24 -0
- package/src/utils/CleanupManager.js +262 -0
- package/src/utils/FileOperations.js +44 -0
- package/src/utils/WatchEventHandler.js +522 -0
- package/supabase/migrations/001_create_initial_schema.sql +366 -0
- package/supabase/migrations/002_align_with_arela_api_schema.sql +145 -0
- package/.envbackup +0 -37
- package/SUPABASE_UPLOAD_FIX.md +0 -157
- package/commands.md +0 -14
|
@@ -0,0 +1,783 @@
|
|
|
1
|
+
import chokidar from 'chokidar';
|
|
2
|
+
import fs from 'fs';
|
|
3
|
+
import path from 'path';
|
|
4
|
+
|
|
5
|
+
import watchEventHandler from '../utils/WatchEventHandler.js';
|
|
6
|
+
import autoProcessingService from './AutoProcessingService.js';
|
|
7
|
+
import { databaseService } from './DatabaseService.js';
|
|
8
|
+
import logger from './LoggingService.js';
|
|
9
|
+
import SmartWatchDatabaseService from './SmartWatchDatabaseService.js';
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* WatchService - Monitors directories for file changes and triggers uploads
|
|
13
|
+
* Provides watch functionality with debouncing, multiple watchers, and event handling
|
|
14
|
+
* Integrated with SmartWatchDatabaseService for intelligent file queuing
|
|
15
|
+
*/
|
|
16
|
+
export class WatchService {
|
|
17
|
+
constructor() {
|
|
18
|
+
this.watchers = new Map(); // Map of directory -> watcher instance
|
|
19
|
+
this.watchedDirs = new Set(); // Set of watched directories
|
|
20
|
+
this.directoryConfigs = new Map(); // Map of directory -> configuration (path, folderStructure)
|
|
21
|
+
this.eventQueue = new Map(); // Queue of pending events with debounce
|
|
22
|
+
this.debounceTimers = new Map(); // Timers for debouncing
|
|
23
|
+
this.isRunning = false;
|
|
24
|
+
this.isWatchModeActive = false; // Flag to prevent direct uploads while watching
|
|
25
|
+
this.autoProcessingEnabled = false;
|
|
26
|
+
this.processingOptions = {};
|
|
27
|
+
this.watcherReady = new Map(); // Map of directory -> ready status (to ignore initial scan)
|
|
28
|
+
this.allWatchersReady = false; // Flag to indicate ALL initial scans are complete
|
|
29
|
+
|
|
30
|
+
// Watch mode state file (for inter-process communication)
|
|
31
|
+
this.watchStateFile = path.join(process.cwd(), '.watch-mode.lock');
|
|
32
|
+
|
|
33
|
+
// Smart Watch Queue Integration
|
|
34
|
+
this.smartWatchDb = new SmartWatchDatabaseService(databaseService);
|
|
35
|
+
this.queueStats = {
|
|
36
|
+
filesPending: 0,
|
|
37
|
+
filesReady: 0,
|
|
38
|
+
filesProcessing: 0,
|
|
39
|
+
filesUploaded: 0,
|
|
40
|
+
filesFailed: 0,
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
// File detection for pedimento simplificado
|
|
44
|
+
this.pedimentoPattern = /simplif|simple/i;
|
|
45
|
+
|
|
46
|
+
// Pipeline debouncing by directory to avoid conflicts
|
|
47
|
+
// When multiple files are detected in same dir, batch them together
|
|
48
|
+
this.pipelineDebounceTimers = new Map(); // Map of dirPath -> debounce timer
|
|
49
|
+
this.pipelineBatchSize = 500; // ms to wait before processing a directory batch
|
|
50
|
+
this.activePipelines = new Set(); // Track which directories have pipelines running
|
|
51
|
+
|
|
52
|
+
this.stats = {
|
|
53
|
+
filesAdded: 0,
|
|
54
|
+
filesModified: 0,
|
|
55
|
+
filesRemoved: 0,
|
|
56
|
+
uploadsTriggered: 0,
|
|
57
|
+
pipelinesTriggered: 0,
|
|
58
|
+
errorsEncountered: 0,
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/**
|
|
63
|
+
* Initialize watcher for a directory
|
|
64
|
+
* @param {string} dirPath - Directory to watch
|
|
65
|
+
* @param {Object} options - Watcher options
|
|
66
|
+
* @param {Object} dirConfig - Directory configuration (path, folderStructure, etc)
|
|
67
|
+
* @returns {Promise<void>}
|
|
68
|
+
*/
|
|
69
|
+
async addWatcher(dirPath, options = {}, dirConfig = {}) {
|
|
70
|
+
try {
|
|
71
|
+
// Validate directory path
|
|
72
|
+
if (!dirPath) {
|
|
73
|
+
throw new Error('Directory path is required');
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const normalizedPath = path.resolve(dirPath);
|
|
77
|
+
|
|
78
|
+
// Check if already watching
|
|
79
|
+
if (this.watchedDirs.has(normalizedPath)) {
|
|
80
|
+
logger.warn(`Directory already being watched: ${normalizedPath}`);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
// Default watcher options
|
|
85
|
+
const watcherOptions = {
|
|
86
|
+
persistent: true,
|
|
87
|
+
ignored: /(^|[\/\\])\.|node_modules|\.git/, // Ignore hidden files and common dirs
|
|
88
|
+
ignoreInitial: true, // Don't trigger events for files found during initial scan
|
|
89
|
+
awaitWriteFinish: {
|
|
90
|
+
stabilityThreshold: options.stabilityThreshold || 300,
|
|
91
|
+
pollInterval: options.pollInterval || 100,
|
|
92
|
+
},
|
|
93
|
+
usePolling: options.usePolling || false,
|
|
94
|
+
interval: options.interval || 100,
|
|
95
|
+
binaryInterval: options.binaryInterval || 300,
|
|
96
|
+
...options,
|
|
97
|
+
};
|
|
98
|
+
|
|
99
|
+
logger.info(`Initializing watcher for: ${normalizedPath}`);
|
|
100
|
+
|
|
101
|
+
// Create watcher
|
|
102
|
+
const watcher = chokidar.watch(normalizedPath, watcherOptions);
|
|
103
|
+
|
|
104
|
+
// Setup event handlers
|
|
105
|
+
watcher
|
|
106
|
+
.on('add', (filePath) => this.#handleFileAdded(filePath))
|
|
107
|
+
.on('change', (filePath) => this.#handleFileChanged(filePath))
|
|
108
|
+
.on('unlink', (filePath) => this.#handleFileRemoved(filePath))
|
|
109
|
+
.on('addDir', (dirPath) => this.#handleDirAdded(dirPath))
|
|
110
|
+
.on('unlinkDir', (dirPath) => this.#handleDirRemoved(dirPath))
|
|
111
|
+
.on('error', (error) => this.#handleWatcherError(error))
|
|
112
|
+
.on('ready', () => {
|
|
113
|
+
logger.info(`Watcher ready for: ${normalizedPath}`);
|
|
114
|
+
this.watcherReady.set(normalizedPath, true);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
// Store watcher and configuration
|
|
118
|
+
this.watchers.set(normalizedPath, watcher);
|
|
119
|
+
this.watchedDirs.add(normalizedPath);
|
|
120
|
+
this.directoryConfigs.set(normalizedPath, {
|
|
121
|
+
path: normalizedPath,
|
|
122
|
+
folderStructure: dirConfig.folderStructure || 'default',
|
|
123
|
+
...dirConfig,
|
|
124
|
+
});
|
|
125
|
+
|
|
126
|
+
logger.info(`✅ Watcher added for: ${normalizedPath}`);
|
|
127
|
+
} catch (error) {
|
|
128
|
+
logger.error(`Failed to add watcher for ${dirPath}: ${error.message}`);
|
|
129
|
+
this.stats.errorsEncountered++;
|
|
130
|
+
throw error;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* Remove watcher for a directory
|
|
136
|
+
* @param {string} dirPath - Directory to stop watching
|
|
137
|
+
* @returns {Promise<void>}
|
|
138
|
+
*/
|
|
139
|
+
async removeWatcher(dirPath) {
|
|
140
|
+
try {
|
|
141
|
+
const normalizedPath = path.resolve(dirPath);
|
|
142
|
+
|
|
143
|
+
if (!this.watchedDirs.has(normalizedPath)) {
|
|
144
|
+
logger.warn(`Directory not being watched: ${normalizedPath}`);
|
|
145
|
+
return;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const watcher = this.watchers.get(normalizedPath);
|
|
149
|
+
if (watcher) {
|
|
150
|
+
await watcher.close();
|
|
151
|
+
this.watchers.delete(normalizedPath);
|
|
152
|
+
this.watchedDirs.delete(normalizedPath);
|
|
153
|
+
this.directoryConfigs.delete(normalizedPath);
|
|
154
|
+
this.watcherReady.delete(normalizedPath);
|
|
155
|
+
logger.info(`✅ Watcher removed for: ${normalizedPath}`);
|
|
156
|
+
}
|
|
157
|
+
} catch (error) {
|
|
158
|
+
logger.error(`Failed to remove watcher for ${dirPath}: ${error.message}`);
|
|
159
|
+
this.stats.errorsEncountered++;
|
|
160
|
+
throw error;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Start watching all configured directories
|
|
166
|
+
* @returns {Promise<void>}
|
|
167
|
+
*/
|
|
168
|
+
async start() {
|
|
169
|
+
try {
|
|
170
|
+
if (this.isRunning) {
|
|
171
|
+
logger.warn('WatchService is already running');
|
|
172
|
+
return;
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
this.isRunning = true;
|
|
176
|
+
this.isWatchModeActive = true; // Prevent direct uploads while watching
|
|
177
|
+
|
|
178
|
+
// Create watch mode state file for inter-process communication
|
|
179
|
+
try {
|
|
180
|
+
fs.writeFileSync(
|
|
181
|
+
this.watchStateFile,
|
|
182
|
+
JSON.stringify({
|
|
183
|
+
started: new Date().toISOString(),
|
|
184
|
+
pid: process.pid,
|
|
185
|
+
timestamp: Date.now(),
|
|
186
|
+
}),
|
|
187
|
+
);
|
|
188
|
+
logger.debug(`📌 Watch state file created at ${this.watchStateFile}`);
|
|
189
|
+
} catch (error) {
|
|
190
|
+
logger.warn(`Could not create watch state file: ${error.message}`);
|
|
191
|
+
}
|
|
192
|
+
|
|
193
|
+
logger.info('🟢 WatchService started');
|
|
194
|
+
logger.info(`📁 Watching ${this.watchedDirs.size} director(y/ies)`);
|
|
195
|
+
|
|
196
|
+
// List watched directories
|
|
197
|
+
this.watchedDirs.forEach((dir) => {
|
|
198
|
+
logger.info(` → ${dir}`);
|
|
199
|
+
});
|
|
200
|
+
} catch (error) {
|
|
201
|
+
logger.error(`Failed to start WatchService: ${error.message}`);
|
|
202
|
+
this.stats.errorsEncountered++;
|
|
203
|
+
this.isRunning = false;
|
|
204
|
+
throw error;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
/**
|
|
209
|
+
* Stop watching all directories
|
|
210
|
+
* @returns {Promise<void>}
|
|
211
|
+
*/
|
|
212
|
+
async stop(reason = 'unknown') {
|
|
213
|
+
try {
|
|
214
|
+
logger.info(`🔴 Stopping WatchService (reason: ${reason})...`);
|
|
215
|
+
|
|
216
|
+
// Clear all timers
|
|
217
|
+
this.debounceTimers.forEach((timer) => clearTimeout(timer));
|
|
218
|
+
this.debounceTimers.clear();
|
|
219
|
+
this.eventQueue.clear();
|
|
220
|
+
|
|
221
|
+
// Close all watchers
|
|
222
|
+
const closePromises = Array.from(this.watchers.values()).map((watcher) =>
|
|
223
|
+
watcher.close(),
|
|
224
|
+
);
|
|
225
|
+
|
|
226
|
+
await Promise.all(closePromises);
|
|
227
|
+
|
|
228
|
+
this.watchers.clear();
|
|
229
|
+
this.watchedDirs.clear();
|
|
230
|
+
this.watcherReady.clear();
|
|
231
|
+
this.allWatchersReady = false;
|
|
232
|
+
this.isWatchModeActive = false; // Allow direct uploads again
|
|
233
|
+
this.isRunning = false;
|
|
234
|
+
|
|
235
|
+
// Clean up watch mode state file for inter-process communication
|
|
236
|
+
try {
|
|
237
|
+
if (fs.existsSync(this.watchStateFile)) {
|
|
238
|
+
fs.unlinkSync(this.watchStateFile);
|
|
239
|
+
logger.debug('🧹 Cleaned up watch state file');
|
|
240
|
+
}
|
|
241
|
+
} catch (error) {
|
|
242
|
+
logger.warn(`Could not clean up watch state file: ${error.message}`);
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
// Save final state
|
|
246
|
+
await this.#saveState();
|
|
247
|
+
|
|
248
|
+
logger.info('✅ WatchService stopped successfully');
|
|
249
|
+
|
|
250
|
+
return {
|
|
251
|
+
success: true,
|
|
252
|
+
reason,
|
|
253
|
+
stats: this.getStats(),
|
|
254
|
+
};
|
|
255
|
+
} catch (error) {
|
|
256
|
+
logger.error(`Error stopping WatchService: ${error.message}`);
|
|
257
|
+
this.stats.errorsEncountered++;
|
|
258
|
+
|
|
259
|
+
return {
|
|
260
|
+
success: false,
|
|
261
|
+
reason,
|
|
262
|
+
error: error.message,
|
|
263
|
+
stats: this.getStats(),
|
|
264
|
+
};
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Handle file added event
|
|
270
|
+
* Integrates with Smart Watch Queue for intelligent file management
|
|
271
|
+
* @private
|
|
272
|
+
* @param {string} filePath - Path to added file
|
|
273
|
+
*/
|
|
274
|
+
async #handleFileAdded(filePath) {
|
|
275
|
+
logger.debug(`📄 File added: ${filePath}`);
|
|
276
|
+
this.stats.filesAdded++;
|
|
277
|
+
|
|
278
|
+
try {
|
|
279
|
+
const parentDir = path.dirname(filePath);
|
|
280
|
+
const fileName = path.basename(filePath);
|
|
281
|
+
const stats = fs.statSync(filePath);
|
|
282
|
+
|
|
283
|
+
// Check if file is a pedimento simplificado
|
|
284
|
+
const isPedimento = this.#isPedimentoSimplificado(fileName);
|
|
285
|
+
|
|
286
|
+
if (isPedimento) {
|
|
287
|
+
// Register pedimento and auto-update pending files
|
|
288
|
+
logger.info(`🎯 Pedimento detected: ${fileName}`);
|
|
289
|
+
await this.smartWatchDb.markPedimentoDetected(parentDir, filePath);
|
|
290
|
+
this.stats.uploadsTriggered++;
|
|
291
|
+
} else {
|
|
292
|
+
// Register file as PENDING, waiting for pedimento
|
|
293
|
+
logger.debug(`⏳ Registering file as PENDING: ${fileName}`);
|
|
294
|
+
await this.smartWatchDb.insertFileToUploader(filePath, {
|
|
295
|
+
processingStatus: 'PENDING',
|
|
296
|
+
dependsOnPath: parentDir,
|
|
297
|
+
size: stats.size,
|
|
298
|
+
fileExtension: path.extname(fileName).slice(1).toLowerCase(),
|
|
299
|
+
});
|
|
300
|
+
}
|
|
301
|
+
} catch (error) {
|
|
302
|
+
logger.error(`Error handling file added event: ${error.message}`);
|
|
303
|
+
this.stats.errorsEncountered++;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// If auto-processing is enabled, trigger the 4-step pipeline
|
|
307
|
+
// BUT only after ALL initial scans are complete (allWatchersReady)
|
|
308
|
+
if (this.autoProcessingEnabled) {
|
|
309
|
+
if (this.allWatchersReady) {
|
|
310
|
+
// Use debouncing to batch multiple files detected in quick succession
|
|
311
|
+
const parentDir = path.dirname(filePath);
|
|
312
|
+
this.#triggerAutoPipelineWithDebounce(filePath, parentDir);
|
|
313
|
+
return;
|
|
314
|
+
} else {
|
|
315
|
+
// During initial scan, silently skip - don't log to avoid noise
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
// Queue for regular upload processing
|
|
320
|
+
this.#queueEvent('add', filePath);
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
/**
|
|
324
|
+
* Trigger auto pipeline with debounce to avoid duplicates
|
|
325
|
+
* @private
|
|
326
|
+
* @param {string} filePath - Path to file
|
|
327
|
+
* @param {string} parentDir - Parent directory
|
|
328
|
+
* @returns {void}
|
|
329
|
+
*/
|
|
330
|
+
#triggerAutoPipelineWithDebounce(filePath, parentDir) {
|
|
331
|
+
// Clear existing timer for this directory
|
|
332
|
+
if (this.pipelineDebounceTimers.has(parentDir)) {
|
|
333
|
+
clearTimeout(this.pipelineDebounceTimers.get(parentDir));
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// Set new timer with increased debounce
|
|
337
|
+
const timer = setTimeout(() => {
|
|
338
|
+
this.#triggerAutoPipeline(filePath);
|
|
339
|
+
this.pipelineDebounceTimers.delete(parentDir);
|
|
340
|
+
}, 1000); // Increased to 1 second for better debouncing
|
|
341
|
+
|
|
342
|
+
this.pipelineDebounceTimers.set(parentDir, timer);
|
|
343
|
+
}
|
|
344
|
+
|
|
345
|
+
/**
|
|
346
|
+
* Handle file changed event
|
|
347
|
+
* @private
|
|
348
|
+
* @param {string} filePath - Path to changed file
|
|
349
|
+
*/
|
|
350
|
+
#handleFileChanged(filePath) {
|
|
351
|
+
logger.debug(`✏️ File changed: ${filePath}`);
|
|
352
|
+
this.stats.filesModified++;
|
|
353
|
+
this.#queueEvent('change', filePath);
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
/**
|
|
357
|
+
* Handle file removed event
|
|
358
|
+
* @private
|
|
359
|
+
* @param {string} filePath - Path to removed file
|
|
360
|
+
*/
|
|
361
|
+
#handleFileRemoved(filePath) {
|
|
362
|
+
logger.debug(`🗑️ File removed: ${filePath}`);
|
|
363
|
+
this.stats.filesRemoved++;
|
|
364
|
+
// Don't queue removal events as they don't need upload
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
/**
|
|
368
|
+
* Handle directory added event
|
|
369
|
+
* @private
|
|
370
|
+
* @param {string} dirPath - Path to added directory
|
|
371
|
+
*/
|
|
372
|
+
#handleDirAdded(dirPath) {
|
|
373
|
+
logger.debug(`📁 Directory added: ${dirPath}`);
|
|
374
|
+
}
|
|
375
|
+
|
|
376
|
+
/**
|
|
377
|
+
* Handle directory removed event
|
|
378
|
+
* @private
|
|
379
|
+
* @param {string} dirPath - Path to removed directory
|
|
380
|
+
*/
|
|
381
|
+
#handleDirRemoved(dirPath) {
|
|
382
|
+
logger.debug(`🗑️ Directory removed: ${dirPath}`);
|
|
383
|
+
}
|
|
384
|
+
|
|
385
|
+
/**
|
|
386
|
+
* Handle watcher error
|
|
387
|
+
* @private
|
|
388
|
+
* @param {Error} error - Error from watcher
|
|
389
|
+
*/
|
|
390
|
+
#handleWatcherError(error) {
|
|
391
|
+
logger.error(`Watcher error: ${error.message}`);
|
|
392
|
+
this.stats.errorsEncountered++;
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
/**
|
|
396
|
+
* Queue an event with debouncing
|
|
397
|
+
* @private
|
|
398
|
+
* @param {string} eventType - Type of event (add, change, etc)
|
|
399
|
+
* @param {string} filePath - File path
|
|
400
|
+
* @param {number} debounceMs - Debounce delay in milliseconds
|
|
401
|
+
*/
|
|
402
|
+
#queueEvent(eventType, filePath, debounceMs = 1000) {
|
|
403
|
+
const eventKey = `${eventType}:${filePath}`;
|
|
404
|
+
|
|
405
|
+
// Clear existing timer if any
|
|
406
|
+
if (this.debounceTimers.has(eventKey)) {
|
|
407
|
+
clearTimeout(this.debounceTimers.get(eventKey));
|
|
408
|
+
}
|
|
409
|
+
|
|
410
|
+
// Set new debounce timer
|
|
411
|
+
const timer = setTimeout(() => {
|
|
412
|
+
this.#processEvent(eventType, filePath);
|
|
413
|
+
this.debounceTimers.delete(eventKey);
|
|
414
|
+
}, debounceMs);
|
|
415
|
+
|
|
416
|
+
this.debounceTimers.set(eventKey, timer);
|
|
417
|
+
}
|
|
418
|
+
|
|
419
|
+
/**
|
|
420
|
+
* Get current watch statistics
|
|
421
|
+
* @returns {Object} Statistics object
|
|
422
|
+
*/
|
|
423
|
+
getStats() {
|
|
424
|
+
return {
|
|
425
|
+
...this.stats,
|
|
426
|
+
watchedDirectories: this.watchedDirs.size,
|
|
427
|
+
activeWatchers: this.watchers.size,
|
|
428
|
+
isRunning: this.isRunning,
|
|
429
|
+
autoProcessingEnabled: this.autoProcessingEnabled,
|
|
430
|
+
};
|
|
431
|
+
}
|
|
432
|
+
|
|
433
|
+
/**
|
|
434
|
+
* Reset statistics
|
|
435
|
+
*/
|
|
436
|
+
resetStats() {
|
|
437
|
+
this.stats = {
|
|
438
|
+
filesAdded: 0,
|
|
439
|
+
filesModified: 0,
|
|
440
|
+
filesRemoved: 0,
|
|
441
|
+
uploadsTriggered: 0,
|
|
442
|
+
errorsEncountered: 0,
|
|
443
|
+
};
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
/**
|
|
447
|
+
* Check if service is running
|
|
448
|
+
* @returns {boolean} Running status
|
|
449
|
+
*/
|
|
450
|
+
isWatching() {
|
|
451
|
+
return this.isRunning;
|
|
452
|
+
}
|
|
453
|
+
|
|
454
|
+
/**
|
|
455
|
+
* Get list of watched directories
|
|
456
|
+
* @returns {Array<string>} Watched directories
|
|
457
|
+
*/
|
|
458
|
+
getWatchedDirs() {
|
|
459
|
+
return Array.from(this.watchedDirs);
|
|
460
|
+
}
|
|
461
|
+
|
|
462
|
+
/**
|
|
463
|
+
* Set custom event handler for processed events
|
|
464
|
+
* Used by WatchCommand to handle upload logic
|
|
465
|
+
* @param {Function} handler - Callback function (eventType, filePath) => void
|
|
466
|
+
*/
|
|
467
|
+
setEventHandler(handler) {
|
|
468
|
+
if (typeof handler !== 'function') {
|
|
469
|
+
throw new Error('Event handler must be a function');
|
|
470
|
+
}
|
|
471
|
+
this.#customEventHandler = handler;
|
|
472
|
+
}
|
|
473
|
+
|
|
474
|
+
/**
|
|
475
|
+
* Custom event handler (to be set by WatchCommand)
|
|
476
|
+
* @private
|
|
477
|
+
*/
|
|
478
|
+
#customEventHandler = null;
|
|
479
|
+
|
|
480
|
+
/**
|
|
481
|
+
* Process event with custom handler if set
|
|
482
|
+
* @private
|
|
483
|
+
* @param {string} eventType - Type of event
|
|
484
|
+
* @param {string} filePath - File path
|
|
485
|
+
*/
|
|
486
|
+
#processEvent(eventType, filePath) {
|
|
487
|
+
logger.info(`⚡ Processing event: ${eventType} - ${filePath}`);
|
|
488
|
+
|
|
489
|
+
// Register event in handler
|
|
490
|
+
watchEventHandler.registerEvent(eventType, filePath);
|
|
491
|
+
|
|
492
|
+
if (this.#customEventHandler) {
|
|
493
|
+
try {
|
|
494
|
+
this.#customEventHandler(eventType, filePath);
|
|
495
|
+
} catch (error) {
|
|
496
|
+
logger.error(`Error in custom event handler: ${error.message}`);
|
|
497
|
+
this.stats.errorsEncountered++;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
/**
|
|
503
|
+
* Trigger the automatic 4-step processing pipeline
|
|
504
|
+
* @private
|
|
505
|
+
* @param {string} filePath - Path to the newly added file
|
|
506
|
+
* @returns {Promise<void>}
|
|
507
|
+
*/
|
|
508
|
+
async #triggerAutoPipeline(filePath) {
|
|
509
|
+
try {
|
|
510
|
+
// Find which watch directory this file belongs to
|
|
511
|
+
const watchDir = this.#findWatchDirectory(filePath);
|
|
512
|
+
if (!watchDir) {
|
|
513
|
+
logger.warn(
|
|
514
|
+
`[AutoPipeline] File not in any watched directory: ${filePath}`,
|
|
515
|
+
);
|
|
516
|
+
return;
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
const dirConfig = this.directoryConfigs.get(watchDir);
|
|
520
|
+
if (!dirConfig) {
|
|
521
|
+
logger.warn(
|
|
522
|
+
`[AutoPipeline] No configuration found for watch directory: ${watchDir}`,
|
|
523
|
+
);
|
|
524
|
+
return;
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
logger.info(
|
|
528
|
+
`[AutoPipeline] Triggering 4-step processing pipeline for: ${filePath}`,
|
|
529
|
+
);
|
|
530
|
+
this.stats.pipelinesTriggered++;
|
|
531
|
+
|
|
532
|
+
// Execute the 4-step pipeline
|
|
533
|
+
const result = await autoProcessingService.executeProcessingPipeline({
|
|
534
|
+
filePath,
|
|
535
|
+
watchDir,
|
|
536
|
+
folderStructure: dirConfig.folderStructure || 'default',
|
|
537
|
+
batchSize: this.processingOptions.batchSize || 10,
|
|
538
|
+
});
|
|
539
|
+
|
|
540
|
+
if (result.summary?.success) {
|
|
541
|
+
logger.info(
|
|
542
|
+
`[AutoPipeline] ✅ Pipeline completed successfully (ID: ${result.pipelineId})`,
|
|
543
|
+
);
|
|
544
|
+
} else {
|
|
545
|
+
const failureMsg = result.summary?.message || 'Unknown error occurred';
|
|
546
|
+
logger.error(`[AutoPipeline] ❌ Pipeline failed: ${failureMsg}`);
|
|
547
|
+
}
|
|
548
|
+
} catch (error) {
|
|
549
|
+
logger.error(
|
|
550
|
+
`[AutoPipeline] Error triggering auto-processing pipeline: ${error.message}`,
|
|
551
|
+
);
|
|
552
|
+
this.stats.errorsEncountered++;
|
|
553
|
+
}
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
/**
|
|
557
|
+
* Find which watch directory a file belongs to
|
|
558
|
+
* @private
|
|
559
|
+
* @param {string} filePath - Path to file
|
|
560
|
+
* @returns {string|null} Watch directory path or null
|
|
561
|
+
*/
|
|
562
|
+
#findWatchDirectory(filePath) {
|
|
563
|
+
const absolutePath = path.resolve(filePath);
|
|
564
|
+
|
|
565
|
+
for (const watchDir of this.watchedDirs) {
|
|
566
|
+
if (absolutePath.startsWith(watchDir)) {
|
|
567
|
+
return watchDir;
|
|
568
|
+
}
|
|
569
|
+
}
|
|
570
|
+
|
|
571
|
+
return null;
|
|
572
|
+
}
|
|
573
|
+
|
|
574
|
+
/**
|
|
575
|
+
* Detect if a file is a "pedimento simplificado"
|
|
576
|
+
* @private
|
|
577
|
+
* @param {string} fileName - File name to check
|
|
578
|
+
* @returns {boolean} True if file matches pedimento pattern
|
|
579
|
+
*/
|
|
580
|
+
#isPedimentoSimplificado(fileName) {
|
|
581
|
+
return this.pedimentoPattern.test(fileName);
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
/**
|
|
585
|
+
* Enable automatic processing pipeline on new files
|
|
586
|
+
* @param {Object} options - Processing options
|
|
587
|
+
* @param {number} options.batchSize - Batch size for processing
|
|
588
|
+
* @returns {void}
|
|
589
|
+
*/
|
|
590
|
+
enableAutoProcessing(options = {}) {
|
|
591
|
+
this.autoProcessingEnabled = true;
|
|
592
|
+
this.processingOptions = options;
|
|
593
|
+
logger.info(
|
|
594
|
+
`[AutoPipeline] Automatic processing enabled with options:`,
|
|
595
|
+
options,
|
|
596
|
+
);
|
|
597
|
+
}
|
|
598
|
+
|
|
599
|
+
/**
|
|
600
|
+
* Disable automatic processing pipeline
|
|
601
|
+
* @returns {void}
|
|
602
|
+
*/
|
|
603
|
+
disableAutoProcessing() {
|
|
604
|
+
this.autoProcessingEnabled = false;
|
|
605
|
+
logger.info('[AutoPipeline] Automatic processing disabled');
|
|
606
|
+
}
|
|
607
|
+
|
|
608
|
+
/**
|
|
609
|
+
* Get current Smart Watch Queue statistics
|
|
610
|
+
* @returns {Object} Queue statistics
|
|
611
|
+
*/
|
|
612
|
+
async getQueueStats() {
|
|
613
|
+
try {
|
|
614
|
+
const stats = await this.smartWatchDb.getProcessingStats();
|
|
615
|
+
const progress = await this.smartWatchDb.getOverallProgress();
|
|
616
|
+
|
|
617
|
+
return {
|
|
618
|
+
queue: {
|
|
619
|
+
pending: stats.pending,
|
|
620
|
+
readyToUpload: stats.readyToUpload,
|
|
621
|
+
processing: stats.processing,
|
|
622
|
+
uploaded: stats.uploaded,
|
|
623
|
+
failed: stats.failed,
|
|
624
|
+
total: stats.total,
|
|
625
|
+
},
|
|
626
|
+
progress: {
|
|
627
|
+
percentComplete: progress.percentComplete,
|
|
628
|
+
estimatedTimeRemaining: progress.estimatedTimeRemaining,
|
|
629
|
+
avgProcessingTimeMs: progress.avgProcessingTimeMs,
|
|
630
|
+
},
|
|
631
|
+
timestamp: new Date().toISOString(),
|
|
632
|
+
};
|
|
633
|
+
} catch (error) {
|
|
634
|
+
logger.error(`Error getting queue stats: ${error.message}`);
|
|
635
|
+
return null;
|
|
636
|
+
}
|
|
637
|
+
}
|
|
638
|
+
|
|
639
|
+
/**
|
|
640
|
+
* Check if auto-processing is enabled
|
|
641
|
+
* @returns {boolean} Auto-processing status
|
|
642
|
+
*/
|
|
643
|
+
isAutoProcessingEnabled() {
|
|
644
|
+
return this.autoProcessingEnabled;
|
|
645
|
+
}
|
|
646
|
+
|
|
647
|
+
/**
|
|
648
|
+
* Wait for all watchers to complete initial scan (ready state)
|
|
649
|
+
* @returns {Promise<void>}
|
|
650
|
+
*/
|
|
651
|
+
async waitForWatchersReady() {
|
|
652
|
+
if (this.watchedDirs.size === 0) {
|
|
653
|
+
logger.info('✅ No watchers to wait for');
|
|
654
|
+
this.allWatchersReady = true;
|
|
655
|
+
return;
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
logger.info(
|
|
659
|
+
`⏳ Waiting for ${this.watchedDirs.size} watcher(s) to complete initial scan...`,
|
|
660
|
+
);
|
|
661
|
+
|
|
662
|
+
// Set a reasonable timeout (30 seconds max)
|
|
663
|
+
const timeout = 30000;
|
|
664
|
+
const startTime = Date.now();
|
|
665
|
+
|
|
666
|
+
while (Date.now() - startTime < timeout) {
|
|
667
|
+
const readyCounts = Array.from(this.watchedDirs).filter(
|
|
668
|
+
(dir) => this.watcherReady.get(dir) === true,
|
|
669
|
+
).length;
|
|
670
|
+
|
|
671
|
+
if (readyCounts === this.watchedDirs.size) {
|
|
672
|
+
logger.info(`✅ All ${this.watchedDirs.size} watcher(s) ready`);
|
|
673
|
+
this.allWatchersReady = true; // Set global flag
|
|
674
|
+
return;
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
// Wait a bit before checking again
|
|
678
|
+
await new Promise((resolve) => setTimeout(resolve, 100));
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
logger.warn(
|
|
682
|
+
'Timeout waiting for watchers to be ready, but setting flag to proceed',
|
|
683
|
+
);
|
|
684
|
+
this.allWatchersReady = true; // Set flag anyway to unblock
|
|
685
|
+
}
|
|
686
|
+
|
|
687
|
+
/**
|
|
688
|
+
* Save the current state before shutdown
|
|
689
|
+
* @private
|
|
690
|
+
* @returns {Promise<void>}
|
|
691
|
+
*/
|
|
692
|
+
async #saveState() {
|
|
693
|
+
try {
|
|
694
|
+
logger.debug('WatchService: Saving current state...');
|
|
695
|
+
|
|
696
|
+
// Log final stats
|
|
697
|
+
const stats = this.getStats();
|
|
698
|
+
logger.info('WatchService: Final Stats');
|
|
699
|
+
logger.info(`├─ Files Added: ${stats.filesAdded}`);
|
|
700
|
+
logger.info(`├─ Files Modified: ${stats.filesModified}`);
|
|
701
|
+
logger.info(`├─ Files Removed: ${stats.filesRemoved}`);
|
|
702
|
+
logger.info(`├─ Uploads Triggered: ${stats.uploadsTriggered}`);
|
|
703
|
+
logger.info(`└─ Errors Encountered: ${stats.errorsEncountered}`);
|
|
704
|
+
|
|
705
|
+
// Flush any pending events
|
|
706
|
+
this.eventQueue.clear();
|
|
707
|
+
|
|
708
|
+
logger.debug('WatchService: State saved successfully');
|
|
709
|
+
} catch (error) {
|
|
710
|
+
logger.error(`WatchService: Error saving state: ${error.message}`);
|
|
711
|
+
}
|
|
712
|
+
}
|
|
713
|
+
|
|
714
|
+
/**
|
|
715
|
+
* Check if watch mode is currently active (prevents direct uploads)
|
|
716
|
+
* @returns {boolean} True if watch mode is active
|
|
717
|
+
*/
|
|
718
|
+
isWatchActive() {
|
|
719
|
+
// Check lock file for inter-process communication
|
|
720
|
+
// This allows upload command to detect watch mode even in separate process
|
|
721
|
+
try {
|
|
722
|
+
if (!fs.existsSync(this.watchStateFile)) {
|
|
723
|
+
return this.isWatchModeActive;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
// Read lock file to check process status
|
|
727
|
+
const lockData = JSON.parse(
|
|
728
|
+
fs.readFileSync(this.watchStateFile, 'utf-8'),
|
|
729
|
+
);
|
|
730
|
+
const lockPid = lockData.pid;
|
|
731
|
+
const lockTimestamp = lockData.timestamp;
|
|
732
|
+
|
|
733
|
+
// Check if process is still alive
|
|
734
|
+
if (lockPid && typeof lockPid === 'number') {
|
|
735
|
+
try {
|
|
736
|
+
// Sending signal 0 checks if process exists without killing it
|
|
737
|
+
process.kill(lockPid, 0);
|
|
738
|
+
return true; // Process exists
|
|
739
|
+
} catch (error) {
|
|
740
|
+
// Process doesn't exist, clean up stale lock file
|
|
741
|
+
logger.warn(
|
|
742
|
+
`🧹 Found stale watch lock (PID ${lockPid} no longer exists), cleaning up...`,
|
|
743
|
+
);
|
|
744
|
+
try {
|
|
745
|
+
fs.unlinkSync(this.watchStateFile);
|
|
746
|
+
logger.debug('✅ Stale watch lock removed');
|
|
747
|
+
} catch (cleanupError) {
|
|
748
|
+
logger.warn(
|
|
749
|
+
`Could not remove stale lock file: ${cleanupError.message}`,
|
|
750
|
+
);
|
|
751
|
+
}
|
|
752
|
+
return false;
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
|
|
756
|
+
// Fallback: check if lock file is too old (>1 hour)
|
|
757
|
+
const ageMs = Date.now() - (lockTimestamp || 0);
|
|
758
|
+
if (ageMs > 3600000) {
|
|
759
|
+
logger.warn(
|
|
760
|
+
'🧹 Found very old watch lock file (>1 hour), cleaning up...',
|
|
761
|
+
);
|
|
762
|
+
try {
|
|
763
|
+
fs.unlinkSync(this.watchStateFile);
|
|
764
|
+
logger.debug('✅ Old watch lock removed');
|
|
765
|
+
} catch (cleanupError) {
|
|
766
|
+
logger.warn(
|
|
767
|
+
`Could not remove old lock file: ${cleanupError.message}`,
|
|
768
|
+
);
|
|
769
|
+
}
|
|
770
|
+
return false;
|
|
771
|
+
}
|
|
772
|
+
|
|
773
|
+
return true;
|
|
774
|
+
} catch (error) {
|
|
775
|
+
// If we can't read the file properly, fall back to memory flag
|
|
776
|
+
logger.debug(`Error checking watch state file: ${error.message}`);
|
|
777
|
+
return this.isWatchModeActive;
|
|
778
|
+
}
|
|
779
|
+
}
|
|
780
|
+
}
|
|
781
|
+
|
|
782
|
+
// Export singleton instance
|
|
783
|
+
export default new WatchService();
|