@arela/uploader 0.2.12 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. package/.env.template +66 -0
  2. package/.vscode/settings.json +1 -0
  3. package/README.md +134 -58
  4. package/SUPABASE_UPLOAD_FIX.md +157 -0
  5. package/package.json +3 -2
  6. package/scripts/cleanup-ds-store.js +109 -0
  7. package/scripts/cleanup-system-files.js +69 -0
  8. package/scripts/tests/phase-7-features.test.js +415 -0
  9. package/scripts/tests/signal-handling.test.js +275 -0
  10. package/scripts/tests/smart-watch-integration.test.js +554 -0
  11. package/scripts/tests/watch-service-integration.test.js +584 -0
  12. package/src/commands/UploadCommand.js +36 -2
  13. package/src/commands/WatchCommand.js +1305 -0
  14. package/src/config/config.js +113 -0
  15. package/src/document-type-shared.js +2 -0
  16. package/src/document-types/support-document.js +201 -0
  17. package/src/file-detection.js +2 -1
  18. package/src/index.js +44 -0
  19. package/src/services/AdvancedFilterService.js +505 -0
  20. package/src/services/AutoProcessingService.js +639 -0
  21. package/src/services/BenchmarkingService.js +381 -0
  22. package/src/services/DatabaseService.js +723 -170
  23. package/src/services/ErrorMonitor.js +275 -0
  24. package/src/services/LoggingService.js +419 -1
  25. package/src/services/MonitoringService.js +401 -0
  26. package/src/services/PerformanceOptimizer.js +511 -0
  27. package/src/services/ReportingService.js +511 -0
  28. package/src/services/SignalHandler.js +255 -0
  29. package/src/services/SmartWatchDatabaseService.js +527 -0
  30. package/src/services/WatchService.js +783 -0
  31. package/src/services/upload/ApiUploadService.js +30 -4
  32. package/src/services/upload/SupabaseUploadService.js +28 -6
  33. package/src/utils/CleanupManager.js +262 -0
  34. package/src/utils/FileOperations.js +41 -0
  35. package/src/utils/WatchEventHandler.js +517 -0
  36. package/supabase/migrations/001_create_initial_schema.sql +366 -0
  37. package/supabase/migrations/002_align_with_arela_api_schema.sql +145 -0
  38. package/commands.md +0 -6
package/.env.template CHANGED
@@ -63,6 +63,72 @@ MAX_CONCURRENT_SOURCES=2
63
63
  # MAX_CONCURRENT_SOURCES=1
64
64
  # BATCH_DELAY=100
65
65
 
66
+ # =============================================================================
67
+ # WATCH MODE CONFIGURATION
68
+ # =============================================================================
69
+
70
+ # Habilitar watch mode (true/false)
71
+ WATCH_ENABLED=false
72
+
73
+ # Configuración de directorios a observar (formato JSON)
74
+ # Cada directorio puede tener su propia folderStructure para organizar en el bucket
75
+ # Formato: {"ruta/directorio1":"estructura-1","ruta/directorio2":"estructura-2"}
76
+ #
77
+ # IMPORTANTE: El folderStructure se usa en el comando:
78
+ # upload --upload-by-rfc --folder-structure <estructura>
79
+ #
80
+ # Ejemplo:
81
+ WATCH_DIRECTORY_CONFIGS={"../../Documents/2022":"estructura-2022","../../Documents/2023":"estructura-2023"}
82
+
83
+ # DEPRECATED: Configuración antigua (se mantiene para compatibilidad hacia atrás)
84
+ # WATCH_DIRECTORIES=/ruta/carpeta1,/ruta/carpeta2
85
+
86
+ # Estrategia de upload (opciones: individual|batch|full-structure)
87
+ # - individual: Sube solo el archivo modificado más reciente
88
+ # - batch: Sube un lote de N archivos recientes
89
+ # - full-structure: Sube la estructura completa de carpetas
90
+ WATCH_STRATEGY=batch
91
+
92
+ # Debouncing en milisegundos (esperar entre eventos antes de procesar)
93
+ WATCH_DEBOUNCE_MS=1000
94
+
95
+ # Tamaño de batch para strategy batch
96
+ WATCH_BATCH_SIZE=10
97
+
98
+ # Usar polling en lugar de eventos nativos del filesystem
99
+ # Útil para sistemas de archivos remotos o NFS
100
+ WATCH_USE_POLLING=false
101
+
102
+ # Interval de polling en milisegundos (solo si WATCH_USE_POLLING=true)
103
+ WATCH_POLL_INTERVAL=100
104
+
105
+ # Umbral de estabilidad en ms (esperar a que el archivo deje de cambiar)
106
+ WATCH_STABILITY_THRESHOLD=300
107
+
108
+ # Patrones a ignorar (separados por coma, se usan como regex)
109
+ WATCH_IGNORE_PATTERNS=*.tmp,*.bak,*.swp
110
+
111
+ # Detección automática de tipos de documento
112
+ WATCH_AUTO_DETECT=false
113
+
114
+ # Organización automática de archivos
115
+ WATCH_AUTO_ORGANIZE=false
116
+
117
+ # =============================================================================
118
+ # WATCH MODE - AUTOMATIC PROCESSING PIPELINE
119
+ # =============================================================================
120
+ #
121
+ # El pipeline automático ejecuta la siguiente secuencia cuando se detecta un archivo nuevo:
122
+ # 1. Stats Collection → stats --stats-only (recopila información del archivo)
123
+ # 2. PDF Detection → detect --detect-pdfs (identifica pedimentos simplificados)
124
+ # 3. Path Propagation → detect --propagate-arela-path (propaga a documentos relacionados)
125
+ # 4. RFC Upload → upload --upload-by-rfc --folder-structure (sube con estructura)
126
+ #
127
+ # El pipeline se habilita automáticamente en watch mode y usa la folderStructure
128
+ # definida para cada WATCH_DIRECTORY_CONFIGS
129
+ #
130
+ # Para deshabilitar en CLI, usa: arela watch --no-auto-processing
131
+
66
132
  # =============================================================================
67
133
  # LOGGING AND MONITORING
68
134
  # =============================================================================
@@ -0,0 +1 @@
1
+ {}
package/README.md CHANGED
@@ -84,6 +84,12 @@ arela --upload-by-rfc # Phase 4: Upload by RFC
84
84
  - 📋 **Upload files by specific RFC values**
85
85
  - 🔍 **Propagate arela_path from pedimento documents to related files**
86
86
  - ⚡ **4-Phase optimized workflow for maximum performance**
87
+ - 👁️ **Watch Mode** ⭐ NEW - Monitor directories for changes and upload automatically
88
+ - Multiple watch strategies (batch, individual, full-structure)
89
+ - Debounce and polling support
90
+ - Auto-processing pipeline
91
+ - Dry-run mode for testing
92
+ - Pattern-based file ignoring
87
93
 
88
94
  ## Installation
89
95
 
@@ -97,78 +103,125 @@ npm install -g @arela/uploader
97
103
 
98
104
  ```bash
99
105
  # Run all phases automatically (most efficient)
100
- arela --run-all-phases --batch-size 20
106
+ arela upload --run-all-phases --batch-size 20
101
107
 
102
108
  # Or run phases individually for fine-grained control
103
- arela --stats-only # Phase 1: Filesystem stats only
104
- arela --detect-pdfs --batch-size 10 # Phase 2: PDF detection
105
- arela --propagate-arela-path # Phase 3: Path propagation
106
- arela --upload-by-rfc --batch-size 5 # Phase 4: RFC-based upload
109
+ arela stats # Phase 1: Filesystem stats only
110
+ arela detect # Phase 2: PDF detection
111
+ arela detect --propagate-arela-path # Phase 3: Path propagation
112
+ arela upload --upload-by-rfc # Phase 4: RFC-based upload
107
113
  ```
108
114
 
109
- ### Traditional Single-Phase Upload (Legacy)
115
+ ### Available Commands
110
116
 
111
- #### Basic Upload with Auto-Processing (API Mode)
117
+ #### 1. **upload** - Upload files to Arela
112
118
  ```bash
113
- arela --batch-size 10 -c 5
114
- ```
119
+ # Basic upload with auto-processing (API Mode)
120
+ arela upload --batch-size 10
115
121
 
116
- ### Upload with Auto-Detection of Year/Pedimento
117
- ```bash
118
- arela --auto-detect-structure --batch-size 10 -c 5
119
- ```
122
+ # Upload with auto-detection of year/pedimento from file paths
123
+ arela upload --auto-detect-structure --batch-size 10
120
124
 
121
- ### Upload with Custom Folder Structure
122
- ```bash
123
- arela --folder-structure "2024/4023260" --batch-size 10 -c 5
125
+ # Upload with custom folder structure
126
+ arela upload --folder-structure "2024/4023260" --batch-size 10
127
+
128
+ # Upload to Supabase directly (skip API)
129
+ arela upload --force-supabase --prefix "my-folder"
130
+
131
+ # Upload files by specific RFC values
132
+ arela upload --upload-by-rfc --batch-size 5
133
+
134
+ # Upload RFC files with custom folder prefix
135
+ arela upload --upload-by-rfc --folder-structure "palco" --batch-size 5
136
+
137
+ # Upload RFC files with nested folder structure
138
+ arela upload --upload-by-rfc --folder-structure "2024/Q1/processed" --batch-size 15
139
+
140
+ # Upload with performance statistics
141
+ arela upload --batch-size 10 --show-stats
142
+
143
+ # Upload with client path tracking
144
+ arela upload --client-path "/client/documents" --batch-size 10
124
145
  ```
125
146
 
126
- ### Upload with Directory Structure Preservation
147
+ #### 2. **stats** - Collect file statistics without uploading
127
148
  ```bash
128
- arela --batch-size 10 -c 5 --preserve-structure
149
+ # Collect filesystem statistics only (Phase 1)
150
+ arela stats --batch-size 10
151
+
152
+ # Stats with custom folder organization
153
+ arela stats --folder-structure "2023/3019796" --batch-size 10
154
+
155
+ # Stats with client path tracking
156
+ arela stats --client-path "/client/documents" --batch-size 10
129
157
  ```
130
158
 
131
- ### Upload to Supabase Directly (Skip API)
159
+ #### 3. **detect** - Run document detection and path propagation
132
160
  ```bash
133
- arela --force-supabase -p "my-folder"
161
+ # Run PDF detection on existing database records (Phase 2)
162
+ arela detect --batch-size 10
163
+
164
+ # Propagate arela_path from pedimento records to related files (Phase 3)
165
+ arela detect --propagate-arela-path
134
166
  ```
135
167
 
136
- ### Upload Files by Specific RFC Values
168
+ #### 4. **watch** - Monitor directories and upload automatically ⭐ NEW
137
169
  ```bash
138
- # Upload all files associated with specific RFCs
139
- arela --upload-by-rfc --batch-size 5
170
+ # Watch directories for changes with automatic upload
171
+ arela watch --directories "/path/to/watch1,/path/to/watch2"
140
172
 
141
- # Upload RFC files with custom folder prefix
142
- arela --upload-by-rfc --folder-structure "palco" --batch-size 5
173
+ # Watch with custom upload strategy (default: batch)
174
+ arela watch --directories "/path/to/watch" --strategy individual
175
+ arela watch --directories "/path/to/watch" --strategy full-structure
143
176
 
144
- # Upload RFC files with nested folder structure
145
- arela --upload-by-rfc --folder-structure "2024/client1/pedimentos" --batch-size 5
146
- ```
177
+ # Watch with custom debounce delay (default: 1000ms)
178
+ arela watch --directories "/path/to/watch" --debounce 2000
147
179
 
148
- ### Propagate Arela Path from Pedimentos to Related Files
149
- ```bash
150
- # Copy arela_path from pedimento_simplificado records to related files
151
- arela --propagate-arela-path
180
+ # Watch with automatic 4-step pipeline
181
+ arela watch --directories "/path/to/watch" --auto-processing --batch-size 10
182
+
183
+ # Watch with polling instead of native file system events
184
+ arela watch --directories "/path/to/watch" --poll 5000
185
+
186
+ # Watch with pattern ignoring
187
+ arela watch --directories "/path/to/watch" --ignore "node_modules,*.log,*.tmp"
188
+
189
+ # Watch in dry-run mode (simulate without uploading)
190
+ arela watch --directories "/path/to/watch" --dry-run
191
+
192
+ # Watch with verbose logging
193
+ arela watch --directories "/path/to/watch" --verbose
152
194
  ```
153
195
 
154
- ### Stats-Only Mode (No Upload)
196
+ **Watch Strategies:**
197
+ - `batch` **(default)**: Groups files and uploads periodically
198
+ - `individual`: Uploads each file immediately as it changes
199
+ - `full-structure`: Preserves directory structure during upload
200
+
201
+ #### 5. **query** - Query database for file status
155
202
  ```bash
156
- # Only process file stats and insert to database, don't upload
157
- arela --stats-only --folder-structure "2023/3019796"
203
+ # Show files ready for upload
204
+ arela query --ready-files
158
205
  ```
159
206
 
160
- ### Upload with Performance Statistics
207
+ #### 6. **config** - Show current configuration
161
208
  ```bash
162
- arela --batch-size 10 -c 5 --show-stats
209
+ # Display all configuration settings
210
+ arela config
163
211
  ```
164
212
 
165
- ### Upload with Client Path Tracking
213
+ ### Legacy Syntax (Still Supported)
214
+
215
+ The old flag-based syntax is still supported for backward compatibility:
216
+
166
217
  ```bash
167
- arela --client-path "/client/documents" --batch-size 10 -c 5
218
+ # These are equivalent to the commands above
219
+ arela --stats-only # Same as: arela stats
220
+ arela --detect-pdfs # Same as: arela detect
221
+ arela --propagate-arela-path # Same as: arela detect --propagate-arela-path
222
+ arela --upload-by-rfc # Same as: arela upload --upload-by-rfc
168
223
  ```
169
224
 
170
- ### Options
171
-
172
225
  #### Phase Control
173
226
  - `--stats-only`: **Phase 1** - Only collect filesystem stats (no file reading)
174
227
  - `--detect-pdfs`: **Phase 2** - Process PDF files for pedimento-simplificado detection
@@ -176,26 +229,49 @@ arela --client-path "/client/documents" --batch-size 10 -c 5
176
229
  - `--upload-by-rfc`: **Phase 4** - Upload files based on RFC values from UPLOAD_RFCS
177
230
  - `--run-all-phases`: **All Phases** - Run complete optimized workflow
178
231
 
179
- #### Performance & Configuration
180
- - `-c, --concurrency <number>`: Files per batch for processing (default: 10)
181
- - `--batch-size <number>`: API batch size (default: 10)
182
- - `--show-stats`: Show detailed processing statistics
232
+ #### Global Options (all commands)
233
+ - `-v, --verbose`: Enable verbose logging
234
+ - `--clear-log`: Clear the log file before starting
235
+ - `-h, --help`: Display help information
236
+ - `--version`: Display version number
183
237
 
184
- #### Upload Configuration
185
- - `-p, --prefix <prefix>`: Prefix path in bucket (default: "")
186
- - `-b, --bucket <bucket>`: Bucket name override
187
- - `--force-supabase`: Force direct Supabase upload (skip API)
188
- - `--no-auto-detect`: Disable automatic file detection (API mode only)
189
- - `--no-auto-organize`: Disable automatic file organization (API mode only)
190
- - `--preserve-structure`: **Preserve original directory structure when using auto-organize**
191
- - `--folder-structure <structure>`: **Custom folder structure** (e.g., "2024/4023260" or "cliente1/pedimentos")
192
- - `--auto-detect-structure`: **Automatically detect year/pedimento from file paths**
238
+ #### Upload Command Options
239
+ - `-b, --batch-size <size>`: API batch size (default: 10)
240
+ - `--folder-structure <structure>`: Custom folder structure (e.g., "2024/4023260")
193
241
  - `--client-path <path>`: Client path for metadata tracking
242
+ - `--auto-detect-structure`: Automatically detect year/pedimento from file paths
243
+ - `--auto-detect`: Enable automatic document type detection
244
+ - `--auto-organize`: Enable automatic file organization
245
+ - `--force-supabase`: Force direct Supabase upload (skip API)
246
+ - `--skip-processed`: Skip files already processed
247
+ - `--show-stats`: Show performance statistics
248
+ - `--upload-by-rfc`: Upload files based on RFC values from UPLOAD_RFCS
249
+ - `--run-all-phases`: Run all processing phases sequentially
194
250
 
195
- #### Legacy Options
196
- - `--no-detect`: Disable document type detection in stats-only mode
197
- - `-v, --version`: Display version number
198
- - `-h, --help`: Display help information
251
+ #### Stats Command Options
252
+ - `-b, --batch-size <size>`: Batch size for processing (default: 10)
253
+ - `--client-path <path>`: Client path for metadata tracking
254
+ - `--show-stats`: Show performance statistics
255
+
256
+ #### Detect Command Options
257
+ - `-b, --batch-size <size>`: Batch size for PDF detection (default: 10)
258
+ - `--propagate-arela-path`: Propagate arela_path from pedimento records to related files
259
+
260
+ #### Watch Command Options
261
+ - `-d, --directories <paths>`: **Comma-separated directories to watch** (required)
262
+ - `-s, --strategy <strategy>`: Upload strategy (default: batch)
263
+ - `batch`: Groups files and uploads periodically
264
+ - `individual`: Uploads each file immediately
265
+ - `full-structure`: Preserves directory structure
266
+ - `--debounce <ms>`: Debounce delay in milliseconds (default: 1000)
267
+ - `-b, --batch-size <size>`: Batch size for uploads (default: 10)
268
+ - `--poll <ms>`: Use polling instead of native file system events (interval in ms)
269
+ - `--ignore <patterns>`: Comma-separated patterns to ignore
270
+ - `--auto-detect`: Enable automatic document type detection
271
+ - `--auto-organize`: Enable automatic file organization
272
+ - `--auto-processing`: Enable automatic 4-step pipeline (stats, detect, propagate, upload)
273
+ - `--dry-run`: Simulate changes without uploading
274
+ - `--verbose`: Enable verbose logging
199
275
 
200
276
  ## Environment Variables
201
277
 
@@ -0,0 +1,157 @@
1
+ # Supabase Upload Issue Fix
2
+
3
+ ## Problem Description
4
+
5
+ When running the CLI with `upload --force-supabase`, the logs displayed that files were being uploaded successfully, but no files appeared in the Supabase bucket.
6
+
7
+ ## Root Cause
8
+
9
+ The issue was in `/src/commands/UploadCommand.js` in the `#processFile` method. The code was **silently ignoring upload failures** from Supabase:
10
+
11
+ ```javascript
12
+ // ❌ BEFORE - Upload result was ignored
13
+ if (uploadService.getServiceName() === 'Arela API') {
14
+ result = await uploadService.upload([fileObject], { ...options, uploadPath });
15
+ } else {
16
+ // Supabase direct upload
17
+ await uploadService.upload([fileObject], { uploadPath });
18
+ // Result ignored! No error checking!
19
+ }
20
+
21
+ logger.info(`SUCCESS: ${path.basename(filePath)} -> ${uploadPath}`);
22
+ ```
23
+
24
+ Even if the Supabase upload failed and returned `{ success: false, error: "..." }`, the code would:
25
+ 1. Ignore the error
26
+ 2. Log "SUCCESS"
27
+ 3. Continue processing
28
+
29
+ This created the illusion that files were being uploaded when they weren't.
30
+
31
+ ## Changes Made
32
+
33
+ ### 1. Fixed Error Handling in `UploadCommand.js`
34
+
35
+ **File:** `/src/commands/UploadCommand.js`
36
+
37
+ Now properly captures and validates the Supabase upload result:
38
+
39
+ ```javascript
40
+ // ✅ AFTER - Upload result is captured and validated
41
+ if (uploadService.getServiceName() === 'Arela API') {
42
+ result = await uploadService.upload([fileObject], { ...options, uploadPath });
43
+ } else {
44
+ // Supabase direct upload
45
+ const uploadResult = await uploadService.upload([fileObject], { uploadPath });
46
+
47
+ // Check if upload was successful
48
+ if (!uploadResult.success) {
49
+ throw new Error(`Supabase upload failed: ${uploadResult.error}`);
50
+ }
51
+
52
+ result = { successCount: 1 };
53
+ }
54
+ ```
55
+
56
+ ### 2. Enhanced Logging in `SupabaseUploadService.js`
57
+
58
+ **File:** `/src/services/upload/SupabaseUploadService.js`
59
+
60
+ Added detailed logging to help diagnose upload issues:
61
+
62
+ #### Connection Initialization
63
+ ```javascript
64
+ console.log(`🔌 Connecting to Supabase: ${appConfig.supabase.url}`);
65
+ console.log(`📦 Using bucket: ${this.bucket}`);
66
+ console.log(`🧪 Testing bucket access...`);
67
+ console.log(`✅ Successfully connected to Supabase bucket: ${this.bucket}`);
68
+ ```
69
+
70
+ #### Upload Operations
71
+ ```javascript
72
+ if (error) {
73
+ console.error(`❌ Supabase upload failed for ${normalizedPath}:`, error.message);
74
+ return { success: false, error: error.message };
75
+ }
76
+
77
+ console.log(`✅ Supabase upload successful: ${normalizedPath}`);
78
+ ```
79
+
80
+ #### Better Error Messages
81
+ ```javascript
82
+ if (!appConfig.supabase.url || !appConfig.supabase.key || !this.bucket) {
83
+ const missing = [];
84
+ if (!appConfig.supabase.url) missing.push('SUPABASE_URL');
85
+ if (!appConfig.supabase.key) missing.push('SUPABASE_KEY');
86
+ if (!this.bucket) missing.push('SUPABASE_BUCKET');
87
+ throw new Error(`Missing Supabase configuration: ${missing.join(', ')}`);
88
+ }
89
+ ```
90
+
91
+ ## Expected Behavior After Fix
92
+
93
+ ### On Successful Upload
94
+ ```
95
+ 🔌 Connecting to Supabase: https://your-project.supabase.co
96
+ 📦 Using bucket: your-bucket-name
97
+ 🧪 Testing bucket access...
98
+ ✅ Successfully connected to Supabase bucket: your-bucket-name
99
+ ✅ Supabase upload successful: 2023/2000601/file.xml
100
+ ```
101
+
102
+ ### On Failed Upload
103
+ ```
104
+ ❌ Supabase upload failed for 2023/2000601/file.xml: [error details]
105
+ Error: Supabase upload failed: [error details]
106
+ ```
107
+
108
+ ### On Configuration Error
109
+ ```
110
+ Error: Missing Supabase configuration: SUPABASE_URL, SUPABASE_KEY
111
+ ```
112
+
113
+ ## Common Issues to Check
114
+
115
+ If uploads still fail after this fix, check:
116
+
117
+ 1. **Environment Variables**
118
+ ```bash
119
+ echo $SUPABASE_URL
120
+ echo $SUPABASE_KEY # Should be anon or service_role key
121
+ echo $SUPABASE_BUCKET
122
+ ```
123
+
124
+ 2. **Bucket Permissions**
125
+ - Check if the bucket exists in Supabase Storage
126
+ - Verify the API key has write permissions to the bucket
127
+ - Check bucket policies (public vs private)
128
+
129
+ 3. **Network/Firewall**
130
+ - Ensure the server can reach Supabase (check firewall rules)
131
+ - Test connection: `curl https://your-project.supabase.co`
132
+
133
+ 4. **File Size Limits**
134
+ - Check Supabase storage limits for your plan
135
+ - Verify file sizes are within allowed limits
136
+
137
+ 5. **Path Format**
138
+ - Ensure paths don't contain invalid characters
139
+ - Paths are normalized (forward slashes only)
140
+
141
+ ## Testing the Fix
142
+
143
+ Run the upload command with verbose logging:
144
+
145
+ ```bash
146
+ arela upload --force-supabase -v
147
+ ```
148
+
149
+ You should now see:
150
+ - Clear connection status messages
151
+ - Per-file upload success/failure messages
152
+ - Proper error messages if uploads fail
153
+ - Accurate success/error counts in the summary
154
+
155
+ ## Migration Notes
156
+
157
+ No database migrations required. This is a code-only fix that improves error handling and logging.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@arela/uploader",
3
- "version": "0.2.12",
3
+ "version": "0.3.0",
4
4
  "description": "CLI to upload files/directories to Arela",
5
5
  "bin": {
6
6
  "arela": "./src/index.js"
@@ -29,6 +29,7 @@
29
29
  "homepage": "https://github.com/inspiraCode/arela-uploader#readme",
30
30
  "dependencies": {
31
31
  "@supabase/supabase-js": "2.49.4",
32
+ "chokidar": "^4.0.3",
32
33
  "cli-progress": "3.12.0",
33
34
  "commander": "13.1.0",
34
35
  "dotenv": "16.5.0",
@@ -43,4 +44,4 @@
43
44
  "@trivago/prettier-plugin-sort-imports": "5.2.2",
44
45
  "prettier": "3.5.3"
45
46
  }
46
- }
47
+ }
@@ -0,0 +1,109 @@
1
+ #!/usr/bin/env node
2
+
3
+ /**
4
+ * cleanup-ds-store.js
5
+ * Remove .DS_Store and other system file records from the uploader table
6
+ */
7
+
8
+ import { fileURLToPath } from 'url';
9
+ import path from 'path';
10
+ import logger from './src/services/LoggingService.js';
11
+ import { databaseService } from './src/services/DatabaseService.js';
12
+
13
+ const __filename = fileURLToPath(import.meta.url);
14
+ const __dirname = path.dirname(__filename);
15
+
16
+ console.log('\n╔════════════════════════════════════════════════════════════════╗');
17
+ console.log('║ Cleaning System Files from Uploader Table ║');
18
+ console.log('╚════════════════════════════════════════════════════════════════╝\n');
19
+
20
+ async function cleanupSystemFiles() {
21
+ try {
22
+ const supabase = await databaseService.getSupabaseClient();
23
+
24
+ // System file patterns to remove
25
+ const systemFiles = [
26
+ '.DS_Store',
27
+ 'Thumbs.db',
28
+ 'desktop.ini',
29
+ '.directory',
30
+ ];
31
+
32
+ let totalDeleted = 0;
33
+
34
+ for (const fileName of systemFiles) {
35
+ console.log(`\n🔍 Searching for ${fileName}...`);
36
+
37
+ // Find records with this filename
38
+ const { data: records, error: selectError } = await supabase
39
+ .from('uploader')
40
+ .select('*')
41
+ .eq('filename', fileName);
42
+
43
+ if (selectError) {
44
+ console.error(`❌ Error searching for ${fileName}: ${selectError.message}`);
45
+ continue;
46
+ }
47
+
48
+ if (!records || records.length === 0) {
49
+ console.log(` ✓ No records found for ${fileName}`);
50
+ continue;
51
+ }
52
+
53
+ console.log(` Found ${records.length} record(s) for ${fileName}`);
54
+
55
+ // Get the IDs to delete
56
+ const idsToDelete = records.map(r => r.id);
57
+
58
+ // Delete records
59
+ const { error: deleteError } = await supabase
60
+ .from('uploader')
61
+ .delete()
62
+ .in('id', idsToDelete);
63
+
64
+ if (deleteError) {
65
+ console.error(` ❌ Error deleting: ${deleteError.message}`);
66
+ continue;
67
+ }
68
+
69
+ console.log(` ✅ Deleted ${records.length} record(s)`);
70
+ totalDeleted += records.length;
71
+ }
72
+
73
+ // Also delete any records with original_path containing .DS_Store
74
+ console.log(`\n🔍 Searching for records with .DS_Store in path...`);
75
+ const { data: pathRecords, error: pathError } = await supabase
76
+ .from('uploader')
77
+ .select('*')
78
+ .ilike('original_path', '%.DS_Store%');
79
+
80
+ if (!pathError && pathRecords && pathRecords.length > 0) {
81
+ console.log(` Found ${pathRecords.length} record(s)`);
82
+ const idsToDelete = pathRecords.map(r => r.id);
83
+
84
+ const { error: deleteError } = await supabase
85
+ .from('uploader')
86
+ .delete()
87
+ .in('id', idsToDelete);
88
+
89
+ if (!deleteError) {
90
+ console.log(` ✅ Deleted ${pathRecords.length} record(s)`);
91
+ totalDeleted += pathRecords.length;
92
+ } else {
93
+ console.error(` ❌ Error deleting: ${deleteError.message}`);
94
+ }
95
+ }
96
+
97
+ console.log('\n╔════════════════════════════════════════════════════════════════╗');
98
+ console.log(`║ ✅ Cleanup Complete: Deleted ${totalDeleted} system file record(s) ║`);
99
+ console.log('╚════════════════════════════════════════════════════════════════╝\n');
100
+
101
+ process.exit(0);
102
+ } catch (error) {
103
+ console.error('\n❌ ERROR');
104
+ console.error(error.message);
105
+ process.exit(1);
106
+ }
107
+ }
108
+
109
+ cleanupSystemFiles();
@@ -0,0 +1,69 @@
1
+ #!/usr/bin/env node
2
+
3
+ /**
4
+ * cleanup-system-files.js
5
+ * Remove system files (.DS_Store, etc) from the uploader table
6
+ */
7
+
8
+ import { fileURLToPath } from 'url';
9
+ import path from 'path';
10
+
11
+ const __filename = fileURLToPath(import.meta.url);
12
+ const __dirname = path.dirname(__filename);
13
+
14
+ console.log('\n╔════════════════════════════════════════════════════════════════╗');
15
+ console.log('║ Cleaning System Files from Database ║');
16
+ console.log('╚════════════════════════════════════════════════════════════════╝\n');
17
+
18
+ try {
19
+ const databaseService = (await import('./src/services/DatabaseService.js')).default;
20
+ const supabase = await databaseService.getSupabaseClient();
21
+
22
+ // System file patterns to remove
23
+ const systemFilePatterns = ['.DS_Store', '__pycache__', '.pyc', '.swp', '.swo', 'Thumbs.db', 'desktop.ini'];
24
+
25
+ console.log('🔍 Searching for system files...\n');
26
+
27
+ let totalRemoved = 0;
28
+
29
+ for (const pattern of systemFilePatterns) {
30
+ const { data: records, error } = await supabase
31
+ .from('uploader')
32
+ .select('id, filename')
33
+ .ilike('filename', `%${pattern}%`);
34
+
35
+ if (!error && records && records.length > 0) {
36
+ console.log(`📝 Found ${records.length} record(s) with "${pattern}"`);
37
+ records.forEach(r => {
38
+ console.log(` - ID: ${r.id}, Filename: ${r.filename}`);
39
+ });
40
+
41
+ // Delete them
42
+ const ids = records.map(r => r.id);
43
+ const { error: deleteError } = await supabase
44
+ .from('uploader')
45
+ .delete()
46
+ .in('id', ids);
47
+
48
+ if (!deleteError) {
49
+ console.log(` ✅ Deleted ${records.length} record(s)\n`);
50
+ totalRemoved += records.length;
51
+ } else {
52
+ console.log(` ❌ Error deleting records: ${deleteError.message}\n`);
53
+ }
54
+ }
55
+ }
56
+
57
+ console.log('╔════════════════════════════════════════════════════════════════╗');
58
+ console.log(`║ ✅ CLEANUP COMPLETED ║`);
59
+ console.log(`║ ║`);
60
+ console.log(`║ Total system files removed: ${totalRemoved}`);
61
+ console.log('╚════════════════════════════════════════════════════════════════╝\n');
62
+
63
+ process.exit(0);
64
+ } catch (error) {
65
+ console.error('\n❌ CLEANUP FAILED');
66
+ console.error(error.message);
67
+ console.error(error.stack);
68
+ process.exit(1);
69
+ }