@arela/uploader 0.2.12 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.template +66 -0
- package/.vscode/settings.json +1 -0
- package/README.md +134 -58
- package/SUPABASE_UPLOAD_FIX.md +157 -0
- package/package.json +3 -2
- package/scripts/cleanup-ds-store.js +109 -0
- package/scripts/cleanup-system-files.js +69 -0
- package/scripts/tests/phase-7-features.test.js +415 -0
- package/scripts/tests/signal-handling.test.js +275 -0
- package/scripts/tests/smart-watch-integration.test.js +554 -0
- package/scripts/tests/watch-service-integration.test.js +584 -0
- package/src/commands/UploadCommand.js +36 -2
- package/src/commands/WatchCommand.js +1305 -0
- package/src/config/config.js +113 -0
- package/src/document-type-shared.js +2 -0
- package/src/document-types/support-document.js +201 -0
- package/src/file-detection.js +2 -1
- package/src/index.js +44 -0
- package/src/services/AdvancedFilterService.js +505 -0
- package/src/services/AutoProcessingService.js +639 -0
- package/src/services/BenchmarkingService.js +381 -0
- package/src/services/DatabaseService.js +723 -170
- package/src/services/ErrorMonitor.js +275 -0
- package/src/services/LoggingService.js +419 -1
- package/src/services/MonitoringService.js +401 -0
- package/src/services/PerformanceOptimizer.js +511 -0
- package/src/services/ReportingService.js +511 -0
- package/src/services/SignalHandler.js +255 -0
- package/src/services/SmartWatchDatabaseService.js +527 -0
- package/src/services/WatchService.js +783 -0
- package/src/services/upload/ApiUploadService.js +30 -4
- package/src/services/upload/SupabaseUploadService.js +28 -6
- package/src/utils/CleanupManager.js +262 -0
- package/src/utils/FileOperations.js +41 -0
- package/src/utils/WatchEventHandler.js +517 -0
- package/supabase/migrations/001_create_initial_schema.sql +366 -0
- package/supabase/migrations/002_align_with_arela_api_schema.sql +145 -0
- package/commands.md +0 -6
package/.env.template
CHANGED
|
@@ -63,6 +63,72 @@ MAX_CONCURRENT_SOURCES=2
|
|
|
63
63
|
# MAX_CONCURRENT_SOURCES=1
|
|
64
64
|
# BATCH_DELAY=100
|
|
65
65
|
|
|
66
|
+
# =============================================================================
|
|
67
|
+
# WATCH MODE CONFIGURATION
|
|
68
|
+
# =============================================================================
|
|
69
|
+
|
|
70
|
+
# Habilitar watch mode (true/false)
|
|
71
|
+
WATCH_ENABLED=false
|
|
72
|
+
|
|
73
|
+
# Configuración de directorios a observar (formato JSON)
|
|
74
|
+
# Cada directorio puede tener su propia folderStructure para organizar en el bucket
|
|
75
|
+
# Formato: {"ruta/directorio1":"estructura-1","ruta/directorio2":"estructura-2"}
|
|
76
|
+
#
|
|
77
|
+
# IMPORTANTE: El folderStructure se usa en el comando:
|
|
78
|
+
# upload --upload-by-rfc --folder-structure <estructura>
|
|
79
|
+
#
|
|
80
|
+
# Ejemplo:
|
|
81
|
+
WATCH_DIRECTORY_CONFIGS={"../../Documents/2022":"estructura-2022","../../Documents/2023":"estructura-2023"}
|
|
82
|
+
|
|
83
|
+
# DEPRECATED: Configuración antigua (se mantiene para compatibilidad hacia atrás)
|
|
84
|
+
# WATCH_DIRECTORIES=/ruta/carpeta1,/ruta/carpeta2
|
|
85
|
+
|
|
86
|
+
# Estrategia de upload (opciones: individual|batch|full-structure)
|
|
87
|
+
# - individual: Sube solo el archivo modificado más reciente
|
|
88
|
+
# - batch: Sube un lote de N archivos recientes
|
|
89
|
+
# - full-structure: Sube la estructura completa de carpetas
|
|
90
|
+
WATCH_STRATEGY=batch
|
|
91
|
+
|
|
92
|
+
# Debouncing en milisegundos (esperar entre eventos antes de procesar)
|
|
93
|
+
WATCH_DEBOUNCE_MS=1000
|
|
94
|
+
|
|
95
|
+
# Tamaño de batch para strategy batch
|
|
96
|
+
WATCH_BATCH_SIZE=10
|
|
97
|
+
|
|
98
|
+
# Usar polling en lugar de eventos nativos del filesystem
|
|
99
|
+
# Útil para sistemas de archivos remotos o NFS
|
|
100
|
+
WATCH_USE_POLLING=false
|
|
101
|
+
|
|
102
|
+
# Interval de polling en milisegundos (solo si WATCH_USE_POLLING=true)
|
|
103
|
+
WATCH_POLL_INTERVAL=100
|
|
104
|
+
|
|
105
|
+
# Umbral de estabilidad en ms (esperar a que el archivo deje de cambiar)
|
|
106
|
+
WATCH_STABILITY_THRESHOLD=300
|
|
107
|
+
|
|
108
|
+
# Patrones a ignorar (separados por coma, se usan como regex)
|
|
109
|
+
WATCH_IGNORE_PATTERNS=*.tmp,*.bak,*.swp
|
|
110
|
+
|
|
111
|
+
# Detección automática de tipos de documento
|
|
112
|
+
WATCH_AUTO_DETECT=false
|
|
113
|
+
|
|
114
|
+
# Organización automática de archivos
|
|
115
|
+
WATCH_AUTO_ORGANIZE=false
|
|
116
|
+
|
|
117
|
+
# =============================================================================
|
|
118
|
+
# WATCH MODE - AUTOMATIC PROCESSING PIPELINE
|
|
119
|
+
# =============================================================================
|
|
120
|
+
#
|
|
121
|
+
# El pipeline automático ejecuta la siguiente secuencia cuando se detecta un archivo nuevo:
|
|
122
|
+
# 1. Stats Collection → stats --stats-only (recopila información del archivo)
|
|
123
|
+
# 2. PDF Detection → detect --detect-pdfs (identifica pedimentos simplificados)
|
|
124
|
+
# 3. Path Propagation → detect --propagate-arela-path (propaga a documentos relacionados)
|
|
125
|
+
# 4. RFC Upload → upload --upload-by-rfc --folder-structure (sube con estructura)
|
|
126
|
+
#
|
|
127
|
+
# El pipeline se habilita automáticamente en watch mode y usa la folderStructure
|
|
128
|
+
# definida para cada WATCH_DIRECTORY_CONFIGS
|
|
129
|
+
#
|
|
130
|
+
# Para deshabilitar en CLI, usa: arela watch --no-auto-processing
|
|
131
|
+
|
|
66
132
|
# =============================================================================
|
|
67
133
|
# LOGGING AND MONITORING
|
|
68
134
|
# =============================================================================
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{}
|
package/README.md
CHANGED
|
@@ -84,6 +84,12 @@ arela --upload-by-rfc # Phase 4: Upload by RFC
|
|
|
84
84
|
- 📋 **Upload files by specific RFC values**
|
|
85
85
|
- 🔍 **Propagate arela_path from pedimento documents to related files**
|
|
86
86
|
- ⚡ **4-Phase optimized workflow for maximum performance**
|
|
87
|
+
- 👁️ **Watch Mode** ⭐ NEW - Monitor directories for changes and upload automatically
|
|
88
|
+
- Multiple watch strategies (batch, individual, full-structure)
|
|
89
|
+
- Debounce and polling support
|
|
90
|
+
- Auto-processing pipeline
|
|
91
|
+
- Dry-run mode for testing
|
|
92
|
+
- Pattern-based file ignoring
|
|
87
93
|
|
|
88
94
|
## Installation
|
|
89
95
|
|
|
@@ -97,78 +103,125 @@ npm install -g @arela/uploader
|
|
|
97
103
|
|
|
98
104
|
```bash
|
|
99
105
|
# Run all phases automatically (most efficient)
|
|
100
|
-
arela --run-all-phases --batch-size 20
|
|
106
|
+
arela upload --run-all-phases --batch-size 20
|
|
101
107
|
|
|
102
108
|
# Or run phases individually for fine-grained control
|
|
103
|
-
arela
|
|
104
|
-
arela
|
|
105
|
-
arela --propagate-arela-path
|
|
106
|
-
arela --upload-by-rfc
|
|
109
|
+
arela stats # Phase 1: Filesystem stats only
|
|
110
|
+
arela detect # Phase 2: PDF detection
|
|
111
|
+
arela detect --propagate-arela-path # Phase 3: Path propagation
|
|
112
|
+
arela upload --upload-by-rfc # Phase 4: RFC-based upload
|
|
107
113
|
```
|
|
108
114
|
|
|
109
|
-
###
|
|
115
|
+
### Available Commands
|
|
110
116
|
|
|
111
|
-
####
|
|
117
|
+
#### 1. **upload** - Upload files to Arela
|
|
112
118
|
```bash
|
|
113
|
-
|
|
114
|
-
|
|
119
|
+
# Basic upload with auto-processing (API Mode)
|
|
120
|
+
arela upload --batch-size 10
|
|
115
121
|
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
arela --auto-detect-structure --batch-size 10 -c 5
|
|
119
|
-
```
|
|
122
|
+
# Upload with auto-detection of year/pedimento from file paths
|
|
123
|
+
arela upload --auto-detect-structure --batch-size 10
|
|
120
124
|
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
125
|
+
# Upload with custom folder structure
|
|
126
|
+
arela upload --folder-structure "2024/4023260" --batch-size 10
|
|
127
|
+
|
|
128
|
+
# Upload to Supabase directly (skip API)
|
|
129
|
+
arela upload --force-supabase --prefix "my-folder"
|
|
130
|
+
|
|
131
|
+
# Upload files by specific RFC values
|
|
132
|
+
arela upload --upload-by-rfc --batch-size 5
|
|
133
|
+
|
|
134
|
+
# Upload RFC files with custom folder prefix
|
|
135
|
+
arela upload --upload-by-rfc --folder-structure "palco" --batch-size 5
|
|
136
|
+
|
|
137
|
+
# Upload RFC files with nested folder structure
|
|
138
|
+
arela upload --upload-by-rfc --folder-structure "2024/Q1/processed" --batch-size 15
|
|
139
|
+
|
|
140
|
+
# Upload with performance statistics
|
|
141
|
+
arela upload --batch-size 10 --show-stats
|
|
142
|
+
|
|
143
|
+
# Upload with client path tracking
|
|
144
|
+
arela upload --client-path "/client/documents" --batch-size 10
|
|
124
145
|
```
|
|
125
146
|
|
|
126
|
-
|
|
147
|
+
#### 2. **stats** - Collect file statistics without uploading
|
|
127
148
|
```bash
|
|
128
|
-
|
|
149
|
+
# Collect filesystem statistics only (Phase 1)
|
|
150
|
+
arela stats --batch-size 10
|
|
151
|
+
|
|
152
|
+
# Stats with custom folder organization
|
|
153
|
+
arela stats --folder-structure "2023/3019796" --batch-size 10
|
|
154
|
+
|
|
155
|
+
# Stats with client path tracking
|
|
156
|
+
arela stats --client-path "/client/documents" --batch-size 10
|
|
129
157
|
```
|
|
130
158
|
|
|
131
|
-
|
|
159
|
+
#### 3. **detect** - Run document detection and path propagation
|
|
132
160
|
```bash
|
|
133
|
-
|
|
161
|
+
# Run PDF detection on existing database records (Phase 2)
|
|
162
|
+
arela detect --batch-size 10
|
|
163
|
+
|
|
164
|
+
# Propagate arela_path from pedimento records to related files (Phase 3)
|
|
165
|
+
arela detect --propagate-arela-path
|
|
134
166
|
```
|
|
135
167
|
|
|
136
|
-
|
|
168
|
+
#### 4. **watch** - Monitor directories and upload automatically ⭐ NEW
|
|
137
169
|
```bash
|
|
138
|
-
#
|
|
139
|
-
arela
|
|
170
|
+
# Watch directories for changes with automatic upload
|
|
171
|
+
arela watch --directories "/path/to/watch1,/path/to/watch2"
|
|
140
172
|
|
|
141
|
-
#
|
|
142
|
-
arela
|
|
173
|
+
# Watch with custom upload strategy (default: batch)
|
|
174
|
+
arela watch --directories "/path/to/watch" --strategy individual
|
|
175
|
+
arela watch --directories "/path/to/watch" --strategy full-structure
|
|
143
176
|
|
|
144
|
-
#
|
|
145
|
-
arela
|
|
146
|
-
```
|
|
177
|
+
# Watch with custom debounce delay (default: 1000ms)
|
|
178
|
+
arela watch --directories "/path/to/watch" --debounce 2000
|
|
147
179
|
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
180
|
+
# Watch with automatic 4-step pipeline
|
|
181
|
+
arela watch --directories "/path/to/watch" --auto-processing --batch-size 10
|
|
182
|
+
|
|
183
|
+
# Watch with polling instead of native file system events
|
|
184
|
+
arela watch --directories "/path/to/watch" --poll 5000
|
|
185
|
+
|
|
186
|
+
# Watch with pattern ignoring
|
|
187
|
+
arela watch --directories "/path/to/watch" --ignore "node_modules,*.log,*.tmp"
|
|
188
|
+
|
|
189
|
+
# Watch in dry-run mode (simulate without uploading)
|
|
190
|
+
arela watch --directories "/path/to/watch" --dry-run
|
|
191
|
+
|
|
192
|
+
# Watch with verbose logging
|
|
193
|
+
arela watch --directories "/path/to/watch" --verbose
|
|
152
194
|
```
|
|
153
195
|
|
|
154
|
-
|
|
196
|
+
**Watch Strategies:**
|
|
197
|
+
- `batch` **(default)**: Groups files and uploads periodically
|
|
198
|
+
- `individual`: Uploads each file immediately as it changes
|
|
199
|
+
- `full-structure`: Preserves directory structure during upload
|
|
200
|
+
|
|
201
|
+
#### 5. **query** - Query database for file status
|
|
155
202
|
```bash
|
|
156
|
-
#
|
|
157
|
-
arela
|
|
203
|
+
# Show files ready for upload
|
|
204
|
+
arela query --ready-files
|
|
158
205
|
```
|
|
159
206
|
|
|
160
|
-
|
|
207
|
+
#### 6. **config** - Show current configuration
|
|
161
208
|
```bash
|
|
162
|
-
|
|
209
|
+
# Display all configuration settings
|
|
210
|
+
arela config
|
|
163
211
|
```
|
|
164
212
|
|
|
165
|
-
###
|
|
213
|
+
### Legacy Syntax (Still Supported)
|
|
214
|
+
|
|
215
|
+
The old flag-based syntax is still supported for backward compatibility:
|
|
216
|
+
|
|
166
217
|
```bash
|
|
167
|
-
|
|
218
|
+
# These are equivalent to the commands above
|
|
219
|
+
arela --stats-only # Same as: arela stats
|
|
220
|
+
arela --detect-pdfs # Same as: arela detect
|
|
221
|
+
arela --propagate-arela-path # Same as: arela detect --propagate-arela-path
|
|
222
|
+
arela --upload-by-rfc # Same as: arela upload --upload-by-rfc
|
|
168
223
|
```
|
|
169
224
|
|
|
170
|
-
### Options
|
|
171
|
-
|
|
172
225
|
#### Phase Control
|
|
173
226
|
- `--stats-only`: **Phase 1** - Only collect filesystem stats (no file reading)
|
|
174
227
|
- `--detect-pdfs`: **Phase 2** - Process PDF files for pedimento-simplificado detection
|
|
@@ -176,26 +229,49 @@ arela --client-path "/client/documents" --batch-size 10 -c 5
|
|
|
176
229
|
- `--upload-by-rfc`: **Phase 4** - Upload files based on RFC values from UPLOAD_RFCS
|
|
177
230
|
- `--run-all-phases`: **All Phases** - Run complete optimized workflow
|
|
178
231
|
|
|
179
|
-
####
|
|
180
|
-
- `-
|
|
181
|
-
- `--
|
|
182
|
-
-
|
|
232
|
+
#### Global Options (all commands)
|
|
233
|
+
- `-v, --verbose`: Enable verbose logging
|
|
234
|
+
- `--clear-log`: Clear the log file before starting
|
|
235
|
+
- `-h, --help`: Display help information
|
|
236
|
+
- `--version`: Display version number
|
|
183
237
|
|
|
184
|
-
#### Upload
|
|
185
|
-
- `-
|
|
186
|
-
-
|
|
187
|
-
- `--force-supabase`: Force direct Supabase upload (skip API)
|
|
188
|
-
- `--no-auto-detect`: Disable automatic file detection (API mode only)
|
|
189
|
-
- `--no-auto-organize`: Disable automatic file organization (API mode only)
|
|
190
|
-
- `--preserve-structure`: **Preserve original directory structure when using auto-organize**
|
|
191
|
-
- `--folder-structure <structure>`: **Custom folder structure** (e.g., "2024/4023260" or "cliente1/pedimentos")
|
|
192
|
-
- `--auto-detect-structure`: **Automatically detect year/pedimento from file paths**
|
|
238
|
+
#### Upload Command Options
|
|
239
|
+
- `-b, --batch-size <size>`: API batch size (default: 10)
|
|
240
|
+
- `--folder-structure <structure>`: Custom folder structure (e.g., "2024/4023260")
|
|
193
241
|
- `--client-path <path>`: Client path for metadata tracking
|
|
242
|
+
- `--auto-detect-structure`: Automatically detect year/pedimento from file paths
|
|
243
|
+
- `--auto-detect`: Enable automatic document type detection
|
|
244
|
+
- `--auto-organize`: Enable automatic file organization
|
|
245
|
+
- `--force-supabase`: Force direct Supabase upload (skip API)
|
|
246
|
+
- `--skip-processed`: Skip files already processed
|
|
247
|
+
- `--show-stats`: Show performance statistics
|
|
248
|
+
- `--upload-by-rfc`: Upload files based on RFC values from UPLOAD_RFCS
|
|
249
|
+
- `--run-all-phases`: Run all processing phases sequentially
|
|
194
250
|
|
|
195
|
-
####
|
|
196
|
-
-
|
|
197
|
-
-
|
|
198
|
-
-
|
|
251
|
+
#### Stats Command Options
|
|
252
|
+
- `-b, --batch-size <size>`: Batch size for processing (default: 10)
|
|
253
|
+
- `--client-path <path>`: Client path for metadata tracking
|
|
254
|
+
- `--show-stats`: Show performance statistics
|
|
255
|
+
|
|
256
|
+
#### Detect Command Options
|
|
257
|
+
- `-b, --batch-size <size>`: Batch size for PDF detection (default: 10)
|
|
258
|
+
- `--propagate-arela-path`: Propagate arela_path from pedimento records to related files
|
|
259
|
+
|
|
260
|
+
#### Watch Command Options
|
|
261
|
+
- `-d, --directories <paths>`: **Comma-separated directories to watch** (required)
|
|
262
|
+
- `-s, --strategy <strategy>`: Upload strategy (default: batch)
|
|
263
|
+
- `batch`: Groups files and uploads periodically
|
|
264
|
+
- `individual`: Uploads each file immediately
|
|
265
|
+
- `full-structure`: Preserves directory structure
|
|
266
|
+
- `--debounce <ms>`: Debounce delay in milliseconds (default: 1000)
|
|
267
|
+
- `-b, --batch-size <size>`: Batch size for uploads (default: 10)
|
|
268
|
+
- `--poll <ms>`: Use polling instead of native file system events (interval in ms)
|
|
269
|
+
- `--ignore <patterns>`: Comma-separated patterns to ignore
|
|
270
|
+
- `--auto-detect`: Enable automatic document type detection
|
|
271
|
+
- `--auto-organize`: Enable automatic file organization
|
|
272
|
+
- `--auto-processing`: Enable automatic 4-step pipeline (stats, detect, propagate, upload)
|
|
273
|
+
- `--dry-run`: Simulate changes without uploading
|
|
274
|
+
- `--verbose`: Enable verbose logging
|
|
199
275
|
|
|
200
276
|
## Environment Variables
|
|
201
277
|
|
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
# Supabase Upload Issue Fix
|
|
2
|
+
|
|
3
|
+
## Problem Description
|
|
4
|
+
|
|
5
|
+
When running the CLI with `upload --force-supabase`, the logs displayed that files were being uploaded successfully, but no files appeared in the Supabase bucket.
|
|
6
|
+
|
|
7
|
+
## Root Cause
|
|
8
|
+
|
|
9
|
+
The issue was in `/src/commands/UploadCommand.js` in the `#processFile` method. The code was **silently ignoring upload failures** from Supabase:
|
|
10
|
+
|
|
11
|
+
```javascript
|
|
12
|
+
// ❌ BEFORE - Upload result was ignored
|
|
13
|
+
if (uploadService.getServiceName() === 'Arela API') {
|
|
14
|
+
result = await uploadService.upload([fileObject], { ...options, uploadPath });
|
|
15
|
+
} else {
|
|
16
|
+
// Supabase direct upload
|
|
17
|
+
await uploadService.upload([fileObject], { uploadPath });
|
|
18
|
+
// Result ignored! No error checking!
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
logger.info(`SUCCESS: ${path.basename(filePath)} -> ${uploadPath}`);
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
Even if the Supabase upload failed and returned `{ success: false, error: "..." }`, the code would:
|
|
25
|
+
1. Ignore the error
|
|
26
|
+
2. Log "SUCCESS"
|
|
27
|
+
3. Continue processing
|
|
28
|
+
|
|
29
|
+
This created the illusion that files were being uploaded when they weren't.
|
|
30
|
+
|
|
31
|
+
## Changes Made
|
|
32
|
+
|
|
33
|
+
### 1. Fixed Error Handling in `UploadCommand.js`
|
|
34
|
+
|
|
35
|
+
**File:** `/src/commands/UploadCommand.js`
|
|
36
|
+
|
|
37
|
+
Now properly captures and validates the Supabase upload result:
|
|
38
|
+
|
|
39
|
+
```javascript
|
|
40
|
+
// ✅ AFTER - Upload result is captured and validated
|
|
41
|
+
if (uploadService.getServiceName() === 'Arela API') {
|
|
42
|
+
result = await uploadService.upload([fileObject], { ...options, uploadPath });
|
|
43
|
+
} else {
|
|
44
|
+
// Supabase direct upload
|
|
45
|
+
const uploadResult = await uploadService.upload([fileObject], { uploadPath });
|
|
46
|
+
|
|
47
|
+
// Check if upload was successful
|
|
48
|
+
if (!uploadResult.success) {
|
|
49
|
+
throw new Error(`Supabase upload failed: ${uploadResult.error}`);
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
result = { successCount: 1 };
|
|
53
|
+
}
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
### 2. Enhanced Logging in `SupabaseUploadService.js`
|
|
57
|
+
|
|
58
|
+
**File:** `/src/services/upload/SupabaseUploadService.js`
|
|
59
|
+
|
|
60
|
+
Added detailed logging to help diagnose upload issues:
|
|
61
|
+
|
|
62
|
+
#### Connection Initialization
|
|
63
|
+
```javascript
|
|
64
|
+
console.log(`🔌 Connecting to Supabase: ${appConfig.supabase.url}`);
|
|
65
|
+
console.log(`📦 Using bucket: ${this.bucket}`);
|
|
66
|
+
console.log(`🧪 Testing bucket access...`);
|
|
67
|
+
console.log(`✅ Successfully connected to Supabase bucket: ${this.bucket}`);
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
#### Upload Operations
|
|
71
|
+
```javascript
|
|
72
|
+
if (error) {
|
|
73
|
+
console.error(`❌ Supabase upload failed for ${normalizedPath}:`, error.message);
|
|
74
|
+
return { success: false, error: error.message };
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
console.log(`✅ Supabase upload successful: ${normalizedPath}`);
|
|
78
|
+
```
|
|
79
|
+
|
|
80
|
+
#### Better Error Messages
|
|
81
|
+
```javascript
|
|
82
|
+
if (!appConfig.supabase.url || !appConfig.supabase.key || !this.bucket) {
|
|
83
|
+
const missing = [];
|
|
84
|
+
if (!appConfig.supabase.url) missing.push('SUPABASE_URL');
|
|
85
|
+
if (!appConfig.supabase.key) missing.push('SUPABASE_KEY');
|
|
86
|
+
if (!this.bucket) missing.push('SUPABASE_BUCKET');
|
|
87
|
+
throw new Error(`Missing Supabase configuration: ${missing.join(', ')}`);
|
|
88
|
+
}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
## Expected Behavior After Fix
|
|
92
|
+
|
|
93
|
+
### On Successful Upload
|
|
94
|
+
```
|
|
95
|
+
🔌 Connecting to Supabase: https://your-project.supabase.co
|
|
96
|
+
📦 Using bucket: your-bucket-name
|
|
97
|
+
🧪 Testing bucket access...
|
|
98
|
+
✅ Successfully connected to Supabase bucket: your-bucket-name
|
|
99
|
+
✅ Supabase upload successful: 2023/2000601/file.xml
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
### On Failed Upload
|
|
103
|
+
```
|
|
104
|
+
❌ Supabase upload failed for 2023/2000601/file.xml: [error details]
|
|
105
|
+
Error: Supabase upload failed: [error details]
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
### On Configuration Error
|
|
109
|
+
```
|
|
110
|
+
Error: Missing Supabase configuration: SUPABASE_URL, SUPABASE_KEY
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
## Common Issues to Check
|
|
114
|
+
|
|
115
|
+
If uploads still fail after this fix, check:
|
|
116
|
+
|
|
117
|
+
1. **Environment Variables**
|
|
118
|
+
```bash
|
|
119
|
+
echo $SUPABASE_URL
|
|
120
|
+
echo $SUPABASE_KEY # Should be anon or service_role key
|
|
121
|
+
echo $SUPABASE_BUCKET
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
2. **Bucket Permissions**
|
|
125
|
+
- Check if the bucket exists in Supabase Storage
|
|
126
|
+
- Verify the API key has write permissions to the bucket
|
|
127
|
+
- Check bucket policies (public vs private)
|
|
128
|
+
|
|
129
|
+
3. **Network/Firewall**
|
|
130
|
+
- Ensure the server can reach Supabase (check firewall rules)
|
|
131
|
+
- Test connection: `curl https://your-project.supabase.co`
|
|
132
|
+
|
|
133
|
+
4. **File Size Limits**
|
|
134
|
+
- Check Supabase storage limits for your plan
|
|
135
|
+
- Verify file sizes are within allowed limits
|
|
136
|
+
|
|
137
|
+
5. **Path Format**
|
|
138
|
+
- Ensure paths don't contain invalid characters
|
|
139
|
+
- Paths are normalized (forward slashes only)
|
|
140
|
+
|
|
141
|
+
## Testing the Fix
|
|
142
|
+
|
|
143
|
+
Run the upload command with verbose logging:
|
|
144
|
+
|
|
145
|
+
```bash
|
|
146
|
+
arela upload --force-supabase -v
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
You should now see:
|
|
150
|
+
- Clear connection status messages
|
|
151
|
+
- Per-file upload success/failure messages
|
|
152
|
+
- Proper error messages if uploads fail
|
|
153
|
+
- Accurate success/error counts in the summary
|
|
154
|
+
|
|
155
|
+
## Migration Notes
|
|
156
|
+
|
|
157
|
+
No database migrations required. This is a code-only fix that improves error handling and logging.
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@arela/uploader",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.3.0",
|
|
4
4
|
"description": "CLI to upload files/directories to Arela",
|
|
5
5
|
"bin": {
|
|
6
6
|
"arela": "./src/index.js"
|
|
@@ -29,6 +29,7 @@
|
|
|
29
29
|
"homepage": "https://github.com/inspiraCode/arela-uploader#readme",
|
|
30
30
|
"dependencies": {
|
|
31
31
|
"@supabase/supabase-js": "2.49.4",
|
|
32
|
+
"chokidar": "^4.0.3",
|
|
32
33
|
"cli-progress": "3.12.0",
|
|
33
34
|
"commander": "13.1.0",
|
|
34
35
|
"dotenv": "16.5.0",
|
|
@@ -43,4 +44,4 @@
|
|
|
43
44
|
"@trivago/prettier-plugin-sort-imports": "5.2.2",
|
|
44
45
|
"prettier": "3.5.3"
|
|
45
46
|
}
|
|
46
|
-
}
|
|
47
|
+
}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* cleanup-ds-store.js
|
|
5
|
+
* Remove .DS_Store and other system file records from the uploader table
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { fileURLToPath } from 'url';
|
|
9
|
+
import path from 'path';
|
|
10
|
+
import logger from './src/services/LoggingService.js';
|
|
11
|
+
import { databaseService } from './src/services/DatabaseService.js';
|
|
12
|
+
|
|
13
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
14
|
+
const __dirname = path.dirname(__filename);
|
|
15
|
+
|
|
16
|
+
console.log('\n╔════════════════════════════════════════════════════════════════╗');
|
|
17
|
+
console.log('║ Cleaning System Files from Uploader Table ║');
|
|
18
|
+
console.log('╚════════════════════════════════════════════════════════════════╝\n');
|
|
19
|
+
|
|
20
|
+
async function cleanupSystemFiles() {
|
|
21
|
+
try {
|
|
22
|
+
const supabase = await databaseService.getSupabaseClient();
|
|
23
|
+
|
|
24
|
+
// System file patterns to remove
|
|
25
|
+
const systemFiles = [
|
|
26
|
+
'.DS_Store',
|
|
27
|
+
'Thumbs.db',
|
|
28
|
+
'desktop.ini',
|
|
29
|
+
'.directory',
|
|
30
|
+
];
|
|
31
|
+
|
|
32
|
+
let totalDeleted = 0;
|
|
33
|
+
|
|
34
|
+
for (const fileName of systemFiles) {
|
|
35
|
+
console.log(`\n🔍 Searching for ${fileName}...`);
|
|
36
|
+
|
|
37
|
+
// Find records with this filename
|
|
38
|
+
const { data: records, error: selectError } = await supabase
|
|
39
|
+
.from('uploader')
|
|
40
|
+
.select('*')
|
|
41
|
+
.eq('filename', fileName);
|
|
42
|
+
|
|
43
|
+
if (selectError) {
|
|
44
|
+
console.error(`❌ Error searching for ${fileName}: ${selectError.message}`);
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
if (!records || records.length === 0) {
|
|
49
|
+
console.log(` ✓ No records found for ${fileName}`);
|
|
50
|
+
continue;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
console.log(` Found ${records.length} record(s) for ${fileName}`);
|
|
54
|
+
|
|
55
|
+
// Get the IDs to delete
|
|
56
|
+
const idsToDelete = records.map(r => r.id);
|
|
57
|
+
|
|
58
|
+
// Delete records
|
|
59
|
+
const { error: deleteError } = await supabase
|
|
60
|
+
.from('uploader')
|
|
61
|
+
.delete()
|
|
62
|
+
.in('id', idsToDelete);
|
|
63
|
+
|
|
64
|
+
if (deleteError) {
|
|
65
|
+
console.error(` ❌ Error deleting: ${deleteError.message}`);
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
console.log(` ✅ Deleted ${records.length} record(s)`);
|
|
70
|
+
totalDeleted += records.length;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Also delete any records with original_path containing .DS_Store
|
|
74
|
+
console.log(`\n🔍 Searching for records with .DS_Store in path...`);
|
|
75
|
+
const { data: pathRecords, error: pathError } = await supabase
|
|
76
|
+
.from('uploader')
|
|
77
|
+
.select('*')
|
|
78
|
+
.ilike('original_path', '%.DS_Store%');
|
|
79
|
+
|
|
80
|
+
if (!pathError && pathRecords && pathRecords.length > 0) {
|
|
81
|
+
console.log(` Found ${pathRecords.length} record(s)`);
|
|
82
|
+
const idsToDelete = pathRecords.map(r => r.id);
|
|
83
|
+
|
|
84
|
+
const { error: deleteError } = await supabase
|
|
85
|
+
.from('uploader')
|
|
86
|
+
.delete()
|
|
87
|
+
.in('id', idsToDelete);
|
|
88
|
+
|
|
89
|
+
if (!deleteError) {
|
|
90
|
+
console.log(` ✅ Deleted ${pathRecords.length} record(s)`);
|
|
91
|
+
totalDeleted += pathRecords.length;
|
|
92
|
+
} else {
|
|
93
|
+
console.error(` ❌ Error deleting: ${deleteError.message}`);
|
|
94
|
+
}
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
console.log('\n╔════════════════════════════════════════════════════════════════╗');
|
|
98
|
+
console.log(`║ ✅ Cleanup Complete: Deleted ${totalDeleted} system file record(s) ║`);
|
|
99
|
+
console.log('╚════════════════════════════════════════════════════════════════╝\n');
|
|
100
|
+
|
|
101
|
+
process.exit(0);
|
|
102
|
+
} catch (error) {
|
|
103
|
+
console.error('\n❌ ERROR');
|
|
104
|
+
console.error(error.message);
|
|
105
|
+
process.exit(1);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
cleanupSystemFiles();
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
|
|
3
|
+
/**
|
|
4
|
+
* cleanup-system-files.js
|
|
5
|
+
* Remove system files (.DS_Store, etc) from the uploader table
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import { fileURLToPath } from 'url';
|
|
9
|
+
import path from 'path';
|
|
10
|
+
|
|
11
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
12
|
+
const __dirname = path.dirname(__filename);
|
|
13
|
+
|
|
14
|
+
console.log('\n╔════════════════════════════════════════════════════════════════╗');
|
|
15
|
+
console.log('║ Cleaning System Files from Database ║');
|
|
16
|
+
console.log('╚════════════════════════════════════════════════════════════════╝\n');
|
|
17
|
+
|
|
18
|
+
try {
|
|
19
|
+
const databaseService = (await import('./src/services/DatabaseService.js')).default;
|
|
20
|
+
const supabase = await databaseService.getSupabaseClient();
|
|
21
|
+
|
|
22
|
+
// System file patterns to remove
|
|
23
|
+
const systemFilePatterns = ['.DS_Store', '__pycache__', '.pyc', '.swp', '.swo', 'Thumbs.db', 'desktop.ini'];
|
|
24
|
+
|
|
25
|
+
console.log('🔍 Searching for system files...\n');
|
|
26
|
+
|
|
27
|
+
let totalRemoved = 0;
|
|
28
|
+
|
|
29
|
+
for (const pattern of systemFilePatterns) {
|
|
30
|
+
const { data: records, error } = await supabase
|
|
31
|
+
.from('uploader')
|
|
32
|
+
.select('id, filename')
|
|
33
|
+
.ilike('filename', `%${pattern}%`);
|
|
34
|
+
|
|
35
|
+
if (!error && records && records.length > 0) {
|
|
36
|
+
console.log(`📝 Found ${records.length} record(s) with "${pattern}"`);
|
|
37
|
+
records.forEach(r => {
|
|
38
|
+
console.log(` - ID: ${r.id}, Filename: ${r.filename}`);
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
// Delete them
|
|
42
|
+
const ids = records.map(r => r.id);
|
|
43
|
+
const { error: deleteError } = await supabase
|
|
44
|
+
.from('uploader')
|
|
45
|
+
.delete()
|
|
46
|
+
.in('id', ids);
|
|
47
|
+
|
|
48
|
+
if (!deleteError) {
|
|
49
|
+
console.log(` ✅ Deleted ${records.length} record(s)\n`);
|
|
50
|
+
totalRemoved += records.length;
|
|
51
|
+
} else {
|
|
52
|
+
console.log(` ❌ Error deleting records: ${deleteError.message}\n`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
console.log('╔════════════════════════════════════════════════════════════════╗');
|
|
58
|
+
console.log(`║ ✅ CLEANUP COMPLETED ║`);
|
|
59
|
+
console.log(`║ ║`);
|
|
60
|
+
console.log(`║ Total system files removed: ${totalRemoved}`);
|
|
61
|
+
console.log('╚════════════════════════════════════════════════════════════════╝\n');
|
|
62
|
+
|
|
63
|
+
process.exit(0);
|
|
64
|
+
} catch (error) {
|
|
65
|
+
console.error('\n❌ CLEANUP FAILED');
|
|
66
|
+
console.error(error.message);
|
|
67
|
+
console.error(error.stack);
|
|
68
|
+
process.exit(1);
|
|
69
|
+
}
|