@arela/uploader 0.2.5 ā 0.2.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/commands/UploadCommand.js +102 -44
- package/src/config/config.js +15 -10
- package/src/errors/ErrorHandler.js +38 -31
- package/src/errors/ErrorTypes.js +2 -2
- package/src/index-old.js +25 -17
- package/src/index.js +124 -49
- package/src/services/DatabaseService.js +403 -191
- package/src/services/LoggingService.js +3 -3
- package/src/services/upload/ApiUploadService.js +16 -10
- package/src/services/upload/BaseUploadService.js +1 -1
- package/src/services/upload/SupabaseUploadService.js +22 -3
- package/src/services/upload/UploadServiceFactory.js +14 -6
- package/src/utils/FileOperations.js +1 -1
- package/src/utils/FileSanitizer.js +1 -1
- package/src/utils/PathDetector.js +9 -7
package/src/index.js
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { Command } from 'commander';
|
|
3
3
|
|
|
4
|
-
import appConfig from './config/config.js';
|
|
5
4
|
import UploadCommand from './commands/UploadCommand.js';
|
|
5
|
+
import appConfig from './config/config.js';
|
|
6
6
|
import ErrorHandler from './errors/ErrorHandler.js';
|
|
7
7
|
import logger from './services/LoggingService.js';
|
|
8
8
|
|
|
@@ -15,7 +15,7 @@ class ArelaUploaderCLI {
|
|
|
15
15
|
this.program = new Command();
|
|
16
16
|
this.errorHandler = new ErrorHandler(logger);
|
|
17
17
|
this.uploadCommand = new UploadCommand();
|
|
18
|
-
|
|
18
|
+
|
|
19
19
|
this.#setupProgram();
|
|
20
20
|
this.#setupCommands();
|
|
21
21
|
this.#setupErrorHandling();
|
|
@@ -28,7 +28,9 @@ class ArelaUploaderCLI {
|
|
|
28
28
|
#setupProgram() {
|
|
29
29
|
this.program
|
|
30
30
|
.name('arela')
|
|
31
|
-
.description(
|
|
31
|
+
.description(
|
|
32
|
+
'CLI to upload files/directories to Arela with automatic processing',
|
|
33
|
+
)
|
|
32
34
|
.version(appConfig.packageVersion)
|
|
33
35
|
.option('-v, --verbose', 'Enable verbose logging')
|
|
34
36
|
.option('--clear-log', 'Clear the log file before starting');
|
|
@@ -43,33 +45,53 @@ class ArelaUploaderCLI {
|
|
|
43
45
|
this.program
|
|
44
46
|
.command('upload')
|
|
45
47
|
.description('Upload files to Arela with automatic processing')
|
|
46
|
-
.option(
|
|
48
|
+
.option(
|
|
49
|
+
'-b, --batch-size <size>',
|
|
50
|
+
'Number of files to process in each batch',
|
|
51
|
+
'10',
|
|
52
|
+
)
|
|
47
53
|
.option('-p, --prefix <prefix>', 'Prefix for uploaded files')
|
|
48
|
-
.option(
|
|
54
|
+
.option(
|
|
55
|
+
'--folder-structure <structure>',
|
|
56
|
+
'Custom folder structure for organization',
|
|
57
|
+
)
|
|
49
58
|
.option('--client-path <path>', 'Override client path for metadata')
|
|
50
|
-
.option(
|
|
59
|
+
.option(
|
|
60
|
+
'--auto-detect-structure',
|
|
61
|
+
'Automatically detect folder structure from file paths',
|
|
62
|
+
)
|
|
51
63
|
.option('--auto-detect', 'Enable automatic document type detection')
|
|
52
64
|
.option('--auto-organize', 'Enable automatic file organization')
|
|
53
65
|
.option('--force-supabase', 'Force direct Supabase upload mode')
|
|
54
66
|
.option('--skip-processed', 'Skip files that have already been processed')
|
|
55
67
|
.option('--show-stats', 'Show performance statistics')
|
|
56
|
-
.option(
|
|
57
|
-
|
|
68
|
+
.option(
|
|
69
|
+
'--upload-by-rfc',
|
|
70
|
+
'Upload files based on specific RFC values from UPLOAD_RFCS',
|
|
71
|
+
)
|
|
72
|
+
.option(
|
|
73
|
+
'--run-all-phases',
|
|
74
|
+
'Run all processing phases (stats, detection, organization)',
|
|
75
|
+
)
|
|
58
76
|
.action(async (options) => {
|
|
59
77
|
try {
|
|
60
78
|
// Handle --upload-by-rfc as a specific operation
|
|
61
79
|
if (options.uploadByRfc) {
|
|
62
|
-
const databaseService = await import(
|
|
80
|
+
const databaseService = await import(
|
|
81
|
+
'./services/DatabaseService.js'
|
|
82
|
+
);
|
|
63
83
|
console.log('šÆ Running RFC-based upload...');
|
|
64
84
|
const result = await databaseService.default.uploadFilesByRfc({
|
|
65
85
|
batchSize: parseInt(options.batchSize) || 10,
|
|
66
86
|
showProgress: true,
|
|
67
87
|
folderStructure: options.folderStructure,
|
|
68
88
|
});
|
|
69
|
-
console.log(
|
|
89
|
+
console.log(
|
|
90
|
+
`ā
RFC upload completed: ${result.processedCount} processed, ${result.uploadedCount} uploaded, ${result.errorCount} errors`,
|
|
91
|
+
);
|
|
70
92
|
return;
|
|
71
93
|
}
|
|
72
|
-
|
|
94
|
+
|
|
73
95
|
await this.uploadCommand.execute(options);
|
|
74
96
|
} catch (error) {
|
|
75
97
|
this.errorHandler.handleFatalError(error, { command: 'upload' });
|
|
@@ -80,10 +102,20 @@ class ArelaUploaderCLI {
|
|
|
80
102
|
this.program
|
|
81
103
|
.command('stats')
|
|
82
104
|
.description('Collect file statistics without uploading')
|
|
83
|
-
.option(
|
|
105
|
+
.option(
|
|
106
|
+
'-b, --batch-size <size>',
|
|
107
|
+
'Number of files to process in each batch',
|
|
108
|
+
'10',
|
|
109
|
+
)
|
|
84
110
|
.option('--client-path <path>', 'Override client path for metadata')
|
|
85
|
-
.option(
|
|
86
|
-
|
|
111
|
+
.option(
|
|
112
|
+
'--stats-only',
|
|
113
|
+
'Collect file statistics without uploading (backward compatibility)',
|
|
114
|
+
)
|
|
115
|
+
.option(
|
|
116
|
+
'--run-all-phases',
|
|
117
|
+
'Run all processing phases (stats, detection, organization)',
|
|
118
|
+
)
|
|
87
119
|
.option('--show-stats', 'Show performance statistics')
|
|
88
120
|
.action(async (options) => {
|
|
89
121
|
try {
|
|
@@ -98,29 +130,46 @@ class ArelaUploaderCLI {
|
|
|
98
130
|
this.program
|
|
99
131
|
.command('detect')
|
|
100
132
|
.description('Run document detection on existing file records')
|
|
101
|
-
.option(
|
|
102
|
-
|
|
103
|
-
|
|
133
|
+
.option(
|
|
134
|
+
'-b, --batch-size <size>',
|
|
135
|
+
'Number of files to process in each batch',
|
|
136
|
+
'10',
|
|
137
|
+
)
|
|
138
|
+
.option(
|
|
139
|
+
'--detect-pdfs',
|
|
140
|
+
'Run PDF detection on existing database records (backward compatibility)',
|
|
141
|
+
)
|
|
142
|
+
.option(
|
|
143
|
+
'--propagate-arela-path',
|
|
144
|
+
'Propagate arela_path from pedimento records to related files',
|
|
145
|
+
)
|
|
104
146
|
.action(async (options) => {
|
|
105
147
|
try {
|
|
106
148
|
const databaseService = await import('./services/DatabaseService.js');
|
|
107
|
-
|
|
149
|
+
|
|
108
150
|
// Handle --propagate-arela-path as a specific operation
|
|
109
151
|
if (options.propagateArelaPath) {
|
|
110
152
|
console.log('š Running arela_path propagation...');
|
|
111
153
|
const result = await databaseService.default.propagateArelaPath({
|
|
112
154
|
showProgress: true,
|
|
113
155
|
});
|
|
114
|
-
console.log(
|
|
156
|
+
console.log(
|
|
157
|
+
`ā
Propagation completed: ${result.processedCount} processed, ${result.updatedCount} updated, ${result.errorCount} errors`,
|
|
158
|
+
);
|
|
115
159
|
return;
|
|
116
160
|
}
|
|
117
|
-
|
|
161
|
+
|
|
118
162
|
// Default behavior: run PDF detection
|
|
119
|
-
console.log(
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
163
|
+
console.log(
|
|
164
|
+
'š Running PDF detection on existing database records...',
|
|
165
|
+
);
|
|
166
|
+
const result =
|
|
167
|
+
await databaseService.default.detectPedimentosInDatabase({
|
|
168
|
+
batchSize: parseInt(options.batchSize) || 10,
|
|
169
|
+
});
|
|
170
|
+
console.log(
|
|
171
|
+
`ā
Detection completed: ${result.detectedCount} detected, ${result.processedCount} processed, ${result.errorCount} errors`,
|
|
172
|
+
);
|
|
124
173
|
} catch (error) {
|
|
125
174
|
this.errorHandler.handleFatalError(error, { command: 'detect' });
|
|
126
175
|
}
|
|
@@ -142,26 +191,36 @@ class ArelaUploaderCLI {
|
|
|
142
191
|
this.program
|
|
143
192
|
.command('query')
|
|
144
193
|
.description('Query database for file status and information')
|
|
145
|
-
.option(
|
|
194
|
+
.option(
|
|
195
|
+
'--ready-files',
|
|
196
|
+
'Show files that are ready for upload (detected but not uploaded)',
|
|
197
|
+
)
|
|
146
198
|
.action(async (options) => {
|
|
147
199
|
try {
|
|
148
200
|
const databaseService = await import('./services/DatabaseService.js');
|
|
149
|
-
|
|
201
|
+
|
|
150
202
|
if (options.readyFiles) {
|
|
151
203
|
console.log('š Querying files ready for upload...');
|
|
152
|
-
|
|
153
|
-
const readyFiles =
|
|
154
|
-
|
|
204
|
+
|
|
205
|
+
const readyFiles =
|
|
206
|
+
await databaseService.default.getFilesReadyForUpload();
|
|
207
|
+
|
|
155
208
|
if (readyFiles.length === 0) {
|
|
156
209
|
console.log('ā¹ļø No files are currently ready for upload');
|
|
157
|
-
console.log(
|
|
210
|
+
console.log(
|
|
211
|
+
' Tip: Run "arela detect" and "arela detect --propagate-arela-path" first to prepare files for upload',
|
|
212
|
+
);
|
|
158
213
|
} else {
|
|
159
|
-
console.log(
|
|
160
|
-
|
|
214
|
+
console.log(
|
|
215
|
+
`\nš ${readyFiles.length} files are ready for upload!`,
|
|
216
|
+
);
|
|
217
|
+
console.log(
|
|
218
|
+
' Use "arela upload --upload-by-rfc" to upload them to Arela API',
|
|
219
|
+
);
|
|
161
220
|
}
|
|
162
221
|
return;
|
|
163
222
|
}
|
|
164
|
-
|
|
223
|
+
|
|
165
224
|
// Default behavior: show help for query command
|
|
166
225
|
console.log('Available query options:');
|
|
167
226
|
console.log(' --ready-files Show files ready for upload');
|
|
@@ -171,7 +230,7 @@ class ArelaUploaderCLI {
|
|
|
171
230
|
});
|
|
172
231
|
|
|
173
232
|
// Version command (already handled by program.version())
|
|
174
|
-
|
|
233
|
+
|
|
175
234
|
// Help command
|
|
176
235
|
this.program
|
|
177
236
|
.command('help')
|
|
@@ -188,15 +247,18 @@ class ArelaUploaderCLI {
|
|
|
188
247
|
#setupErrorHandling() {
|
|
189
248
|
// Handle uncaught exceptions
|
|
190
249
|
process.on('uncaughtException', (error) => {
|
|
191
|
-
this.errorHandler.handleFatalError(error, {
|
|
250
|
+
this.errorHandler.handleFatalError(error, {
|
|
251
|
+
context: 'uncaughtException',
|
|
252
|
+
});
|
|
192
253
|
});
|
|
193
254
|
|
|
194
255
|
// Handle unhandled promise rejections
|
|
195
256
|
process.on('unhandledRejection', (reason, promise) => {
|
|
196
|
-
const error =
|
|
197
|
-
|
|
257
|
+
const error =
|
|
258
|
+
reason instanceof Error ? reason : new Error(String(reason));
|
|
259
|
+
this.errorHandler.handleFatalError(error, {
|
|
198
260
|
context: 'unhandledRejection',
|
|
199
|
-
promise: promise.toString()
|
|
261
|
+
promise: promise.toString(),
|
|
200
262
|
});
|
|
201
263
|
});
|
|
202
264
|
|
|
@@ -232,19 +294,33 @@ class ArelaUploaderCLI {
|
|
|
232
294
|
console.log(` Key: ${appConfig.supabase.key ? 'ā
Set' : 'ā Not set'}`);
|
|
233
295
|
console.log(` Bucket: ${appConfig.supabase.bucket || 'Not configured'}`);
|
|
234
296
|
console.log('\nš Upload Configuration:');
|
|
235
|
-
console.log(
|
|
236
|
-
|
|
237
|
-
|
|
297
|
+
console.log(
|
|
298
|
+
` Base Path: ${appConfig.upload.basePath || 'Not configured'}`,
|
|
299
|
+
);
|
|
300
|
+
console.log(
|
|
301
|
+
` Sources: ${appConfig.upload.sources?.join(', ') || 'Not configured'}`,
|
|
302
|
+
);
|
|
303
|
+
console.log(
|
|
304
|
+
` RFCs: ${appConfig.upload.rfcs?.join(', ') || 'Not configured'}`,
|
|
305
|
+
);
|
|
238
306
|
console.log('\nā” Performance Configuration:');
|
|
239
307
|
console.log(` Batch Delay: ${appConfig.performance.batchDelay}ms`);
|
|
240
|
-
console.log(
|
|
308
|
+
console.log(
|
|
309
|
+
` Progress Update Interval: ${appConfig.performance.progressUpdateInterval}`,
|
|
310
|
+
);
|
|
241
311
|
console.log(` Log Buffer Size: ${appConfig.performance.logBufferSize}`);
|
|
242
312
|
console.log('\nš Logging Configuration:');
|
|
243
|
-
console.log(
|
|
313
|
+
console.log(
|
|
314
|
+
` Verbose: ${appConfig.logging.verbose ? 'ā
Enabled' : 'ā Disabled'}`,
|
|
315
|
+
);
|
|
244
316
|
console.log(` Log File: ${appConfig.logging.logFilePath}`);
|
|
245
317
|
console.log('\nšÆ Service Availability:');
|
|
246
|
-
console.log(
|
|
247
|
-
|
|
318
|
+
console.log(
|
|
319
|
+
` API Mode: ${appConfig.isApiModeAvailable() ? 'ā
Available' : 'ā Not available'}`,
|
|
320
|
+
);
|
|
321
|
+
console.log(
|
|
322
|
+
` Supabase Mode: ${appConfig.isSupabaseModeAvailable() ? 'ā
Available' : 'ā Not available'}`,
|
|
323
|
+
);
|
|
248
324
|
}
|
|
249
325
|
|
|
250
326
|
/**
|
|
@@ -257,7 +333,7 @@ class ArelaUploaderCLI {
|
|
|
257
333
|
if (args.includes('-v') || args.includes('--verbose')) {
|
|
258
334
|
logger.setVerbose(true);
|
|
259
335
|
}
|
|
260
|
-
|
|
336
|
+
|
|
261
337
|
// Clear log if requested globally
|
|
262
338
|
if (args.includes('--clear-log')) {
|
|
263
339
|
logger.clearLogFile();
|
|
@@ -270,7 +346,6 @@ class ArelaUploaderCLI {
|
|
|
270
346
|
|
|
271
347
|
// Parse and execute commands
|
|
272
348
|
await this.program.parseAsync();
|
|
273
|
-
|
|
274
349
|
} catch (error) {
|
|
275
350
|
this.errorHandler.handleFatalError(error, { context: 'cli-execution' });
|
|
276
351
|
}
|
|
@@ -279,4 +354,4 @@ class ArelaUploaderCLI {
|
|
|
279
354
|
|
|
280
355
|
// Create and run the CLI application
|
|
281
356
|
const cli = new ArelaUploaderCLI();
|
|
282
|
-
await cli.run();
|
|
357
|
+
await cli.run();
|