@danielarndt0/cnpj-db-loader 2.4.0-beta.2 → 2.4.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -6
- package/dist/cli.js +1037 -296
- package/dist/cli.js.map +1 -1
- package/dist/index.d.ts +337 -297
- package/dist/index.js +879 -290
- package/dist/index.js.map +1 -1
- package/docs/commands.md +11 -1
- package/docs/federal-revenue.md +36 -2
- package/docs/postgres-direct.md +235 -41
- package/package.json +1 -1
package/dist/index.d.ts
CHANGED
|
@@ -20,8 +20,14 @@ type AppConfig = {
|
|
|
20
20
|
description: string;
|
|
21
21
|
};
|
|
22
22
|
type DatasetBlock = "companies" | "establishments" | "partners" | "simples_options" | "countries" | "cities" | "partner_qualifications" | "legal_natures" | "cnaes";
|
|
23
|
+
type FederalRevenueConfig = {
|
|
24
|
+
shareToken?: string;
|
|
25
|
+
webdavUrl?: string;
|
|
26
|
+
userAgent?: string;
|
|
27
|
+
};
|
|
23
28
|
type DatabaseConfig = {
|
|
24
29
|
defaultDbUrl?: string;
|
|
30
|
+
federalRevenue?: FederalRevenueConfig;
|
|
25
31
|
};
|
|
26
32
|
|
|
27
33
|
type Result<T> = {
|
|
@@ -46,11 +52,74 @@ declare function defaultExtractedOutputPath(inputPath: string): string;
|
|
|
46
52
|
type SupportedOs = "windows" | "macos" | "linux" | "unknown";
|
|
47
53
|
declare function detectOs(): SupportedOs;
|
|
48
54
|
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
55
|
+
type ExtractionEntry = {
|
|
56
|
+
archivePath: string;
|
|
57
|
+
archiveName: string;
|
|
58
|
+
destinationPath: string;
|
|
59
|
+
success: boolean;
|
|
60
|
+
sizeInBytes: number;
|
|
61
|
+
errorMessage?: string;
|
|
62
|
+
};
|
|
63
|
+
type ExtractionSummary = {
|
|
64
|
+
inputPath: string;
|
|
65
|
+
outputPath: string;
|
|
66
|
+
operatingSystem: string;
|
|
67
|
+
zipFilesFound: number;
|
|
68
|
+
extractedArchives: string[];
|
|
69
|
+
skippedEntries: string[];
|
|
70
|
+
failedArchives: string[];
|
|
71
|
+
totalArchiveBytes: number;
|
|
72
|
+
extractedArchiveBytes: number;
|
|
73
|
+
entries: ExtractionEntry[];
|
|
74
|
+
};
|
|
75
|
+
type ExtractionProgressEvent = {
|
|
76
|
+
kind: "start";
|
|
77
|
+
totalArchives: number;
|
|
78
|
+
totalBytes: number;
|
|
79
|
+
inputPath: string;
|
|
80
|
+
outputPath: string;
|
|
81
|
+
} | {
|
|
82
|
+
kind: "archive-start";
|
|
83
|
+
currentArchiveName: string;
|
|
84
|
+
currentArchivePath: string;
|
|
85
|
+
archiveSizeInBytes: number;
|
|
86
|
+
archiveIndex: number;
|
|
87
|
+
totalArchives: number;
|
|
88
|
+
completedArchives: number;
|
|
89
|
+
extractedBytes: number;
|
|
90
|
+
totalBytes: number;
|
|
91
|
+
} | {
|
|
92
|
+
kind: "archive-complete";
|
|
93
|
+
currentArchiveName: string;
|
|
94
|
+
currentArchivePath: string;
|
|
95
|
+
archiveSizeInBytes: number;
|
|
96
|
+
archiveIndex: number;
|
|
97
|
+
totalArchives: number;
|
|
98
|
+
completedArchives: number;
|
|
99
|
+
extractedBytes: number;
|
|
100
|
+
totalBytes: number;
|
|
101
|
+
} | {
|
|
102
|
+
kind: "archive-failed";
|
|
103
|
+
currentArchiveName: string;
|
|
104
|
+
currentArchivePath: string;
|
|
105
|
+
archiveSizeInBytes: number;
|
|
106
|
+
archiveIndex: number;
|
|
107
|
+
totalArchives: number;
|
|
108
|
+
completedArchives: number;
|
|
109
|
+
extractedBytes: number;
|
|
110
|
+
totalBytes: number;
|
|
111
|
+
errorMessage: string;
|
|
112
|
+
} | {
|
|
113
|
+
kind: "finish";
|
|
114
|
+
totalArchives: number;
|
|
115
|
+
completedArchives: number;
|
|
116
|
+
failedArchives: number;
|
|
117
|
+
extractedBytes: number;
|
|
118
|
+
totalBytes: number;
|
|
119
|
+
outputPath: string;
|
|
120
|
+
};
|
|
121
|
+
type ExtractionProgressListener = (event: ExtractionProgressEvent) => void;
|
|
122
|
+
declare function extractArchives(inputPath: string, outputPath?: string, onProgress?: ExtractionProgressListener): Promise<ExtractionSummary>;
|
|
54
123
|
|
|
55
124
|
type FieldDefinition = {
|
|
56
125
|
sourceLabel: string;
|
|
@@ -288,299 +357,48 @@ type ImportProgressEvent = {
|
|
|
288
357
|
totalRows: number;
|
|
289
358
|
committedBatches: number;
|
|
290
359
|
totalBatches: number;
|
|
291
|
-
quarantinedRows: number;
|
|
292
|
-
};
|
|
293
|
-
type ImportProgressListener = (event: ImportProgressEvent) => void;
|
|
294
|
-
type ImportOptions = {
|
|
295
|
-
dbUrl?: string;
|
|
296
|
-
dataset?: ImportDatasetType | undefined;
|
|
297
|
-
batchSize?: number | undefined;
|
|
298
|
-
loadBatchSize?: number | undefined;
|
|
299
|
-
materializeBatchSize?: number | undefined;
|
|
300
|
-
verboseProgress?: boolean | undefined;
|
|
301
|
-
onProgress?: ImportProgressListener | undefined;
|
|
302
|
-
};
|
|
303
|
-
type ImportExecutionMode = "full" | "load" | "materialize";
|
|
304
|
-
type ImportSummary = {
|
|
305
|
-
executionMode: ImportExecutionMode;
|
|
306
|
-
inputPath: string;
|
|
307
|
-
validatedPath: string;
|
|
308
|
-
targetDatabase: string;
|
|
309
|
-
importPlanId: number | null;
|
|
310
|
-
reusedImportPlan: boolean;
|
|
311
|
-
importedDatasets: ImportDatasetType[];
|
|
312
|
-
importedFiles: number;
|
|
313
|
-
processedRows: number;
|
|
314
|
-
plannedRows: number;
|
|
315
|
-
committedBatches: number;
|
|
316
|
-
plannedBatches: number;
|
|
317
|
-
quarantinedRows: number;
|
|
318
|
-
resumedFiles: number;
|
|
319
|
-
skippedCompletedFiles: number;
|
|
320
|
-
datasetSummaries: Array<{
|
|
321
|
-
dataset: ImportDatasetType;
|
|
322
|
-
files: number;
|
|
323
|
-
rows: number;
|
|
324
|
-
}>;
|
|
325
|
-
performance: ImportPerformanceSummary;
|
|
326
|
-
warnings: string[];
|
|
327
|
-
progressLogPath: string;
|
|
328
|
-
};
|
|
329
|
-
|
|
330
|
-
type DatabaseCleanupScope = "staging" | "materialized" | "checkpoints" | "plans";
|
|
331
|
-
type CheckpointCleanupPhase = "load" | "materialization" | "all";
|
|
332
|
-
type DatabaseCleanupSummary = {
|
|
333
|
-
scope: DatabaseCleanupScope;
|
|
334
|
-
targetDatabase: string;
|
|
335
|
-
dataset?: ImportDatasetType | undefined;
|
|
336
|
-
phase?: CheckpointCleanupPhase | undefined;
|
|
337
|
-
validatedPath?: string | undefined;
|
|
338
|
-
planId?: number | undefined;
|
|
339
|
-
truncatedTables: string[];
|
|
340
|
-
deletedLoadCheckpoints: number;
|
|
341
|
-
deletedMaterializationCheckpoints: number;
|
|
342
|
-
deletedPlans: number;
|
|
343
|
-
notes: string[];
|
|
344
|
-
};
|
|
345
|
-
|
|
346
|
-
declare function resolveDatabaseUrl(override?: string): Promise<string>;
|
|
347
|
-
declare function testDatabaseConnection(url: string): Promise<void>;
|
|
348
|
-
declare function cleanupDatabaseStagingData(options?: {
|
|
349
|
-
dbUrl?: string;
|
|
350
|
-
dataset?: ImportDatasetType | undefined;
|
|
351
|
-
validatedPath?: string | undefined;
|
|
352
|
-
}): Promise<DatabaseCleanupSummary>;
|
|
353
|
-
declare function cleanupDatabaseMaterializedData(options?: {
|
|
354
|
-
dbUrl?: string;
|
|
355
|
-
dataset?: ImportDatasetType | undefined;
|
|
356
|
-
}): Promise<DatabaseCleanupSummary>;
|
|
357
|
-
declare function cleanupDatabaseCheckpointsData(options?: {
|
|
358
|
-
dbUrl?: string;
|
|
359
|
-
phase?: CheckpointCleanupPhase | undefined;
|
|
360
|
-
dataset?: ImportDatasetType | undefined;
|
|
361
|
-
validatedPath?: string | undefined;
|
|
362
|
-
planId?: number | undefined;
|
|
363
|
-
}): Promise<DatabaseCleanupSummary>;
|
|
364
|
-
declare function cleanupDatabasePlansData(options?: {
|
|
365
|
-
dbUrl?: string;
|
|
366
|
-
validatedPath?: string | undefined;
|
|
367
|
-
planId?: number | undefined;
|
|
368
|
-
}): Promise<DatabaseCleanupSummary>;
|
|
369
|
-
declare function resolveDbUrl(override?: string): Promise<string>;
|
|
370
|
-
|
|
371
|
-
declare function getAllLayouts(): TableLayout[];
|
|
372
|
-
declare function getLayoutSummary(): Array<Pick<TableLayout, "key" | "tableName" | "sourceName">>;
|
|
373
|
-
|
|
374
|
-
declare function runDoctor(inputPath?: string, dbUrl?: string): Promise<string[]>;
|
|
375
|
-
|
|
376
|
-
declare const SCHEMA_PROFILES: readonly ["full", "final", "staging"];
|
|
377
|
-
type SchemaProfile = (typeof SCHEMA_PROFILES)[number];
|
|
378
|
-
type SchemaGenerationOptions = {
|
|
379
|
-
profile?: SchemaProfile;
|
|
380
|
-
};
|
|
381
|
-
|
|
382
|
-
declare function generateSchemaSql(options?: SchemaGenerationOptions): string;
|
|
383
|
-
declare function writeSchemaFile(outFile: string, options?: SchemaGenerationOptions): Promise<void>;
|
|
384
|
-
declare function resolveSchemaProfile(profile?: string): SchemaProfile;
|
|
385
|
-
|
|
386
|
-
type ValidationSummary = {
|
|
387
|
-
ok: boolean;
|
|
388
|
-
errors: string[];
|
|
389
|
-
warnings: string[];
|
|
390
|
-
inspected: InspectSummary;
|
|
391
|
-
validatedPath: string;
|
|
392
|
-
presentDatasets: DatasetType[];
|
|
393
|
-
missingDatasets: DatasetType[];
|
|
394
|
-
nextStep?: string | undefined;
|
|
395
|
-
};
|
|
396
|
-
declare function validateInputDirectory(inputPath: string): Promise<ValidationSummary>;
|
|
397
|
-
|
|
398
|
-
type ExtractionEntry = {
|
|
399
|
-
archivePath: string;
|
|
400
|
-
archiveName: string;
|
|
401
|
-
destinationPath: string;
|
|
402
|
-
success: boolean;
|
|
403
|
-
sizeInBytes: number;
|
|
404
|
-
errorMessage?: string;
|
|
405
|
-
};
|
|
406
|
-
type ExtractionSummary = {
|
|
407
|
-
inputPath: string;
|
|
408
|
-
outputPath: string;
|
|
409
|
-
operatingSystem: string;
|
|
410
|
-
zipFilesFound: number;
|
|
411
|
-
extractedArchives: string[];
|
|
412
|
-
skippedEntries: string[];
|
|
413
|
-
failedArchives: string[];
|
|
414
|
-
totalArchiveBytes: number;
|
|
415
|
-
extractedArchiveBytes: number;
|
|
416
|
-
entries: ExtractionEntry[];
|
|
417
|
-
};
|
|
418
|
-
type ExtractionProgressEvent = {
|
|
419
|
-
kind: "start";
|
|
420
|
-
totalArchives: number;
|
|
421
|
-
totalBytes: number;
|
|
422
|
-
inputPath: string;
|
|
423
|
-
outputPath: string;
|
|
424
|
-
} | {
|
|
425
|
-
kind: "archive-start";
|
|
426
|
-
currentArchiveName: string;
|
|
427
|
-
currentArchivePath: string;
|
|
428
|
-
archiveSizeInBytes: number;
|
|
429
|
-
archiveIndex: number;
|
|
430
|
-
totalArchives: number;
|
|
431
|
-
completedArchives: number;
|
|
432
|
-
extractedBytes: number;
|
|
433
|
-
totalBytes: number;
|
|
434
|
-
} | {
|
|
435
|
-
kind: "archive-complete";
|
|
436
|
-
currentArchiveName: string;
|
|
437
|
-
currentArchivePath: string;
|
|
438
|
-
archiveSizeInBytes: number;
|
|
439
|
-
archiveIndex: number;
|
|
440
|
-
totalArchives: number;
|
|
441
|
-
completedArchives: number;
|
|
442
|
-
extractedBytes: number;
|
|
443
|
-
totalBytes: number;
|
|
444
|
-
} | {
|
|
445
|
-
kind: "archive-failed";
|
|
446
|
-
currentArchiveName: string;
|
|
447
|
-
currentArchivePath: string;
|
|
448
|
-
archiveSizeInBytes: number;
|
|
449
|
-
archiveIndex: number;
|
|
450
|
-
totalArchives: number;
|
|
451
|
-
completedArchives: number;
|
|
452
|
-
extractedBytes: number;
|
|
453
|
-
totalBytes: number;
|
|
454
|
-
errorMessage: string;
|
|
455
|
-
} | {
|
|
456
|
-
kind: "finish";
|
|
457
|
-
totalArchives: number;
|
|
458
|
-
completedArchives: number;
|
|
459
|
-
failedArchives: number;
|
|
460
|
-
extractedBytes: number;
|
|
461
|
-
totalBytes: number;
|
|
462
|
-
outputPath: string;
|
|
463
|
-
};
|
|
464
|
-
type ExtractionProgressListener = (event: ExtractionProgressEvent) => void;
|
|
465
|
-
declare function extractArchives(inputPath: string, outputPath?: string, onProgress?: ExtractionProgressListener): Promise<ExtractionSummary>;
|
|
466
|
-
|
|
467
|
-
type InputMode = "unzip" | "already-extracted";
|
|
468
|
-
declare function resolveInputMode(options: {
|
|
469
|
-
unzip?: boolean;
|
|
470
|
-
alreadyExtracted?: boolean;
|
|
471
|
-
}): InputMode;
|
|
472
|
-
|
|
473
|
-
declare const LOG_LEVELS: readonly ["trace", "debug", "info", "warning", "error", "critical", "fatal"];
|
|
474
|
-
type LogLevel = (typeof LOG_LEVELS)[number];
|
|
475
|
-
declare const LOG_STATUSES: readonly ["success", "failure"];
|
|
476
|
-
type LogStatus = (typeof LOG_STATUSES)[number];
|
|
477
|
-
type LogRecord = Record<string, unknown>;
|
|
478
|
-
type StructuredLogEntry = LogRecord & {
|
|
479
|
-
timestamp: string;
|
|
480
|
-
level: LogLevel;
|
|
481
|
-
severity: LogLevel;
|
|
482
|
-
event: string;
|
|
483
|
-
kind: string;
|
|
484
|
-
command?: string;
|
|
485
|
-
status?: LogStatus;
|
|
486
|
-
message?: string;
|
|
487
|
-
};
|
|
488
|
-
|
|
489
|
-
type WriteLogOptions = {
|
|
490
|
-
baseDirectory?: string;
|
|
491
|
-
event?: string;
|
|
492
|
-
level?: LogLevel;
|
|
493
|
-
status?: LogStatus;
|
|
494
|
-
message?: string;
|
|
495
|
-
};
|
|
496
|
-
declare function getUserAppDirectoryPath(): string;
|
|
497
|
-
declare function getLogsDirectoryPath(baseDirectory?: string): string;
|
|
498
|
-
declare function writeCommandLog(commandName: string, payload: unknown, baseDirectoryOrOptions?: string | WriteLogOptions, maybeOptions?: Omit<WriteLogOptions, "baseDirectory">): Promise<string>;
|
|
499
|
-
declare function writeCommandFailureLog(commandName: string, error: unknown, input?: {
|
|
500
|
-
argv?: string[];
|
|
501
|
-
context?: Record<string, unknown>;
|
|
502
|
-
baseDirectory?: string;
|
|
503
|
-
fatal?: boolean;
|
|
504
|
-
}): Promise<string>;
|
|
505
|
-
declare function createJsonLinesLog(commandName: string, baseDirectory?: string): Promise<string>;
|
|
506
|
-
declare function appendJsonLinesLog(filePath: string, payload: unknown, options?: Omit<WriteLogOptions, "baseDirectory" | "status"> & {
|
|
507
|
-
command?: string;
|
|
508
|
-
status?: LogStatus;
|
|
509
|
-
}): Promise<void>;
|
|
510
|
-
|
|
511
|
-
declare function importDataToDatabase(inputPath: string, options?: ImportOptions): Promise<ImportSummary>;
|
|
512
|
-
declare function loadImportDataToStaging(inputPath: string, options?: ImportOptions): Promise<ImportSummary>;
|
|
513
|
-
declare function materializeImportedData(inputPath: string, options?: ImportOptions): Promise<ImportSummary>;
|
|
514
|
-
|
|
515
|
-
type QuarantineStatsFilters = {
|
|
516
|
-
dataset?: string;
|
|
517
|
-
category?: string;
|
|
518
|
-
stage?: string;
|
|
519
|
-
retryable?: boolean;
|
|
520
|
-
terminal?: boolean;
|
|
521
|
-
};
|
|
522
|
-
type QuarantineListFilters = QuarantineStatsFilters & {
|
|
523
|
-
limit: number;
|
|
524
|
-
afterId?: number;
|
|
525
|
-
};
|
|
526
|
-
type QuarantineStatsCount = {
|
|
527
|
-
key: string;
|
|
528
|
-
count: number;
|
|
529
|
-
};
|
|
530
|
-
type QuarantineStatsSummary = {
|
|
531
|
-
totalRows: number;
|
|
532
|
-
retryableRows: number;
|
|
533
|
-
terminalRows: number;
|
|
534
|
-
rowsByDataset: QuarantineStatsCount[];
|
|
535
|
-
rowsByCategory: QuarantineStatsCount[];
|
|
536
|
-
rowsByStage: QuarantineStatsCount[];
|
|
537
|
-
appliedFilters: QuarantineStatsFilters;
|
|
538
|
-
};
|
|
539
|
-
type QuarantineListRow = {
|
|
540
|
-
id: number;
|
|
541
|
-
dataset: string;
|
|
542
|
-
filePath: string;
|
|
543
|
-
rowNumber: number | null;
|
|
544
|
-
checkpointOffset: number | null;
|
|
545
|
-
errorCode: string | null;
|
|
546
|
-
errorCategory: string | null;
|
|
547
|
-
errorStage: string | null;
|
|
548
|
-
errorMessage: string;
|
|
549
|
-
retryCount: number;
|
|
550
|
-
canRetryLater: boolean;
|
|
551
|
-
createdAt: string;
|
|
552
|
-
};
|
|
553
|
-
type QuarantineListSummary = {
|
|
554
|
-
rows: QuarantineListRow[];
|
|
555
|
-
appliedFilters: QuarantineListFilters;
|
|
360
|
+
quarantinedRows: number;
|
|
556
361
|
};
|
|
557
|
-
type
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
362
|
+
type ImportProgressListener = (event: ImportProgressEvent) => void;
|
|
363
|
+
type ImportOptions = {
|
|
364
|
+
dbUrl?: string;
|
|
365
|
+
dataset?: ImportDatasetType | undefined;
|
|
366
|
+
batchSize?: number | undefined;
|
|
367
|
+
loadBatchSize?: number | undefined;
|
|
368
|
+
materializeBatchSize?: number | undefined;
|
|
369
|
+
verboseProgress?: boolean | undefined;
|
|
370
|
+
onProgress?: ImportProgressListener | undefined;
|
|
371
|
+
};
|
|
372
|
+
type ImportExecutionMode = "full" | "load" | "materialize";
|
|
373
|
+
type ImportSummary = {
|
|
374
|
+
executionMode: ImportExecutionMode;
|
|
375
|
+
inputPath: string;
|
|
376
|
+
validatedPath: string;
|
|
377
|
+
targetDatabase: string;
|
|
378
|
+
importPlanId: number | null;
|
|
379
|
+
reusedImportPlan: boolean;
|
|
380
|
+
importedDatasets: ImportDatasetType[];
|
|
381
|
+
importedFiles: number;
|
|
382
|
+
processedRows: number;
|
|
383
|
+
plannedRows: number;
|
|
384
|
+
committedBatches: number;
|
|
385
|
+
plannedBatches: number;
|
|
386
|
+
quarantinedRows: number;
|
|
387
|
+
resumedFiles: number;
|
|
388
|
+
skippedCompletedFiles: number;
|
|
389
|
+
datasetSummaries: Array<{
|
|
390
|
+
dataset: ImportDatasetType;
|
|
391
|
+
files: number;
|
|
392
|
+
rows: number;
|
|
393
|
+
}>;
|
|
394
|
+
performance: ImportPerformanceSummary;
|
|
395
|
+
warnings: string[];
|
|
396
|
+
progressLogPath: string;
|
|
573
397
|
};
|
|
574
398
|
|
|
575
|
-
declare function
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
declare function listQuarantineRows(filters: QuarantineListFilters & {
|
|
579
|
-
dbUrl?: string;
|
|
580
|
-
}): Promise<QuarantineListSummary>;
|
|
581
|
-
declare function showQuarantineRow(id: number, options?: {
|
|
582
|
-
dbUrl?: string;
|
|
583
|
-
}): Promise<QuarantineRecord>;
|
|
399
|
+
declare function importDataToDatabase(inputPath: string, options?: ImportOptions): Promise<ImportSummary>;
|
|
400
|
+
declare function loadImportDataToStaging(inputPath: string, options?: ImportOptions): Promise<ImportSummary>;
|
|
401
|
+
declare function materializeImportedData(inputPath: string, options?: ImportOptions): Promise<ImportSummary>;
|
|
584
402
|
|
|
585
403
|
type SanitizeSourceEncoding = "WIN1252" | "LATIN1" | "UTF8";
|
|
586
404
|
|
|
@@ -671,6 +489,18 @@ type SanitizeOptions = {
|
|
|
671
489
|
|
|
672
490
|
declare function sanitizeInputDirectory(inputPath: string, options?: SanitizeOptions): Promise<SanitizeSummary>;
|
|
673
491
|
|
|
492
|
+
type ValidationSummary = {
|
|
493
|
+
ok: boolean;
|
|
494
|
+
errors: string[];
|
|
495
|
+
warnings: string[];
|
|
496
|
+
inspected: InspectSummary;
|
|
497
|
+
validatedPath: string;
|
|
498
|
+
presentDatasets: DatasetType[];
|
|
499
|
+
missingDatasets: DatasetType[];
|
|
500
|
+
nextStep?: string | undefined;
|
|
501
|
+
};
|
|
502
|
+
declare function validateInputDirectory(inputPath: string): Promise<ValidationSummary>;
|
|
503
|
+
|
|
674
504
|
type FederalRevenueReferenceMode = "latest" | "current" | "explicit";
|
|
675
505
|
type FederalRevenueClientOptions = {
|
|
676
506
|
baseUrl?: string | undefined;
|
|
@@ -917,7 +747,198 @@ type FederalRevenueLockFile = {
|
|
|
917
747
|
startedAt: string;
|
|
918
748
|
};
|
|
919
749
|
|
|
920
|
-
|
|
750
|
+
type FederalRevenueConfigKey = "share-token" | "webdav-url" | "user-agent";
|
|
751
|
+
type FederalRevenueEffectiveConfig = {
|
|
752
|
+
webdavUrl: string;
|
|
753
|
+
userAgent: string;
|
|
754
|
+
shareToken?: string | undefined;
|
|
755
|
+
configured: {
|
|
756
|
+
webdavUrl: boolean;
|
|
757
|
+
userAgent: boolean;
|
|
758
|
+
shareToken: boolean;
|
|
759
|
+
};
|
|
760
|
+
};
|
|
761
|
+
declare function readDatabaseConfig(): Promise<DatabaseConfig>;
|
|
762
|
+
declare function writeDatabaseConfig(config: DatabaseConfig): Promise<void>;
|
|
763
|
+
declare function assertPostgresUrl(url: string): void;
|
|
764
|
+
declare function setDefaultDbUrl(url: string): Promise<void>;
|
|
765
|
+
declare function resetDefaultDbUrl(): Promise<void>;
|
|
766
|
+
declare function setFederalRevenueConfigValue(key: string, value: string): Promise<FederalRevenueEffectiveConfig>;
|
|
767
|
+
declare function resetFederalRevenueConfig(key?: string | undefined): Promise<FederalRevenueEffectiveConfig>;
|
|
768
|
+
declare function getFederalRevenueEffectiveConfig(config?: FederalRevenueConfig): FederalRevenueEffectiveConfig;
|
|
769
|
+
declare function readFederalRevenueEffectiveConfig(): Promise<FederalRevenueEffectiveConfig>;
|
|
770
|
+
declare function resolveFederalRevenueClientOptions(overrides?: FederalRevenueClientOptions): Promise<FederalRevenueClientOptions>;
|
|
771
|
+
|
|
772
|
+
type DatabaseCleanupScope = "staging" | "materialized" | "checkpoints" | "plans";
|
|
773
|
+
type CheckpointCleanupPhase = "load" | "materialization" | "all";
|
|
774
|
+
type DatabaseCleanupSummary = {
|
|
775
|
+
scope: DatabaseCleanupScope;
|
|
776
|
+
targetDatabase: string;
|
|
777
|
+
dataset?: ImportDatasetType | undefined;
|
|
778
|
+
phase?: CheckpointCleanupPhase | undefined;
|
|
779
|
+
validatedPath?: string | undefined;
|
|
780
|
+
planId?: number | undefined;
|
|
781
|
+
truncatedTables: string[];
|
|
782
|
+
deletedLoadCheckpoints: number;
|
|
783
|
+
deletedMaterializationCheckpoints: number;
|
|
784
|
+
deletedPlans: number;
|
|
785
|
+
notes: string[];
|
|
786
|
+
};
|
|
787
|
+
|
|
788
|
+
declare function resolveDatabaseUrl(override?: string): Promise<string>;
|
|
789
|
+
declare function testDatabaseConnection(url: string): Promise<void>;
|
|
790
|
+
declare function cleanupDatabaseStagingData(options?: {
|
|
791
|
+
dbUrl?: string;
|
|
792
|
+
dataset?: ImportDatasetType | undefined;
|
|
793
|
+
validatedPath?: string | undefined;
|
|
794
|
+
}): Promise<DatabaseCleanupSummary>;
|
|
795
|
+
declare function cleanupDatabaseMaterializedData(options?: {
|
|
796
|
+
dbUrl?: string;
|
|
797
|
+
dataset?: ImportDatasetType | undefined;
|
|
798
|
+
}): Promise<DatabaseCleanupSummary>;
|
|
799
|
+
declare function cleanupDatabaseCheckpointsData(options?: {
|
|
800
|
+
dbUrl?: string;
|
|
801
|
+
phase?: CheckpointCleanupPhase | undefined;
|
|
802
|
+
dataset?: ImportDatasetType | undefined;
|
|
803
|
+
validatedPath?: string | undefined;
|
|
804
|
+
planId?: number | undefined;
|
|
805
|
+
}): Promise<DatabaseCleanupSummary>;
|
|
806
|
+
declare function cleanupDatabasePlansData(options?: {
|
|
807
|
+
dbUrl?: string;
|
|
808
|
+
validatedPath?: string | undefined;
|
|
809
|
+
planId?: number | undefined;
|
|
810
|
+
}): Promise<DatabaseCleanupSummary>;
|
|
811
|
+
declare function resolveDbUrl(override?: string): Promise<string>;
|
|
812
|
+
|
|
813
|
+
declare function getAllLayouts(): TableLayout[];
|
|
814
|
+
declare function getLayoutSummary(): Array<Pick<TableLayout, "key" | "tableName" | "sourceName">>;
|
|
815
|
+
|
|
816
|
+
declare function runDoctor(inputPath?: string, dbUrl?: string): Promise<string[]>;
|
|
817
|
+
|
|
818
|
+
declare const SCHEMA_PROFILES: readonly ["full", "final", "staging"];
|
|
819
|
+
type SchemaProfile = (typeof SCHEMA_PROFILES)[number];
|
|
820
|
+
type SchemaGenerationOptions = {
|
|
821
|
+
profile?: SchemaProfile;
|
|
822
|
+
};
|
|
823
|
+
|
|
824
|
+
declare function generateSchemaSql(options?: SchemaGenerationOptions): string;
|
|
825
|
+
declare function writeSchemaFile(outFile: string, options?: SchemaGenerationOptions): Promise<void>;
|
|
826
|
+
declare function resolveSchemaProfile(profile?: string): SchemaProfile;
|
|
827
|
+
|
|
828
|
+
type InputMode = "unzip" | "already-extracted";
|
|
829
|
+
declare function resolveInputMode(options: {
|
|
830
|
+
unzip?: boolean;
|
|
831
|
+
alreadyExtracted?: boolean;
|
|
832
|
+
}): InputMode;
|
|
833
|
+
|
|
834
|
+
declare const LOG_LEVELS: readonly ["trace", "debug", "info", "warning", "error", "critical", "fatal"];
|
|
835
|
+
type LogLevel = (typeof LOG_LEVELS)[number];
|
|
836
|
+
declare const LOG_STATUSES: readonly ["success", "failure"];
|
|
837
|
+
type LogStatus = (typeof LOG_STATUSES)[number];
|
|
838
|
+
type LogRecord = Record<string, unknown>;
|
|
839
|
+
type StructuredLogEntry = LogRecord & {
|
|
840
|
+
timestamp: string;
|
|
841
|
+
level: LogLevel;
|
|
842
|
+
severity: LogLevel;
|
|
843
|
+
event: string;
|
|
844
|
+
kind: string;
|
|
845
|
+
command?: string;
|
|
846
|
+
status?: LogStatus;
|
|
847
|
+
message?: string;
|
|
848
|
+
};
|
|
849
|
+
|
|
850
|
+
type WriteLogOptions = {
|
|
851
|
+
baseDirectory?: string;
|
|
852
|
+
event?: string;
|
|
853
|
+
level?: LogLevel;
|
|
854
|
+
status?: LogStatus;
|
|
855
|
+
message?: string;
|
|
856
|
+
};
|
|
857
|
+
declare function getUserAppDirectoryPath(): string;
|
|
858
|
+
declare function getLogsDirectoryPath(baseDirectory?: string): string;
|
|
859
|
+
declare function writeCommandLog(commandName: string, payload: unknown, baseDirectoryOrOptions?: string | WriteLogOptions, maybeOptions?: Omit<WriteLogOptions, "baseDirectory">): Promise<string>;
|
|
860
|
+
declare function writeCommandFailureLog(commandName: string, error: unknown, input?: {
|
|
861
|
+
argv?: string[];
|
|
862
|
+
context?: Record<string, unknown>;
|
|
863
|
+
baseDirectory?: string;
|
|
864
|
+
fatal?: boolean;
|
|
865
|
+
}): Promise<string>;
|
|
866
|
+
declare function createJsonLinesLog(commandName: string, baseDirectory?: string): Promise<string>;
|
|
867
|
+
declare function appendJsonLinesLog(filePath: string, payload: unknown, options?: Omit<WriteLogOptions, "baseDirectory" | "status"> & {
|
|
868
|
+
command?: string;
|
|
869
|
+
status?: LogStatus;
|
|
870
|
+
}): Promise<void>;
|
|
871
|
+
|
|
872
|
+
type QuarantineStatsFilters = {
|
|
873
|
+
dataset?: string;
|
|
874
|
+
category?: string;
|
|
875
|
+
stage?: string;
|
|
876
|
+
retryable?: boolean;
|
|
877
|
+
terminal?: boolean;
|
|
878
|
+
};
|
|
879
|
+
type QuarantineListFilters = QuarantineStatsFilters & {
|
|
880
|
+
limit: number;
|
|
881
|
+
afterId?: number;
|
|
882
|
+
};
|
|
883
|
+
type QuarantineStatsCount = {
|
|
884
|
+
key: string;
|
|
885
|
+
count: number;
|
|
886
|
+
};
|
|
887
|
+
type QuarantineStatsSummary = {
|
|
888
|
+
totalRows: number;
|
|
889
|
+
retryableRows: number;
|
|
890
|
+
terminalRows: number;
|
|
891
|
+
rowsByDataset: QuarantineStatsCount[];
|
|
892
|
+
rowsByCategory: QuarantineStatsCount[];
|
|
893
|
+
rowsByStage: QuarantineStatsCount[];
|
|
894
|
+
appliedFilters: QuarantineStatsFilters;
|
|
895
|
+
};
|
|
896
|
+
type QuarantineListRow = {
|
|
897
|
+
id: number;
|
|
898
|
+
dataset: string;
|
|
899
|
+
filePath: string;
|
|
900
|
+
rowNumber: number | null;
|
|
901
|
+
checkpointOffset: number | null;
|
|
902
|
+
errorCode: string | null;
|
|
903
|
+
errorCategory: string | null;
|
|
904
|
+
errorStage: string | null;
|
|
905
|
+
errorMessage: string;
|
|
906
|
+
retryCount: number;
|
|
907
|
+
canRetryLater: boolean;
|
|
908
|
+
createdAt: string;
|
|
909
|
+
};
|
|
910
|
+
type QuarantineListSummary = {
|
|
911
|
+
rows: QuarantineListRow[];
|
|
912
|
+
appliedFilters: QuarantineListFilters;
|
|
913
|
+
};
|
|
914
|
+
type QuarantineRecord = {
|
|
915
|
+
id: number;
|
|
916
|
+
dataset: string;
|
|
917
|
+
filePath: string;
|
|
918
|
+
rowNumber: number | null;
|
|
919
|
+
checkpointOffset: number | null;
|
|
920
|
+
errorCode: string | null;
|
|
921
|
+
errorCategory: string | null;
|
|
922
|
+
errorStage: string | null;
|
|
923
|
+
errorMessage: string;
|
|
924
|
+
rawLine: string;
|
|
925
|
+
parsedPayload: Record<string, unknown> | null;
|
|
926
|
+
sanitizationsApplied: unknown[];
|
|
927
|
+
retryCount: number;
|
|
928
|
+
canRetryLater: boolean;
|
|
929
|
+
createdAt: string;
|
|
930
|
+
};
|
|
931
|
+
|
|
932
|
+
declare function getQuarantineStats(filters: QuarantineStatsFilters & {
|
|
933
|
+
dbUrl?: string;
|
|
934
|
+
}): Promise<QuarantineStatsSummary>;
|
|
935
|
+
declare function listQuarantineRows(filters: QuarantineListFilters & {
|
|
936
|
+
dbUrl?: string;
|
|
937
|
+
}): Promise<QuarantineListSummary>;
|
|
938
|
+
declare function showQuarantineRow(id: number, options?: {
|
|
939
|
+
dbUrl?: string;
|
|
940
|
+
}): Promise<QuarantineRecord>;
|
|
941
|
+
|
|
921
942
|
declare const DEFAULT_FEDERAL_REVENUE_WEBDAV_URL = "https://arquivos.receitafederal.gov.br/public.php/webdav";
|
|
922
943
|
declare const DEFAULT_FEDERAL_REVENUE_USER_AGENT = "cnpj-db-loader federal-revenue-client";
|
|
923
944
|
declare function validateFederalRevenueReference(reference: string): void;
|
|
@@ -1052,6 +1073,14 @@ type PostgresDirectScriptDatasetSummary = {
|
|
|
1052
1073
|
totalBytes: number;
|
|
1053
1074
|
sourceFiles: string[];
|
|
1054
1075
|
};
|
|
1076
|
+
type PostgresDirectTransactionMode = "single" | "phase" | "none";
|
|
1077
|
+
type PostgresDirectIncludeTarget = "domains" | "companies" | "establishments" | "partners" | "simples" | "secondary-cnaes" | "indexes" | "analyze";
|
|
1078
|
+
type PostgresDirectScriptStep = {
|
|
1079
|
+
name: string;
|
|
1080
|
+
file: string;
|
|
1081
|
+
dependsOn: string[];
|
|
1082
|
+
included: boolean;
|
|
1083
|
+
};
|
|
1055
1084
|
type PostgresDirectScriptProgressEvent = {
|
|
1056
1085
|
kind: "start";
|
|
1057
1086
|
inputPath: string;
|
|
@@ -1060,6 +1089,10 @@ type PostgresDirectScriptProgressEvent = {
|
|
|
1060
1089
|
totalFiles: number;
|
|
1061
1090
|
datasets: ImportDatasetType[];
|
|
1062
1091
|
sourceEncoding: string;
|
|
1092
|
+
transactionMode: PostgresDirectTransactionMode;
|
|
1093
|
+
include: PostgresDirectIncludeTarget[];
|
|
1094
|
+
skipIndexes: boolean;
|
|
1095
|
+
skipAnalyze: boolean;
|
|
1063
1096
|
} | {
|
|
1064
1097
|
kind: "file_registered";
|
|
1065
1098
|
dataset: ImportDatasetType;
|
|
@@ -1080,6 +1113,10 @@ type PostgresDirectScriptOptions = {
|
|
|
1080
1113
|
dataset?: ImportDatasetType | undefined;
|
|
1081
1114
|
scriptName?: string | undefined;
|
|
1082
1115
|
sourceEncoding?: string | undefined;
|
|
1116
|
+
transactionMode?: PostgresDirectTransactionMode | undefined;
|
|
1117
|
+
include?: PostgresDirectIncludeTarget[] | undefined;
|
|
1118
|
+
skipIndexes?: boolean | undefined;
|
|
1119
|
+
skipAnalyze?: boolean | undefined;
|
|
1083
1120
|
onProgress?: PostgresDirectScriptProgressListener | undefined;
|
|
1084
1121
|
};
|
|
1085
1122
|
type PostgresDirectScriptSummary = {
|
|
@@ -1089,9 +1126,12 @@ type PostgresDirectScriptSummary = {
|
|
|
1089
1126
|
scriptPath: string;
|
|
1090
1127
|
manifestPath: string;
|
|
1091
1128
|
sourceEncoding: string;
|
|
1129
|
+
transactionMode: PostgresDirectTransactionMode;
|
|
1092
1130
|
totalFiles: number;
|
|
1093
1131
|
totalBytes: number;
|
|
1094
1132
|
datasets: PostgresDirectScriptDatasetSummary[];
|
|
1133
|
+
scriptFiles: string[];
|
|
1134
|
+
steps: PostgresDirectScriptStep[];
|
|
1095
1135
|
warnings: string[];
|
|
1096
1136
|
nextStep?: string | undefined;
|
|
1097
1137
|
};
|
|
@@ -1100,4 +1140,4 @@ declare function exportPostgresCsvDataset(inputPath: string, options?: PostgresC
|
|
|
1100
1140
|
|
|
1101
1141
|
declare function generatePostgresDirectScript(inputPath: string, options?: PostgresDirectScriptOptions): Promise<PostgresDirectScriptSummary>;
|
|
1102
1142
|
|
|
1103
|
-
export { type AppConfig, type AppEnvironment, AppError, type CheckpointCleanupPhase, DEFAULT_FEDERAL_REVENUE_DOWNLOAD_ROOT,
|
|
1143
|
+
export { type AppConfig, type AppEnvironment, AppError, type CheckpointCleanupPhase, DEFAULT_FEDERAL_REVENUE_DOWNLOAD_ROOT, DEFAULT_FEDERAL_REVENUE_USER_AGENT, DEFAULT_FEDERAL_REVENUE_WEBDAV_URL, type DatabaseCleanupSummary, type DatabaseConfig, type DatasetBlock, type DatasetType, type ExtractionEntry, type ExtractionProgressEvent, type ExtractionProgressListener, type ExtractionSummary, FEDERAL_REVENUE_CONTROL_DIR, FEDERAL_REVENUE_CONTROL_SCOPE, FEDERAL_REVENUE_MANIFEST_VERSION, type Failure, type FederalRevenueCheckOptions, type FederalRevenueCheckSummary, type FederalRevenueCleanMode, type FederalRevenueCleanOptions, type FederalRevenueCleanSummary, type FederalRevenueClientOptions, type FederalRevenueConfig, type FederalRevenueConfigKey, type FederalRevenueDownloadEntry, type FederalRevenueDownloadOptions, type FederalRevenueDownloadProgressEvent, type FederalRevenueDownloadProgressListener, type FederalRevenueDownloadStatus, type FederalRevenueDownloadSummary, type FederalRevenueEffectiveConfig, type FederalRevenueFile, type FederalRevenueLocalFileStatus, type FederalRevenueLocalStatusEntry, type FederalRevenueLockFile, type FederalRevenueManifest, type FederalRevenueManifestFile, type FederalRevenueManifestLastCommand, type FederalRevenueManifestLastStatus, type FederalRevenueReference, type FederalRevenueReferenceMode, type FederalRevenueReferenceSelection, type FederalRevenueRetryOptions, type FederalRevenueStatusOptions, type FederalRevenueStatusSummary, type FederalRevenueSyncLockOptions, type FederalRevenueSyncOptions, type FederalRevenueSyncSummary, type FileInspection, type ImportCheckpointRecord, type ImportCheckpointStatus, type ImportDatasetPlan, type ImportDatasetType, type ImportFilePlan, type ImportOptions, type ImportPerformanceSummary, type ImportPhaseStatus, type ImportPlanRecord, type ImportProgressEvent, type ImportProgressListener, type ImportSchemaCapabilities, type ImportSummary, type InputDetectionMode, type InputMode, type InspectSummary, type LogLevel, type LogStatus, type PostgresCsvDatasetSummary, type PostgresCsvExportOptions, type PostgresCsvExportProgressEvent, type PostgresCsvExportProgressListener, type PostgresCsvExportSummary, type PostgresCsvFile, type PostgresDirectIncludeTarget, type PostgresDirectScriptDatasetSummary, type PostgresDirectScriptOptions, type PostgresDirectScriptProgressEvent, type PostgresDirectScriptProgressListener, type PostgresDirectScriptStep, type PostgresDirectScriptSummary, type PostgresDirectSourceFile, type PostgresDirectTransactionMode, type QuarantineListFilters, type QuarantineListSummary, type QuarantineRecord, type QuarantineStatsFilters, type QuarantineStatsSummary, type Result, type SanitizeDatasetType, type SanitizeOptions, type SanitizePlan, type SanitizeProgressEvent, type SanitizeProgressListener, type SanitizeSourceEncoding, type SanitizeSummary, type SchemaGenerationOptions, type SchemaProfile, ServiceError, type StructuredLogEntry, type SupportedOs, ValidationError, type ValidationSummary, appendJsonLinesLog, assertPostgresUrl, buildFederalRevenueDownloadHeaders, buildFederalRevenueReferenceOutputPath, checkFederalRevenueDataset, cleanFederalRevenueDataset, cleanupDatabaseCheckpointsData, cleanupDatabaseMaterializedData, cleanupDatabasePlansData, cleanupDatabaseStagingData, createFederalRevenueManifest, createJsonLinesLog, defaultExtractedOutputPath, detectOs, downloadFederalRevenueDataset, ensureDirectory, evaluateFederalRevenueManifestFile, evaluateFederalRevenueManifestFiles, exportPostgresCsvDataset, extractArchives, finalizeFederalRevenueManifest, generatePostgresDirectScript, generateSchemaSql, getAllLayouts, getCurrentFederalRevenueReference, getFederalRevenueControlDirectory, getFederalRevenueEffectiveConfig, getFederalRevenueManifestPath, getFederalRevenueStatus, getFederalRevenueSyncLockPath, getLayoutSummary, getLogsDirectoryPath, getQuarantineStats, getUserAppDirectoryPath, importDataToDatabase, inspectFiles, listFederalRevenueFiles, listFederalRevenueReferences, listQuarantineRows, loadImportDataToStaging, materializeImportedData, prettyJson, readDatabaseConfig, readFederalRevenueEffectiveConfig, readFederalRevenueManifest, resetDefaultDbUrl, resetFederalRevenueConfig, resolveDatabaseUrl, resolveDbUrl, resolveFederalRevenueClientOptions, resolveFederalRevenueReference, resolveInputMode, resolveSchemaProfile, retryFederalRevenueDataset, runDoctor, safeReadText, safeWriteText, sanitizeInputDirectory, setDefaultDbUrl, setFederalRevenueConfigValue, showQuarantineRow, syncFederalRevenueDataset, testDatabaseConnection, toTitleCase, updateFederalRevenueManifestFile, validateFederalRevenueReference, validateInputDirectory, withFederalRevenueSyncLock, writeCommandFailureLog, writeCommandLog, writeDatabaseConfig, writeFederalRevenueManifest, writeSchemaFile };
|