@danielarndt0/cnpj-db-loader 2.4.0-beta.1 → 2.4.0-beta.3

package/dist/index.d.ts

@@ -582,6 +582,8 @@ declare function showQuarantineRow(id: number, options?: {
     dbUrl?: string;
 }): Promise<QuarantineRecord>;
 
+type SanitizeSourceEncoding = "WIN1252" | "LATIN1" | "UTF8";
+
 type SanitizeDatasetType = Exclude<DatasetType, "zip-archive" | "unknown">;
 type SanitizeFilePlan = {
     dataset: SanitizeDatasetType;
@@ -607,7 +609,10 @@ type SanitizeSummary = {
     totalBytes: number;
     processedFiles: number;
     processedRows: number;
+    sourceEncoding: SanitizeSourceEncoding;
     nulBytesRemoved: number;
+    invalidBytesRemoved: number;
+    controlCharsRemoved: number;
     changedFiles: number;
     unchangedFiles: number;
     datasets: SanitizeDatasetType[];
@@ -618,6 +623,8 @@ type SanitizeSummary = {
         lineCount: number;
         changed: boolean;
         nulBytesRemoved: number;
+        invalidBytesRemoved: number;
+        controlCharsRemoved: number;
     }>;
     warnings: string[];
     nextStep?: string | undefined;
@@ -629,6 +636,7 @@ type SanitizeProgressEvent = {
     totalFiles: number;
     totalBytes: number;
     datasets: SanitizeDatasetType[];
+    sourceEncoding: SanitizeSourceEncoding;
 } | {
     kind: "progress";
     currentFileDisplayPath: string;
@@ -640,12 +648,16 @@ type SanitizeProgressEvent = {
     currentFileSize: number;
     processedRows: number;
     nulBytesRemoved: number;
+    invalidBytesRemoved: number;
+    controlCharsRemoved: number;
     changedFiles: number;
 } | {
     kind: "finish";
     totalFiles: number;
     processedRows: number;
     nulBytesRemoved: number;
+    invalidBytesRemoved: number;
+    controlCharsRemoved: number;
     changedFiles: number;
     totalBytes: number;
 };
@@ -653,6 +665,7 @@ type SanitizeProgressListener = (event: SanitizeProgressEvent) => void;
 type SanitizeOptions = {
     outputPath?: string | undefined;
     dataset?: SanitizeDatasetType | undefined;
+    sourceEncoding?: string | undefined;
     onProgress?: SanitizeProgressListener | undefined;
 };
 
@@ -1039,6 +1052,14 @@ type PostgresDirectScriptDatasetSummary = {
     totalBytes: number;
     sourceFiles: string[];
 };
+type PostgresDirectTransactionMode = "single" | "phase" | "none";
+type PostgresDirectIncludeTarget = "domains" | "companies" | "establishments" | "partners" | "simples" | "secondary-cnaes" | "indexes" | "analyze";
+type PostgresDirectScriptStep = {
+    name: string;
+    file: string;
+    dependsOn: string[];
+    included: boolean;
+};
 type PostgresDirectScriptProgressEvent = {
     kind: "start";
     inputPath: string;
@@ -1047,6 +1068,10 @@ type PostgresDirectScriptProgressEvent = {
     totalFiles: number;
     datasets: ImportDatasetType[];
     sourceEncoding: string;
+    transactionMode: PostgresDirectTransactionMode;
+    include: PostgresDirectIncludeTarget[];
+    skipIndexes: boolean;
+    skipAnalyze: boolean;
 } | {
     kind: "file_registered";
     dataset: ImportDatasetType;
@@ -1067,6 +1092,10 @@ type PostgresDirectScriptOptions = {
     dataset?: ImportDatasetType | undefined;
     scriptName?: string | undefined;
     sourceEncoding?: string | undefined;
+    transactionMode?: PostgresDirectTransactionMode | undefined;
+    include?: PostgresDirectIncludeTarget[] | undefined;
+    skipIndexes?: boolean | undefined;
+    skipAnalyze?: boolean | undefined;
     onProgress?: PostgresDirectScriptProgressListener | undefined;
 };
 type PostgresDirectScriptSummary = {
@@ -1076,9 +1105,12 @@ type PostgresDirectScriptSummary = {
     scriptPath: string;
     manifestPath: string;
     sourceEncoding: string;
+    transactionMode: PostgresDirectTransactionMode;
     totalFiles: number;
     totalBytes: number;
     datasets: PostgresDirectScriptDatasetSummary[];
+    scriptFiles: string[];
+    steps: PostgresDirectScriptStep[];
     warnings: string[];
     nextStep?: string | undefined;
 };
@@ -1087,4 +1119,4 @@ declare function exportPostgresCsvDataset(inputPath: string, options?: PostgresC
 
 declare function generatePostgresDirectScript(inputPath: string, options?: PostgresDirectScriptOptions): Promise<PostgresDirectScriptSummary>;
 
-export { type AppConfig, type AppEnvironment, AppError, type CheckpointCleanupPhase, DEFAULT_FEDERAL_REVENUE_DOWNLOAD_ROOT, DEFAULT_FEDERAL_REVENUE_SHARE_TOKEN, DEFAULT_FEDERAL_REVENUE_USER_AGENT, DEFAULT_FEDERAL_REVENUE_WEBDAV_URL, type DatabaseCleanupSummary, type DatabaseConfig, type DatasetBlock, type DatasetType, type ExtractionEntry, type ExtractionProgressEvent, type ExtractionProgressListener, type ExtractionSummary, FEDERAL_REVENUE_CONTROL_DIR, FEDERAL_REVENUE_CONTROL_SCOPE, FEDERAL_REVENUE_MANIFEST_VERSION, type Failure, type FederalRevenueCheckOptions, type FederalRevenueCheckSummary, type FederalRevenueCleanMode, type FederalRevenueCleanOptions, type FederalRevenueCleanSummary, type FederalRevenueClientOptions, type FederalRevenueDownloadEntry, type FederalRevenueDownloadOptions, type FederalRevenueDownloadProgressEvent, type FederalRevenueDownloadProgressListener, type FederalRevenueDownloadStatus, type FederalRevenueDownloadSummary, type FederalRevenueFile, type FederalRevenueLocalFileStatus, type FederalRevenueLocalStatusEntry, type FederalRevenueLockFile, type FederalRevenueManifest, type FederalRevenueManifestFile, type FederalRevenueManifestLastCommand, type FederalRevenueManifestLastStatus, type FederalRevenueReference, type FederalRevenueReferenceMode, type FederalRevenueReferenceSelection, type FederalRevenueRetryOptions, type FederalRevenueStatusOptions, type FederalRevenueStatusSummary, type FederalRevenueSyncLockOptions, type FederalRevenueSyncOptions, type FederalRevenueSyncSummary, type FileInspection, type ImportCheckpointRecord, type ImportCheckpointStatus, type ImportDatasetPlan, type ImportDatasetType, type ImportFilePlan, type ImportOptions, type ImportPerformanceSummary, type ImportPhaseStatus, type ImportPlanRecord, type ImportProgressEvent, type ImportProgressListener, type ImportSchemaCapabilities, type ImportSummary, type InputDetectionMode, type InputMode, type InspectSummary, type LogLevel, type LogStatus, type PostgresCsvDatasetSummary, type PostgresCsvExportOptions, type PostgresCsvExportProgressEvent, type PostgresCsvExportProgressListener, type PostgresCsvExportSummary, type PostgresCsvFile, type PostgresDirectScriptDatasetSummary, type PostgresDirectScriptOptions, type PostgresDirectScriptProgressEvent, type PostgresDirectScriptProgressListener, type PostgresDirectScriptSummary, type PostgresDirectSourceFile, type QuarantineListFilters, type QuarantineListSummary, type QuarantineRecord, type QuarantineStatsFilters, type QuarantineStatsSummary, type Result, type SanitizeDatasetType, type SanitizeOptions, type SanitizePlan, type SanitizeProgressEvent, type SanitizeProgressListener, type SanitizeSummary, type SchemaGenerationOptions, type SchemaProfile, ServiceError, type StructuredLogEntry, type SupportedOs, ValidationError, type ValidationSummary, appendJsonLinesLog, assertPostgresUrl, buildFederalRevenueDownloadHeaders, buildFederalRevenueReferenceOutputPath, checkFederalRevenueDataset, cleanFederalRevenueDataset, cleanupDatabaseCheckpointsData, cleanupDatabaseMaterializedData, cleanupDatabasePlansData, cleanupDatabaseStagingData, createFederalRevenueManifest, createJsonLinesLog, defaultExtractedOutputPath, detectOs, downloadFederalRevenueDataset, ensureDirectory, evaluateFederalRevenueManifestFile, evaluateFederalRevenueManifestFiles, exportPostgresCsvDataset, extractArchives, finalizeFederalRevenueManifest, generatePostgresDirectScript, generateSchemaSql, getAllLayouts, getCurrentFederalRevenueReference, getFederalRevenueControlDirectory, getFederalRevenueManifestPath, getFederalRevenueStatus, getFederalRevenueSyncLockPath, getLayoutSummary, getLogsDirectoryPath, getQuarantineStats, getUserAppDirectoryPath, importDataToDatabase, inspectFiles, listFederalRevenueFiles, listFederalRevenueReferences, listQuarantineRows, loadImportDataToStaging, materializeImportedData, prettyJson, readDatabaseConfig, readFederalRevenueManifest, resetDefaultDbUrl, resolveDatabaseUrl, resolveDbUrl, resolveFederalRevenueReference, resolveInputMode, resolveSchemaProfile, retryFederalRevenueDataset, runDoctor, safeReadText, safeWriteText, sanitizeInputDirectory, setDefaultDbUrl, showQuarantineRow, syncFederalRevenueDataset, testDatabaseConnection, toTitleCase, updateFederalRevenueManifestFile, validateFederalRevenueReference, validateInputDirectory, withFederalRevenueSyncLock, writeCommandFailureLog, writeCommandLog, writeDatabaseConfig, writeFederalRevenueManifest, writeSchemaFile };
+export { type AppConfig, type AppEnvironment, AppError, type CheckpointCleanupPhase, DEFAULT_FEDERAL_REVENUE_DOWNLOAD_ROOT, DEFAULT_FEDERAL_REVENUE_SHARE_TOKEN, DEFAULT_FEDERAL_REVENUE_USER_AGENT, DEFAULT_FEDERAL_REVENUE_WEBDAV_URL, type DatabaseCleanupSummary, type DatabaseConfig, type DatasetBlock, type DatasetType, type ExtractionEntry, type ExtractionProgressEvent, type ExtractionProgressListener, type ExtractionSummary, FEDERAL_REVENUE_CONTROL_DIR, FEDERAL_REVENUE_CONTROL_SCOPE, FEDERAL_REVENUE_MANIFEST_VERSION, type Failure, type FederalRevenueCheckOptions, type FederalRevenueCheckSummary, type FederalRevenueCleanMode, type FederalRevenueCleanOptions, type FederalRevenueCleanSummary, type FederalRevenueClientOptions, type FederalRevenueDownloadEntry, type FederalRevenueDownloadOptions, type FederalRevenueDownloadProgressEvent, type FederalRevenueDownloadProgressListener, type FederalRevenueDownloadStatus, type FederalRevenueDownloadSummary, type FederalRevenueFile, type FederalRevenueLocalFileStatus, type FederalRevenueLocalStatusEntry, type FederalRevenueLockFile, type FederalRevenueManifest, type FederalRevenueManifestFile, type FederalRevenueManifestLastCommand, type FederalRevenueManifestLastStatus, type FederalRevenueReference, type FederalRevenueReferenceMode, type FederalRevenueReferenceSelection, type FederalRevenueRetryOptions, type FederalRevenueStatusOptions, type FederalRevenueStatusSummary, type FederalRevenueSyncLockOptions, type FederalRevenueSyncOptions, type FederalRevenueSyncSummary, type FileInspection, type ImportCheckpointRecord, type ImportCheckpointStatus, type ImportDatasetPlan, type ImportDatasetType, type ImportFilePlan, type ImportOptions, type ImportPerformanceSummary, type ImportPhaseStatus, type ImportPlanRecord, type ImportProgressEvent, type ImportProgressListener, type ImportSchemaCapabilities, type ImportSummary, type InputDetectionMode, type InputMode, type InspectSummary, type LogLevel, type LogStatus, type PostgresCsvDatasetSummary, type PostgresCsvExportOptions, type PostgresCsvExportProgressEvent, type PostgresCsvExportProgressListener, type PostgresCsvExportSummary, type PostgresCsvFile, type PostgresDirectIncludeTarget, type PostgresDirectScriptDatasetSummary, type PostgresDirectScriptOptions, type PostgresDirectScriptProgressEvent, type PostgresDirectScriptProgressListener, type PostgresDirectScriptStep, type PostgresDirectScriptSummary, type PostgresDirectSourceFile, type PostgresDirectTransactionMode, type QuarantineListFilters, type QuarantineListSummary, type QuarantineRecord, type QuarantineStatsFilters, type QuarantineStatsSummary, type Result, type SanitizeDatasetType, type SanitizeOptions, type SanitizePlan, type SanitizeProgressEvent, type SanitizeProgressListener, type SanitizeSourceEncoding, type SanitizeSummary, type SchemaGenerationOptions, type SchemaProfile, ServiceError, type StructuredLogEntry, type SupportedOs, ValidationError, type ValidationSummary, appendJsonLinesLog, assertPostgresUrl, buildFederalRevenueDownloadHeaders, buildFederalRevenueReferenceOutputPath, checkFederalRevenueDataset, cleanFederalRevenueDataset, cleanupDatabaseCheckpointsData, cleanupDatabaseMaterializedData, cleanupDatabasePlansData, cleanupDatabaseStagingData, createFederalRevenueManifest, createJsonLinesLog, defaultExtractedOutputPath, detectOs, downloadFederalRevenueDataset, ensureDirectory, evaluateFederalRevenueManifestFile, evaluateFederalRevenueManifestFiles, exportPostgresCsvDataset, extractArchives, finalizeFederalRevenueManifest, generatePostgresDirectScript, generateSchemaSql, getAllLayouts, getCurrentFederalRevenueReference, getFederalRevenueControlDirectory, getFederalRevenueManifestPath, getFederalRevenueStatus, getFederalRevenueSyncLockPath, getLayoutSummary, getLogsDirectoryPath, getQuarantineStats, getUserAppDirectoryPath, importDataToDatabase, inspectFiles, listFederalRevenueFiles, listFederalRevenueReferences, listQuarantineRows, loadImportDataToStaging, materializeImportedData, prettyJson, readDatabaseConfig, readFederalRevenueManifest, resetDefaultDbUrl, resolveDatabaseUrl, resolveDbUrl, resolveFederalRevenueReference, resolveInputMode, resolveSchemaProfile, retryFederalRevenueDataset, runDoctor, safeReadText, safeWriteText, sanitizeInputDirectory, setDefaultDbUrl, showQuarantineRow, syncFederalRevenueDataset, testDatabaseConnection, toTitleCase, updateFederalRevenueManifestFile, validateFederalRevenueReference, validateInputDirectory, withFederalRevenueSyncLock, writeCommandFailureLog, writeCommandLog, writeDatabaseConfig, writeFederalRevenueManifest, writeSchemaFile };