datera 0.1.0

package/dist/pipeline/index.js

@@ -0,0 +1,17 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatDuration = exports.formatReport = exports.buildModeStep = exports.parseMode = exports.MODES = exports.buildSteps = exports.DEFAULT_PREVIEW_SIZE = exports.countPendingReasons = exports.applySteps = exports.runEtl = void 0;
+var runEtl_1 = require("./runEtl");
+Object.defineProperty(exports, "runEtl", { enumerable: true, get: function () { return runEtl_1.runEtl; } });
+Object.defineProperty(exports, "applySteps", { enumerable: true, get: function () { return runEtl_1.applySteps; } });
+Object.defineProperty(exports, "countPendingReasons", { enumerable: true, get: function () { return runEtl_1.countPendingReasons; } });
+Object.defineProperty(exports, "DEFAULT_PREVIEW_SIZE", { enumerable: true, get: function () { return runEtl_1.DEFAULT_PREVIEW_SIZE; } });
+var steps_1 = require("./steps");
+Object.defineProperty(exports, "buildSteps", { enumerable: true, get: function () { return steps_1.buildSteps; } });
+var modes_1 = require("./modes");
+Object.defineProperty(exports, "MODES", { enumerable: true, get: function () { return modes_1.MODES; } });
+Object.defineProperty(exports, "parseMode", { enumerable: true, get: function () { return modes_1.parseMode; } });
+Object.defineProperty(exports, "buildModeStep", { enumerable: true, get: function () { return modes_1.buildModeStep; } });
+var report_1 = require("./report");
+Object.defineProperty(exports, "formatReport", { enumerable: true, get: function () { return report_1.formatReport; } });
+Object.defineProperty(exports, "formatDuration", { enumerable: true, get: function () { return report_1.formatDuration; } });

package/dist/pipeline/modes.d.ts

@@ -0,0 +1,6 @@
+import { EtlConfig } from "../config";
+import { Step } from "./types";
+export declare const MODES: readonly ["raw", "dedupe", "merge"];
+export type Mode = (typeof MODES)[number];
+export declare function parseMode(value: string): Mode;
+export declare function buildModeStep(mode: Mode, config: EtlConfig): Step;

package/dist/pipeline/modes.js

@@ -0,0 +1,48 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.MODES = void 0;
+exports.parseMode = parseMode;
+exports.buildModeStep = buildModeStep;
+const dedupe_1 = require("../filters/dedupe");
+const merge_1 = require("../filters/merge");
+const registry_1 = require("../normalizers/registry");
+exports.MODES = ["raw", "dedupe", "merge"];
+function parseMode(value) {
+    const mode = exports.MODES.find((known) => known === value);
+    if (mode === undefined) {
+        throw new Error(`Modo inválido: "${value}". Modos aceitos: ${exports.MODES.join(", ")}.`);
+    }
+    return mode;
+}
+function buildModeStep(mode, config) {
+    switch (mode) {
+        case "raw":
+            return { name: "raw", run: (rows) => ({ rows }) };
+        case "dedupe": {
+            if (!config.dedupeColumn) {
+                throw new Error("dedupeColumn não definido na config para o modo dedupe.");
+            }
+            const filter = new dedupe_1.DedupeFilter(config.dedupeColumn, config.dedupeStrategy, config.dedupeKeyNormalizer === undefined
+                ? undefined
+                : (0, registry_1.getKeyNormalizer)(config.dedupeKeyNormalizer));
+            return {
+                name: "dedupe",
+                run: (rows) => ({ rows: filter.apply(rows) }),
+            };
+        }
+        case "merge": {
+            if (!config.mergeKeyColumn || !config.mergeColumns) {
+                throw new Error("mergeKeyColumn/mergeColumns não definidos na config para o modo merge.");
+            }
+            const filter = new merge_1.MergeFilter(config.mergeKeyColumn, config.mergeColumns, {
+                emptyKeyLabel: config.mergeEmptyKeyLabel,
+                rejectedKeyLabel: config.mergeRejectedKeyLabel,
+                keyNormalizer: config.mergeKeyNormalizer,
+            });
+            return {
+                name: "merge",
+                run: (rows) => ({ rows: filter.apply(rows) }),
+            };
+        }
+    }
+}

package/dist/pipeline/report.d.ts

@@ -0,0 +1,3 @@
+import { EtlReport } from "./types";
+export declare function formatDuration(ms: number): string;
+export declare function formatReport(report: EtlReport): string[];

package/dist/pipeline/report.js

@@ -0,0 +1,32 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.formatDuration = formatDuration;
+exports.formatReport = formatReport;
+function formatDuration(ms) {
+    if (ms < 1000)
+        return `${Math.round(ms)} ms`;
+    return `${(ms / 1000).toFixed(1).replace(".", ",")} s`;
+}
+function plural(count, one, many) {
+    return `${count} ${count === 1 ? one : many}`;
+}
+function formatReport(report) {
+    const lines = [
+        "Resumo da execução",
+        `  Modo: ${report.mode}`,
+        `  Linhas lidas: ${report.rowsRead}`,
+    ];
+    for (const step of report.steps) {
+        const pending = step.pending > 0
+            ? `, ${plural(step.pending, "pendência", "pendências")}`
+            : "";
+        lines.push(`  ${step.name}: de ${step.rowsIn} para ${step.rowsOut}${pending} (${formatDuration(step.durationMs)})`);
+    }
+    lines.push(`  Linhas no resultado: ${report.rowsOut}`);
+    lines.push(`  Pendências: ${report.pendingRows}`);
+    lines.push(`  Tempo total: ${formatDuration(report.durationMs)}`);
+    if (report.dryRun) {
+        lines.push("  Nada foi gravado, porque era só um teste (--dry-run).");
+    }
+    return lines;
+}

package/dist/pipeline/runEtl.d.ts

@@ -0,0 +1,24 @@
+import { EtlConfig } from "../config";
+import { Sink, Source } from "../io/types";
+import { Logger } from "../logger";
+import { TableRow } from "../types";
+import { Mode } from "./modes";
+import { EtlReport, PendingReason, Step, StepReport } from "./types";
+export declare const DEFAULT_PREVIEW_SIZE = 5;
+export declare function countPendingReasons(pending: TableRow[], reasonColumn?: string): PendingReason[];
+export interface AppliedSteps {
+    rows: TableRow[];
+    pending: TableRow[];
+    steps: StepReport[];
+}
+export declare function applySteps(steps: Step[], input: TableRow[], onStep?: (report: StepReport) => void): AppliedSteps;
+export interface RunEtlOptions {
+    mode?: Mode | undefined;
+    dryRun?: boolean | undefined;
+    logger?: Logger | undefined;
+    source?: Source | undefined;
+    sink?: Sink | undefined;
+    previewSize?: number | undefined;
+    onStep?: ((report: StepReport) => void) | undefined;
+}
+export declare function runEtl(config: EtlConfig, options?: RunEtlOptions): Promise<EtlReport>;

package/dist/pipeline/runEtl.js

@@ -0,0 +1,98 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_PREVIEW_SIZE = void 0;
+exports.countPendingReasons = countPendingReasons;
+exports.applySteps = applySteps;
+exports.runEtl = runEtl;
+const factory_1 = require("../io/factory");
+const loader_1 = require("../io/custom/loader");
+const validate_1 = require("../filters/validate");
+const normalizers_1 = require("../normalizers");
+const logger_1 = require("../logger");
+const steps_1 = require("./steps");
+exports.DEFAULT_PREVIEW_SIZE = 5;
+function countPendingReasons(pending, reasonColumn = validate_1.DEFAULT_REASON_COLUMN) {
+    const counts = new Map();
+    for (const row of pending) {
+        const text = row[reasonColumn];
+        if (text === null || text === undefined)
+            continue;
+        for (const part of String(text).split(";")) {
+            const reason = part.trim();
+            if (reason === "")
+                continue;
+            counts.set(reason, (counts.get(reason) ?? 0) + 1);
+        }
+    }
+    return Array.from(counts, ([reason, count]) => ({ reason, count })).sort((a, b) => b.count - a.count || a.reason.localeCompare(b.reason));
+}
+function applySteps(steps, input, onStep) {
+    let rows = input;
+    const pending = [];
+    const reports = [];
+    for (const step of steps) {
+        const startedAt = Date.now();
+        const output = step.run(rows);
+        const stepPending = output.pending ?? [];
+        pending.push(...stepPending);
+        const report = {
+            name: step.name,
+            rowsIn: rows.length,
+            rowsOut: output.rows.length,
+            pending: stepPending.length,
+            durationMs: Date.now() - startedAt,
+        };
+        reports.push(report);
+        onStep?.(report);
+        rows = output.rows;
+    }
+    return { rows, pending, steps: reports };
+}
+async function runEtl(config, options = {}) {
+    const startedAt = Date.now();
+    const logger = options.logger ?? logger_1.consoleLogger;
+    const dryRun = options.dryRun ?? false;
+    const mode = options.mode ?? config.mode;
+    (0, normalizers_1.registerBuiltinKeyNormalizers)();
+    (0, normalizers_1.loadNormalizerModules)(config.normalizerModules ?? []);
+    (0, loader_1.loadAdapterModules)(config.adapterModules ?? []);
+    const steps = (0, steps_1.buildSteps)(config, mode);
+    const sink = options.sink ?? (0, factory_1.createSink)(config, logger);
+    const ownsSource = options.source === undefined;
+    const source = options.source ?? (0, factory_1.createSource)(config, logger);
+    try {
+        logger.info(dryRun ? `Modo em uso: ${mode} (só teste)` : `Modo em uso: ${mode}`);
+        const rawRows = await source.read();
+        logger.info(`${rawRows.length} linhas lidas.`);
+        const { rows, pending, steps: stepReports, } = applySteps(steps, rawRows, options.onStep);
+        if (!dryRun) {
+            await sink.write(rows);
+            if (config.validation) {
+                await sink.write(pending, {
+                    name: config.validation.pendingSheet ?? validate_1.DEFAULT_PENDING_SHEET,
+                });
+            }
+        }
+        return {
+            mode,
+            dryRun,
+            written: !dryRun,
+            rowsRead: rawRows.length,
+            rowsOut: rows.length,
+            pendingRows: pending.length,
+            pendingByReason: countPendingReasons(pending, config.validation?.reasonColumn),
+            steps: stepReports,
+            durationMs: Date.now() - startedAt,
+            preview: dryRun
+                ? rows.slice(0, options.previewSize ?? exports.DEFAULT_PREVIEW_SIZE)
+                : [],
+            pendingPreview: dryRun
+                ? pending.slice(0, options.previewSize ?? exports.DEFAULT_PREVIEW_SIZE)
+                : [],
+        };
+    }
+    finally {
+        if (ownsSource)
+            await source.close?.();
+    }
+}

package/dist/pipeline/steps.d.ts

@@ -0,0 +1,4 @@
+import { EtlConfig } from "../config";
+import { Mode } from "./modes";
+import { Step } from "./types";
+export declare function buildSteps(config: EtlConfig, mode: Mode): Step[];

package/dist/pipeline/steps.js

@@ -0,0 +1,36 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildSteps = buildSteps;
+const fillEmpty_1 = require("../filters/fillEmpty");
+const combine_1 = require("../filters/combine");
+const validate_1 = require("../filters/validate");
+const modes_1 = require("./modes");
+function buildSteps(config, mode) {
+    const steps = [];
+    if (config.fillEmpty) {
+        const filter = new fillEmpty_1.FillEmptyFilter(config.fillEmpty);
+        steps.push({
+            name: "fillEmpty",
+            run: (rows) => ({ rows: filter.apply(rows) }),
+        });
+    }
+    if (config.combineColumns) {
+        const filter = new combine_1.CombineFilter(config.combineColumns);
+        steps.push({
+            name: "combineColumns",
+            run: (rows) => ({ rows: filter.apply(rows) }),
+        });
+    }
+    if (config.validation) {
+        const validator = new validate_1.RowValidator(config.validation.rules, config.validation.reasonColumn);
+        steps.push({
+            name: "validation",
+            run: (rows) => {
+                const { valid, pending } = validator.split(rows);
+                return { rows: valid, pending };
+            },
+        });
+    }
+    steps.push((0, modes_1.buildModeStep)(mode, config));
+    return steps;
+}

package/dist/pipeline/types.d.ts

@@ -0,0 +1,34 @@
+import { TableRow } from "../types";
+import { Mode } from "./modes";
+export interface StepOutput {
+    rows: TableRow[];
+    pending?: TableRow[] | undefined;
+}
+export interface Step {
+    name: string;
+    run(rows: TableRow[]): StepOutput;
+}
+export interface StepReport {
+    name: string;
+    rowsIn: number;
+    rowsOut: number;
+    pending: number;
+    durationMs: number;
+}
+export interface PendingReason {
+    reason: string;
+    count: number;
+}
+export interface EtlReport {
+    mode: Mode;
+    dryRun: boolean;
+    written: boolean;
+    rowsRead: number;
+    rowsOut: number;
+    pendingRows: number;
+    pendingByReason: PendingReason[];
+    steps: StepReport[];
+    durationMs: number;
+    preview: TableRow[];
+    pendingPreview: TableRow[];
+}

package/dist/pipeline/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/types.d.ts

@@ -0,0 +1,3 @@
+export interface TableRow {
+    [key: string]: string | number | null;
+}

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,128 @@
+{
+    "name": "datera",
+    "version": "0.1.0",
+    "description": "Limpa, unifica e organiza dados bagunçados de bancos de dados, planilhas, CSV, JSON, XML e Parquet, e exporta pro Google Sheets, Excel, CSV, JSON, XML ou Parquet.",
+    "main": "dist/index.js",
+    "types": "dist/index.d.ts",
+    "bin": {
+        "datera": "dist/main.js"
+    },
+    "files": [
+        "dist"
+    ],
+    "workspaces": [
+        "apps/*"
+    ],
+    "scripts": {
+        "build": "tsc -p tsconfig.build.json",
+        "prepublishOnly": "npm run build && npm test",
+        "start": "tsx src/main.ts",
+        "validate": "tsx src/main.ts validate",
+        "example": "tsx src/main.ts --config examples/config.csv.json",
+        "example:excel": "tsx src/main.ts --config examples/config.excel.json",
+        "example:xml": "tsx src/main.ts --config examples/config.xml.json",
+        "desktop": "npm run dev -w apps/desktop",
+        "desktop:dev": "npm run dev -w apps/dev",
+        "test": "vitest run",
+        "test:watch": "vitest",
+        "typecheck": "tsc --noEmit",
+        "test:connection": "tsx scripts/testConnection.ts",
+        "test:config": "tsx scripts/testConfig.ts",
+        "test:sheet": "tsx scripts/testSheetFake.ts",
+        "prepare": "husky"
+    },
+    "keywords": [
+        "etl",
+        "csv",
+        "excel",
+        "xml",
+        "parquet",
+        "sqlite",
+        "postgres",
+        "mysql",
+        "dedupe",
+        "merge",
+        "data-cleaning",
+        "google-sheets"
+    ],
+    "repository": {
+        "type": "git",
+        "url": "git+https://github.com/alessandravieiradev-blip/datera.git"
+    },
+    "homepage": "https://github.com/alessandravieiradev-blip/datera#readme",
+    "bugs": "https://github.com/alessandravieiradev-blip/datera/issues",
+    "engines": {
+        "node": ">=22.13"
+    },
+    "author": "Alessandra Vieira",
+    "license": "MIT",
+    "type": "commonjs",
+    "devDependencies": {
+        "@commitlint/cli": "^21.2.3",
+        "@commitlint/config-conventional": "^21.2.3",
+        "@googleapis/sheets": "^18.0.1",
+        "@types/node": "^22.20.3",
+        "exceljs": "^4.4.0",
+        "husky": "^9.1.7",
+        "hyparquet": "^1.31.2",
+        "hyparquet-compressors": "^1.1.2",
+        "hyparquet-writer": "^0.16.10",
+        "lint-staged": "^17.5.1",
+        "mssql": "^12.7.2",
+        "mysql2": "^3.24.4",
+        "pg": "^8.23.0",
+        "prettier": "^3.9.9",
+        "tsx": "^4.23.13",
+        "typescript": "^7.0.2",
+        "vitest": "^5.0.1"
+    },
+    "dependencies": {
+        "commander": "^15.0.0",
+        "dotenv": "^17.4.2",
+        "zod": "^4.6.5"
+    },
+    "peerDependencies": {
+        "@googleapis/sheets": "^18.0.1",
+        "exceljs": "^4.4.0",
+        "hyparquet": "^1.31.2",
+        "hyparquet-compressors": "^1.1.2",
+        "hyparquet-writer": "^0.16.10",
+        "mssql": "^12.7.2",
+        "mysql2": "^3.24.4",
+        "pg": "^8.23.0"
+    },
+    "peerDependenciesMeta": {
+        "@googleapis/sheets": {
+            "optional": true
+        },
+        "exceljs": {
+            "optional": true
+        },
+        "hyparquet": {
+            "optional": true
+        },
+        "hyparquet-compressors": {
+            "optional": true
+        },
+        "hyparquet-writer": {
+            "optional": true
+        },
+        "mssql": {
+            "optional": true
+        },
+        "mysql2": {
+            "optional": true
+        },
+        "pg": {
+            "optional": true
+        }
+    },
+    "lint-staged": {
+        "**/*.{ts,tsx,json,md}": [
+            "prettier --write"
+        ]
+    },
+    "overrides": {
+        "uuid": "^11.1.1"
+    }
+}