datera 0.1.0

package/dist/io/sqlserver/sqlServerSource.js

@@ -0,0 +1,49 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SqlServerSource = exports.createSqlServerClient = void 0;
+const optional_1 = require("../optional");
+const names_1 = require("../sql/names");
+const cell_1 = require("../sql/cell");
+const createSqlServerClient = async (connection) => {
+    const mssql = (0, optional_1.requireOptional)(() => require("mssql"), ["mssql"], "SQL Server");
+    const pool = new mssql.ConnectionPool({
+        server: connection.host,
+        port: connection.port,
+        user: connection.user,
+        ...(connection.password !== undefined
+            ? { password: connection.password }
+            : {}),
+        database: connection.database,
+        options: {
+            encrypt: connection.encrypt ?? true,
+            trustServerCertificate: connection.trustServerCertificate ?? false,
+        },
+    });
+    await pool.connect();
+    return {
+        query: async (text) => (await pool.request().query(text)).recordset,
+        close: () => pool.close(),
+    };
+};
+exports.createSqlServerClient = createSqlServerClient;
+class SqlServerSource {
+    connection;
+    table;
+    createClient;
+    constructor(connection, table, createClient = exports.createSqlServerClient) {
+        this.connection = connection;
+        this.table = table;
+        this.createClient = createClient;
+    }
+    async read() {
+        const client = await this.createClient(this.connection);
+        try {
+            const records = await client.query(`SELECT * FROM ${(0, names_1.quoteTable)(this.table, "[", "]")}`);
+            return records.map(cell_1.toRow);
+        }
+        finally {
+            await client.close();
+        }
+    }
+}
+exports.SqlServerSource = SqlServerSource;

package/dist/io/types.d.ts

@@ -0,0 +1,11 @@
+import { TableRow } from "../types";
+export interface Source {
+    read(): Promise<TableRow[]>;
+    close?(): Promise<void>;
+}
+export interface SinkWriteOptions {
+    name?: string | undefined;
+}
+export interface Sink {
+    write(rows: TableRow[], options?: SinkWriteOptions): Promise<void>;
+}

package/dist/io/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/dist/io/xml/xmlParse.d.ts

@@ -0,0 +1,9 @@
+export interface XmlElement {
+    name: string;
+    attributes: Record<string, string>;
+    children: XmlElement[];
+    text: string;
+}
+export declare class XmlError extends Error {
+}
+export declare function parseXml(text: string): XmlElement;

package/dist/io/xml/xmlParse.js

@@ -0,0 +1,178 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.XmlError = void 0;
+exports.parseXml = parseXml;
+const NAMED_ENTITIES = {
+    lt: "<",
+    gt: ">",
+    amp: "&",
+    quot: '"',
+    apos: "'",
+};
+const NAME = /^[^\s/>=<"'!?]+/;
+class XmlError extends Error {
+}
+exports.XmlError = XmlError;
+class Reader {
+    text;
+    position = 0;
+    constructor(text) {
+        this.text = text;
+    }
+    get done() {
+        return this.position >= this.text.length;
+    }
+    fail(message, at = this.position) {
+        const line = this.text.slice(0, at).split("\n").length;
+        throw new XmlError(`${message} (linha ${line})`);
+    }
+    startsWith(value) {
+        return this.text.startsWith(value, this.position);
+    }
+    skip(value) {
+        if (!this.startsWith(value))
+            this.fail(`Esperava "${value}"`);
+        this.position += value.length;
+    }
+    skipSpaces() {
+        while (!this.done && /\s/.test(this.text[this.position] ?? "")) {
+            this.position++;
+        }
+    }
+    until(end, what) {
+        const found = this.text.indexOf(end, this.position);
+        if (found === -1)
+            this.fail(`${what} sem fechar`);
+        const value = this.text.slice(this.position, found);
+        this.position = found + end.length;
+        return value;
+    }
+    textUntilTag() {
+        const found = this.text.indexOf("<", this.position);
+        const end = found === -1 ? this.text.length : found;
+        const start = this.position;
+        this.position = end;
+        return decode(this.text.slice(start, end), this, start);
+    }
+    name() {
+        const match = this.text.slice(this.position).match(NAME);
+        if (!match)
+            this.fail("Esperava o nome de um elemento ou atributo");
+        this.position += match[0].length;
+        return match[0];
+    }
+    quoted() {
+        const quote = this.text[this.position];
+        if (quote !== '"' && quote !== "'") {
+            this.fail("O valor do atributo precisa estar entre aspas");
+        }
+        this.position++;
+        const start = this.position;
+        const value = this.until(quote, "Valor de atributo");
+        if (value.includes("<"))
+            this.fail("Atributo com < dentro", start);
+        return decode(value, this, start);
+    }
+}
+function decode(value, reader, at) {
+    return value.replace(/&([^;&\s]*);?/g, (whole, entity) => {
+        if (!whole.endsWith(";"))
+            reader.fail("Tem um & solto no texto", at);
+        const named = NAMED_ENTITIES[entity];
+        if (named !== undefined)
+            return named;
+        const code = entity.startsWith("#x")
+            ? parseInt(entity.slice(2), 16)
+            : entity.startsWith("#")
+                ? parseInt(entity.slice(1), 10)
+                : NaN;
+        if (Number.isNaN(code) || code > 0x10ffff) {
+            reader.fail(`A entidade &${entity}; não existe`, at);
+        }
+        return String.fromCodePoint(code);
+    });
+}
+function skipMarkup(reader) {
+    if (reader.startsWith("<?")) {
+        reader.until("?>", "Instrução <?");
+        return true;
+    }
+    if (reader.startsWith("<!--")) {
+        reader.until("-->", "Comentário");
+        return true;
+    }
+    if (reader.startsWith("<!DOCTYPE") || reader.startsWith("<!ENTITY")) {
+        reader.fail("O XML tem um <!DOCTYPE>, e o Datera não lê isso por segurança. Tire essa parte do arquivo");
+    }
+    return false;
+}
+function readElement(reader) {
+    reader.skip("<");
+    const element = {
+        name: reader.name(),
+        attributes: {},
+        children: [],
+        text: "",
+    };
+    for (;;) {
+        reader.skipSpaces();
+        if (reader.startsWith("/>")) {
+            reader.skip("/>");
+            return element;
+        }
+        if (reader.startsWith(">")) {
+            reader.skip(">");
+            break;
+        }
+        const attribute = reader.name();
+        reader.skipSpaces();
+        reader.skip("=");
+        reader.skipSpaces();
+        element.attributes[attribute] = reader.quoted();
+    }
+    const texts = [];
+    for (;;) {
+        if (reader.done)
+            reader.fail(`O elemento <${element.name}> não fecha`);
+        if (reader.startsWith("</")) {
+            reader.skip("</");
+            const closing = reader.name();
+            if (closing !== element.name) {
+                reader.fail(`<${element.name}> foi fechado com </${closing}>`);
+            }
+            reader.skipSpaces();
+            reader.skip(">");
+            element.text = texts.join("").trim();
+            return element;
+        }
+        if (reader.startsWith("<![CDATA[")) {
+            reader.skip("<![CDATA[");
+            texts.push(reader.until("]]>", "Bloco CDATA"));
+        }
+        else if (!skipMarkup(reader)) {
+            if (reader.startsWith("<"))
+                element.children.push(readElement(reader));
+            else
+                texts.push(reader.textUntilTag());
+        }
+    }
+}
+function parseXml(text) {
+    const reader = new Reader(text.replace(/^/, ""));
+    let root;
+    for (;;) {
+        reader.skipSpaces();
+        if (reader.done)
+            break;
+        if (skipMarkup(reader))
+            continue;
+        if (!reader.startsWith("<"))
+            reader.fail("Tem texto fora do elemento principal");
+        if (root)
+            reader.fail("O XML tem mais de um elemento principal");
+        root = readElement(reader);
+    }
+    if (!root)
+        throw new XmlError("O arquivo não tem nenhum elemento XML.");
+    return root;
+}

package/dist/io/xml/xmlSink.d.ts

@@ -0,0 +1,18 @@
+import { TableRow } from "../../types";
+import { Sink, SinkWriteOptions } from "../types";
+import { Logger } from "../../logger";
+export interface XmlSinkOptions {
+    path: string;
+    root?: string | undefined;
+    record?: string | undefined;
+}
+export declare const DEFAULT_XML_ROOT = "registros";
+export declare const DEFAULT_XML_RECORD = "registro";
+export declare function xmlNameFor(column: string): string;
+export declare function toXml(rows: TableRow[], root: string, record: string, names?: Map<string, string>): string;
+export declare class XmlSink implements Sink {
+    private readonly options;
+    private readonly logger;
+    constructor(options: XmlSinkOptions, logger?: Logger);
+    write(rows: TableRow[], options?: SinkWriteOptions): Promise<void>;
+}

package/dist/io/xml/xmlSink.js

@@ -0,0 +1,79 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.XmlSink = exports.DEFAULT_XML_RECORD = exports.DEFAULT_XML_ROOT = void 0;
+exports.xmlNameFor = xmlNameFor;
+exports.toXml = toXml;
+const files_1 = require("../files");
+const logger_1 = require("../../logger");
+exports.DEFAULT_XML_ROOT = "registros";
+exports.DEFAULT_XML_RECORD = "registro";
+const INDENT = "  ";
+function escapeText(value) {
+    return value
+        .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F]/g, "")
+        .replace(/&/g, "&")
+        .replace(/</g, "&lt;")
+        .replace(/>/g, "&gt;");
+}
+function xmlNameFor(column) {
+    const cleaned = column.trim().replace(/[^\p{L}\p{N}_.-]+/gu, "_");
+    if (cleaned === "")
+        return "coluna";
+    return /^[\p{L}_]/u.test(cleaned) ? cleaned : `_${cleaned}`;
+}
+function elementNames(columns) {
+    const names = new Map();
+    const used = new Set();
+    for (const column of columns) {
+        const base = xmlNameFor(column);
+        let name = base;
+        for (let count = 2; used.has(name); count++)
+            name = `${base}_${count}`;
+        used.add(name);
+        names.set(column, name);
+    }
+    return names;
+}
+function columnsOf(rows) {
+    const columns = new Set();
+    for (const row of rows) {
+        for (const column of Object.keys(row))
+            columns.add(column);
+    }
+    return [...columns];
+}
+function toXml(rows, root, record, names = elementNames(columnsOf(rows))) {
+    const lines = ['<?xml version="1.0" encoding="UTF-8"?>', `<${root}>`];
+    for (const row of rows) {
+        lines.push(`${INDENT}<${record}>`);
+        for (const [column, value] of Object.entries(row)) {
+            const name = names.get(column) ?? xmlNameFor(column);
+            lines.push(value === null || value === ""
+                ? `${INDENT}${INDENT}<${name}/>`
+                : `${INDENT}${INDENT}<${name}>${escapeText(String(value))}</${name}>`);
+        }
+        lines.push(`${INDENT}</${record}>`);
+    }
+    lines.push(`</${root}>`);
+    return lines.join("\n") + "\n";
+}
+class XmlSink {
+    options;
+    logger;
+    constructor(options, logger = logger_1.consoleLogger) {
+        this.options = options;
+        this.logger = logger;
+    }
+    async write(rows, options = {}) {
+        const filePath = (0, files_1.extraOutputPath)(this.options.path, options.name);
+        const names = elementNames(columnsOf(rows));
+        for (const [column, name] of names) {
+            if (column !== name) {
+                this.logger.warn(`A coluna "${column}" virou <${name}> no XML, porque o nome original não vale como elemento.`);
+            }
+        }
+        (0, files_1.writeTextFile)(filePath, toXml(rows, this.options.root ?? exports.DEFAULT_XML_ROOT, this.options.record ?? exports.DEFAULT_XML_RECORD, names));
+        this.logger.info(`${rows.length} linhas escritas em ${filePath}.`);
+    }
+}
+exports.XmlSink = XmlSink;

package/dist/io/xml/xmlSource.d.ts

@@ -0,0 +1,11 @@
+import { TableRow } from "../../types";
+import { Source } from "../types";
+export interface XmlSourceOptions {
+    path: string;
+    recordsPath?: string | undefined;
+}
+export declare class XmlSource implements Source {
+    private readonly options;
+    constructor(options: XmlSourceOptions);
+    read(): Promise<TableRow[]>;
+}

package/dist/io/xml/xmlSource.js

@@ -0,0 +1,93 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.XmlSource = void 0;
+const files_1 = require("../files");
+const xmlParse_1 = require("./xmlParse");
+function join(prefix, name) {
+    return prefix ? `${prefix}.${name}` : name;
+}
+function isSimple(element) {
+    return (element.children.length === 0 &&
+        Object.keys(element.attributes).length === 0);
+}
+function groupByName(elements) {
+    const groups = new Map();
+    for (const element of elements) {
+        const group = groups.get(element.name) ?? [];
+        group.push(element);
+        groups.set(element.name, group);
+    }
+    return groups;
+}
+function flatten(element, prefix, row) {
+    for (const [name, value] of Object.entries(element.attributes)) {
+        row[join(prefix, name)] = value === "" ? null : value;
+    }
+    if (prefix && element.children.length === 0) {
+        if (element.text !== "")
+            row[prefix] = element.text;
+        else if (isSimple(element))
+            row[prefix] = null;
+    }
+    for (const [name, group] of groupByName(element.children)) {
+        const key = join(prefix, name);
+        const [first] = group;
+        if (group.length === 1 && first) {
+            flatten(first, key, row);
+        }
+        else if (group.every(isSimple)) {
+            row[key] = group.map((item) => item.text).join(", ");
+        }
+        else {
+            row[key] = JSON.stringify(group.map((item) => {
+                const inner = {};
+                flatten(item, "", inner);
+                return inner;
+            }));
+        }
+    }
+}
+function findRecords(root, recordsPath) {
+    const [first, ...rest] = recordsPath.split(".");
+    if (first !== root.name) {
+        throw new Error(`O XML começa em <${root.name}>, então o recordsPath precisa começar com "${root.name}".`);
+    }
+    let current = [root];
+    for (const name of rest) {
+        current = current.flatMap((element) => element.children.filter((child) => child.name === name));
+    }
+    return current;
+}
+class XmlSource {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    async read() {
+        let root;
+        try {
+            root = (0, xmlParse_1.parseXml)((0, files_1.readTextFile)(this.options.path));
+        }
+        catch (error) {
+            if (error instanceof xmlParse_1.XmlError) {
+                throw new Error(`XML inválido em ${this.options.path}: ${error.message}`);
+            }
+            throw error;
+        }
+        const { recordsPath } = this.options;
+        const records = recordsPath
+            ? findRecords(root, recordsPath)
+            : root.children;
+        if (records.length === 0) {
+            throw new Error(recordsPath
+                ? `Não achei nenhum registro em "${recordsPath}" no XML ${this.options.path}.`
+                : `O elemento <${root.name}> em ${this.options.path} não tem nenhum registro dentro. Se os registros estão mais para dentro, use recordsPath.`);
+        }
+        return records.map((record) => {
+            const row = {};
+            flatten(record, "", row);
+            return row;
+        });
+    }
+}
+exports.XmlSource = XmlSource;

package/dist/logger.d.ts

@@ -0,0 +1,11 @@
+export interface Logger {
+    info(message: string): void;
+    warn(message: string): void;
+    error(message: string, error?: unknown): void;
+}
+export declare const consoleLogger: Logger;
+export declare const silentLogger: Logger;
+export interface MemoryLogger extends Logger {
+    messages: string[];
+}
+export declare function createMemoryLogger(): MemoryLogger;

package/dist/logger.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.silentLogger = exports.consoleLogger = void 0;
+exports.createMemoryLogger = createMemoryLogger;
+function withReason(message, error) {
+    if (error === undefined)
+        return message;
+    const reason = error instanceof Error ? error.message : String(error);
+    return `${message} ${reason}`;
+}
+exports.consoleLogger = {
+    info: (message) => console.log(message),
+    warn: (message) => console.warn(message),
+    error: (message, error) => console.error(withReason(message, error)),
+};
+exports.silentLogger = {
+    info: () => { },
+    warn: () => { },
+    error: () => { },
+};
+function createMemoryLogger() {
+    const messages = [];
+    return {
+        messages,
+        info: (message) => messages.push(message),
+        warn: (message) => messages.push(message),
+        error: (message, error) => messages.push(withReason(message, error)),
+    };
+}

package/dist/main.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+import "dotenv/config";

package/dist/main.js

@@ -0,0 +1,18 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+require("dotenv/config");
+const cli_1 = require("./cli");
+const commands_1 = require("./commands");
+const logger_1 = require("./logger");
+async function main() {
+    const cli = (0, cli_1.parseCli)();
+    try {
+        process.exitCode = await (0, commands_1.runCommand)(cli, logger_1.consoleLogger);
+    }
+    catch (error) {
+        logger_1.consoleLogger.error(cli.command === "run" ? "Deu erro no ETL:" : "Deu erro:", error);
+        process.exitCode = 1;
+    }
+}
+main();

package/dist/normalizers/builtin.d.ts

@@ -0,0 +1,5 @@
+import { KeyNormalizer } from "./registry";
+export declare const trimNormalizer: KeyNormalizer;
+export declare const lowercaseNormalizer: KeyNormalizer;
+export declare const digitsOnlyNormalizer: KeyNormalizer;
+export declare const alphanumericNormalizer: KeyNormalizer;

package/dist/normalizers/builtin.js

@@ -0,0 +1,23 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.alphanumericNormalizer = exports.digitsOnlyNormalizer = exports.lowercaseNormalizer = exports.trimNormalizer = void 0;
+const trimNormalizer = (raw) => ({
+    key: typeof raw === "string" ? raw.trim() : raw,
+});
+exports.trimNormalizer = trimNormalizer;
+const lowercaseNormalizer = (raw) => ({
+    key: String(raw).trim().toLowerCase(),
+});
+exports.lowercaseNormalizer = lowercaseNormalizer;
+const digitsOnlyNormalizer = (raw) => {
+    const digits = String(raw).replace(/\D/g, "");
+    return digits === "" ? null : { key: digits };
+};
+exports.digitsOnlyNormalizer = digitsOnlyNormalizer;
+const alphanumericNormalizer = (raw) => {
+    const clean = String(raw)
+        .replace(/[^\p{L}\p{N}]/gu, "")
+        .toLowerCase();
+    return clean === "" ? null : { key: clean };
+};
+exports.alphanumericNormalizer = alphanumericNormalizer;

package/dist/normalizers/index.d.ts

@@ -0,0 +1,2 @@
+export declare function registerBuiltinKeyNormalizers(): void;
+export { loadNormalizerModules } from "./loader";

package/dist/normalizers/index.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadNormalizerModules = void 0;
+exports.registerBuiltinKeyNormalizers = registerBuiltinKeyNormalizers;
+const registry_1 = require("./registry");
+const builtin_1 = require("./builtin");
+function registerBuiltinKeyNormalizers() {
+    (0, registry_1.registerKeyNormalizer)("trim", builtin_1.trimNormalizer);
+    (0, registry_1.registerKeyNormalizer)("lowercase", builtin_1.lowercaseNormalizer);
+    (0, registry_1.registerKeyNormalizer)("digitsOnly", builtin_1.digitsOnlyNormalizer);
+    (0, registry_1.registerKeyNormalizer)("alphanumeric", builtin_1.alphanumericNormalizer);
+}
+var loader_1 = require("./loader");
+Object.defineProperty(exports, "loadNormalizerModules", { enumerable: true, get: function () { return loader_1.loadNormalizerModules; } });

package/dist/normalizers/loader.d.ts

@@ -0,0 +1 @@
+export declare function loadNormalizerModules(modulePaths: string[]): void;

package/dist/normalizers/loader.js

@@ -0,0 +1,30 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadNormalizerModules = loadNormalizerModules;
+const path_1 = __importDefault(require("path"));
+const registry_1 = require("./registry");
+function loadNormalizerModules(modulePaths) {
+    for (const modulePath of modulePaths) {
+        let loaded;
+        try {
+            loaded = require(path_1.default.resolve(modulePath));
+        }
+        catch (error) {
+            const reason = error instanceof Error ? error.message : String(error);
+            throw new Error(`Não consegui carregar o módulo de normalizadores "${modulePath}": ${reason}`);
+        }
+        const normalizers = loaded?.normalizers;
+        if (!normalizers || typeof normalizers !== "object") {
+            throw new Error(`O módulo "${modulePath}" precisa exportar um objeto chamado "normalizers". Exemplo: export const normalizers = { meuNormalizador };`);
+        }
+        for (const [name, normalizer] of Object.entries(normalizers)) {
+            if (typeof normalizer !== "function") {
+                throw new Error(`Em "${modulePath}", "${name}" não é uma função.`);
+            }
+            (0, registry_1.registerKeyNormalizer)(name, normalizer);
+        }
+    }
+}

package/dist/normalizers/registry.d.ts

@@ -0,0 +1,9 @@
+export interface NormalizedKey {
+    key: string | number;
+    group?: string;
+}
+export type KeyNormalizer = (raw: string | number) => NormalizedKey | null;
+export declare function registerKeyNormalizer(name: string, normalizer: KeyNormalizer): void;
+export declare function getKeyNormalizer(name: string): KeyNormalizer;
+export declare function listKeyNormalizers(): string[];
+export declare function clearKeyNormalizers(): void;

package/dist/normalizers/registry.js

@@ -0,0 +1,29 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.registerKeyNormalizer = registerKeyNormalizer;
+exports.getKeyNormalizer = getKeyNormalizer;
+exports.listKeyNormalizers = listKeyNormalizers;
+exports.clearKeyNormalizers = clearKeyNormalizers;
+const registry = new Map();
+function registerKeyNormalizer(name, normalizer) {
+    const existing = registry.get(name);
+    if (existing && existing !== normalizer) {
+        throw new Error(`Já existe um normalizador de chave registrado com o nome "${name}".`);
+    }
+    registry.set(name, normalizer);
+}
+function getKeyNormalizer(name) {
+    const normalizer = registry.get(name);
+    if (!normalizer) {
+        const available = listKeyNormalizers();
+        const hint = available.length > 0 ? available.join(", ") : "nenhum registrado";
+        throw new Error(`Normalizador de chave "${name}" não encontrado. Disponíveis: ${hint}`);
+    }
+    return normalizer;
+}
+function listKeyNormalizers() {
+    return Array.from(registry.keys());
+}
+function clearKeyNormalizers() {
+    registry.clear();
+}

package/dist/pipeline/index.d.ts

@@ -0,0 +1,7 @@
+export { runEtl, applySteps, countPendingReasons, DEFAULT_PREVIEW_SIZE, } from "./runEtl";
+export type { RunEtlOptions, AppliedSteps } from "./runEtl";
+export { buildSteps } from "./steps";
+export { MODES, parseMode, buildModeStep } from "./modes";
+export type { Mode } from "./modes";
+export { formatReport, formatDuration } from "./report";
+export type { EtlReport, PendingReason, Step, StepOutput, StepReport, } from "./types";