datera 0.1.0

package/dist/io/parquet/parquetSource.js

@@ -0,0 +1,68 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ParquetSource = void 0;
+exports.parquetRow = parquetRow;
+const fs_1 = __importDefault(require("fs"));
+const cell_1 = require("../sql/cell");
+const parquetLibrary_1 = require("./parquetLibrary");
+function join(prefix, name) {
+    return prefix ? `${prefix}.${name}` : name;
+}
+function isPlainObject(value) {
+    return (typeof value === "object" &&
+        value !== null &&
+        !Array.isArray(value) &&
+        !(value instanceof Date) &&
+        !(value instanceof Uint8Array));
+}
+function isSimple(value) {
+    return value === null || (typeof value !== "object" && value !== undefined);
+}
+function flatten(value, prefix, row) {
+    if (isPlainObject(value)) {
+        for (const [name, inner] of Object.entries(value)) {
+            flatten(inner, join(prefix, name), row);
+        }
+    }
+    else if (Array.isArray(value) && value.every(isSimple)) {
+        row[prefix] = value
+            .map((item) => String((0, cell_1.toCell)(item) ?? ""))
+            .join(", ");
+    }
+    else {
+        row[prefix] = (0, cell_1.toCell)(value);
+    }
+}
+function parquetRow(record) {
+    const row = {};
+    for (const [name, value] of Object.entries(record)) {
+        flatten(value, name, row);
+    }
+    return row;
+}
+class ParquetSource {
+    options;
+    library;
+    constructor(options, library = parquetLibrary_1.parquetLibrary) {
+        this.options = options;
+        this.library = library;
+    }
+    async read() {
+        if (!fs_1.default.existsSync(this.options.path)) {
+            throw new Error(`Arquivo não encontrado: ${this.options.path}`);
+        }
+        let records;
+        try {
+            records = await this.library.read(this.options.path);
+        }
+        catch (error) {
+            const reason = error instanceof Error ? error.message : String(error);
+            throw new Error(`Não consegui ler o Parquet em ${this.options.path}: ${reason}`);
+        }
+        return records.map(parquetRow);
+    }
+}
+exports.ParquetSource = ParquetSource;

package/dist/io/postgres/postgresSource.d.ts

@@ -0,0 +1,26 @@
+import { TableRow } from "../../types";
+import { Source } from "../types";
+export interface PostgresConnection {
+    host: string;
+    port: number;
+    user: string;
+    password?: string | undefined;
+    database: string;
+    ssl?: boolean | undefined;
+}
+export interface PostgresClient {
+    connect(): Promise<unknown>;
+    query(text: string): Promise<{
+        rows: Record<string, unknown>[];
+    }>;
+    end(): Promise<unknown>;
+}
+export type PostgresClientFactory = (connection: PostgresConnection) => PostgresClient;
+export declare const createPostgresClient: PostgresClientFactory;
+export declare class PostgresSource implements Source {
+    private readonly connection;
+    private readonly table;
+    private readonly createClient;
+    constructor(connection: PostgresConnection, table: string, createClient?: PostgresClientFactory);
+    read(): Promise<TableRow[]>;
+}

package/dist/io/postgres/postgresSource.js

@@ -0,0 +1,42 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostgresSource = exports.createPostgresClient = void 0;
+const optional_1 = require("../optional");
+const names_1 = require("../sql/names");
+const cell_1 = require("../sql/cell");
+const createPostgresClient = (connection) => {
+    const pg = (0, optional_1.requireOptional)(() => require("pg"), ["pg"], "PostgreSQL");
+    return new pg.Client({
+        host: connection.host,
+        port: connection.port,
+        user: connection.user,
+        ...(connection.password !== undefined
+            ? { password: connection.password }
+            : {}),
+        database: connection.database,
+        ...(connection.ssl ? { ssl: true } : {}),
+    });
+};
+exports.createPostgresClient = createPostgresClient;
+class PostgresSource {
+    connection;
+    table;
+    createClient;
+    constructor(connection, table, createClient = exports.createPostgresClient) {
+        this.connection = connection;
+        this.table = table;
+        this.createClient = createClient;
+    }
+    async read() {
+        const client = this.createClient(this.connection);
+        await client.connect();
+        try {
+            const result = await client.query(`SELECT * FROM ${(0, names_1.quoteTable)(this.table, '"')}`);
+            return result.rows.map(cell_1.toRow);
+        }
+        finally {
+            await client.end();
+        }
+    }
+}
+exports.PostgresSource = PostgresSource;

package/dist/io/safety.d.ts

@@ -0,0 +1,4 @@
+import { EtlConfig } from "../config";
+export declare function isSameFile(a: string, b: string): boolean;
+export declare function assertDifferentFiles(input: string, output: string): void;
+export declare function assertSourceIsSafe(config: EtlConfig): void;

package/dist/io/safety.js

@@ -0,0 +1,83 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.isSameFile = isSameFile;
+exports.assertDifferentFiles = assertDifferentFiles;
+exports.assertSourceIsSafe = assertSourceIsSafe;
+const fs_1 = __importDefault(require("fs"));
+const path_1 = __importDefault(require("path"));
+const validate_1 = require("../filters/validate");
+function sameName(a, b) {
+    return a.trim().toLowerCase() === b.trim().toLowerCase();
+}
+function filePath(side) {
+    return "path" in side && typeof side.path === "string"
+        ? side.path
+        : undefined;
+}
+function assertSheetsAreSafe(config) {
+    const source = config.source;
+    const destination = config.destination ?? { type: "sheets" };
+    if (source?.type !== "sheets" || destination.type !== "sheets")
+        return;
+    const destinationId = destination.spreadsheetId ?? config.spreadsheetId;
+    if (source.spreadsheetId !== destinationId)
+        return;
+    if (source.sheet === undefined) {
+        throw new Error('A fonte e o destino são a mesma planilha. Coloque "sheet" na fonte com o nome da aba dos dados originais, senão ele pode ler e apagar a mesma aba.');
+    }
+    if (destination.sheet === undefined) {
+        throw new Error('A fonte e o destino são a mesma planilha. Coloque "sheet" no destino com o nome da aba do resultado, senão ele grava na primeira aba, que pode ser a dos dados originais.');
+    }
+    const pendingSheet = config.validation
+        ? (config.validation.pendingSheet ?? validate_1.DEFAULT_PENDING_SHEET)
+        : undefined;
+    for (const target of [destination.sheet, pendingSheet]) {
+        if (target !== undefined && sameName(target, source.sheet)) {
+            throw new Error(`A aba "${source.sheet}" é de onde o Datera lê, então ele não pode gravar nela. Escolha outro nome pra aba do resultado ou das pendências.`);
+        }
+    }
+}
+function comparablePath(file) {
+    const resolved = path_1.default.resolve(file);
+    let real = resolved;
+    try {
+        real = fs_1.default.realpathSync.native(resolved);
+    }
+    catch {
+        try {
+            real = path_1.default.join(fs_1.default.realpathSync.native(path_1.default.dirname(resolved)), path_1.default.basename(resolved));
+        }
+        catch {
+            real = resolved;
+        }
+    }
+    return process.platform === "win32" || process.platform === "darwin"
+        ? real.toLowerCase()
+        : real;
+}
+function isSameFile(a, b) {
+    return comparablePath(a) === comparablePath(b);
+}
+function assertDifferentFiles(input, output) {
+    if (isSameFile(input, output)) {
+        throw new Error(`A entrada e a saída são o mesmo arquivo (${input}). A saída é apagada antes de gravar, então escolha outro arquivo pra ela.`);
+    }
+}
+function assertFilesAreSafe(config) {
+    const sourcePath = config.source ? filePath(config.source) : undefined;
+    const destinationPath = config.destination
+        ? filePath(config.destination)
+        : undefined;
+    if (sourcePath === undefined || destinationPath === undefined)
+        return;
+    if (isSameFile(sourcePath, destinationPath)) {
+        throw new Error(`A fonte e o destino são o mesmo arquivo (${sourcePath}). O destino é apagado antes de gravar, então escolha outro arquivo pra saída.`);
+    }
+}
+function assertSourceIsSafe(config) {
+    assertSheetsAreSafe(config);
+    assertFilesAreSafe(config);
+}

package/dist/io/sheets/client.d.ts

@@ -0,0 +1,10 @@
+import type { sheets_v4 } from "@googleapis/sheets";
+import { TableRow } from "../../types";
+import { Logger } from "../../logger";
+export declare function createSheetsClient(credentialsPath: string): sheets_v4.Sheets;
+export interface WriteOptions {
+    sheetName?: string | undefined;
+    logger?: Logger | undefined;
+}
+export declare function readSheet(sheets: sheets_v4.Sheets, spreadsheetId: string, sheetName?: string): Promise<unknown[][]>;
+export declare function writeData(sheets: sheets_v4.Sheets, spreadsheetId: string, data: TableRow[], options?: WriteOptions): Promise<void>;

package/dist/io/sheets/client.js

@@ -0,0 +1,141 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSheetsClient = createSheetsClient;
+exports.readSheet = readSheet;
+exports.writeData = writeData;
+const optional_1 = require("../optional");
+const header_1 = require("../header");
+const logger_1 = require("../../logger");
+function createSheetsClient(credentialsPath) {
+    const { auth, sheets: sheetsApi } = (0, optional_1.requireOptional)(() => require("@googleapis/sheets"), ["@googleapis/sheets"], "Google Sheets");
+    const credentials = new auth.GoogleAuth({
+        keyFile: credentialsPath,
+        scopes: ["https://www.googleapis.com/auth/spreadsheets"],
+    });
+    return sheetsApi({ version: "v4", auth: credentials });
+}
+function quoteSheetTitle(title) {
+    return `'${title.replace(/'/g, "''")}'`;
+}
+async function listSheets(sheets, spreadsheetId) {
+    const response = await sheets.spreadsheets.get({
+        spreadsheetId,
+        fields: "sheets.properties(sheetId,title)",
+    });
+    const found = [];
+    for (const sheet of response.data.sheets ?? []) {
+        const { title, sheetId } = sheet.properties ?? {};
+        if (title && sheetId !== null && sheetId !== undefined) {
+            found.push({ title, sheetId });
+        }
+    }
+    return found;
+}
+async function resolveSheet(sheets, spreadsheetId, sheetName, logger = logger_1.consoleLogger) {
+    const existing = await listSheets(sheets, spreadsheetId);
+    const found = sheetName === undefined
+        ? existing[0]
+        : existing.find((sheet) => sheet.title === sheetName);
+    if (found)
+        return found;
+    if (sheetName === undefined) {
+        throw new Error("A planilha não tem nenhuma aba pra escrever.");
+    }
+    const created = await sheets.spreadsheets.batchUpdate({
+        spreadsheetId,
+        requestBody: {
+            requests: [{ addSheet: { properties: { title: sheetName } } }],
+        },
+    });
+    const sheetId = created.data.replies?.[0]?.addSheet?.properties?.sheetId;
+    if (sheetId === null || sheetId === undefined) {
+        throw new Error(`Não consegui criar a aba "${sheetName}".`);
+    }
+    logger.info(`Aba "${sheetName}" criada.`);
+    return { title: sheetName, sheetId };
+}
+async function readSheet(sheets, spreadsheetId, sheetName) {
+    const existing = await listSheets(sheets, spreadsheetId);
+    const target = sheetName === undefined
+        ? existing[0]
+        : existing.find((sheet) => sheet.title === sheetName);
+    if (!target) {
+        const names = existing.map((sheet) => sheet.title).join(", ");
+        throw new Error(sheetName === undefined
+            ? "A planilha não tem nenhuma aba pra ler."
+            : `Aba "${sheetName}" não encontrada na planilha. Abas: ${names}`);
+    }
+    const response = await sheets.spreadsheets.values.get({
+        spreadsheetId,
+        range: quoteSheetTitle(target.title),
+        valueRenderOption: "UNFORMATTED_VALUE",
+        dateTimeRenderOption: "FORMATTED_STRING",
+    });
+    return response.data.values ?? [];
+}
+async function formatSheet(sheets, spreadsheetId, sheetId, columnCount) {
+    await sheets.spreadsheets.batchUpdate({
+        spreadsheetId,
+        requestBody: {
+            requests: [
+                {
+                    updateSheetProperties: {
+                        properties: {
+                            sheetId,
+                            gridProperties: { frozenRowCount: 1 },
+                        },
+                        fields: "gridProperties.frozenRowCount",
+                    },
+                },
+                {
+                    repeatCell: {
+                        range: { sheetId, startRowIndex: 0, endRowIndex: 1 },
+                        cell: {
+                            userEnteredFormat: {
+                                textFormat: { bold: true },
+                                backgroundColor: {
+                                    red: 0.9,
+                                    green: 0.9,
+                                    blue: 0.9,
+                                },
+                            },
+                        },
+                        fields: "userEnteredFormat(textFormat,backgroundColor)",
+                    },
+                },
+                {
+                    autoResizeDimensions: {
+                        dimensions: {
+                            sheetId,
+                            dimension: "COLUMNS",
+                            startIndex: 0,
+                            endIndex: columnCount,
+                        },
+                    },
+                },
+            ],
+        },
+    });
+}
+async function writeData(sheets, spreadsheetId, data, options = {}) {
+    const logger = options.logger ?? logger_1.consoleLogger;
+    const target = await resolveSheet(sheets, spreadsheetId, options.sheetName, logger);
+    const range = quoteSheetTitle(target.title);
+    await sheets.spreadsheets.values.clear({ spreadsheetId, range });
+    if (data.length === 0) {
+        logger.info(`Nenhum dado pra escrever na aba "${target.title}".`);
+        return;
+    }
+    const header = (0, header_1.buildHeader)(data);
+    const rows = data.map((row) => header.map((column) => row[column] ?? ""));
+    await sheets.spreadsheets.values.update({
+        spreadsheetId,
+        range: `${range}!A1`,
+        valueInputOption: "RAW",
+        requestBody: {
+            values: [header, ...rows],
+        },
+    });
+    await formatSheet(sheets, spreadsheetId, target.sheetId, header.length);
+    logger.info(`${data.length} linhas escritas na aba "${target.title}".`);
+}

package/dist/io/sheets/sheetsSink.d.ts

@@ -0,0 +1,12 @@
+import type { sheets_v4 } from "@googleapis/sheets";
+import { TableRow } from "../../types";
+import { Sink, SinkWriteOptions } from "../types";
+import { Logger } from "../../logger";
+export declare class SheetsSink implements Sink {
+    private readonly sheets;
+    private readonly spreadsheetId;
+    private readonly sheet?;
+    private readonly logger;
+    constructor(sheets: sheets_v4.Sheets, spreadsheetId: string, sheet?: string | undefined, logger?: Logger);
+    write(rows: TableRow[], options?: SinkWriteOptions): Promise<void>;
+}

package/dist/io/sheets/sheetsSink.js

@@ -0,0 +1,24 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SheetsSink = void 0;
+const client_1 = require("./client");
+const logger_1 = require("../../logger");
+class SheetsSink {
+    sheets;
+    spreadsheetId;
+    sheet;
+    logger;
+    constructor(sheets, spreadsheetId, sheet, logger = logger_1.consoleLogger) {
+        this.sheets = sheets;
+        this.spreadsheetId = spreadsheetId;
+        this.sheet = sheet;
+        this.logger = logger;
+    }
+    async write(rows, options = {}) {
+        await (0, client_1.writeData)(this.sheets, this.spreadsheetId, rows, {
+            sheetName: options.name ?? this.sheet,
+            logger: this.logger,
+        });
+    }
+}
+exports.SheetsSink = SheetsSink;

package/dist/io/sheets/sheetsSource.d.ts

@@ -0,0 +1,10 @@
+import type { sheets_v4 } from "@googleapis/sheets";
+import { TableRow } from "../../types";
+import { Source } from "../types";
+export declare class SheetsSource implements Source {
+    private readonly sheets;
+    private readonly spreadsheetId;
+    private readonly sheet?;
+    constructor(sheets: sheets_v4.Sheets, spreadsheetId: string, sheet?: string | undefined);
+    read(): Promise<TableRow[]>;
+}

package/dist/io/sheets/sheetsSource.js

@@ -0,0 +1,47 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SheetsSource = void 0;
+const client_1 = require("./client");
+function fromSheetCell(value) {
+    if (value === null || value === undefined || value === "")
+        return null;
+    if (typeof value === "number" || typeof value === "string")
+        return value;
+    return String(value);
+}
+class SheetsSource {
+    sheets;
+    spreadsheetId;
+    sheet;
+    constructor(sheets, spreadsheetId, sheet) {
+        this.sheets = sheets;
+        this.spreadsheetId = spreadsheetId;
+        this.sheet = sheet;
+    }
+    async read() {
+        const [header, ...lines] = await (0, client_1.readSheet)(this.sheets, this.spreadsheetId, this.sheet);
+        if (!header)
+            return [];
+        const columns = header.map((cell) => {
+            const name = fromSheetCell(cell);
+            return name === null ? null : String(name).trim();
+        });
+        const rows = [];
+        for (const values of lines) {
+            const row = {};
+            let hasValue = false;
+            columns.forEach((column, index) => {
+                if (column === null)
+                    return;
+                const value = fromSheetCell(values[index]);
+                row[column] = value;
+                if (value !== null)
+                    hasValue = true;
+            });
+            if (hasValue)
+                rows.push(row);
+        }
+        return rows;
+    }
+}
+exports.SheetsSource = SheetsSource;

package/dist/io/sql/cell.d.ts

@@ -0,0 +1,5 @@
+import { TableRow } from "../../types";
+type Cell = TableRow[string];
+export declare function toCell(value: unknown): Cell;
+export declare function toRow(record: Record<string, unknown>): TableRow;
+export {};

package/dist/io/sql/cell.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toCell = toCell;
+exports.toRow = toRow;
+function isoDate(date) {
+    if (Number.isNaN(date.getTime()))
+        return "";
+    const iso = date.toISOString();
+    return iso.endsWith("T00:00:00.000Z") ? iso.slice(0, 10) : iso;
+}
+function toCell(value) {
+    if (value === null || value === undefined)
+        return null;
+    if (typeof value === "number")
+        return Number.isFinite(value) ? value : null;
+    if (typeof value === "string")
+        return value === "" ? null : value;
+    if (typeof value === "bigint") {
+        return Number.isSafeInteger(Number(value))
+            ? Number(value)
+            : value.toString();
+    }
+    if (typeof value === "boolean")
+        return String(value);
+    if (value instanceof Date)
+        return isoDate(value) || null;
+    if (value instanceof Uint8Array) {
+        return Buffer.from(value).toString("base64");
+    }
+    return JSON.stringify(value, (_key, inner) => typeof inner === "bigint" ? inner.toString() : inner);
+}
+function toRow(record) {
+    const row = {};
+    for (const [column, value] of Object.entries(record)) {
+        row[column] = toCell(value);
+    }
+    return row;
+}

package/dist/io/sql/driver.d.ts

@@ -0,0 +1 @@
+export declare function loadDriver<T>(load: () => unknown, packageName: string, label: string): T;

package/dist/io/sql/driver.js

@@ -0,0 +1,15 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.loadDriver = loadDriver;
+function loadDriver(load, packageName, label) {
+    try {
+        return load();
+    }
+    catch (error) {
+        const code = error?.code;
+        if (code === "MODULE_NOT_FOUND") {
+            throw new Error(`Pra ler de ${label}, o pacote "${packageName}" precisa estar instalado. Rode: npm install ${packageName}`);
+        }
+        throw error;
+    }
+}

package/dist/io/sql/names.d.ts

@@ -0,0 +1 @@
+export declare function quoteTable(table: string, open: string, close?: string): string;

package/dist/io/sql/names.js

@@ -0,0 +1,12 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.quoteTable = quoteTable;
+function quoteTable(table, open, close = open) {
+    const parts = table.split(".").map((part) => part.trim());
+    if (parts.some((part) => part === "")) {
+        throw new Error(`Nome de tabela inválido: "${table}".`);
+    }
+    return parts
+        .map((part) => `${open}${part.split(close).join(close + close)}${close}`)
+        .join(".");
+}

package/dist/io/sqlite/sqliteSource.d.ts

@@ -0,0 +1,11 @@
+import { TableRow } from "../../types";
+import { Source } from "../types";
+export interface SqliteSourceOptions {
+    path: string;
+    table: string;
+}
+export declare class SqliteSource implements Source {
+    private readonly options;
+    constructor(options: SqliteSourceOptions);
+    read(): Promise<TableRow[]>;
+}

package/dist/io/sqlite/sqliteSource.js

@@ -0,0 +1,52 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.SqliteSource = void 0;
+const fs_1 = __importDefault(require("fs"));
+const names_1 = require("../sql/names");
+const cell_1 = require("../sql/cell");
+function loadSqlite() {
+    try {
+        return require("node:sqlite");
+    }
+    catch {
+        throw new Error("Essa versão do Node não sabe ler SQLite. Atualize pro Node 22.13 ou mais novo.");
+    }
+}
+class SqliteSource {
+    options;
+    constructor(options) {
+        this.options = options;
+    }
+    async read() {
+        if (!fs_1.default.existsSync(this.options.path)) {
+            throw new Error(`Arquivo não encontrado: ${this.options.path}`);
+        }
+        const { DatabaseSync } = loadSqlite();
+        const database = new DatabaseSync(this.options.path, {
+            readOnly: true,
+        });
+        try {
+            const tables = database
+                .prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY name")
+                .all()
+                .map((row) => String(row.name));
+            if (!tables.includes(this.options.table)) {
+                throw new Error(`A tabela "${this.options.table}" não existe em ${this.options.path}.` +
+                    (tables.length > 0
+                        ? ` As que existem são: ${tables.join(", ")}.`
+                        : " O arquivo não tem nenhuma tabela."));
+            }
+            return database
+                .prepare(`SELECT * FROM ${(0, names_1.quoteTable)(this.options.table, '"')}`)
+                .all()
+                .map((record) => (0, cell_1.toRow)(record));
+        }
+        finally {
+            database.close();
+        }
+    }
+}
+exports.SqliteSource = SqliteSource;

package/dist/io/sqlserver/sqlServerSource.d.ts

@@ -0,0 +1,24 @@
+import { TableRow } from "../../types";
+import { Source } from "../types";
+export interface SqlServerConnection {
+    host: string;
+    port: number;
+    user: string;
+    password?: string | undefined;
+    database: string;
+    encrypt?: boolean | undefined;
+    trustServerCertificate?: boolean | undefined;
+}
+export interface SqlServerClient {
+    query(text: string): Promise<Record<string, unknown>[]>;
+    close(): Promise<unknown>;
+}
+export type SqlServerClientFactory = (connection: SqlServerConnection) => Promise<SqlServerClient>;
+export declare const createSqlServerClient: SqlServerClientFactory;
+export declare class SqlServerSource implements Source {
+    private readonly connection;
+    private readonly table;
+    private readonly createClient;
+    constructor(connection: SqlServerConnection, table: string, createClient?: SqlServerClientFactory);
+    read(): Promise<TableRow[]>;
+}