datera 0.1.0 → 0.2.1

package/dist/io/postgres/postgresSink.js

@@ -0,0 +1,82 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PostgresSink = exports.POSTGRES_DIALECT = void 0;
+exports.postgresSession = postgresSession;
+const logger_1 = require("../../logger");
+const writer_1 = require("../sql/writer");
+const target_1 = require("../sql/target");
+const postgresSource_1 = require("./postgresSource");
+exports.POSTGRES_DIALECT = {
+    label: "PostgreSQL",
+    open: '"',
+    close: '"',
+    schemas: true,
+    maxNameLength: 63,
+    nameLength: (name) => Buffer.byteLength(name, "utf8"),
+    maxParams: 65000,
+    maxRowsPerInsert: 1000,
+    transactionalDdl: true,
+    types: {
+        integer: "BIGINT",
+        number: "DOUBLE PRECISION",
+        text: "TEXT",
+    },
+    keyType: "TEXT",
+    tableSuffix: "",
+    param: (index) => `$${index}`,
+    tableMark: () => ({ inside: "", after: "" }),
+    tableId: async (session, table) => {
+        const [found] = await session.all("SELECT to_regclass($1)::oid::text AS id", [(0, writer_1.quoteTableName)(exports.POSTGRES_DIALECT, table)]);
+        return typeof found?.id === "string" ? found.id : null;
+    },
+    ownsTable: async (session, table) => {
+        const [found] = await session.all("SELECT pg_get_userbyid(c.relowner) = current_user AS dono FROM pg_class c WHERE c.oid = to_regclass($1)", [(0, writer_1.quoteTableName)(exports.POSTGRES_DIALECT, table)]);
+        return found?.dono === true;
+    },
+    tableExists: async (session, table) => {
+        const [found] = await session.all("SELECT to_regclass($1) AS nome", [
+            (0, writer_1.quoteTableName)(exports.POSTGRES_DIALECT, table),
+        ]);
+        return found?.nome !== null && found?.nome !== undefined;
+    },
+};
+function postgresSession(client) {
+    return {
+        run: async (sql, values) => {
+            await client.query(sql, values);
+        },
+        all: async (sql, values) => (await client.query(sql, values)).rows,
+        begin: async () => {
+            await client.query("BEGIN");
+        },
+        commit: async () => {
+            await client.query("COMMIT");
+        },
+        rollback: async () => {
+            await client.query("ROLLBACK");
+        },
+    };
+}
+class PostgresSink {
+    connection;
+    target;
+    logger;
+    createClient;
+    constructor(connection, target, logger = logger_1.consoleLogger, createClient = postgresSource_1.createPostgresClient) {
+        this.connection = connection;
+        this.target = target;
+        this.logger = logger;
+        this.createClient = createClient;
+    }
+    async write(rows, options = {}) {
+        const client = this.createClient(this.connection);
+        await client.connect();
+        try {
+            await (0, writer_1.writeTable)(postgresSession(client), exports.POSTGRES_DIALECT, (0, target_1.tableFor)(this.target, options), rows, this.logger);
+        }
+        finally {
+            await client.end();
+        }
+    }
+}
+exports.PostgresSink = PostgresSink;

package/dist/io/postgres/postgresSource.d.ts

@@ -10,7 +10,7 @@ export interface PostgresConnection {
 }
 export interface PostgresClient {
     connect(): Promise<unknown>;
-    query(text: string): Promise<{
+    query(text: string, values?: unknown[]): Promise<{
         rows: Record<string, unknown>[];
     }>;
     end(): Promise<unknown>;

package/dist/io/safety.d.ts

@@ -1,4 +1,4 @@
 import { EtlConfig } from "../config";
 export declare function isSameFile(a: string, b: string): boolean;
-export declare function assertDifferentFiles(input: string, output: string): void;
+export declare function tableProblem(config: EtlConfig): string | null;
 export declare function assertSourceIsSafe(config: EtlConfig): void;

package/dist/io/safety.js

@@ -4,11 +4,18 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
 };
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.isSameFile = isSameFile;
-exports.assertDifferentFiles = assertDifferentFiles;
+exports.tableProblem = tableProblem;
 exports.assertSourceIsSafe = assertSourceIsSafe;
 const fs_1 = __importDefault(require("fs"));
 const path_1 = __importDefault(require("path"));
 const validate_1 = require("../filters/validate");
+const writer_1 = require("./sql/writer");
+const sqliteSink_1 = require("./sqlite/sqliteSink");
+const mysqlSink_1 = require("./mysql/mysqlSink");
+const postgresSink_1 = require("./postgres/postgresSink");
+const sqlServerSink_1 = require("./sqlserver/sqlServerSink");
+const target_1 = require("./sql/target");
+const servers_1 = require("./servers");
 function sameName(a, b) {
     return a.trim().toLowerCase() === b.trim().toLowerCase();
 }
@@ -61,12 +68,10 @@ function comparablePath(file) {
 function isSameFile(a, b) {
     return comparablePath(a) === comparablePath(b);
 }
-function assertDifferentFiles(input, output) {
-    if (isSameFile(input, output)) {
-        throw new Error(`A entrada e a saída são o mesmo arquivo (${input}). A saída é apagada antes de gravar, então escolha outro arquivo pra ela.`);
-    }
-}
 function assertFilesAreSafe(config) {
+    if (config.source?.type === "sqlite" &&
+        config.destination?.type === "sqlite")
+        return;
     const sourcePath = config.source ? filePath(config.source) : undefined;
     const destinationPath = config.destination
         ? filePath(config.destination)
@@ -77,7 +82,101 @@ function assertFilesAreSafe(config) {
         throw new Error(`A fonte e o destino são o mesmo arquivo (${sourcePath}). O destino é apagado antes de gravar, então escolha outro arquivo pra saída.`);
     }
 }
+function destinationTable(config) {
+    const destination = config.destination;
+    switch (destination?.type) {
+        case "sqlite":
+        case "mysql":
+        case "postgres":
+        case "sqlserver":
+            return destination.table?.trim() || undefined;
+        default:
+            return undefined;
+    }
+}
+function sameDatabase(config) {
+    const { source, destination } = config;
+    if (source?.type === "sqlite" && destination?.type === "sqlite") {
+        return isSameFile(source.path, destination.path);
+    }
+    const from = (0, servers_1.serverOf)(config, "source");
+    const to = (0, servers_1.serverOf)(config, "destination");
+    if (!from || !to || from.type !== to.type)
+        return false;
+    return (sameName(from.host ?? "", to.host ?? "") &&
+        from.port === to.port &&
+        sameName(from.database ?? "", to.database ?? ""));
+}
+function sourceTable(config) {
+    const source = config.source;
+    if (source?.type === "sqlite")
+        return source.table;
+    return (0, servers_1.serverOf)(config, "source")?.table;
+}
+const DIALECTS = {
+    sqlite: sqliteSink_1.SQLITE_DIALECT,
+    mysql: mysqlSink_1.MYSQL_DIALECT,
+    postgres: postgresSink_1.POSTGRES_DIALECT,
+    sqlserver: sqlServerSink_1.SQLSERVER_DIALECT,
+};
+function nameProblem(type, targets) {
+    const dialect = DIALECTS[type];
+    if (!dialect)
+        return null;
+    for (const target of targets) {
+        try {
+            (0, writer_1.checkTableName)(dialect, target, (0, writer_1.nameRoom)(dialect));
+        }
+        catch (error) {
+            return error instanceof Error ? error.message : String(error);
+        }
+    }
+    return null;
+}
+function tableProblem(config) {
+    const table = destinationTable(config);
+    const destination = config.destination;
+    if (table === undefined || destination === undefined)
+        return null;
+    const pending = config.validation
+        ? (0, target_1.pendingTableOf)({
+            table,
+            pendingTable: "pendingTable" in destination
+                ? destination.pendingTable
+                : undefined,
+        })
+        : undefined;
+    const targets = pending === undefined ? [table] : [table, pending];
+    const names = nameProblem(destination.type, targets);
+    if (names !== null)
+        return names;
+    const lastPart = (target) => target.split(".").pop()?.trim().toLowerCase() ?? "";
+    if (targets.some((target) => {
+        const last = lastPart(target);
+        return (last === writer_1.OWNED_TABLES ||
+            last.endsWith(writer_1.NEW_SUFFIX) ||
+            last.endsWith(writer_1.OLD_SUFFIX));
+    })) {
+        return `"${writer_1.OWNED_TABLES}" e os nomes terminados em ${writer_1.NEW_SUFFIX} ou ${writer_1.OLD_SUFFIX} são usados pelo próprio Datera. Escolha outro nome.`;
+    }
+    if (pending !== undefined && sameName(pending, table)) {
+        return `A tabela das pendências precisa ter um nome diferente da tabela do resultado ("${table}").`;
+    }
+    const from = sourceTable(config);
+    if (from !== undefined && sameDatabase(config)) {
+        if (targets.some((target) => sameName(target, from))) {
+            return `A tabela "${from}" é de onde o Datera lê, então ele não pode gravar nela. Escolha outro nome pra tabela do resultado ou das pendências.`;
+        }
+    }
+    return null;
+}
+function assertTablesAreSafe(config) {
+    const problem = tableProblem(config);
+    if (problem !== null)
+        throw new Error(problem);
+}
 function assertSourceIsSafe(config) {
     assertSheetsAreSafe(config);
     assertFilesAreSafe(config);
+    assertTablesAreSafe(config);
 }

package/dist/io/servers.d.ts

@@ -0,0 +1,14 @@
+import { EtlConfig } from "../config";
+export type ServerType = "mysql" | "postgres" | "sqlserver";
+export declare const DEFAULT_PORTS: Record<ServerType, number>;
+export interface ServerPlace {
+    type: ServerType;
+    host: string | undefined;
+    port: number;
+    user: string | undefined;
+    password: string | undefined;
+    database: string | undefined;
+    table: string | undefined;
+}
+export declare function serverOf(config: EtlConfig, side: "source" | "destination"): ServerPlace | undefined;
+export declare function destinationSharesSource(config: EtlConfig): boolean;

package/dist/io/servers.js

@@ -0,0 +1,72 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_PORTS = void 0;
+exports.serverOf = serverOf;
+exports.destinationSharesSource = destinationSharesSource;
+exports.DEFAULT_PORTS = {
+    mysql: 3306,
+    postgres: 5432,
+    sqlserver: 1433,
+};
+function sourceServer(config) {
+    const source = config.source ?? { type: "mysql" };
+    if (source.type !== "mysql" &&
+        source.type !== "postgres" &&
+        source.type !== "sqlserver")
+        return undefined;
+    const legacy = source.type === "mysql";
+    return {
+        type: source.type,
+        host: source.host ?? (legacy ? config.dbHost : undefined),
+        port: source.port ??
+            (legacy ? config.dbPort : undefined) ??
+            exports.DEFAULT_PORTS[source.type],
+        user: source.user ?? (legacy ? config.dbUser : undefined),
+        password: source.password ?? (legacy ? config.dbPassword : undefined),
+        database: source.database ?? (legacy ? config.dbName : undefined),
+        table: source.table ?? (legacy ? config.tableName : undefined),
+    };
+}
+function sameHost(a, b) {
+    return (a ?? "").trim().toLowerCase() === (b ?? "").trim().toLowerCase();
+}
+function isSameServer(source, destination) {
+    if (source.type !== destination.type)
+        return false;
+    if (destination.host === undefined)
+        return destination.port === undefined;
+    return (sameHost(source.host, destination.host) &&
+        (destination.port ?? exports.DEFAULT_PORTS[destination.type]) === source.port);
+}
+function serverOf(config, side) {
+    const source = sourceServer(config);
+    if (side === "source")
+        return source;
+    const destination = config.destination;
+    if (destination?.type !== "mysql" &&
+        destination?.type !== "postgres" &&
+        destination?.type !== "sqlserver")
+        return undefined;
+    const same = source !== undefined && isSameServer(source, destination)
+        ? source
+        : undefined;
+    return {
+        type: destination.type,
+        host: destination.host ?? same?.host,
+        port: destination.port ?? same?.port ?? exports.DEFAULT_PORTS[destination.type],
+        user: destination.user ?? same?.user,
+        password: destination.password ?? same?.password,
+        database: destination.database ?? same?.database,
+        table: destination.table,
+    };
+}
+function destinationSharesSource(config) {
+    const source = sourceServer(config);
+    const destination = config.destination;
+    if (source === undefined ||
+        (destination?.type !== "mysql" &&
+            destination?.type !== "postgres" &&
+            destination?.type !== "sqlserver"))
+        return false;
+    return isSameServer(source, destination);
+}

package/dist/io/sheets/client.js

@@ -6,6 +6,7 @@ exports.writeData = writeData;
 const optional_1 = require("../optional");
 const header_1 = require("../header");
 const logger_1 = require("../../logger");
+const cells_1 = require("../../cells");
 function createSheetsClient(credentialsPath) {
     const { auth, sheets: sheetsApi } = (0, optional_1.requireOptional)(() => require("@googleapis/sheets"), ["@googleapis/sheets"], "Google Sheets");
     const credentials = new auth.GoogleAuth({
@@ -127,7 +128,7 @@ async function writeData(sheets, spreadsheetId, data, options = {}) {
         return;
     }
     const header = (0, header_1.buildHeader)(data);
-    const rows = data.map((row) => header.map((column) => row[column] ?? ""));
+    const rows = data.map((row) => header.map((column) => (0, cells_1.cellOf)(row, column) ?? ""));
     await sheets.spreadsheets.values.update({
         spreadsheetId,
         range: `${range}!A1`,

package/dist/io/sheets/sheetsSource.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.SheetsSource = void 0;
 const client_1 = require("./client");
+const cells_1 = require("../../cells");
 function fromSheetCell(value) {
     if (value === null || value === undefined || value === "")
         return null;
@@ -34,7 +35,7 @@ class SheetsSource {
                 if (column === null)
                     return;
                 const value = fromSheetCell(values[index]);
-                row[column] = value;
+                (0, cells_1.setCell)(row, column, value);
                 if (value !== null)
                     hasValue = true;
             });

package/dist/io/sql/cell.js

@@ -2,6 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.toCell = toCell;
 exports.toRow = toRow;
+const cells_1 = require("../../cells");
 function isoDate(date) {
     if (Number.isNaN(date.getTime()))
         return "";
@@ -32,7 +33,7 @@ function toCell(value) {
 function toRow(record) {
     const row = {};
     for (const [column, value] of Object.entries(record)) {
-        row[column] = toCell(value);
+        (0, cells_1.setCell)(row, column, toCell(value));
     }
     return row;
 }

package/dist/io/sql/target.d.ts

@@ -0,0 +1,8 @@
+import { SinkWriteOptions } from "../types";
+export declare const PENDING_SUFFIX = "_pendencias";
+export interface TableTarget {
+    table: string;
+    pendingTable?: string | undefined;
+}
+export declare function pendingTableOf(target: TableTarget): string;
+export declare function tableFor(target: TableTarget, options: SinkWriteOptions): string;

package/dist/io/sql/target.js

@@ -0,0 +1,14 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.PENDING_SUFFIX = void 0;
+exports.pendingTableOf = pendingTableOf;
+exports.tableFor = tableFor;
+exports.PENDING_SUFFIX = "_pendencias";
+function pendingTableOf(target) {
+    return (target.pendingTable?.trim() || `${target.table.trim()}${exports.PENDING_SUFFIX}`);
+}
+function tableFor(target, options) {
+    return options.name === undefined
+        ? target.table.trim()
+        : pendingTableOf(target);
+}

package/dist/io/sql/writer.d.ts

@@ -0,0 +1,46 @@
+import { TableRow } from "../../types";
+import { Logger } from "../../logger";
+export type ColumnKind = "integer" | "number" | "text";
+export type SqlValue = string | number | null;
+export interface SqlSession {
+    run(sql: string, params?: SqlValue[]): Promise<void>;
+    all(sql: string, params?: SqlValue[]): Promise<Record<string, unknown>[]>;
+    begin(): Promise<void>;
+    commit(): Promise<void>;
+    rollback(): Promise<void>;
+}
+export interface SqlDialect {
+    label: string;
+    open: string;
+    close: string;
+    schemas: boolean;
+    maxNameLength: number;
+    nameLength(name: string): number;
+    maxParams: number;
+    maxRowsPerInsert: number;
+    transactionalDdl: boolean;
+    types: Record<ColumnKind, string>;
+    keyType: string;
+    tableSuffix: string;
+    param(index: number): string;
+    tableExists(session: SqlSession, table: string): Promise<boolean>;
+    tableMark(token: string): {
+        inside: string;
+        after: string;
+    };
+    tableId(session: SqlSession, table: string): Promise<string | null>;
+    ownsTable?(session: SqlSession, table: string): Promise<boolean>;
+}
+export declare const OWNED_TABLES = "datera_tabelas";
+export declare const NEW_SUFFIX = "__datera_novo";
+export declare const OLD_SUFFIX = "__datera_velho";
+export declare const MARK_PREFIX = "datera:";
+export declare function newToken(): string;
+export declare function quoteName(dialect: SqlDialect, name: string): string;
+export declare function tableParts(dialect: SqlDialect, table: string): string[];
+export declare function quoteTableName(dialect: SqlDialect, table: string): string;
+export declare function nameRoom(dialect: SqlDialect): number;
+export declare function checkTableName(dialect: SqlDialect, table: string, extra?: number): void;
+export declare function checkColumns(dialect: SqlDialect, header: string[]): void;
+export declare function columnKind(rows: TableRow[], column: string): ColumnKind;
+export declare function writeTable(session: SqlSession, dialect: SqlDialect, table: string, rows: TableRow[], logger: Logger): Promise<void>;