datera 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (159) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +132 -0
  3. package/dist/api/clean.d.ts +138 -0
  4. package/dist/api/clean.js +138 -0
  5. package/dist/api/formats.d.ts +19 -0
  6. package/dist/api/formats.js +106 -0
  7. package/dist/api/index.d.ts +8 -0
  8. package/dist/api/index.js +27 -0
  9. package/dist/api/inspect.d.ts +13 -0
  10. package/dist/api/inspect.js +44 -0
  11. package/dist/api/parse.d.ts +2 -0
  12. package/dist/api/parse.js +19 -0
  13. package/dist/api/rows.d.ts +23 -0
  14. package/dist/api/rows.js +63 -0
  15. package/dist/cli.d.ts +62 -0
  16. package/dist/cli.js +225 -0
  17. package/dist/commands.d.ts +3 -0
  18. package/dist/commands.js +169 -0
  19. package/dist/config/check.d.ts +11 -0
  20. package/dist/config/check.js +99 -0
  21. package/dist/config/index.d.ts +7 -0
  22. package/dist/config/index.js +16 -0
  23. package/dist/config/ioSchema.d.ts +97 -0
  24. package/dist/config/ioSchema.js +111 -0
  25. package/dist/config/load.d.ts +5 -0
  26. package/dist/config/load.js +68 -0
  27. package/dist/config/mergeSchema.d.ts +36 -0
  28. package/dist/config/mergeSchema.js +46 -0
  29. package/dist/config/messages.d.ts +7 -0
  30. package/dist/config/messages.js +33 -0
  31. package/dist/config/prepareSchema.d.ts +12 -0
  32. package/dist/config/prepareSchema.js +15 -0
  33. package/dist/config/schema.d.ts +195 -0
  34. package/dist/config/schema.js +44 -0
  35. package/dist/config/validationSchema.d.ts +27 -0
  36. package/dist/config/validationSchema.js +50 -0
  37. package/dist/env.d.ts +2 -0
  38. package/dist/env.js +15 -0
  39. package/dist/filters/combine.d.ts +10 -0
  40. package/dist/filters/combine.js +60 -0
  41. package/dist/filters/combineTypes.d.ts +6 -0
  42. package/dist/filters/combineTypes.js +2 -0
  43. package/dist/filters/dedupe.d.ts +12 -0
  44. package/dist/filters/dedupe.js +45 -0
  45. package/dist/filters/fillEmpty.d.ts +9 -0
  46. package/dist/filters/fillEmpty.js +32 -0
  47. package/dist/filters/fillEmptyTypes.d.ts +5 -0
  48. package/dist/filters/fillEmptyTypes.js +2 -0
  49. package/dist/filters/merge.d.ts +15 -0
  50. package/dist/filters/merge.js +113 -0
  51. package/dist/filters/mergeStrategies.d.ts +10 -0
  52. package/dist/filters/mergeStrategies.js +69 -0
  53. package/dist/filters/mergeTypes.d.ts +32 -0
  54. package/dist/filters/mergeTypes.js +2 -0
  55. package/dist/filters/mergeUnkeyed.d.ts +3 -0
  56. package/dist/filters/mergeUnkeyed.js +21 -0
  57. package/dist/filters/types.d.ts +3 -0
  58. package/dist/filters/types.js +2 -0
  59. package/dist/filters/validate.d.ts +16 -0
  60. package/dist/filters/validate.js +79 -0
  61. package/dist/filters/validateTypes.d.ts +28 -0
  62. package/dist/filters/validateTypes.js +2 -0
  63. package/dist/index.d.ts +13 -0
  64. package/dist/index.js +40 -0
  65. package/dist/io/csv/csvFormat.d.ts +4 -0
  66. package/dist/io/csv/csvFormat.js +94 -0
  67. package/dist/io/csv/csvSink.d.ts +14 -0
  68. package/dist/io/csv/csvSink.js +32 -0
  69. package/dist/io/csv/csvSource.d.ts +13 -0
  70. package/dist/io/csv/csvSource.js +29 -0
  71. package/dist/io/custom/loader.d.ts +1 -0
  72. package/dist/io/custom/loader.js +36 -0
  73. package/dist/io/custom/registry.d.ts +13 -0
  74. package/dist/io/custom/registry.js +50 -0
  75. package/dist/io/excel/excelCell.d.ts +4 -0
  76. package/dist/io/excel/excelCell.js +29 -0
  77. package/dist/io/excel/excelSink.d.ts +14 -0
  78. package/dist/io/excel/excelSink.js +57 -0
  79. package/dist/io/excel/excelSource.d.ts +11 -0
  80. package/dist/io/excel/excelSource.js +47 -0
  81. package/dist/io/excel/workbook.d.ts +4 -0
  82. package/dist/io/excel/workbook.js +20 -0
  83. package/dist/io/factory.d.ts +5 -0
  84. package/dist/io/factory.js +106 -0
  85. package/dist/io/files.d.ts +7 -0
  86. package/dist/io/files.js +48 -0
  87. package/dist/io/header.d.ts +2 -0
  88. package/dist/io/header.js +42 -0
  89. package/dist/io/json/jsonSink.d.ts +12 -0
  90. package/dist/io/json/jsonSink.js +19 -0
  91. package/dist/io/json/jsonSource.d.ts +11 -0
  92. package/dist/io/json/jsonSource.js +66 -0
  93. package/dist/io/mysql/client.d.ts +13 -0
  94. package/dist/io/mysql/client.js +73 -0
  95. package/dist/io/mysql/mysqlSource.d.ts +13 -0
  96. package/dist/io/mysql/mysqlSource.js +25 -0
  97. package/dist/io/optional.d.ts +2 -0
  98. package/dist/io/optional.js +36 -0
  99. package/dist/io/parquet/parquetLibrary.d.ts +11 -0
  100. package/dist/io/parquet/parquetLibrary.js +20 -0
  101. package/dist/io/parquet/parquetSink.d.ts +15 -0
  102. package/dist/io/parquet/parquetSink.js +58 -0
  103. package/dist/io/parquet/parquetSource.d.ts +13 -0
  104. package/dist/io/parquet/parquetSource.js +68 -0
  105. package/dist/io/postgres/postgresSource.d.ts +26 -0
  106. package/dist/io/postgres/postgresSource.js +42 -0
  107. package/dist/io/safety.d.ts +4 -0
  108. package/dist/io/safety.js +83 -0
  109. package/dist/io/sheets/client.d.ts +10 -0
  110. package/dist/io/sheets/client.js +141 -0
  111. package/dist/io/sheets/sheetsSink.d.ts +12 -0
  112. package/dist/io/sheets/sheetsSink.js +24 -0
  113. package/dist/io/sheets/sheetsSource.d.ts +10 -0
  114. package/dist/io/sheets/sheetsSource.js +47 -0
  115. package/dist/io/sql/cell.d.ts +5 -0
  116. package/dist/io/sql/cell.js +38 -0
  117. package/dist/io/sql/driver.d.ts +1 -0
  118. package/dist/io/sql/driver.js +15 -0
  119. package/dist/io/sql/names.d.ts +1 -0
  120. package/dist/io/sql/names.js +12 -0
  121. package/dist/io/sqlite/sqliteSource.d.ts +11 -0
  122. package/dist/io/sqlite/sqliteSource.js +52 -0
  123. package/dist/io/sqlserver/sqlServerSource.d.ts +24 -0
  124. package/dist/io/sqlserver/sqlServerSource.js +49 -0
  125. package/dist/io/types.d.ts +11 -0
  126. package/dist/io/types.js +2 -0
  127. package/dist/io/xml/xmlParse.d.ts +9 -0
  128. package/dist/io/xml/xmlParse.js +178 -0
  129. package/dist/io/xml/xmlSink.d.ts +18 -0
  130. package/dist/io/xml/xmlSink.js +79 -0
  131. package/dist/io/xml/xmlSource.d.ts +11 -0
  132. package/dist/io/xml/xmlSource.js +93 -0
  133. package/dist/logger.d.ts +11 -0
  134. package/dist/logger.js +29 -0
  135. package/dist/main.d.ts +2 -0
  136. package/dist/main.js +18 -0
  137. package/dist/normalizers/builtin.d.ts +5 -0
  138. package/dist/normalizers/builtin.js +23 -0
  139. package/dist/normalizers/index.d.ts +2 -0
  140. package/dist/normalizers/index.js +14 -0
  141. package/dist/normalizers/loader.d.ts +1 -0
  142. package/dist/normalizers/loader.js +30 -0
  143. package/dist/normalizers/registry.d.ts +9 -0
  144. package/dist/normalizers/registry.js +29 -0
  145. package/dist/pipeline/index.d.ts +7 -0
  146. package/dist/pipeline/index.js +17 -0
  147. package/dist/pipeline/modes.d.ts +6 -0
  148. package/dist/pipeline/modes.js +48 -0
  149. package/dist/pipeline/report.d.ts +3 -0
  150. package/dist/pipeline/report.js +32 -0
  151. package/dist/pipeline/runEtl.d.ts +24 -0
  152. package/dist/pipeline/runEtl.js +98 -0
  153. package/dist/pipeline/steps.d.ts +4 -0
  154. package/dist/pipeline/steps.js +36 -0
  155. package/dist/pipeline/types.d.ts +34 -0
  156. package/dist/pipeline/types.js +2 -0
  157. package/dist/types.d.ts +3 -0
  158. package/dist/types.js +2 -0
  159. package/package.json +128 -0
@@ -0,0 +1,5 @@
1
+ import { EtlConfig } from "../config";
2
+ import { Sink, Source } from "./types";
3
+ import { Logger } from "../logger";
4
+ export declare function createSource(config: EtlConfig, logger?: Logger): Source;
5
+ export declare function createSink(config: EtlConfig, logger?: Logger): Sink;
@@ -0,0 +1,106 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createSource = createSource;
4
+ exports.createSink = createSink;
5
+ const client_1 = require("./sheets/client");
6
+ const mysqlSource_1 = require("./mysql/mysqlSource");
7
+ const csvSource_1 = require("./csv/csvSource");
8
+ const jsonSource_1 = require("./json/jsonSource");
9
+ const sheetsSink_1 = require("./sheets/sheetsSink");
10
+ const csvSink_1 = require("./csv/csvSink");
11
+ const jsonSink_1 = require("./json/jsonSink");
12
+ const xmlSource_1 = require("./xml/xmlSource");
13
+ const postgresSource_1 = require("./postgres/postgresSource");
14
+ const sqlServerSource_1 = require("./sqlserver/sqlServerSource");
15
+ const sqliteSource_1 = require("./sqlite/sqliteSource");
16
+ const xmlSink_1 = require("./xml/xmlSink");
17
+ const parquetSource_1 = require("./parquet/parquetSource");
18
+ const parquetSink_1 = require("./parquet/parquetSink");
19
+ const excelSource_1 = require("./excel/excelSource");
20
+ const excelSink_1 = require("./excel/excelSink");
21
+ const sheetsSource_1 = require("./sheets/sheetsSource");
22
+ const registry_1 = require("./custom/registry");
23
+ const logger_1 = require("../logger");
24
+ const safety_1 = require("./safety");
25
+ const DEFAULT_MYSQL_PORT = 3306;
26
+ const DEFAULT_POSTGRES_PORT = 5432;
27
+ const DEFAULT_SQLSERVER_PORT = 1433;
28
+ function required(value, what) {
29
+ if (value === undefined || value === "") {
30
+ throw new Error(`Faltou configurar ${what}.`);
31
+ }
32
+ return value;
33
+ }
34
+ function createSource(config, logger = logger_1.consoleLogger) {
35
+ const source = config.source ?? { type: "mysql" };
36
+ switch (source.type) {
37
+ case "mysql":
38
+ return new mysqlSource_1.MysqlSource({
39
+ host: required(source.host ?? config.dbHost, "o host do MySQL"),
40
+ port: source.port ?? config.dbPort ?? DEFAULT_MYSQL_PORT,
41
+ user: required(source.user ?? config.dbUser, "o usuário do MySQL"),
42
+ password: source.password ?? config.dbPassword,
43
+ database: required(source.database ?? config.dbName, "o banco do MySQL"),
44
+ }, required(source.table ?? config.tableName, "a tabela do MySQL"), logger);
45
+ case "postgres":
46
+ return new postgresSource_1.PostgresSource({
47
+ host: required(source.host, "o host do PostgreSQL"),
48
+ port: source.port ?? DEFAULT_POSTGRES_PORT,
49
+ user: required(source.user, "o usuário do PostgreSQL"),
50
+ password: source.password,
51
+ database: required(source.database, "o banco do PostgreSQL"),
52
+ ssl: source.ssl,
53
+ }, required(source.table, "a tabela do PostgreSQL"));
54
+ case "sqlserver":
55
+ return new sqlServerSource_1.SqlServerSource({
56
+ host: required(source.host, "o servidor do SQL Server"),
57
+ port: source.port ?? DEFAULT_SQLSERVER_PORT,
58
+ user: required(source.user, "o usuário do SQL Server"),
59
+ password: source.password,
60
+ database: required(source.database, "o banco do SQL Server"),
61
+ encrypt: source.encrypt,
62
+ trustServerCertificate: source.trustServerCertificate,
63
+ }, required(source.table, "a tabela do SQL Server"));
64
+ case "sqlite":
65
+ return new sqliteSource_1.SqliteSource(source);
66
+ case "csv":
67
+ return new csvSource_1.CsvSource(source);
68
+ case "json":
69
+ return new jsonSource_1.JsonSource(source);
70
+ case "xml":
71
+ return new xmlSource_1.XmlSource(source);
72
+ case "parquet":
73
+ return new parquetSource_1.ParquetSource(source);
74
+ case "excel":
75
+ return new excelSource_1.ExcelSource(source);
76
+ case "sheets": {
77
+ const credentialsPath = required(source.credentialsPath ?? config.credentialsPath, "o caminho das credenciais do Google");
78
+ return new sheetsSource_1.SheetsSource((0, client_1.createSheetsClient)(credentialsPath), source.spreadsheetId, source.sheet);
79
+ }
80
+ case "custom":
81
+ return (0, registry_1.createCustomSource)(source.adapter, source.options ?? {}, logger);
82
+ }
83
+ }
84
+ function createSink(config, logger = logger_1.consoleLogger) {
85
+ (0, safety_1.assertSourceIsSafe)(config);
86
+ const destination = config.destination ?? { type: "sheets" };
87
+ switch (destination.type) {
88
+ case "sheets": {
89
+ const credentialsPath = required(destination.credentialsPath ?? config.credentialsPath, "o caminho das credenciais do Google");
90
+ const spreadsheetId = required(destination.spreadsheetId ?? config.spreadsheetId, "o id da planilha");
91
+ return new sheetsSink_1.SheetsSink((0, client_1.createSheetsClient)(credentialsPath), spreadsheetId, destination.sheet, logger);
92
+ }
93
+ case "csv":
94
+ return new csvSink_1.CsvSink(destination, logger);
95
+ case "json":
96
+ return new jsonSink_1.JsonSink(destination, logger);
97
+ case "xml":
98
+ return new xmlSink_1.XmlSink(destination, logger);
99
+ case "parquet":
100
+ return new parquetSink_1.ParquetSink(destination, logger);
101
+ case "excel":
102
+ return new excelSink_1.ExcelSink(destination, logger);
103
+ case "custom":
104
+ return (0, registry_1.createCustomSink)(destination.adapter, destination.options ?? {}, logger);
105
+ }
106
+ }
@@ -0,0 +1,7 @@
1
+ export type FileEncoding = "utf-8" | "latin1";
2
+ export declare function slugify(name: string): string;
3
+ export declare function extraOutputPath(mainPath: string, name: string | undefined): string;
4
+ export declare function readTextFile(filePath: string, encoding?: FileEncoding): string;
5
+ export declare function writeTextFile(filePath: string, content: string): void;
6
+ export declare function readBinaryFile(filePath: string): Buffer;
7
+ export declare function writeBinaryFile(filePath: string, content: Buffer): void;
@@ -0,0 +1,48 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.slugify = slugify;
7
+ exports.extraOutputPath = extraOutputPath;
8
+ exports.readTextFile = readTextFile;
9
+ exports.writeTextFile = writeTextFile;
10
+ exports.readBinaryFile = readBinaryFile;
11
+ exports.writeBinaryFile = writeBinaryFile;
12
+ const fs_1 = __importDefault(require("fs"));
13
+ const path_1 = __importDefault(require("path"));
14
+ function slugify(name) {
15
+ return name
16
+ .normalize("NFD")
17
+ .replace(/[\u0300-\u036f]/g, "")
18
+ .toLowerCase()
19
+ .replace(/[^a-z0-9]+/g, "-")
20
+ .replace(/^-+|-+$/g, "");
21
+ }
22
+ function extraOutputPath(mainPath, name) {
23
+ if (name === undefined)
24
+ return mainPath;
25
+ const ext = path_1.default.extname(mainPath);
26
+ const base = mainPath.slice(0, mainPath.length - ext.length);
27
+ return `${base}.${slugify(name) || "extra"}${ext}`;
28
+ }
29
+ function readTextFile(filePath, encoding = "utf-8") {
30
+ if (!fs_1.default.existsSync(filePath)) {
31
+ throw new Error(`Arquivo não encontrado: ${filePath}`);
32
+ }
33
+ return fs_1.default.readFileSync(filePath, encoding === "latin1" ? "latin1" : "utf8");
34
+ }
35
+ function writeTextFile(filePath, content) {
36
+ fs_1.default.mkdirSync(path_1.default.dirname(filePath), { recursive: true });
37
+ fs_1.default.writeFileSync(filePath, content, "utf8");
38
+ }
39
+ function readBinaryFile(filePath) {
40
+ if (!fs_1.default.existsSync(filePath)) {
41
+ throw new Error(`Arquivo não encontrado: ${filePath}`);
42
+ }
43
+ return fs_1.default.readFileSync(filePath);
44
+ }
45
+ function writeBinaryFile(filePath, content) {
46
+ fs_1.default.mkdirSync(path_1.default.dirname(filePath), { recursive: true });
47
+ fs_1.default.writeFileSync(filePath, content);
48
+ }
@@ -0,0 +1,2 @@
1
+ import { TableRow } from "../types";
2
+ export declare function buildHeader(data: TableRow[]): string[];
@@ -0,0 +1,42 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.buildHeader = buildHeader;
4
+ function buildHeader(data) {
5
+ const seen = new Set();
6
+ const orderedKeys = [];
7
+ for (const row of data) {
8
+ for (const key of Object.keys(row)) {
9
+ if (!seen.has(key)) {
10
+ seen.add(key);
11
+ orderedKeys.push(key);
12
+ }
13
+ }
14
+ }
15
+ const suffixPattern = /^(.+)_(\d+)$/;
16
+ const variantsByBase = new Map();
17
+ const baseKeys = [];
18
+ for (const key of orderedKeys) {
19
+ const match = key.match(suffixPattern);
20
+ if (match && seen.has(match[1])) {
21
+ const base = match[1];
22
+ const n = Number(match[2]);
23
+ if (!variantsByBase.has(base))
24
+ variantsByBase.set(base, []);
25
+ variantsByBase.get(base).push(n);
26
+ }
27
+ else {
28
+ baseKeys.push(key);
29
+ }
30
+ }
31
+ const header = [];
32
+ for (const base of baseKeys) {
33
+ header.push(base);
34
+ const variants = variantsByBase.get(base);
35
+ if (variants) {
36
+ variants.sort((a, b) => a - b);
37
+ for (const n of variants)
38
+ header.push(`${base}_${n}`);
39
+ }
40
+ }
41
+ return header;
42
+ }
@@ -0,0 +1,12 @@
1
+ import { TableRow } from "../../types";
2
+ import { Sink, SinkWriteOptions } from "../types";
3
+ import { Logger } from "../../logger";
4
+ export interface JsonSinkOptions {
5
+ path: string;
6
+ }
7
+ export declare class JsonSink implements Sink {
8
+ private readonly options;
9
+ private readonly logger;
10
+ constructor(options: JsonSinkOptions, logger?: Logger);
11
+ write(rows: TableRow[], options?: SinkWriteOptions): Promise<void>;
12
+ }
@@ -0,0 +1,19 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.JsonSink = void 0;
4
+ const files_1 = require("../files");
5
+ const logger_1 = require("../../logger");
6
+ class JsonSink {
7
+ options;
8
+ logger;
9
+ constructor(options, logger = logger_1.consoleLogger) {
10
+ this.options = options;
11
+ this.logger = logger;
12
+ }
13
+ async write(rows, options = {}) {
14
+ const filePath = (0, files_1.extraOutputPath)(this.options.path, options.name);
15
+ (0, files_1.writeTextFile)(filePath, JSON.stringify(rows, null, 2) + "\n");
16
+ this.logger.info(`${rows.length} linhas escritas em ${filePath}.`);
17
+ }
18
+ }
19
+ exports.JsonSink = JsonSink;
@@ -0,0 +1,11 @@
1
+ import { TableRow } from "../../types";
2
+ import { Source } from "../types";
3
+ export interface JsonSourceOptions {
4
+ path: string;
5
+ recordsPath?: string | undefined;
6
+ }
7
+ export declare class JsonSource implements Source {
8
+ private readonly options;
9
+ constructor(options: JsonSourceOptions);
10
+ read(): Promise<TableRow[]>;
11
+ }
@@ -0,0 +1,66 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.JsonSource = void 0;
4
+ const files_1 = require("../files");
5
+ function flatten(value, prefix, row) {
6
+ if (value === null || value === undefined) {
7
+ row[prefix] = null;
8
+ }
9
+ else if (Array.isArray(value)) {
10
+ row[prefix] = value.every((item) => typeof item !== "object" || item === null)
11
+ ? value.map((item) => String(item ?? "")).join(", ")
12
+ : JSON.stringify(value);
13
+ }
14
+ else if (typeof value === "object") {
15
+ for (const [key, inner] of Object.entries(value)) {
16
+ flatten(inner, prefix ? `${prefix}.${key}` : key, row);
17
+ }
18
+ }
19
+ else if (typeof value === "number" || typeof value === "string") {
20
+ row[prefix] = value;
21
+ }
22
+ else {
23
+ row[prefix] = String(value);
24
+ }
25
+ }
26
+ class JsonSource {
27
+ options;
28
+ constructor(options) {
29
+ this.options = options;
30
+ }
31
+ async read() {
32
+ let data;
33
+ try {
34
+ data = JSON.parse((0, files_1.readTextFile)(this.options.path));
35
+ }
36
+ catch (error) {
37
+ if (error instanceof SyntaxError) {
38
+ throw new Error(`JSON inválido em ${this.options.path}: ${error.message}`);
39
+ }
40
+ throw error;
41
+ }
42
+ for (const key of this.options.recordsPath?.split(".") ?? []) {
43
+ data =
44
+ typeof data === "object" && data !== null
45
+ ? data[key]
46
+ : undefined;
47
+ }
48
+ if (!Array.isArray(data)) {
49
+ throw new Error(`O JSON em ${this.options.path} precisa ser uma lista de objetos` +
50
+ (this.options.recordsPath
51
+ ? ` em "${this.options.recordsPath}".`
52
+ : "."));
53
+ }
54
+ return data.map((item, index) => {
55
+ if (typeof item !== "object" ||
56
+ item === null ||
57
+ Array.isArray(item)) {
58
+ throw new Error(`O item ${index} do JSON não é um objeto.`);
59
+ }
60
+ const row = {};
61
+ flatten(item, "", row);
62
+ return row;
63
+ });
64
+ }
65
+ }
66
+ exports.JsonSource = JsonSource;
@@ -0,0 +1,13 @@
1
+ import type { Pool } from "mysql2/promise";
2
+ import { TableRow } from "../../types";
3
+ import { Logger } from "../../logger";
4
+ export interface MysqlConnection {
5
+ host: string;
6
+ port: number;
7
+ user: string;
8
+ password?: string | undefined;
9
+ database: string;
10
+ }
11
+ export declare function createPool(connection: MysqlConnection): Pool;
12
+ export declare function readTableInBatches(pool: Pool, tableName: string, batchSize?: number, logger?: Logger): AsyncGenerator<TableRow[]>;
13
+ export declare function readTable(pool: Pool, tableName: string, logger?: Logger): Promise<TableRow[]>;
@@ -0,0 +1,73 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.createPool = createPool;
4
+ exports.readTableInBatches = readTableInBatches;
5
+ exports.readTable = readTable;
6
+ const optional_1 = require("../optional");
7
+ const logger_1 = require("../../logger");
8
+ function createPool(connection) {
9
+ const mysql = (0, optional_1.requireOptional)(() => require("mysql2/promise"), ["mysql2"], "MySQL");
10
+ return mysql.createPool({
11
+ host: connection.host,
12
+ port: connection.port,
13
+ user: connection.user,
14
+ ...(connection.password !== undefined
15
+ ? { password: connection.password }
16
+ : {}),
17
+ database: connection.database,
18
+ });
19
+ }
20
+ const RETRYABLE_ERROR_CODES = new Set([
21
+ "PROTOCOL_CONNECTION_LOST",
22
+ "ECONNREFUSED",
23
+ "ETIMEDOUT",
24
+ "ECONNRESET",
25
+ ]);
26
+ function isRetryableError(error) {
27
+ const code = error?.code;
28
+ return typeof code === "string" && RETRYABLE_ERROR_CODES.has(code);
29
+ }
30
+ async function withRetry(fn, logger, maxRetries = 3, baseDelayMs = 500) {
31
+ for (let attempt = 0; attempt <= maxRetries; attempt++) {
32
+ try {
33
+ return await fn();
34
+ }
35
+ catch (error) {
36
+ const isLastAttempt = attempt === maxRetries;
37
+ if (!isRetryableError(error) || isLastAttempt) {
38
+ throw error;
39
+ }
40
+ const delay = baseDelayMs * 2 ** attempt;
41
+ logger.warn(`Erro de conexão com o banco. Tentando de novo em ${delay}ms (tentativa ${attempt + 1}/${maxRetries})...`);
42
+ await new Promise((resolve) => setTimeout(resolve, delay));
43
+ }
44
+ }
45
+ throw new Error("withRetry: número de tentativas esgotado sem sucesso e sem erro capturado.");
46
+ }
47
+ const DEFAULT_BATCH_SIZE = 1000;
48
+ async function* readTableInBatches(pool, tableName, batchSize = DEFAULT_BATCH_SIZE, logger = logger_1.consoleLogger) {
49
+ let offset = 0;
50
+ while (true) {
51
+ const [rows] = await withRetry(() => pool.query(`SELECT * FROM ?? LIMIT ? OFFSET ?`, [
52
+ tableName,
53
+ batchSize,
54
+ offset,
55
+ ]), logger);
56
+ const batch = rows;
57
+ if (batch.length === 0) {
58
+ break;
59
+ }
60
+ yield batch;
61
+ if (batch.length < batchSize) {
62
+ break;
63
+ }
64
+ offset += batchSize;
65
+ }
66
+ }
67
+ async function readTable(pool, tableName, logger = logger_1.consoleLogger) {
68
+ const allRows = [];
69
+ for await (const batch of readTableInBatches(pool, tableName, DEFAULT_BATCH_SIZE, logger)) {
70
+ allRows.push(...batch);
71
+ }
72
+ return allRows;
73
+ }
@@ -0,0 +1,13 @@
1
+ import { MysqlConnection } from "./client";
2
+ import { TableRow } from "../../types";
3
+ import { Source } from "../types";
4
+ import { Logger } from "../../logger";
5
+ export declare class MysqlSource implements Source {
6
+ private readonly connection;
7
+ private readonly table;
8
+ private readonly logger;
9
+ private pool;
10
+ constructor(connection: MysqlConnection, table: string, logger?: Logger);
11
+ read(): Promise<TableRow[]>;
12
+ close(): Promise<void>;
13
+ }
@@ -0,0 +1,25 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.MysqlSource = void 0;
4
+ const client_1 = require("./client");
5
+ const logger_1 = require("../../logger");
6
+ class MysqlSource {
7
+ connection;
8
+ table;
9
+ logger;
10
+ pool;
11
+ constructor(connection, table, logger = logger_1.consoleLogger) {
12
+ this.connection = connection;
13
+ this.table = table;
14
+ this.logger = logger;
15
+ }
16
+ async read() {
17
+ this.pool ??= (0, client_1.createPool)(this.connection);
18
+ return (0, client_1.readTable)(this.pool, this.table, this.logger);
19
+ }
20
+ async close() {
21
+ await this.pool?.end();
22
+ this.pool = undefined;
23
+ }
24
+ }
25
+ exports.MysqlSource = MysqlSource;
@@ -0,0 +1,2 @@
1
+ export declare function requireOptional<T>(load: () => unknown, packages: string[], feature: string): T;
2
+ export declare function importOptional<T>(load: () => Promise<T>, packages: string[], feature: string): Promise<T>;
@@ -0,0 +1,36 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.requireOptional = requireOptional;
4
+ exports.importOptional = importOptional;
5
+ const NOT_FOUND = new Set(["MODULE_NOT_FOUND", "ERR_MODULE_NOT_FOUND"]);
6
+ function missing(packages, feature) {
7
+ const list = packages.join(" ");
8
+ return new Error(`Pra usar ${feature}, o Datera precisa do pacote ${packages.map((name) => `"${name}"`).join(" e ")}. Rode "npm install ${list}" no seu projeto (ou "npm install -g ${list}" se você usa o comando datera instalado com -g).`);
9
+ }
10
+ function isNotFound(error, packages) {
11
+ const code = error?.code;
12
+ const message = error instanceof Error ? error.message : "";
13
+ return (typeof code === "string" &&
14
+ NOT_FOUND.has(code) &&
15
+ packages.some((name) => message.includes(name)));
16
+ }
17
+ function requireOptional(load, packages, feature) {
18
+ try {
19
+ return load();
20
+ }
21
+ catch (error) {
22
+ if (isNotFound(error, packages))
23
+ throw missing(packages, feature);
24
+ throw error;
25
+ }
26
+ }
27
+ async function importOptional(load, packages, feature) {
28
+ try {
29
+ return await load();
30
+ }
31
+ catch (error) {
32
+ if (isNotFound(error, packages))
33
+ throw missing(packages, feature);
34
+ throw error;
35
+ }
36
+ }
@@ -0,0 +1,11 @@
1
+ export type ParquetColumnType = "STRING" | "INT32" | "DOUBLE";
2
+ export interface ParquetColumn {
3
+ name: string;
4
+ type: ParquetColumnType;
5
+ data: (string | number | null)[];
6
+ }
7
+ export interface ParquetLibrary {
8
+ read(filePath: string): Promise<Record<string, unknown>[]>;
9
+ write(filePath: string, columns: ParquetColumn[]): Promise<void>;
10
+ }
11
+ export declare const parquetLibrary: ParquetLibrary;
@@ -0,0 +1,20 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.parquetLibrary = void 0;
4
+ const files_1 = require("../files");
5
+ const optional_1 = require("../optional");
6
+ const READ_PACKAGES = ["hyparquet", "hyparquet-compressors"];
7
+ const WRITE_PACKAGES = ["hyparquet-writer"];
8
+ exports.parquetLibrary = {
9
+ async read(filePath) {
10
+ const { asyncBufferFromFile, parquetReadObjects } = await (0, optional_1.importOptional)(() => import("hyparquet"), READ_PACKAGES, "Parquet");
11
+ const { compressors } = await (0, optional_1.importOptional)(() => import("hyparquet-compressors"), READ_PACKAGES, "Parquet");
12
+ const file = await asyncBufferFromFile(filePath);
13
+ return (await parquetReadObjects({ file, compressors }));
14
+ },
15
+ async write(filePath, columns) {
16
+ const { parquetWriteBuffer } = await (0, optional_1.importOptional)(() => import("hyparquet-writer"), WRITE_PACKAGES, "Parquet");
17
+ const buffer = parquetWriteBuffer({ columnData: columns });
18
+ (0, files_1.writeBinaryFile)(filePath, Buffer.from(buffer));
19
+ },
20
+ };
@@ -0,0 +1,15 @@
1
+ import { TableRow } from "../../types";
2
+ import { Sink, SinkWriteOptions } from "../types";
3
+ import { Logger } from "../../logger";
4
+ import { ParquetColumn, ParquetLibrary } from "./parquetLibrary";
5
+ export interface ParquetSinkOptions {
6
+ path: string;
7
+ }
8
+ export declare function parquetColumns(rows: TableRow[]): ParquetColumn[];
9
+ export declare class ParquetSink implements Sink {
10
+ private readonly options;
11
+ private readonly logger;
12
+ private readonly library;
13
+ constructor(options: ParquetSinkOptions, logger?: Logger, library?: ParquetLibrary);
14
+ write(rows: TableRow[], options?: SinkWriteOptions): Promise<void>;
15
+ }
@@ -0,0 +1,58 @@
1
+ "use strict";
2
+ var __importDefault = (this && this.__importDefault) || function (mod) {
3
+ return (mod && mod.__esModule) ? mod : { "default": mod };
4
+ };
5
+ Object.defineProperty(exports, "__esModule", { value: true });
6
+ exports.ParquetSink = void 0;
7
+ exports.parquetColumns = parquetColumns;
8
+ const fs_1 = __importDefault(require("fs"));
9
+ const files_1 = require("../files");
10
+ const header_1 = require("../header");
11
+ const logger_1 = require("../../logger");
12
+ const parquetLibrary_1 = require("./parquetLibrary");
13
+ const INT32_MIN = -(2 ** 31);
14
+ const INT32_MAX = 2 ** 31 - 1;
15
+ function typeOf(values) {
16
+ const present = values.filter((value) => value !== null);
17
+ if (present.length === 0)
18
+ return "STRING";
19
+ if (!present.every((value) => typeof value === "number"))
20
+ return "STRING";
21
+ const integers = present.every((value) => Number.isInteger(value) && value >= INT32_MIN && value <= INT32_MAX);
22
+ return integers ? "INT32" : "DOUBLE";
23
+ }
24
+ function parquetColumns(rows) {
25
+ return (0, header_1.buildHeader)(rows).map((name) => {
26
+ const values = rows.map((row) => row[name] ?? null);
27
+ const type = typeOf(values);
28
+ return {
29
+ name,
30
+ type,
31
+ data: type === "STRING"
32
+ ? values.map((value) => value === null ? null : String(value))
33
+ : values,
34
+ };
35
+ });
36
+ }
37
+ class ParquetSink {
38
+ options;
39
+ logger;
40
+ library;
41
+ constructor(options, logger = logger_1.consoleLogger, library = parquetLibrary_1.parquetLibrary) {
42
+ this.options = options;
43
+ this.logger = logger;
44
+ this.library = library;
45
+ }
46
+ async write(rows, options = {}) {
47
+ const filePath = (0, files_1.extraOutputPath)(this.options.path, options.name);
48
+ const columns = parquetColumns(rows);
49
+ if (columns.length === 0) {
50
+ fs_1.default.rmSync(filePath, { force: true });
51
+ this.logger.info(`Nenhuma linha pra escrever em ${filePath}.`);
52
+ return;
53
+ }
54
+ await this.library.write(filePath, columns);
55
+ this.logger.info(`${rows.length} linhas escritas em ${filePath}.`);
56
+ }
57
+ }
58
+ exports.ParquetSink = ParquetSink;
@@ -0,0 +1,13 @@
1
+ import { TableRow } from "../../types";
2
+ import { Source } from "../types";
3
+ import { ParquetLibrary } from "./parquetLibrary";
4
+ export interface ParquetSourceOptions {
5
+ path: string;
6
+ }
7
+ export declare function parquetRow(record: Record<string, unknown>): TableRow;
8
+ export declare class ParquetSource implements Source {
9
+ private readonly options;
10
+ private readonly library;
11
+ constructor(options: ParquetSourceOptions, library?: ParquetLibrary);
12
+ read(): Promise<TableRow[]>;
13
+ }