datera 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +132 -0
- package/dist/api/clean.d.ts +138 -0
- package/dist/api/clean.js +138 -0
- package/dist/api/formats.d.ts +19 -0
- package/dist/api/formats.js +106 -0
- package/dist/api/index.d.ts +8 -0
- package/dist/api/index.js +27 -0
- package/dist/api/inspect.d.ts +13 -0
- package/dist/api/inspect.js +44 -0
- package/dist/api/parse.d.ts +2 -0
- package/dist/api/parse.js +19 -0
- package/dist/api/rows.d.ts +23 -0
- package/dist/api/rows.js +63 -0
- package/dist/cli.d.ts +62 -0
- package/dist/cli.js +225 -0
- package/dist/commands.d.ts +3 -0
- package/dist/commands.js +169 -0
- package/dist/config/check.d.ts +11 -0
- package/dist/config/check.js +99 -0
- package/dist/config/index.d.ts +7 -0
- package/dist/config/index.js +16 -0
- package/dist/config/ioSchema.d.ts +97 -0
- package/dist/config/ioSchema.js +111 -0
- package/dist/config/load.d.ts +5 -0
- package/dist/config/load.js +68 -0
- package/dist/config/mergeSchema.d.ts +36 -0
- package/dist/config/mergeSchema.js +46 -0
- package/dist/config/messages.d.ts +7 -0
- package/dist/config/messages.js +33 -0
- package/dist/config/prepareSchema.d.ts +12 -0
- package/dist/config/prepareSchema.js +15 -0
- package/dist/config/schema.d.ts +195 -0
- package/dist/config/schema.js +44 -0
- package/dist/config/validationSchema.d.ts +27 -0
- package/dist/config/validationSchema.js +50 -0
- package/dist/env.d.ts +2 -0
- package/dist/env.js +15 -0
- package/dist/filters/combine.d.ts +10 -0
- package/dist/filters/combine.js +60 -0
- package/dist/filters/combineTypes.d.ts +6 -0
- package/dist/filters/combineTypes.js +2 -0
- package/dist/filters/dedupe.d.ts +12 -0
- package/dist/filters/dedupe.js +45 -0
- package/dist/filters/fillEmpty.d.ts +9 -0
- package/dist/filters/fillEmpty.js +32 -0
- package/dist/filters/fillEmptyTypes.d.ts +5 -0
- package/dist/filters/fillEmptyTypes.js +2 -0
- package/dist/filters/merge.d.ts +15 -0
- package/dist/filters/merge.js +113 -0
- package/dist/filters/mergeStrategies.d.ts +10 -0
- package/dist/filters/mergeStrategies.js +69 -0
- package/dist/filters/mergeTypes.d.ts +32 -0
- package/dist/filters/mergeTypes.js +2 -0
- package/dist/filters/mergeUnkeyed.d.ts +3 -0
- package/dist/filters/mergeUnkeyed.js +21 -0
- package/dist/filters/types.d.ts +3 -0
- package/dist/filters/types.js +2 -0
- package/dist/filters/validate.d.ts +16 -0
- package/dist/filters/validate.js +79 -0
- package/dist/filters/validateTypes.d.ts +28 -0
- package/dist/filters/validateTypes.js +2 -0
- package/dist/index.d.ts +13 -0
- package/dist/index.js +40 -0
- package/dist/io/csv/csvFormat.d.ts +4 -0
- package/dist/io/csv/csvFormat.js +94 -0
- package/dist/io/csv/csvSink.d.ts +14 -0
- package/dist/io/csv/csvSink.js +32 -0
- package/dist/io/csv/csvSource.d.ts +13 -0
- package/dist/io/csv/csvSource.js +29 -0
- package/dist/io/custom/loader.d.ts +1 -0
- package/dist/io/custom/loader.js +36 -0
- package/dist/io/custom/registry.d.ts +13 -0
- package/dist/io/custom/registry.js +50 -0
- package/dist/io/excel/excelCell.d.ts +4 -0
- package/dist/io/excel/excelCell.js +29 -0
- package/dist/io/excel/excelSink.d.ts +14 -0
- package/dist/io/excel/excelSink.js +57 -0
- package/dist/io/excel/excelSource.d.ts +11 -0
- package/dist/io/excel/excelSource.js +47 -0
- package/dist/io/excel/workbook.d.ts +4 -0
- package/dist/io/excel/workbook.js +20 -0
- package/dist/io/factory.d.ts +5 -0
- package/dist/io/factory.js +106 -0
- package/dist/io/files.d.ts +7 -0
- package/dist/io/files.js +48 -0
- package/dist/io/header.d.ts +2 -0
- package/dist/io/header.js +42 -0
- package/dist/io/json/jsonSink.d.ts +12 -0
- package/dist/io/json/jsonSink.js +19 -0
- package/dist/io/json/jsonSource.d.ts +11 -0
- package/dist/io/json/jsonSource.js +66 -0
- package/dist/io/mysql/client.d.ts +13 -0
- package/dist/io/mysql/client.js +73 -0
- package/dist/io/mysql/mysqlSource.d.ts +13 -0
- package/dist/io/mysql/mysqlSource.js +25 -0
- package/dist/io/optional.d.ts +2 -0
- package/dist/io/optional.js +36 -0
- package/dist/io/parquet/parquetLibrary.d.ts +11 -0
- package/dist/io/parquet/parquetLibrary.js +20 -0
- package/dist/io/parquet/parquetSink.d.ts +15 -0
- package/dist/io/parquet/parquetSink.js +58 -0
- package/dist/io/parquet/parquetSource.d.ts +13 -0
- package/dist/io/parquet/parquetSource.js +68 -0
- package/dist/io/postgres/postgresSource.d.ts +26 -0
- package/dist/io/postgres/postgresSource.js +42 -0
- package/dist/io/safety.d.ts +4 -0
- package/dist/io/safety.js +83 -0
- package/dist/io/sheets/client.d.ts +10 -0
- package/dist/io/sheets/client.js +141 -0
- package/dist/io/sheets/sheetsSink.d.ts +12 -0
- package/dist/io/sheets/sheetsSink.js +24 -0
- package/dist/io/sheets/sheetsSource.d.ts +10 -0
- package/dist/io/sheets/sheetsSource.js +47 -0
- package/dist/io/sql/cell.d.ts +5 -0
- package/dist/io/sql/cell.js +38 -0
- package/dist/io/sql/driver.d.ts +1 -0
- package/dist/io/sql/driver.js +15 -0
- package/dist/io/sql/names.d.ts +1 -0
- package/dist/io/sql/names.js +12 -0
- package/dist/io/sqlite/sqliteSource.d.ts +11 -0
- package/dist/io/sqlite/sqliteSource.js +52 -0
- package/dist/io/sqlserver/sqlServerSource.d.ts +24 -0
- package/dist/io/sqlserver/sqlServerSource.js +49 -0
- package/dist/io/types.d.ts +11 -0
- package/dist/io/types.js +2 -0
- package/dist/io/xml/xmlParse.d.ts +9 -0
- package/dist/io/xml/xmlParse.js +178 -0
- package/dist/io/xml/xmlSink.d.ts +18 -0
- package/dist/io/xml/xmlSink.js +79 -0
- package/dist/io/xml/xmlSource.d.ts +11 -0
- package/dist/io/xml/xmlSource.js +93 -0
- package/dist/logger.d.ts +11 -0
- package/dist/logger.js +29 -0
- package/dist/main.d.ts +2 -0
- package/dist/main.js +18 -0
- package/dist/normalizers/builtin.d.ts +5 -0
- package/dist/normalizers/builtin.js +23 -0
- package/dist/normalizers/index.d.ts +2 -0
- package/dist/normalizers/index.js +14 -0
- package/dist/normalizers/loader.d.ts +1 -0
- package/dist/normalizers/loader.js +30 -0
- package/dist/normalizers/registry.d.ts +9 -0
- package/dist/normalizers/registry.js +29 -0
- package/dist/pipeline/index.d.ts +7 -0
- package/dist/pipeline/index.js +17 -0
- package/dist/pipeline/modes.d.ts +6 -0
- package/dist/pipeline/modes.js +48 -0
- package/dist/pipeline/report.d.ts +3 -0
- package/dist/pipeline/report.js +32 -0
- package/dist/pipeline/runEtl.d.ts +24 -0
- package/dist/pipeline/runEtl.js +98 -0
- package/dist/pipeline/steps.d.ts +4 -0
- package/dist/pipeline/steps.js +36 -0
- package/dist/pipeline/types.d.ts +34 -0
- package/dist/pipeline/types.js +2 -0
- package/dist/types.d.ts +3 -0
- package/dist/types.js +2 -0
- package/package.json +128 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parseWith = parseWith;
|
|
4
|
+
const messages_1 = require("../config/messages");
|
|
5
|
+
function where(path) {
|
|
6
|
+
return path
|
|
7
|
+
.filter((part) => typeof part === "string" || typeof part === "number")
|
|
8
|
+
.join(".");
|
|
9
|
+
}
|
|
10
|
+
function parseWith(schema, value, label) {
|
|
11
|
+
const result = schema.safeParse(value);
|
|
12
|
+
if (result.success)
|
|
13
|
+
return result.data;
|
|
14
|
+
const problems = result.error.issues.map((issue) => {
|
|
15
|
+
const place = where(issue.path);
|
|
16
|
+
return `${place ? `${place}: ` : ""}${(0, messages_1.translateIssue)(issue)}`;
|
|
17
|
+
});
|
|
18
|
+
throw new Error(`${label} com problema: ${problems.join(" ")}`);
|
|
19
|
+
}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { DestinationConfig, SourceConfig } from "../config/ioSchema";
|
|
2
|
+
import { Logger } from "../logger";
|
|
3
|
+
import { TableRow } from "../types";
|
|
4
|
+
import { ReadOptions, WriteOptions } from "./formats";
|
|
5
|
+
export type Input = string | SourceConfig;
|
|
6
|
+
export type Output = string | DestinationConfig;
|
|
7
|
+
export interface ReadRowsOptions extends ReadOptions {
|
|
8
|
+
logger?: Logger | undefined;
|
|
9
|
+
}
|
|
10
|
+
export interface WriteRowsOptions extends WriteOptions {
|
|
11
|
+
pending?: TableRow[] | undefined;
|
|
12
|
+
pendingName?: string | undefined;
|
|
13
|
+
logger?: Logger | undefined;
|
|
14
|
+
}
|
|
15
|
+
export declare const DEFAULT_PENDING_NAME = "Pend\u00EAncias";
|
|
16
|
+
export declare function readRows(input: Input, options?: ReadRowsOptions): Promise<TableRow[]>;
|
|
17
|
+
export declare function writeRows(rows: TableRow[], output: Output, options?: WriteRowsOptions): Promise<void>;
|
|
18
|
+
export interface ConvertOptions {
|
|
19
|
+
read?: ReadOptions | undefined;
|
|
20
|
+
write?: WriteOptions | undefined;
|
|
21
|
+
logger?: Logger | undefined;
|
|
22
|
+
}
|
|
23
|
+
export declare function convert(input: Input, output: Output, options?: ConvertOptions): Promise<number>;
|
package/dist/api/rows.js
ADDED
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.DEFAULT_PENDING_NAME = void 0;
|
|
4
|
+
exports.readRows = readRows;
|
|
5
|
+
exports.writeRows = writeRows;
|
|
6
|
+
exports.convert = convert;
|
|
7
|
+
const ioSchema_1 = require("../config/ioSchema");
|
|
8
|
+
const factory_1 = require("../io/factory");
|
|
9
|
+
const normalizers_1 = require("../normalizers");
|
|
10
|
+
const logger_1 = require("../logger");
|
|
11
|
+
const formats_1 = require("./formats");
|
|
12
|
+
const parse_1 = require("./parse");
|
|
13
|
+
exports.DEFAULT_PENDING_NAME = "Pendências";
|
|
14
|
+
function sourceOf(input, options) {
|
|
15
|
+
return typeof input === "string"
|
|
16
|
+
? (0, formats_1.sourceFromPath)(input, options)
|
|
17
|
+
: (0, parse_1.parseWith)(ioSchema_1.sourceSchema, input, "A fonte");
|
|
18
|
+
}
|
|
19
|
+
function destinationOf(output, options) {
|
|
20
|
+
return typeof output === "string"
|
|
21
|
+
? (0, formats_1.destinationFromPath)(output, options)
|
|
22
|
+
: (0, parse_1.parseWith)(ioSchema_1.destinationSchema, output, "O destino");
|
|
23
|
+
}
|
|
24
|
+
async function readRows(input, options = {}) {
|
|
25
|
+
(0, normalizers_1.registerBuiltinKeyNormalizers)();
|
|
26
|
+
const config = { mode: "raw", source: sourceOf(input, options) };
|
|
27
|
+
const source = (0, factory_1.createSource)(config, options.logger ?? logger_1.silentLogger);
|
|
28
|
+
try {
|
|
29
|
+
return await source.read();
|
|
30
|
+
}
|
|
31
|
+
finally {
|
|
32
|
+
await source.close?.();
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
async function writeRows(rows, output, options = {}) {
|
|
36
|
+
const config = {
|
|
37
|
+
mode: "raw",
|
|
38
|
+
destination: destinationOf(output, options),
|
|
39
|
+
};
|
|
40
|
+
const sink = (0, factory_1.createSink)(config, options.logger ?? logger_1.silentLogger);
|
|
41
|
+
await sink.write(rows);
|
|
42
|
+
if (options.pending !== undefined) {
|
|
43
|
+
await sink.write(options.pending, {
|
|
44
|
+
name: options.pendingName ?? exports.DEFAULT_PENDING_NAME,
|
|
45
|
+
});
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
async function convert(input, output, options = {}) {
|
|
49
|
+
const source = sourceOf(input, options.read ?? {});
|
|
50
|
+
const destination = destinationOf(output, options.write ?? {});
|
|
51
|
+
const config = { mode: "raw", source, destination };
|
|
52
|
+
const logger = options.logger ?? logger_1.silentLogger;
|
|
53
|
+
const reader = (0, factory_1.createSource)(config, logger);
|
|
54
|
+
const sink = (0, factory_1.createSink)(config, logger);
|
|
55
|
+
try {
|
|
56
|
+
const rows = await reader.read();
|
|
57
|
+
await sink.write(rows);
|
|
58
|
+
return rows.length;
|
|
59
|
+
}
|
|
60
|
+
finally {
|
|
61
|
+
await reader.close?.();
|
|
62
|
+
}
|
|
63
|
+
}
|
package/dist/cli.d.ts
ADDED
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { Command } from "commander";
|
|
2
|
+
import { ReadOptions, WriteOptions } from "./api/formats";
|
|
3
|
+
export type CliOptions = {
|
|
4
|
+
command: "run";
|
|
5
|
+
config: string;
|
|
6
|
+
mode?: string | undefined;
|
|
7
|
+
dryRun: boolean;
|
|
8
|
+
input?: string | undefined;
|
|
9
|
+
output?: string | undefined;
|
|
10
|
+
} | {
|
|
11
|
+
command: "validate";
|
|
12
|
+
config: string;
|
|
13
|
+
} | {
|
|
14
|
+
command: "init";
|
|
15
|
+
folder: string;
|
|
16
|
+
from?: string | undefined;
|
|
17
|
+
force: boolean;
|
|
18
|
+
} | {
|
|
19
|
+
command: "convert";
|
|
20
|
+
input: string;
|
|
21
|
+
output: string;
|
|
22
|
+
read: ReadOptions;
|
|
23
|
+
write: WriteOptions;
|
|
24
|
+
} | {
|
|
25
|
+
command: "dedupe";
|
|
26
|
+
input: string;
|
|
27
|
+
output: string;
|
|
28
|
+
by: string;
|
|
29
|
+
keep: "first" | "last";
|
|
30
|
+
normalizer?: string | undefined;
|
|
31
|
+
read: ReadOptions;
|
|
32
|
+
write: WriteOptions;
|
|
33
|
+
} | {
|
|
34
|
+
command: "merge";
|
|
35
|
+
input: string;
|
|
36
|
+
output: string;
|
|
37
|
+
key: string;
|
|
38
|
+
normalizer?: string | undefined;
|
|
39
|
+
separator?: string | undefined;
|
|
40
|
+
overwrite: string[];
|
|
41
|
+
extraColumn: string[];
|
|
42
|
+
read: ReadOptions;
|
|
43
|
+
write: WriteOptions;
|
|
44
|
+
} | {
|
|
45
|
+
command: "columns";
|
|
46
|
+
input: string;
|
|
47
|
+
read: ReadOptions;
|
|
48
|
+
} | {
|
|
49
|
+
command: "preview";
|
|
50
|
+
input: string;
|
|
51
|
+
rows: number;
|
|
52
|
+
read: ReadOptions;
|
|
53
|
+
} | {
|
|
54
|
+
command: "normalizers";
|
|
55
|
+
} | {
|
|
56
|
+
command: "normalize";
|
|
57
|
+
name: string;
|
|
58
|
+
value: string;
|
|
59
|
+
};
|
|
60
|
+
export declare function dateraVersion(): string;
|
|
61
|
+
export declare function buildProgram(onCommand: (options: CliOptions) => void): Command;
|
|
62
|
+
export declare function parseCli(argv?: string[]): CliOptions;
|
package/dist/cli.js
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.dateraVersion = dateraVersion;
|
|
7
|
+
exports.buildProgram = buildProgram;
|
|
8
|
+
exports.parseCli = parseCli;
|
|
9
|
+
const fs_1 = __importDefault(require("fs"));
|
|
10
|
+
const path_1 = __importDefault(require("path"));
|
|
11
|
+
const commander_1 = require("commander");
|
|
12
|
+
const DEFAULT_CONFIG = "./config.json";
|
|
13
|
+
const DEFAULT_PREVIEW_ROWS = 10;
|
|
14
|
+
function dateraVersion() {
|
|
15
|
+
const packageJson = JSON.parse(fs_1.default.readFileSync(path_1.default.join(__dirname, "..", "package.json"), "utf-8"));
|
|
16
|
+
return packageJson.version ?? "0.0.0";
|
|
17
|
+
}
|
|
18
|
+
function withReadFlags(command) {
|
|
19
|
+
return command
|
|
20
|
+
.option("--table <tabela>", "Tabela a ler, quando a entrada é SQLite.")
|
|
21
|
+
.option("--sheet <aba>", "Aba a ler, quando a entrada é Excel.")
|
|
22
|
+
.option("--records-path <caminho>", "Onde estão os registros no JSON ou XML (ex.: escola.alunos.aluno).")
|
|
23
|
+
.option("--delimiter <separador>", "Separador do CSV de entrada.")
|
|
24
|
+
.addOption(new commander_1.Option("--encoding <codificação>", "Codificação do CSV de entrada.").choices(["utf-8", "latin1"]));
|
|
25
|
+
}
|
|
26
|
+
function withWriteFlags(command) {
|
|
27
|
+
return command
|
|
28
|
+
.option("--output-sheet <aba>", "Nome da aba, quando a saída é Excel.")
|
|
29
|
+
.option("--output-delimiter <separador>", "Separador do CSV de saída.")
|
|
30
|
+
.option("--root <nome>", "Elemento de fora, quando a saída é XML.")
|
|
31
|
+
.option("--record <nome>", "Elemento de cada linha, quando a saída é XML.");
|
|
32
|
+
}
|
|
33
|
+
function readOptions(flags) {
|
|
34
|
+
return {
|
|
35
|
+
table: flags.table,
|
|
36
|
+
sheet: flags.sheet,
|
|
37
|
+
recordsPath: flags.recordsPath,
|
|
38
|
+
delimiter: flags.delimiter,
|
|
39
|
+
encoding: flags.encoding,
|
|
40
|
+
};
|
|
41
|
+
}
|
|
42
|
+
function writeOptions(flags) {
|
|
43
|
+
return {
|
|
44
|
+
sheet: flags.outputSheet,
|
|
45
|
+
delimiter: flags.outputDelimiter,
|
|
46
|
+
root: flags.root,
|
|
47
|
+
record: flags.record,
|
|
48
|
+
};
|
|
49
|
+
}
|
|
50
|
+
function list(value, previous = []) {
|
|
51
|
+
return [
|
|
52
|
+
...previous,
|
|
53
|
+
...value
|
|
54
|
+
.split(",")
|
|
55
|
+
.map((item) => item.trim())
|
|
56
|
+
.filter(Boolean),
|
|
57
|
+
];
|
|
58
|
+
}
|
|
59
|
+
function positiveNumber(value) {
|
|
60
|
+
const parsed = Number(value);
|
|
61
|
+
if (!Number.isInteger(parsed) || parsed <= 0) {
|
|
62
|
+
throw new commander_1.InvalidArgumentError("precisa ser um número inteiro maior que zero.");
|
|
63
|
+
}
|
|
64
|
+
return parsed;
|
|
65
|
+
}
|
|
66
|
+
const TITLES = {
|
|
67
|
+
"Usage:": "Uso:",
|
|
68
|
+
"Options:": "Opções:",
|
|
69
|
+
"Commands:": "Comandos:",
|
|
70
|
+
"Arguments:": "Argumentos:",
|
|
71
|
+
};
|
|
72
|
+
const ERRORS = [
|
|
73
|
+
[/^error: /, "erro: "],
|
|
74
|
+
[/missing required argument/, "faltou o argumento"],
|
|
75
|
+
[/required option (.+) not specified/, "faltou a opção $1"],
|
|
76
|
+
[/unknown command/, "comando desconhecido"],
|
|
77
|
+
[/unknown option/, "opção desconhecida"],
|
|
78
|
+
[/too many arguments/, "argumentos demais"],
|
|
79
|
+
[/Did you mean (.+)\?/, "Você quis dizer $1?"],
|
|
80
|
+
[/Allowed choices are/, "As opções aceitas são"],
|
|
81
|
+
[
|
|
82
|
+
/option '(.+)' argument '(.+)' is invalid\./,
|
|
83
|
+
"o valor '$2' da opção '$1' não vale:",
|
|
84
|
+
],
|
|
85
|
+
[/is invalid\./, "não vale."],
|
|
86
|
+
];
|
|
87
|
+
function translateError(text) {
|
|
88
|
+
return ERRORS.reduce((current, [pattern, replacement]) => current.replace(pattern, replacement), text);
|
|
89
|
+
}
|
|
90
|
+
function buildProgram(onCommand) {
|
|
91
|
+
const program = new commander_1.Command();
|
|
92
|
+
program
|
|
93
|
+
.configureHelp({ styleTitle: (title) => TITLES[title] ?? title })
|
|
94
|
+
.configureOutput({
|
|
95
|
+
outputError: (text, write) => write(translateError(text)),
|
|
96
|
+
})
|
|
97
|
+
.name("datera")
|
|
98
|
+
.description("Limpa, junta, converte e organiza dados bagunçados de banco, planilha ou arquivo.")
|
|
99
|
+
.version(dateraVersion(), "-v, --version", "Mostra a versão.")
|
|
100
|
+
.helpOption("-h, --help", "Mostra essa ajuda.")
|
|
101
|
+
.helpCommand("help [comando]", "Mostra a ajuda de um comando.")
|
|
102
|
+
.showSuggestionAfterError();
|
|
103
|
+
program
|
|
104
|
+
.command("run", { isDefault: true })
|
|
105
|
+
.description("Roda uma config inteira: lê, aplica as regras e grava.")
|
|
106
|
+
.argument("[config]", "Caminho da config.")
|
|
107
|
+
.option("--config <caminho>", "Caminho da config (jeito antigo).")
|
|
108
|
+
.option("--mode <modo>", "Troca o modo da config (raw, dedupe ou merge).")
|
|
109
|
+
.option("--dry-run", "Roda tudo e mostra o resultado, sem gravar nada.")
|
|
110
|
+
.option("--input <arquivo>", "Lê desse arquivo em vez da fonte da config.")
|
|
111
|
+
.option("--output <arquivo>", "Grava nesse arquivo em vez do destino da config.")
|
|
112
|
+
.action((config, flags) => onCommand({
|
|
113
|
+
command: "run",
|
|
114
|
+
config: config ?? flags.config ?? DEFAULT_CONFIG,
|
|
115
|
+
mode: flags.mode,
|
|
116
|
+
dryRun: flags.dryRun ?? false,
|
|
117
|
+
input: flags.input,
|
|
118
|
+
output: flags.output,
|
|
119
|
+
}));
|
|
120
|
+
program
|
|
121
|
+
.command("validate")
|
|
122
|
+
.description("Confere a config e os arquivos que ela cita, sem ler dados.")
|
|
123
|
+
.argument("[config]", "Caminho da config.")
|
|
124
|
+
.action((config) => onCommand({
|
|
125
|
+
command: "validate",
|
|
126
|
+
config: config ?? DEFAULT_CONFIG,
|
|
127
|
+
}));
|
|
128
|
+
program
|
|
129
|
+
.command("init")
|
|
130
|
+
.description("Cria uma config.json pra começar.")
|
|
131
|
+
.argument("[pasta]", "Pasta onde criar a config.", ".")
|
|
132
|
+
.option("--from <arquivo>", "Já monta a config lendo desse arquivo.")
|
|
133
|
+
.option("--force", "Troca a config.json se ela já existir.")
|
|
134
|
+
.action((folder, flags) => onCommand({
|
|
135
|
+
command: "init",
|
|
136
|
+
folder,
|
|
137
|
+
from: flags.from,
|
|
138
|
+
force: flags.force ?? false,
|
|
139
|
+
}));
|
|
140
|
+
withWriteFlags(withReadFlags(program
|
|
141
|
+
.command("convert")
|
|
142
|
+
.description("Converte de um formato pra outro, sem mexer nos dados.")
|
|
143
|
+
.argument("<entrada>", "Arquivo de entrada.")
|
|
144
|
+
.argument("<saida>", "Arquivo de saída. O formato vem da extensão."))).action((input, output, flags) => onCommand({
|
|
145
|
+
command: "convert",
|
|
146
|
+
input,
|
|
147
|
+
output,
|
|
148
|
+
read: readOptions(flags),
|
|
149
|
+
write: writeOptions(flags),
|
|
150
|
+
}));
|
|
151
|
+
withWriteFlags(withReadFlags(program
|
|
152
|
+
.command("dedupe")
|
|
153
|
+
.description("Tira as linhas repetidas olhando uma coluna.")
|
|
154
|
+
.argument("<entrada>", "Arquivo de entrada.")
|
|
155
|
+
.argument("<saida>", "Arquivo de saída.")
|
|
156
|
+
.requiredOption("--by <coluna>", "Coluna que diz se a linha é repetida.")
|
|
157
|
+
.addOption(new commander_1.Option("--keep <qual>", "Qual linha manter.")
|
|
158
|
+
.choices(["first", "last"])
|
|
159
|
+
.default("first"))
|
|
160
|
+
.option("--normalizer <nome>", "Normalizador aplicado antes de comparar (ex.: lowercase)."))).action((input, output, flags) => onCommand({
|
|
161
|
+
command: "dedupe",
|
|
162
|
+
input,
|
|
163
|
+
output,
|
|
164
|
+
by: flags.by,
|
|
165
|
+
keep: flags.keep,
|
|
166
|
+
normalizer: flags.normalizer,
|
|
167
|
+
read: readOptions(flags),
|
|
168
|
+
write: writeOptions(flags),
|
|
169
|
+
}));
|
|
170
|
+
withWriteFlags(withReadFlags(program
|
|
171
|
+
.command("merge")
|
|
172
|
+
.description("Junta as linhas com a mesma chave sem perder informação.")
|
|
173
|
+
.argument("<entrada>", "Arquivo de entrada.")
|
|
174
|
+
.argument("<saida>", "Arquivo de saída.")
|
|
175
|
+
.requiredOption("--key <coluna>", "Coluna que identifica cada registro.")
|
|
176
|
+
.option("--normalizer <nome>", "Normalizador da chave (ex.: digitsOnly).")
|
|
177
|
+
.option("--separator <texto>", 'Separador quando junta valores diferentes (padrão " | ").')
|
|
178
|
+
.option("--overwrite <colunas>", "Colunas em que vale o último valor, separadas por vírgula.", list)
|
|
179
|
+
.option("--extra-column <colunas>", "Colunas em que cada valor diferente vira uma coluna nova.", list))).action((input, output, flags) => onCommand({
|
|
180
|
+
command: "merge",
|
|
181
|
+
input,
|
|
182
|
+
output,
|
|
183
|
+
key: flags.key,
|
|
184
|
+
normalizer: flags.normalizer,
|
|
185
|
+
separator: flags.separator,
|
|
186
|
+
overwrite: flags.overwrite ?? [],
|
|
187
|
+
extraColumn: flags.extraColumn ?? [],
|
|
188
|
+
read: readOptions(flags),
|
|
189
|
+
write: writeOptions(flags),
|
|
190
|
+
}));
|
|
191
|
+
withReadFlags(program
|
|
192
|
+
.command("columns")
|
|
193
|
+
.description("Mostra as colunas com quantas estão preenchidas, vazias e diferentes.")
|
|
194
|
+
.argument("<entrada>", "Arquivo de entrada.")).action((input, flags) => onCommand({ command: "columns", input, read: readOptions(flags) }));
|
|
195
|
+
withReadFlags(program
|
|
196
|
+
.command("preview")
|
|
197
|
+
.description("Mostra as primeiras linhas de um arquivo.")
|
|
198
|
+
.argument("<entrada>", "Arquivo de entrada.")
|
|
199
|
+
.option("-n, --rows <quantidade>", "Quantas linhas mostrar.", positiveNumber, DEFAULT_PREVIEW_ROWS)).action((input, flags) => onCommand({
|
|
200
|
+
command: "preview",
|
|
201
|
+
input,
|
|
202
|
+
rows: flags.rows,
|
|
203
|
+
read: readOptions(flags),
|
|
204
|
+
}));
|
|
205
|
+
program
|
|
206
|
+
.command("normalizers")
|
|
207
|
+
.description("Lista os normalizadores prontos.")
|
|
208
|
+
.action(() => onCommand({ command: "normalizers" }));
|
|
209
|
+
program
|
|
210
|
+
.command("normalize")
|
|
211
|
+
.description("Mostra como um normalizador transforma um valor.")
|
|
212
|
+
.argument("<nome>", "Nome do normalizador.")
|
|
213
|
+
.argument("<valor>", "Valor pra testar.")
|
|
214
|
+
.action((name, value) => onCommand({ command: "normalize", name, value }));
|
|
215
|
+
return program;
|
|
216
|
+
}
|
|
217
|
+
function parseCli(argv = process.argv) {
|
|
218
|
+
let result;
|
|
219
|
+
buildProgram((options) => {
|
|
220
|
+
result = options;
|
|
221
|
+
}).parse(argv);
|
|
222
|
+
if (!result)
|
|
223
|
+
throw new Error("Comando não reconhecido.");
|
|
224
|
+
return result;
|
|
225
|
+
}
|
package/dist/commands.js
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.runCommand = runCommand;
|
|
7
|
+
const fs_1 = __importDefault(require("fs"));
|
|
8
|
+
const path_1 = __importDefault(require("path"));
|
|
9
|
+
const dotenv_1 = __importDefault(require("dotenv"));
|
|
10
|
+
const config_1 = require("./config");
|
|
11
|
+
const api_1 = require("./api");
|
|
12
|
+
const safety_1 = require("./io/safety");
|
|
13
|
+
const pipeline_1 = require("./pipeline");
|
|
14
|
+
function plural(count, one, many) {
|
|
15
|
+
return `${count} ${count === 1 ? one : many}`;
|
|
16
|
+
}
|
|
17
|
+
async function runConfig(cli, logger) {
|
|
18
|
+
const configPath = path_1.default.resolve(cli.config);
|
|
19
|
+
if (!fs_1.default.existsSync(configPath)) {
|
|
20
|
+
throw new Error(`Não achei a config ${cli.config}. Se era pra ser um comando, veja a lista com "datera --help".`);
|
|
21
|
+
}
|
|
22
|
+
const folder = path_1.default.dirname(configPath);
|
|
23
|
+
const input = cli.input === undefined ? undefined : path_1.default.resolve(cli.input);
|
|
24
|
+
const output = cli.output === undefined ? undefined : path_1.default.resolve(cli.output);
|
|
25
|
+
dotenv_1.default.config({ path: path_1.default.join(folder, ".env"), quiet: true });
|
|
26
|
+
const raw = JSON.parse(fs_1.default.readFileSync(configPath, "utf-8"));
|
|
27
|
+
if (input)
|
|
28
|
+
raw.source = (0, api_1.sourceFromPath)(input);
|
|
29
|
+
if (output)
|
|
30
|
+
raw.destination = (0, api_1.destinationFromPath)(output);
|
|
31
|
+
process.chdir(folder);
|
|
32
|
+
const config = (0, config_1.parseConfig)(raw);
|
|
33
|
+
const report = await (0, pipeline_1.runEtl)(config, {
|
|
34
|
+
mode: cli.mode === undefined ? undefined : (0, pipeline_1.parseMode)(cli.mode),
|
|
35
|
+
dryRun: cli.dryRun,
|
|
36
|
+
logger,
|
|
37
|
+
});
|
|
38
|
+
if (report.dryRun && report.preview.length > 0) {
|
|
39
|
+
logger.info(`Primeiras ${report.preview.length} linhas do resultado:`);
|
|
40
|
+
console.table(report.preview);
|
|
41
|
+
}
|
|
42
|
+
logger.info("");
|
|
43
|
+
for (const line of (0, pipeline_1.formatReport)(report))
|
|
44
|
+
logger.info(line);
|
|
45
|
+
logger.info(report.written ? "ETL concluído com sucesso!" : "Teste concluído.");
|
|
46
|
+
return 0;
|
|
47
|
+
}
|
|
48
|
+
function relativeTo(folder, file) {
|
|
49
|
+
const relative = path_1.default.relative(folder, path_1.default.resolve(file));
|
|
50
|
+
if (path_1.default.isAbsolute(relative))
|
|
51
|
+
return relative;
|
|
52
|
+
const posix = relative.split(path_1.default.sep).join("/");
|
|
53
|
+
return posix.startsWith("..") ? posix : `./${posix}`;
|
|
54
|
+
}
|
|
55
|
+
function starterConfig(from) {
|
|
56
|
+
if (from === undefined) {
|
|
57
|
+
return {
|
|
58
|
+
source: { type: "csv", path: "./dados.csv" },
|
|
59
|
+
destination: {
|
|
60
|
+
type: "csv",
|
|
61
|
+
path: "./saida/resultado.csv",
|
|
62
|
+
delimiter: ";",
|
|
63
|
+
},
|
|
64
|
+
mode: "raw",
|
|
65
|
+
};
|
|
66
|
+
}
|
|
67
|
+
const format = (0, api_1.detectFormat)(from);
|
|
68
|
+
const extension = format === "sqlite" ? ".csv" : path_1.default.extname(from);
|
|
69
|
+
return {
|
|
70
|
+
source: (0, api_1.sourceFromPath)(from, { table: "NOME_DA_TABELA" }),
|
|
71
|
+
destination: (0, api_1.destinationFromPath)(`./saida/resultado${extension}`),
|
|
72
|
+
mode: "raw",
|
|
73
|
+
};
|
|
74
|
+
}
|
|
75
|
+
function init(cli, logger) {
|
|
76
|
+
const target = path_1.default.resolve(cli.folder, "config.json");
|
|
77
|
+
if (fs_1.default.existsSync(target) && !cli.force) {
|
|
78
|
+
logger.error(`Já existe uma config em ${target}. Use --force se quiser trocar.`);
|
|
79
|
+
return 1;
|
|
80
|
+
}
|
|
81
|
+
const from = cli.from === undefined
|
|
82
|
+
? undefined
|
|
83
|
+
: relativeTo(path_1.default.dirname(target), cli.from);
|
|
84
|
+
fs_1.default.mkdirSync(path_1.default.dirname(target), { recursive: true });
|
|
85
|
+
fs_1.default.writeFileSync(target, JSON.stringify(starterConfig(from), null, 4) + "\n");
|
|
86
|
+
logger.info(`Config criada em ${target}.`);
|
|
87
|
+
logger.info('Confira com "datera validate", teste com "datera run --dry-run" e depois rode "datera run".');
|
|
88
|
+
return 0;
|
|
89
|
+
}
|
|
90
|
+
async function runCommand(cli, logger) {
|
|
91
|
+
switch (cli.command) {
|
|
92
|
+
case "run":
|
|
93
|
+
return runConfig(cli, logger);
|
|
94
|
+
case "validate": {
|
|
95
|
+
const check = (0, config_1.checkConfigFile)(cli.config);
|
|
96
|
+
for (const line of (0, config_1.formatCheck)(cli.config, check)) {
|
|
97
|
+
if (check.ok)
|
|
98
|
+
logger.info(line);
|
|
99
|
+
else
|
|
100
|
+
logger.error(line);
|
|
101
|
+
}
|
|
102
|
+
return check.ok ? 0 : 1;
|
|
103
|
+
}
|
|
104
|
+
case "init":
|
|
105
|
+
return init(cli, logger);
|
|
106
|
+
case "convert": {
|
|
107
|
+
const count = await (0, api_1.convert)(cli.input, cli.output, {
|
|
108
|
+
read: cli.read,
|
|
109
|
+
write: cli.write,
|
|
110
|
+
});
|
|
111
|
+
logger.info(`${plural(count, "linha convertida", "linhas convertidas")} de ${cli.input} pra ${cli.output}.`);
|
|
112
|
+
return 0;
|
|
113
|
+
}
|
|
114
|
+
case "dedupe": {
|
|
115
|
+
(0, safety_1.assertDifferentFiles)(cli.input, cli.output);
|
|
116
|
+
const rows = await (0, api_1.readRows)(cli.input, cli.read);
|
|
117
|
+
const result = (0, api_1.dedupe)(rows, cli.by, {
|
|
118
|
+
keep: cli.keep,
|
|
119
|
+
normalizer: cli.normalizer,
|
|
120
|
+
});
|
|
121
|
+
await (0, api_1.writeRows)(result, cli.output, cli.write);
|
|
122
|
+
logger.info(`${plural(rows.length, "linha lida", "linhas lidas")}, ${plural(rows.length - result.length, "repetida tirada", "repetidas tiradas")}, ${plural(result.length, "linha gravada", "linhas gravadas")} em ${cli.output}.`);
|
|
123
|
+
return 0;
|
|
124
|
+
}
|
|
125
|
+
case "merge": {
|
|
126
|
+
(0, safety_1.assertDifferentFiles)(cli.input, cli.output);
|
|
127
|
+
const rows = await (0, api_1.readRows)(cli.input, cli.read);
|
|
128
|
+
const result = (0, api_1.merge)(rows, cli.key, {
|
|
129
|
+
normalizer: cli.normalizer,
|
|
130
|
+
separator: cli.separator,
|
|
131
|
+
overwrite: cli.overwrite,
|
|
132
|
+
extraColumn: cli.extraColumn,
|
|
133
|
+
});
|
|
134
|
+
await (0, api_1.writeRows)(result, cli.output, cli.write);
|
|
135
|
+
logger.info(`${plural(rows.length, "linha lida", "linhas lidas")}, juntadas em ${plural(result.length, "registro", "registros")} gravados em ${cli.output}.`);
|
|
136
|
+
return 0;
|
|
137
|
+
}
|
|
138
|
+
case "columns": {
|
|
139
|
+
const rows = await (0, api_1.readRows)(cli.input, cli.read);
|
|
140
|
+
logger.info(`${cli.input}: ${plural(rows.length, "linha", "linhas")}.`);
|
|
141
|
+
console.table((0, api_1.describeColumns)(rows).map((column) => ({
|
|
142
|
+
coluna: column.name,
|
|
143
|
+
preenchidas: column.filled,
|
|
144
|
+
vazias: column.empty,
|
|
145
|
+
diferentes: column.distinct,
|
|
146
|
+
tipo: column.kind,
|
|
147
|
+
exemplos: column.examples.join(" · "),
|
|
148
|
+
})));
|
|
149
|
+
return 0;
|
|
150
|
+
}
|
|
151
|
+
case "preview": {
|
|
152
|
+
const rows = await (0, api_1.readRows)(cli.input, cli.read);
|
|
153
|
+
logger.info(`${cli.input}: ${plural(rows.length, "linha", "linhas")}. Mostrando ${Math.min(cli.rows, rows.length)}.`);
|
|
154
|
+
console.table(rows.slice(0, cli.rows));
|
|
155
|
+
return 0;
|
|
156
|
+
}
|
|
157
|
+
case "normalizers":
|
|
158
|
+
for (const name of (0, api_1.listNormalizers)())
|
|
159
|
+
logger.info(name);
|
|
160
|
+
return 0;
|
|
161
|
+
case "normalize": {
|
|
162
|
+
const key = (0, api_1.normalize)(cli.name, cli.value);
|
|
163
|
+
logger.info(key === null
|
|
164
|
+
? `"${cli.value}" não vira chave nenhuma com ${cli.name} (a linha seria tratada como sem chave).`
|
|
165
|
+
: `"${cli.value}" → "${key}"`);
|
|
166
|
+
return 0;
|
|
167
|
+
}
|
|
168
|
+
}
|
|
169
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export interface ConfigProblem {
|
|
2
|
+
where: string;
|
|
3
|
+
message: string;
|
|
4
|
+
}
|
|
5
|
+
export interface ConfigCheck {
|
|
6
|
+
ok: boolean;
|
|
7
|
+
problems: ConfigProblem[];
|
|
8
|
+
}
|
|
9
|
+
export declare function checkConfigText(text: string, folder: string): ConfigCheck;
|
|
10
|
+
export declare function checkConfigFile(configPath: string): ConfigCheck;
|
|
11
|
+
export declare function formatCheck(configPath: string, check: ConfigCheck): string[];
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.checkConfigText = checkConfigText;
|
|
7
|
+
exports.checkConfigFile = checkConfigFile;
|
|
8
|
+
exports.formatCheck = formatCheck;
|
|
9
|
+
const fs_1 = __importDefault(require("fs"));
|
|
10
|
+
const path_1 = __importDefault(require("path"));
|
|
11
|
+
const messages_1 = require("./messages");
|
|
12
|
+
const schema_1 = require("./schema");
|
|
13
|
+
function pathText(parts) {
|
|
14
|
+
return parts
|
|
15
|
+
.filter((part) => typeof part === "string" || typeof part === "number")
|
|
16
|
+
.map((part, index) => typeof part === "number"
|
|
17
|
+
? `[${part}]`
|
|
18
|
+
: index === 0
|
|
19
|
+
? part
|
|
20
|
+
: `.${part}`)
|
|
21
|
+
.join("");
|
|
22
|
+
}
|
|
23
|
+
function filePathOf(side) {
|
|
24
|
+
return typeof side === "object" &&
|
|
25
|
+
side !== null &&
|
|
26
|
+
"path" in side &&
|
|
27
|
+
typeof side.path === "string"
|
|
28
|
+
? side.path
|
|
29
|
+
: undefined;
|
|
30
|
+
}
|
|
31
|
+
function missingFiles(config, folder) {
|
|
32
|
+
const files = [];
|
|
33
|
+
const sourcePath = filePathOf(config.source);
|
|
34
|
+
if (sourcePath)
|
|
35
|
+
files.push({ where: "source.path", file: sourcePath });
|
|
36
|
+
const credentials = (config.source?.type === "sheets"
|
|
37
|
+
? config.source.credentialsPath
|
|
38
|
+
: undefined) ??
|
|
39
|
+
(config.destination?.type === "sheets"
|
|
40
|
+
? config.destination.credentialsPath
|
|
41
|
+
: undefined) ??
|
|
42
|
+
config.credentialsPath;
|
|
43
|
+
if (credentials)
|
|
44
|
+
files.push({ where: "credentialsPath", file: credentials });
|
|
45
|
+
(config.normalizerModules ?? []).forEach((file, index) => files.push({ where: `normalizerModules[${index}]`, file }));
|
|
46
|
+
(config.adapterModules ?? []).forEach((file, index) => files.push({ where: `adapterModules[${index}]`, file }));
|
|
47
|
+
return files
|
|
48
|
+
.filter(({ file }) => !fs_1.default.existsSync(path_1.default.resolve(folder, file)))
|
|
49
|
+
.map(({ where, file }) => ({
|
|
50
|
+
where,
|
|
51
|
+
message: `Arquivo não encontrado: ${file}`,
|
|
52
|
+
}));
|
|
53
|
+
}
|
|
54
|
+
function checkConfigText(text, folder) {
|
|
55
|
+
let value;
|
|
56
|
+
try {
|
|
57
|
+
value = JSON.parse(text);
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
const reason = error instanceof Error ? error.message : String(error);
|
|
61
|
+
return {
|
|
62
|
+
ok: false,
|
|
63
|
+
problems: [{ where: "", message: `JSON inválido: ${reason}` }],
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
const result = schema_1.etlConfigSchema.safeParse(value);
|
|
67
|
+
if (!result.success) {
|
|
68
|
+
return {
|
|
69
|
+
ok: false,
|
|
70
|
+
problems: result.error.issues.map((issue) => ({
|
|
71
|
+
where: pathText(issue.path),
|
|
72
|
+
message: (0, messages_1.translateIssue)(issue),
|
|
73
|
+
})),
|
|
74
|
+
};
|
|
75
|
+
}
|
|
76
|
+
const problems = missingFiles(result.data, folder);
|
|
77
|
+
return { ok: problems.length === 0, problems };
|
|
78
|
+
}
|
|
79
|
+
function checkConfigFile(configPath) {
|
|
80
|
+
const resolved = path_1.default.resolve(configPath);
|
|
81
|
+
if (!fs_1.default.existsSync(resolved)) {
|
|
82
|
+
return {
|
|
83
|
+
ok: false,
|
|
84
|
+
problems: [
|
|
85
|
+
{ where: "", message: `Arquivo não encontrado: ${configPath}` },
|
|
86
|
+
],
|
|
87
|
+
};
|
|
88
|
+
}
|
|
89
|
+
return checkConfigText(fs_1.default.readFileSync(resolved, "utf-8"), path_1.default.dirname(resolved));
|
|
90
|
+
}
|
|
91
|
+
function formatCheck(configPath, check) {
|
|
92
|
+
if (check.ok)
|
|
93
|
+
return [`${configPath}: configuração válida.`];
|
|
94
|
+
const count = check.problems.length;
|
|
95
|
+
return [
|
|
96
|
+
`${configPath}: ${count} ${count === 1 ? "problema" : "problemas"}.`,
|
|
97
|
+
...check.problems.map(({ where, message }) => where ? ` ${where}: ${message}` : ` ${message}`),
|
|
98
|
+
];
|
|
99
|
+
}
|