csv-to-pg 0.6.2 → 2.0.2

package/LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (c) 2020 Dan Lynch <pyramation@gmail.com>
+Copyright (c) 2024 Dan Lynch <pyramation@gmail.com>
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package/cli.d.ts

@@ -0,0 +1,2 @@
+#!/usr/bin/env node
+export {};

package/cli.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+// @ts-nocheck
+const prompt_1 = require("@pyramation/prompt");
+const parse_1 = require("./parse");
+const parser_1 = require("./parser");
+const utils_1 = require("./utils");
+const path_1 = require("path");
+const fs_1 = require("fs");
+const argv = process.argv.slice(2);
+(async () => {
+    let { config } = await (0, prompt_1.prompt)([
+        {
+            _: true,
+            name: 'config',
+            type: 'config',
+            required: true
+        }
+    ], argv);
+    config = (0, utils_1.normalizePath)(config);
+    const dir = (0, path_1.dirname)(config);
+    config = (0, parse_1.readConfig)(config);
+    if (config.input) {
+        if (!argv.includes('--input')) {
+            argv.push('--input');
+            config.input = (0, utils_1.normalizePath)(config.input, dir);
+            argv.push(config.input);
+        }
+    }
+    if (config.output) {
+        if (!argv.includes('--output')) {
+            argv.push('--output');
+            config.output = (0, utils_1.normalizePath)(config.output, dir);
+            argv.push(config.output);
+        }
+    }
+    const results = await (0, prompt_1.prompt)([
+        {
+            name: 'input',
+            type: 'path',
+            required: true
+        },
+        {
+            name: 'output',
+            type: 'path',
+            required: true
+        }
+    ], argv);
+    config.input = results.input;
+    let outFile = results.output;
+    if (!outFile.endsWith('.sql'))
+        outFile = outFile + '.sql';
+    if (argv.includes('--debug')) {
+        config.debug = true;
+    }
+    const parser = new parser_1.Parser(config);
+    const sql = await parser.parse();
+    (0, fs_1.writeFileSync)(config.output, sql);
+})();

package/esm/cli.js

@@ -0,0 +1,58 @@
+#!/usr/bin/env node
+// @ts-nocheck
+import { prompt } from '@pyramation/prompt';
+import { readConfig } from './parse';
+import { Parser } from './parser';
+import { normalizePath } from './utils';
+import { dirname } from 'path';
+import { writeFileSync } from 'fs';
+const argv = process.argv.slice(2);
+(async () => {
+    let { config } = await prompt([
+        {
+            _: true,
+            name: 'config',
+            type: 'config',
+            required: true
+        }
+    ], argv);
+    config = normalizePath(config);
+    const dir = dirname(config);
+    config = readConfig(config);
+    if (config.input) {
+        if (!argv.includes('--input')) {
+            argv.push('--input');
+            config.input = normalizePath(config.input, dir);
+            argv.push(config.input);
+        }
+    }
+    if (config.output) {
+        if (!argv.includes('--output')) {
+            argv.push('--output');
+            config.output = normalizePath(config.output, dir);
+            argv.push(config.output);
+        }
+    }
+    const results = await prompt([
+        {
+            name: 'input',
+            type: 'path',
+            required: true
+        },
+        {
+            name: 'output',
+            type: 'path',
+            required: true
+        }
+    ], argv);
+    config.input = results.input;
+    let outFile = results.output;
+    if (!outFile.endsWith('.sql'))
+        outFile = outFile + '.sql';
+    if (argv.includes('--debug')) {
+        config.debug = true;
+    }
+    const parser = new Parser(config);
+    const sql = await parser.parse();
+    writeFileSync(config.output, sql);
+})();

package/esm/index.js

@@ -0,0 +1,3 @@
+// @ts-nocheck
+export * from './parse';
+export * from './parser';

package/esm/parse.js

@@ -0,0 +1,257 @@
+// @ts-nocheck
+import csv from 'csv-parser';
+import { createReadStream, readFileSync } from 'fs';
+import { safeLoad as parseYAML } from 'js-yaml';
+import * as ast from 'pg-ast';
+import { makeBoundingBox, makeLocation, getRelatedField, wrapValue } from './utils';
+function isNumeric(str) {
+    if (typeof str === 'number')
+        return true;
+    if (typeof str !== 'string')
+        return false; // we only process strings!
+    return (!isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
+        !isNaN(parseFloat(str))); // ...and ensure strings of whitespace fail
+}
+const parseJson = (value) => {
+    if (typeof value === 'string')
+        return value;
+    return value && JSON.stringify(value);
+};
+const psqlArray = (value) => {
+    if (value && value.length) {
+        return `{${value.map((v) => v)}}`;
+    }
+};
+export const parse = (path, opts) => new Promise((resolve, reject) => {
+    const results = [];
+    createReadStream(path)
+        .pipe(csv(opts))
+        // TODO check if 'data' is guaranteed to have a full row,
+        // if so, make a hook to use the stream properly
+        .on('data', (data) => results.push(data))
+        .on('error', (er) => {
+        reject(er);
+    })
+        .on('end', () => {
+        resolve(results);
+    });
+});
+export const readConfig = (config) => {
+    let configValue;
+    if (config.endsWith('.js')) {
+        configValue = require(config);
+    }
+    else if (config.endsWith('json')) {
+        configValue = JSON.parse(readFileSync(config, 'utf-8'));
+    }
+    else if (config.endsWith('yaml') || config.endsWith('yml')) {
+        configValue = parseYAML(readFileSync(config, 'utf-8'));
+    }
+    else {
+        throw new Error('unsupported config!');
+    }
+    return configValue;
+};
+const getFromValue = (from) => {
+    if (Array.isArray(from))
+        return from;
+    return [from];
+};
+// looks like the CSV library gives us empty strings?
+const cleanseEmptyStrings = (str) => {
+    if (typeof str === 'string') {
+        if (str.trim() === '')
+            return null;
+        return str;
+    }
+    else {
+        return str;
+    }
+};
+const parseBoolean = (str) => {
+    if (typeof str === 'boolean') {
+        return str;
+    }
+    else if (typeof str === 'string') {
+        const s = str.toLowerCase();
+        if (s === 'true') {
+            return true;
+        }
+        else if (s === 't') {
+            return true;
+        }
+        else if (s === 'f') {
+            return false;
+        }
+        else if (s === 'false') {
+            return false;
+        }
+        return null;
+    }
+    else {
+        return null;
+    }
+};
+const getValuesFromKeys = (object, keys) => keys.map((key) => object[key]);
+const identity = (a) => a;
+const isEmpty = (value) => value === null || typeof value === 'undefined';
+// type (int, text, etc)
+// from Array of keys that map to records found (e.g., ['lon', 'lat'])
+const getCoercionFunc = (type, from, opts) => {
+    const parse = (opts.parse = opts.parse || identity);
+    switch (type) {
+        case 'int':
+            return (record) => {
+                const value = parse(record[from[0]]);
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                if (!isNumeric(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.Integer({ ival: value })
+                });
+                return wrapValue(val, opts);
+            };
+        case 'float':
+            return (record) => {
+                const value = parse(record[from[0]]);
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                if (!isNumeric(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.Float({ str: value })
+                });
+                return wrapValue(val, opts);
+            };
+        case 'boolean':
+        case 'bool':
+            return (record) => {
+                const value = parse(parseBoolean(record[from[0]]));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.String({
+                    str: value ? 'TRUE' : 'FALSE'
+                });
+                return wrapValue(val, opts);
+            };
+        case 'bbox':
+            // do bbox magic with args from the fields
+            return (record) => {
+                const val = makeBoundingBox(parse(record[from[0]]));
+                return wrapValue(val, opts);
+            };
+        case 'location':
+            return (record) => {
+                const [lon, lat] = getValuesFromKeys(record, from);
+                if (typeof lon === 'undefined') {
+                    return ast.Null({});
+                }
+                if (typeof lat === 'undefined') {
+                    return ast.Null({});
+                }
+                if (!isNumeric(lon) || !isNumeric(lat)) {
+                    return ast.Null({});
+                }
+                // NO parse here...
+                const val = makeLocation(lon, lat);
+                return wrapValue(val, opts);
+            };
+        case 'related':
+            return (record) => {
+                return getRelatedField({
+                    ...opts,
+                    record,
+                    from
+                });
+            };
+        case 'uuid':
+            return (record) => {
+                const value = parse(record[from[0]]);
+                if (isEmpty(value) ||
+                    !/^([0-9a-fA-F]{8})-(([0-9a-fA-F]{4}-){3})([0-9a-fA-F]{12})$/i.test(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return wrapValue(val, opts);
+            };
+        case 'text':
+            return (record) => {
+                const value = parse(cleanseEmptyStrings(record[from[0]]));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return wrapValue(val, opts);
+            };
+        case 'text[]':
+            return (record) => {
+                const value = parse(psqlArray(cleanseEmptyStrings(record[from[0]])));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return wrapValue(val, opts);
+            };
+        case 'image':
+        case 'attachment':
+        case 'json':
+        case 'jsonb':
+            return (record) => {
+                const value = parse(parseJson(cleanseEmptyStrings(record[from[0]])));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return wrapValue(val, opts);
+            };
+        default:
+            return (record) => {
+                const value = parse(cleanseEmptyStrings(record[from[0]]));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return wrapValue(val, opts);
+            };
+    }
+};
+export const parseTypes = (config) => {
+    return Object.entries(config.fields).reduce((m, v) => {
+        let [key, value] = v;
+        let type;
+        let from;
+        if (typeof value === 'string') {
+            type = value;
+            from = [key];
+            if (['related', 'location'].includes(type)) {
+                throw new Error('must use object for ' + type + ' type');
+            }
+            value = {
+                type,
+                from
+            };
+        }
+        else {
+            type = value.type;
+            from = getFromValue(value.from || key);
+        }
+        m[key] = getCoercionFunc(type, from, value);
+        return m;
+    }, {});
+};

package/esm/parser.js

@@ -0,0 +1,59 @@
+// @ts-nocheck
+import { readFileSync } from 'fs';
+import { parse, parseTypes } from './index';
+import { deparse } from 'pgsql-deparser';
+import { InsertOne, InsertMany } from './utils';
+export class Parser {
+    constructor(config) {
+        this.config = config;
+    }
+    async parse(data) {
+        const config = this.config;
+        const { schema, table, singleStmts, conflict, headers, delimeter } = config;
+        const opts = {};
+        if (headers)
+            opts.headers = headers;
+        if (delimeter)
+            opts.separator = delimeter;
+        let records;
+        if (typeof data === 'undefined') {
+            if (config.json || config.input.endsWith('.json')) {
+                records = JSON.parse(readFileSync(config.input, 'utf-8'));
+            }
+            else {
+                records = await parse(config.input, opts);
+            }
+        }
+        else {
+            if (!Array.isArray(data)) {
+                throw new Error('data is not an array');
+            }
+            records = data;
+        }
+        if (config.debug) {
+            console.log(records);
+            return;
+        }
+        const types = parseTypes(config);
+        if (singleStmts) {
+            const stmts = records.map((record) => InsertOne({
+                schema,
+                table,
+                types,
+                record,
+                conflict
+            }));
+            return deparse(stmts);
+        }
+        else {
+            const stmt = InsertMany({
+                schema,
+                table,
+                types,
+                records,
+                conflict
+            });
+            return deparse([stmt]);
+        }
+    }
+}