csv-to-pg 0.6.2 → 2.0.2

package/esm/utils.js

@@ -0,0 +1,267 @@
+// @ts-nocheck
+import * as ast from 'pg-ast';
+import { join, resolve } from 'path';
+export const normalizePath = (path, cwd) => path.startsWith('/') ? path : resolve(join(cwd ? cwd : process.cwd(), path));
+const lstr = (bbox) => {
+    const [lng1, lat1, lng2, lat2] = bbox.split(',').map((a) => a.trim());
+    return `LINESTRING(${lng1} ${lat1}, ${lng1} ${lat2}, ${lng2} ${lat2}, ${lng2} ${lat1}, ${lng1} ${lat1})`;
+};
+const funcCall = (name, args) => {
+    return ast.FuncCall({
+        funcname: [ast.String({ str: name })],
+        args
+    });
+};
+const aflt = (num) => ast.A_Const({ val: ast.Float({ str: num }) });
+const aint = (num) => ast.A_Const({ val: ast.Integer({ ival: num }) });
+export const makeLocation = (longitude, latitude) => {
+    if (!longitude || !latitude) {
+        return ast.Null();
+    }
+    return funcCall('st_setsrid', [
+        funcCall('st_makepoint', [aflt(longitude), aflt(latitude)]),
+        aint(4326)
+    ]);
+};
+// a string in the form of lon,lat,lon,lat
+// -118.587533,34.024999,-118.495177,34.13165
+export const makeBoundingBox = (bbox) => {
+    return funcCall('st_setsrid', [
+        funcCall('st_makepolygon', [
+            funcCall('st_geomfromtext', [
+                ast.A_Const({ val: ast.String({ str: lstr(bbox) }) })
+            ])
+        ]),
+        aint(4326)
+    ]);
+};
+const ValuesLists = ({ types, record }) => Object.entries(types).map(([field, type]) => {
+    if (typeof type === 'function') {
+        return type(record);
+    }
+    throw new Error('coercion function missing');
+});
+const makeCast = (arg, type) => ({
+    TypeCast: {
+        arg,
+        typeName: {
+            TypeName: {
+                names: [
+                    {
+                        String: {
+                            str: type
+                        }
+                    }
+                ],
+                typemod: -1
+            }
+        }
+    }
+});
+const ref = (name) => ({
+    ResTarget: {
+        name,
+        val: {
+            ColumnRef: {
+                fields: [
+                    {
+                        String: {
+                            str: 'excluded'
+                        }
+                    },
+                    {
+                        String: {
+                            str: name
+                        }
+                    }
+                ]
+            }
+        }
+    }
+});
+const indexElem = (name) => ({
+    IndexElem: {
+        name,
+        ordering: 0,
+        nulls_ordering: 0
+    }
+});
+const makeConflictClause = (conflictElems, fields) => {
+    if (!conflictElems || !conflictElems.length)
+        return undefined;
+    const setElems = fields.filter((el) => !conflictElems.includes(el));
+    if (setElems.length) {
+        return {
+            OnConflictClause: {
+                action: 2,
+                infer: {
+                    InferClause: {
+                        indexElems: conflictElems.map((a) => indexElem(a))
+                    }
+                },
+                targetList: setElems.map((a) => ref(a))
+            }
+        };
+    }
+    else {
+        return {
+            OnConflictClause: {
+                action: 1,
+                infer: {
+                    InferClause: {
+                        indexElems: conflictElems.map((a) => indexElem(a))
+                    }
+                }
+            }
+        };
+    }
+};
+export const InsertOne = ({ schema = 'public', table, types, record, conflict }) => ({
+    RawStmt: {
+        stmt: {
+            InsertStmt: {
+                relation: {
+                    RangeVar: {
+                        schemaname: schema,
+                        relname: table,
+                        inh: true,
+                        relpersistence: 'p'
+                    }
+                },
+                cols: Object.keys(types).map((field) => ast.ResTarget({ name: field })),
+                selectStmt: {
+                    SelectStmt: {
+                        valuesLists: [ValuesLists({ types, record })],
+                        op: 0
+                    }
+                },
+                onConflictClause: makeConflictClause(conflict, Object.keys(types)),
+                override: 0
+            }
+        },
+        stmt_len: 1
+    }
+});
+export const InsertMany = ({ schema = 'public', table, types, records, conflict }) => ({
+    RawStmt: {
+        stmt: {
+            InsertStmt: {
+                relation: {
+                    RangeVar: {
+                        schemaname: schema,
+                        relname: table,
+                        inh: true,
+                        relpersistence: 'p'
+                    }
+                },
+                cols: Object.keys(types).map((field) => ast.ResTarget({ name: field })),
+                selectStmt: {
+                    SelectStmt: {
+                        valuesLists: records.map((record) => ValuesLists({ types, record })),
+                        op: 0
+                    }
+                },
+                onConflictClause: makeConflictClause(conflict, Object.keys(types)),
+                override: 0
+            }
+        },
+        stmt_len: 1
+    }
+});
+export const wrapValue = (val, { wrap, wrapAst, cast } = {}) => {
+    if (Array.isArray(wrap)) {
+        val = ast.FuncCall({
+            funcname: wrap.map((n) => ast.String({ str: n })),
+            args: [val]
+        });
+    }
+    if (wrapAst)
+        return wrapAst(val);
+    if (cast)
+        return makeCast(val, cast);
+    return val;
+};
+export const getRelatedField = ({ schema = 'public', table, refType, refKey, refField, wrap, wrapAst, cast, record, parse, from }) => {
+    let val;
+    const value = parse(record[from[0]]);
+    if (typeof value === 'undefined') {
+        return ast.Null({});
+    }
+    switch (refType) {
+        case 'int':
+            val = ast.A_Const({ val: ast.Integer({ ival: value }) });
+            break;
+        case 'float':
+            val = ast.A_Const({ val: ast.Float({ str: value }) });
+            break;
+        case 'boolean':
+        case 'bool':
+            val = ast.String({ str: value ? 'TRUE' : 'FALSE' });
+            break;
+        case 'text':
+        default:
+            val = ast.A_Const({ val: ast.String({ str: value }) });
+    }
+    val = wrapValue(val, { wrap, wrapAst });
+    return wrapValue({
+        SubLink: {
+            subLinkType: 4,
+            subselect: {
+                SelectStmt: {
+                    targetList: [
+                        {
+                            ResTarget: {
+                                val: {
+                                    ColumnRef: {
+                                        fields: [
+                                            {
+                                                String: {
+                                                    str: refKey
+                                                }
+                                            }
+                                        ]
+                                    }
+                                }
+                            }
+                        }
+                    ],
+                    fromClause: [
+                        {
+                            RangeVar: {
+                                schemaname: schema,
+                                relname: table,
+                                inh: true,
+                                relpersistence: 'p'
+                            }
+                        }
+                    ],
+                    whereClause: {
+                        A_Expr: {
+                            kind: 0,
+                            name: [
+                                {
+                                    String: {
+                                        str: '='
+                                    }
+                                }
+                            ],
+                            lexpr: {
+                                ColumnRef: {
+                                    fields: [
+                                        {
+                                            String: {
+                                                str: refField
+                                            }
+                                        }
+                                    ]
+                                }
+                            },
+                            rexpr: val
+                        }
+                    },
+                    op: 0
+                }
+            }
+        }
+    }, { cast });
+};

package/index.d.ts

@@ -0,0 +1,2 @@
+export * from './parse';
+export * from './parser';

package/index.js

@@ -0,0 +1,19 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+// @ts-nocheck
+__exportStar(require("./parse"), exports);
+__exportStar(require("./parser"), exports);

package/package.json

@@ -1,74 +1,44 @@
 {
   "name": "csv-to-pg",
-  "version": "0.6.2",
-  "description": "csv to pg statements",
+  "version": "2.0.2",
   "author": "Dan Lynch <pyramation@gmail.com>",
-  "homepage": "https://github.com/pyramation/csv-to-pg#readme",
+  "description": "csv to pg statements",
+  "main": "index.js",
+  "module": "esm/index.js",
+  "types": "index.d.ts",
+  "homepage": "https://github.com/launchql/launchql",
   "license": "SEE LICENSE IN LICENSE",
-  "main": "main/index.js",
-  "module": "module/index.js",
-  "directories": {
-    "lib": "src",
-    "test": "__tests__"
+  "publishConfig": {
+    "access": "public",
+    "directory": "dist"
   },
-  "files": [
-    "main",
-    "module"
-  ],
   "bin": {
-    "csv2pg": "main/cli.js"
-  },
-  "scripts": {
-    "build:main": "cross-env BABEL_ENV=production babel src --out-dir main --delete-dir-on-start",
-    "build:module": "cross-env MODULE=true babel src --out-dir module --delete-dir-on-start",
-    "build": "npm run build:module && npm run build:main",
-    "prepublish": "npm run build",
-    "dev": "cross-env NODE_ENV=development babel-node src/cli",
-    "watch": "cross-env NODE_ENV=development babel-watch src/index",
-    "lint": "eslint src --fix",
-    "test": "jest",
-    "test:watch": "jest --watch",
-    "test:debug": "node --inspect node_modules/.bin/jest --runInBand"
-  },
-  "publishConfig": {
-    "access": "public"
+    "csv2pg": "cli.js"
   },
   "repository": {
     "type": "git",
-    "url": "https://github.com/pyramation/csv-to-pg"
+    "url": "https://github.com/launchql/launchql"
   },
-  "keywords": [],
   "bugs": {
-    "url": "https://github.com/pyramation/csv-to-pg/issues"
+    "url": "https://github.com/launchql/launchql/issues"
   },
-  "devDependencies": {
-    "@babel/cli": "7.11.6",
-    "@babel/core": "7.11.6",
-    "@babel/node": "^7.10.5",
-    "@babel/plugin-proposal-class-properties": "7.10.4",
-    "@babel/plugin-proposal-export-default-from": "7.10.4",
-    "@babel/plugin-proposal-object-rest-spread": "7.11.0",
-    "@babel/plugin-transform-runtime": "7.11.5",
-    "@babel/preset-env": "7.11.5",
-    "babel-core": "7.0.0-bridge.0",
-    "babel-eslint": "10.1.0",
-    "babel-jest": "25.1.0",
-    "babel-watch": "^7.0.0",
-    "cross-env": "^7.0.2",
-    "eslint": "6.8.0",
-    "eslint-config-prettier": "^6.10.0",
-    "eslint-plugin-prettier": "^3.1.2",
-    "jest": "^24.5.0",
-    "jest-in-case": "^1.0.2",
-    "prettier": "^2.1.2",
-    "regenerator-runtime": "^0.13.7"
+  "scripts": {
+    "copy": "copyfiles -f ../../LICENSE README.md package.json dist",
+    "clean": "rimraf dist/**",
+    "prepare": "npm run build",
+    "build": "npm run clean; tsc; tsc -p tsconfig.esm.json; npm run copy",
+    "build:dev": "npm run clean; tsc --declarationMap; tsc -p tsconfig.esm.json; npm run copy",
+    "lint": "eslint . --fix",
+    "test": "jest",
+    "test:watch": "jest --watch"
   },
+  "keywords": [],
   "dependencies": {
-    "@babel/runtime": "^7.11.2",
     "@pyramation/prompt": "^0.0.1",
     "csv-parser": "^2.3.3",
     "js-yaml": "^3.14.0",
-    "pg-ast": "^0.1.0",
+    "pg-ast": "^2.0.2",
     "pgsql-deparser": "^1.2.3"
-  }
+  },
+  "gitHead": "d4ef195aec8d4c5a470f507b5a57c6c8e81124f4"
 }

package/parse.d.ts

@@ -0,0 +1,3 @@
+export declare const parse: (path: any, opts: any) => Promise<unknown>;
+export declare const readConfig: (config: any) => any;
+export declare const parseTypes: (config: any) => {};

package/parse.js

@@ -0,0 +1,289 @@
+"use strict";
+// @ts-nocheck
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.parseTypes = exports.readConfig = exports.parse = void 0;
+const csv_parser_1 = __importDefault(require("csv-parser"));
+const fs_1 = require("fs");
+const js_yaml_1 = require("js-yaml");
+const ast = __importStar(require("pg-ast"));
+const utils_1 = require("./utils");
+function isNumeric(str) {
+    if (typeof str === 'number')
+        return true;
+    if (typeof str !== 'string')
+        return false; // we only process strings!
+    return (!isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
+        !isNaN(parseFloat(str))); // ...and ensure strings of whitespace fail
+}
+const parseJson = (value) => {
+    if (typeof value === 'string')
+        return value;
+    return value && JSON.stringify(value);
+};
+const psqlArray = (value) => {
+    if (value && value.length) {
+        return `{${value.map((v) => v)}}`;
+    }
+};
+const parse = (path, opts) => new Promise((resolve, reject) => {
+    const results = [];
+    (0, fs_1.createReadStream)(path)
+        .pipe((0, csv_parser_1.default)(opts))
+        // TODO check if 'data' is guaranteed to have a full row,
+        // if so, make a hook to use the stream properly
+        .on('data', (data) => results.push(data))
+        .on('error', (er) => {
+        reject(er);
+    })
+        .on('end', () => {
+        resolve(results);
+    });
+});
+exports.parse = parse;
+const readConfig = (config) => {
+    let configValue;
+    if (config.endsWith('.js')) {
+        configValue = require(config);
+    }
+    else if (config.endsWith('json')) {
+        configValue = JSON.parse((0, fs_1.readFileSync)(config, 'utf-8'));
+    }
+    else if (config.endsWith('yaml') || config.endsWith('yml')) {
+        configValue = (0, js_yaml_1.safeLoad)((0, fs_1.readFileSync)(config, 'utf-8'));
+    }
+    else {
+        throw new Error('unsupported config!');
+    }
+    return configValue;
+};
+exports.readConfig = readConfig;
+const getFromValue = (from) => {
+    if (Array.isArray(from))
+        return from;
+    return [from];
+};
+// looks like the CSV library gives us empty strings?
+const cleanseEmptyStrings = (str) => {
+    if (typeof str === 'string') {
+        if (str.trim() === '')
+            return null;
+        return str;
+    }
+    else {
+        return str;
+    }
+};
+const parseBoolean = (str) => {
+    if (typeof str === 'boolean') {
+        return str;
+    }
+    else if (typeof str === 'string') {
+        const s = str.toLowerCase();
+        if (s === 'true') {
+            return true;
+        }
+        else if (s === 't') {
+            return true;
+        }
+        else if (s === 'f') {
+            return false;
+        }
+        else if (s === 'false') {
+            return false;
+        }
+        return null;
+    }
+    else {
+        return null;
+    }
+};
+const getValuesFromKeys = (object, keys) => keys.map((key) => object[key]);
+const identity = (a) => a;
+const isEmpty = (value) => value === null || typeof value === 'undefined';
+// type (int, text, etc)
+// from Array of keys that map to records found (e.g., ['lon', 'lat'])
+const getCoercionFunc = (type, from, opts) => {
+    const parse = (opts.parse = opts.parse || identity);
+    switch (type) {
+        case 'int':
+            return (record) => {
+                const value = parse(record[from[0]]);
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                if (!isNumeric(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.Integer({ ival: value })
+                });
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+        case 'float':
+            return (record) => {
+                const value = parse(record[from[0]]);
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                if (!isNumeric(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.Float({ str: value })
+                });
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+        case 'boolean':
+        case 'bool':
+            return (record) => {
+                const value = parse(parseBoolean(record[from[0]]));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.String({
+                    str: value ? 'TRUE' : 'FALSE'
+                });
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+        case 'bbox':
+            // do bbox magic with args from the fields
+            return (record) => {
+                const val = (0, utils_1.makeBoundingBox)(parse(record[from[0]]));
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+        case 'location':
+            return (record) => {
+                const [lon, lat] = getValuesFromKeys(record, from);
+                if (typeof lon === 'undefined') {
+                    return ast.Null({});
+                }
+                if (typeof lat === 'undefined') {
+                    return ast.Null({});
+                }
+                if (!isNumeric(lon) || !isNumeric(lat)) {
+                    return ast.Null({});
+                }
+                // NO parse here...
+                const val = (0, utils_1.makeLocation)(lon, lat);
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+        case 'related':
+            return (record) => {
+                return (0, utils_1.getRelatedField)({
+                    ...opts,
+                    record,
+                    from
+                });
+            };
+        case 'uuid':
+            return (record) => {
+                const value = parse(record[from[0]]);
+                if (isEmpty(value) ||
+                    !/^([0-9a-fA-F]{8})-(([0-9a-fA-F]{4}-){3})([0-9a-fA-F]{12})$/i.test(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+        case 'text':
+            return (record) => {
+                const value = parse(cleanseEmptyStrings(record[from[0]]));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+        case 'text[]':
+            return (record) => {
+                const value = parse(psqlArray(cleanseEmptyStrings(record[from[0]])));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+        case 'image':
+        case 'attachment':
+        case 'json':
+        case 'jsonb':
+            return (record) => {
+                const value = parse(parseJson(cleanseEmptyStrings(record[from[0]])));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+        default:
+            return (record) => {
+                const value = parse(cleanseEmptyStrings(record[from[0]]));
+                if (isEmpty(value)) {
+                    return ast.Null({});
+                }
+                const val = ast.A_Const({
+                    val: ast.String({ str: value })
+                });
+                return (0, utils_1.wrapValue)(val, opts);
+            };
+    }
+};
+const parseTypes = (config) => {
+    return Object.entries(config.fields).reduce((m, v) => {
+        let [key, value] = v;
+        let type;
+        let from;
+        if (typeof value === 'string') {
+            type = value;
+            from = [key];
+            if (['related', 'location'].includes(type)) {
+                throw new Error('must use object for ' + type + ' type');
+            }
+            value = {
+                type,
+                from
+            };
+        }
+        else {
+            type = value.type;
+            from = getFromValue(value.from || key);
+        }
+        m[key] = getCoercionFunc(type, from, value);
+        return m;
+    }, {});
+};
+exports.parseTypes = parseTypes;

package/parser.d.ts

@@ -0,0 +1,4 @@
+export declare class Parser {
+    constructor(config: any);
+    parse(data: any): Promise<any>;
+}