csv-to-pg 0.6.2 → 2.0.2

package/module/parse.js

@@ -1,308 +0,0 @@
-import _defineProperty from "@babel/runtime/helpers/esm/defineProperty";
-
-function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); keys.push.apply(keys, symbols); } return keys; }
-
-function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { _defineProperty(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }
-
-import csv from 'csv-parser';
-import { createReadStream, readFileSync } from 'fs';
-import { safeLoad as parseYAML } from 'js-yaml';
-import * as ast from 'pg-ast';
-import { makeBoundingBox, makeLocation, getRelatedField, wrapValue } from './utils';
-
-function isNumeric(str) {
-  if (typeof str != 'string') return false; // we only process strings!
-
-  return !isNaN(str) && // use type coercion to parse the _entirety_ of the string (`parseFloat` alone does not do this)...
-  !isNaN(parseFloat(str)); // ...and ensure strings of whitespace fail
-}
-
-const parseJson = value => {
-  if (typeof value === 'string') return value;
-  return value && JSON.stringify(value);
-};
-
-const psqlArray = value => {
-  if (value && value.length) {
-    return `{${value.map(v => v)}}`;
-  }
-};
-
-export const parse = (path, opts) => new Promise((resolve, reject) => {
-  const results = [];
-  createReadStream(path).pipe(csv(opts)) // TODO check if 'data' is guaranteed to have a full row,
-  // if so, make a hook to use the stream properly
-  .on('data', data => results.push(data)).on('error', er => {
-    reject(er);
-  }).on('end', () => {
-    resolve(results);
-  });
-});
-export const readConfig = config => {
-  let configValue;
-
-  if (config.endsWith('.js')) {
-    configValue = require(config);
-  } else if (config.endsWith('json')) {
-    configValue = JSON.parse(readFileSync(config, 'utf-8'));
-  } else if (config.endsWith('yaml') || config.endsWith('yml')) {
-    configValue = parseYAML(readFileSync(config, 'utf-8'));
-  } else {
-    throw new Error('unsupported config!');
-  }
-
-  return configValue;
-};
-
-const getFromValue = from => {
-  if (Array.isArray(from)) return from;
-  return [from];
-}; // looks like the CSV library gives us empty strings?
-
-
-const cleanseEmptyStrings = str => {
-  if (typeof str === 'string') {
-    if (str.trim() === '') return null;
-    return str;
-  } else {
-    return str;
-  }
-};
-
-const parseBoolean = str => {
-  if (typeof str === 'boolean') {
-    return str;
-  } else if (typeof str === 'string') {
-    const s = str.toLowerCase();
-
-    if (s === 'true') {
-      return true;
-    } else if (s === 't') {
-      return true;
-    } else if (s === 'f') {
-      return false;
-    } else if (s === 'false') {
-      return false;
-    }
-
-    return null;
-  } else {
-    return null;
-  }
-};
-
-const getValuesFromKeys = (object, keys) => keys.map(key => object[key]);
-
-const identity = a => a;
-
-const isEmpty = value => value === null || typeof value === 'undefined'; // type (int, text, etc)
-// from Array of keys that map to records found (e.g., ['lon', 'lat'])
-
-
-const getCoercionFunc = (type, from, opts) => {
-  const parse = opts.parse = opts.parse || identity;
-
-  switch (type) {
-    case 'int':
-      return record => {
-        const value = parse(record[from[0]]);
-
-        if (isEmpty(value)) {
-          return ast.Null({});
-        }
-
-        if (!isNumeric(value)) {
-          return ast.Null({});
-        }
-
-        const val = ast.A_Const({
-          val: ast.Integer({
-            ival: value
-          })
-        });
-        return wrapValue(val, opts);
-      };
-
-    case 'float':
-      return record => {
-        const value = parse(record[from[0]]);
-
-        if (isEmpty(value)) {
-          return ast.Null({});
-        }
-
-        if (!isNumeric(value)) {
-          return ast.Null({});
-        }
-
-        const val = ast.A_Const({
-          val: ast.Float({
-            str: value
-          })
-        });
-        return wrapValue(val, opts);
-      };
-
-    case 'boolean':
-    case 'bool':
-      return record => {
-        const value = parse(parseBoolean(record[from[0]]));
-
-        if (isEmpty(value)) {
-          return ast.Null({});
-        }
-
-        const val = ast.String({
-          str: value ? 'TRUE' : 'FALSE'
-        });
-        return wrapValue(val, opts);
-      };
-
-    case 'bbox':
-      // do bbox magic with args from the fields
-      return record => {
-        const val = makeBoundingBox(parse(record[from[0]]));
-        return wrapValue(val, opts);
-      };
-
-    case 'location':
-      return record => {
-        const [lon, lat] = getValuesFromKeys(record, from);
-
-        if (typeof lon === 'undefined') {
-          return ast.Null({});
-        }
-
-        if (typeof lat === 'undefined') {
-          return ast.Null({});
-        }
-
-        if (!isNumeric(lon) || !isNumeric(lat)) {
-          return ast.Null({});
-        } // NO parse here...
-
-
-        const val = makeLocation(lon, lat);
-        return wrapValue(val, opts);
-      };
-
-    case 'related':
-      return record => {
-        return getRelatedField(_objectSpread(_objectSpread({}, opts), {}, {
-          record,
-          from
-        }));
-      };
-
-    case 'uuid':
-      return record => {
-        const value = parse(record[from[0]]);
-
-        if (isEmpty(value) || !/^([0-9a-fA-F]{8})-(([0-9a-fA-F]{4}-){3})([0-9a-fA-F]{12})$/i.test(value)) {
-          return ast.Null({});
-        }
-
-        const val = ast.A_Const({
-          val: ast.String({
-            str: value
-          })
-        });
-        return wrapValue(val, opts);
-      };
-
-    case 'text':
-      return record => {
-        const value = parse(cleanseEmptyStrings(record[from[0]]));
-
-        if (isEmpty(value)) {
-          return ast.Null({});
-        }
-
-        const val = ast.A_Const({
-          val: ast.String({
-            str: value
-          })
-        });
-        return wrapValue(val, opts);
-      };
-
-    case 'text[]':
-      return record => {
-        const value = parse(psqlArray(cleanseEmptyStrings(record[from[0]])));
-
-        if (isEmpty(value)) {
-          return ast.Null({});
-        }
-
-        const val = ast.A_Const({
-          val: ast.String({
-            str: value
-          })
-        });
-        return wrapValue(val, opts);
-      };
-
-    case 'image':
-    case 'attachment':
-    case 'json':
-    case 'jsonb':
-      return record => {
-        const value = parse(parseJson(cleanseEmptyStrings(record[from[0]])));
-
-        if (isEmpty(value)) {
-          return ast.Null({});
-        }
-
-        const val = ast.A_Const({
-          val: ast.String({
-            str: value
-          })
-        });
-        return wrapValue(val, opts);
-      };
-
-    default:
-      return record => {
-        const value = parse(cleanseEmptyStrings(record[from[0]]));
-
-        if (isEmpty(value)) {
-          return ast.Null({});
-        }
-
-        const val = ast.A_Const({
-          val: ast.String({
-            str: value
-          })
-        });
-        return wrapValue(val, opts);
-      };
-  }
-};
-
-export const parseTypes = config => {
-  return Object.entries(config.fields).reduce((m, v) => {
-    let [key, value] = v;
-    let type;
-    let from;
-
-    if (typeof value === 'string') {
-      type = value;
-      from = [key];
-
-      if (['related', 'location'].includes(type)) {
-        throw new Error('must use object for ' + type + ' type');
-      }
-
-      value = {
-        type,
-        from
-      };
-    } else {
-      type = value.type;
-      from = getFromValue(value.from || key);
-    }
-
-    m[key] = getCoercionFunc(type, from, value);
-    return m;
-  }, {});
-};

package/module/parser.js

@@ -1,67 +0,0 @@
-import { readFileSync, writeFileSync } from 'fs';
-import { parse, parseTypes } from './index';
-import { deparse } from 'pgsql-deparser';
-import { InsertOne, InsertMany } from './utils';
-export class Parser {
-  constructor(config) {
-    this.config = config;
-  }
-
-  async parse(data) {
-    const config = this.config;
-    const {
-      schema,
-      table,
-      singleStmts,
-      conflict,
-      headers,
-      delimeter
-    } = config;
-    const opts = {};
-    if (headers) opts.headers = headers;
-    if (delimeter) opts.separator = delimeter;
-    let records;
-
-    if (typeof data === 'undefined') {
-      if (config.json || config.input.endsWith('.json')) {
-        records = JSON.parse(readFileSync(config.input, 'utf-8'));
-      } else {
-        records = await parse(config.input, opts);
-      }
-    } else {
-      if (!Array.isArray(data)) {
-        throw new Error('data is not an array');
-      }
-
-      records = data;
-    }
-
-    if (config.debug) {
-      console.log(records);
-      return;
-    }
-
-    const types = parseTypes(config);
-
-    if (singleStmts) {
-      const stmts = records.map(record => InsertOne({
-        schema,
-        table,
-        types,
-        record,
-        conflict
-      }));
-      return deparse(stmts);
-    } else {
-      const stmt = InsertMany({
-        schema,
-        table,
-        types,
-        records,
-        conflict
-      });
-      return deparse([stmt]);
-    }
-  }
-
-}

package/module/utils.js

@@ -1,334 +0,0 @@
-import * as ast from 'pg-ast';
-import { join, resolve } from 'path';
-export const normalizePath = (path, cwd) => path.startsWith('/') ? path : resolve(join(cwd ? cwd : process.cwd(), path));
-
-const lstr = bbox => {
-  const [lng1, lat1, lng2, lat2] = bbox.split(',').map(a => a.trim());
-  return `LINESTRING(${lng1} ${lat1}, ${lng1} ${lat2}, ${lng2} ${lat2}, ${lng2} ${lat1}, ${lng1} ${lat1})`;
-};
-
-const funcCall = (name, args) => {
-  return ast.FuncCall({
-    funcname: [ast.String({
-      str: name
-    })],
-    args
-  });
-};
-
-const aflt = num => ast.A_Const({
-  val: ast.Float({
-    str: num
-  })
-});
-
-const aint = num => ast.A_Const({
-  val: ast.Integer({
-    ival: num
-  })
-});
-
-export const makeLocation = (longitude, latitude) => {
-  if (!longitude || !latitude) {
-    return ast.Null();
-  }
-
-  return funcCall('st_setsrid', [funcCall('st_makepoint', [aflt(longitude), aflt(latitude)]), aint(4326)]);
-}; // a string in the form of lon,lat,lon,lat
-// -118.587533,34.024999,-118.495177,34.13165
-
-export const makeBoundingBox = bbox => {
-  return funcCall('st_setsrid', [funcCall('st_makepolygon', [funcCall('st_geomfromtext', [ast.A_Const({
-    val: ast.String({
-      str: lstr(bbox)
-    })
-  })])]), aint(4326)]);
-};
-
-const ValuesLists = ({
-  types,
-  record
-}) => Object.entries(types).map(([field, type]) => {
-  if (typeof type === 'function') {
-    return type(record);
-  }
-
-  throw new Error('coercion function missing');
-});
-
-const makeCast = (arg, type) => ({
-  TypeCast: {
-    arg,
-    typeName: {
-      TypeName: {
-        names: [{
-          String: {
-            str: type
-          }
-        }],
-        typemod: -1
-      }
-    }
-  }
-});
-
-const ref = name => ({
-  ResTarget: {
-    name,
-    val: {
-      ColumnRef: {
-        fields: [{
-          String: {
-            str: 'excluded'
-          }
-        }, {
-          String: {
-            str: name
-          }
-        }]
-      }
-    }
-  }
-});
-
-const indexElem = name => ({
-  IndexElem: {
-    name,
-    ordering: 0,
-    nulls_ordering: 0
-  }
-});
-
-const makeConflictClause = (conflictElems, fields) => {
-  if (!conflictElems || !conflictElems.length) return undefined;
-  const setElems = fields.filter(el => !conflictElems.includes(el));
-
-  if (setElems.length) {
-    return {
-      OnConflictClause: {
-        action: 2,
-        infer: {
-          InferClause: {
-            indexElems: conflictElems.map(a => indexElem(a))
-          }
-        },
-        targetList: setElems.map(a => ref(a))
-      }
-    };
-  } else {
-    return {
-      OnConflictClause: {
-        action: 1,
-        infer: {
-          InferClause: {
-            indexElems: conflictElems.map(a => indexElem(a))
-          }
-        }
-      }
-    };
-  }
-};
-
-export const InsertOne = ({
-  schema = 'public',
-  table,
-  types,
-  record,
-  conflict
-}) => ({
-  RawStmt: {
-    stmt: {
-      InsertStmt: {
-        relation: {
-          RangeVar: {
-            schemaname: schema,
-            relname: table,
-            inh: true,
-            relpersistence: 'p'
-          }
-        },
-        cols: Object.keys(types).map(field => ast.ResTarget({
-          name: field
-        })),
-        selectStmt: {
-          SelectStmt: {
-            valuesLists: [ValuesLists({
-              types,
-              record
-            })],
-            op: 0
-          }
-        },
-        onConflictClause: makeConflictClause(conflict, Object.keys(types)),
-        override: 0
-      }
-    },
-    stmt_len: 1
-  }
-});
-export const InsertMany = ({
-  schema = 'public',
-  table,
-  types,
-  records,
-  conflict
-}) => ({
-  RawStmt: {
-    stmt: {
-      InsertStmt: {
-        relation: {
-          RangeVar: {
-            schemaname: schema,
-            relname: table,
-            inh: true,
-            relpersistence: 'p'
-          }
-        },
-        cols: Object.keys(types).map(field => ast.ResTarget({
-          name: field
-        })),
-        selectStmt: {
-          SelectStmt: {
-            valuesLists: records.map(record => ValuesLists({
-              types,
-              record
-            })),
-            op: 0
-          }
-        },
-        onConflictClause: makeConflictClause(conflict, Object.keys(types)),
-        override: 0
-      }
-    },
-    stmt_len: 1
-  }
-});
-export const wrapValue = (val, {
-  wrap,
-  wrapAst,
-  cast
-} = {}) => {
-  if (Array.isArray(wrap)) {
-    val = ast.FuncCall({
-      funcname: wrap.map(n => ast.String({
-        str: n
-      })),
-      args: [val]
-    });
-  }
-
-  if (wrapAst) return wrapAst(val);
-  if (cast) return makeCast(val, cast);
-  return val;
-};
-export const getRelatedField = ({
-  schema = 'public',
-  table,
-  refType,
-  refKey,
-  refField,
-  wrap,
-  wrapAst,
-  cast,
-  record,
-  parse,
-  from
-}) => {
-  let val;
-  const value = parse(record[from[0]]);
-
-  if (typeof value === 'undefined') {
-    return ast.Null({});
-  }
-
-  switch (refType) {
-    case 'int':
-      val = ast.A_Const({
-        val: ast.Integer({
-          ival: value
-        })
-      });
-      break;
-
-    case 'float':
-      val = ast.A_Const({
-        val: ast.Float({
-          str: value
-        })
-      });
-      break;
-
-    case 'boolean':
-    case 'bool':
-      val = ast.String({
-        str: value ? 'TRUE' : 'FALSE'
-      });
-      break;
-
-    case 'text':
-    default:
-      val = ast.A_Const({
-        val: ast.String({
-          str: value
-        })
-      });
-  }
-
-  val = wrapValue(val, {
-    wrap,
-    wrapAst
-  });
-  return wrapValue({
-    SubLink: {
-      subLinkType: 4,
-      subselect: {
-        SelectStmt: {
-          targetList: [{
-            ResTarget: {
-              val: {
-                ColumnRef: {
-                  fields: [{
-                    String: {
-                      str: refKey
-                    }
-                  }]
-                }
-              }
-            }
-          }],
-          fromClause: [{
-            RangeVar: {
-              schemaname: schema,
-              relname: table,
-              inh: true,
-              relpersistence: 'p'
-            }
-          }],
-          whereClause: {
-            A_Expr: {
-              kind: 0,
-              name: [{
-                String: {
-                  str: '='
-                }
-              }],
-              lexpr: {
-                ColumnRef: {
-                  fields: [{
-                    String: {
-                      str: refField
-                    }
-                  }]
-                }
-              },
-              rexpr: val
-            }
-          },
-          op: 0
-        }
-      }
-    }
-  }, {
-    cast
-  });
-};