dbx 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c52bc907eb88537752612d4c288654e737048082
-  data.tar.gz: d4fb5d7f94bf24c126cd896b142eb17fc585fd93
+  metadata.gz: b769b3b6e3be916660bd21df6ccbdaaedf31558c
+  data.tar.gz: e67cdb52aca07a0035884c3866697d32327dd956
 SHA512:
-  metadata.gz: 1755fa1dd6c372745ed3f9234fe1918bfb01f3ef76bff45d5626ff1e0be9300f6ab21fadb096cac23950a27bd2461d2f5130b28c3367e4ed5a7eb5293db0cb1b
-  data.tar.gz: 45cd468ce1ead1093b22fc8fb50171c45fc5f51e7c18ff3bd55b4c703f764d35d1cc950535cc53491238bd47139c92182d89819489db923e6e9c20b14791a113
+  metadata.gz: c2b8a54503f2b6487553d7e7551268de810eb15c7b80632ec1d3b7c3b30d2b9b037384b4c506e6be5263dfe6c6ca871503470d64ee48c3dc42a00546b4fc19d1
+  data.tar.gz: 25024c1049ba266994d6113d21f4fb0fcddb6a40e108bf687a5ba37d72702d58fdb493db376c33beedc345bca2f7edc5d11382d07ffde9d6f8224845ecd9e575

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    dbx (0.1.1)
+    dbx (0.1.2)
       activerecord (~> 4.0)
       activesupport (~> 4.0)
       thor

data/README.md

@@ -1,22 +1,72 @@
 # DBX
 
-Database eXtras for working with CSV files in a database.
+Database eXtras for working with CSV files in a Postgres database.
 
-## Commands
+We currently only support Postgres database, but others will be supported soon.
+
+## Usage Examples
+
+### Import CSV file into database
+
+Column type detection is performed based on column contents. By default the new table name will be the file name minus its extension, indexes are added to column ending with `_id$`.
+
+```sh
+dbx import path/to/data.csv --name data_v1 --db postgres://localhost/scratch --column-patterns _ref$:string
+# --db   : Used to define where to put the table.
+# --column-patterns : [] Override the detected column type.
+# --name : [data] Optional override to default table name of file's base name without extension.
+# --sample : [100] Number of rows to sample during column detection.
+# --force : [false] Drops the destination table if it exists.
+# --auto-index-pattern : [^(\w+_id|id)$] Creates indexes for columns matching the pattern
+```
+
+### Create diff table of two tables in a database
+
++ The new table will be named `diff_data_v1_data_v2`.
++ It will contains column_a, column_b, column_diff. Where column is every `column_` from `data_v1` and `column_diff` is a simple difference representation of columns `_a` and `_b`.
+  + Columns `_a` and `_b` can be omitted with `--no-a-b`
+
+```sh
+dbx diff data_v1 data_v2 --db postgres://localhost/scratch --using id
+# --db   : Used to define where to put the table.
+# --using : Space delimited list of join columns.
+# --no-a-b: [false] Omit the `_a` and `_b` columns showing the source data.
+# --force : [false] Drops the destination diff table if it exists.
+```
+
+### Import and diff two CSV files
+
+Do the import and diff all at once!!!
+
+```sh
+dbx import_diff /path/to/data_v1.csv /path/to/data_v2.csv --db postgres://localhost/scratch --using id
+# --db   : Used to define where to put the table.
+# --column-patterns : [] Override the detected column type.
+# --sample : [100] Number of rows to sample during column detection.
+# --auto-index-pattern : [^(\w+_id|id)$] Creates indexes for columns matching the pattern
+# --using : Space delimited list of join columns.
+# --no-a-b: [false] Omit the `_a` and `_b` columns showing the source data.
+# --force : [false] Drops the destination diff table if it exists.
+```
+
+### List of Commands `dbx help`
 
 ```sh
 Commands:
-  dbx create SRC               # create a table with types from SRC CSV file
-  dbx diff TABLE_A TABLE_B     # create diff table between TABLE_A and TABLE_B.
+  dbx create SRC               # Create a table with types from SRC CSV file
+  dbx diff TABLE_A TABLE_B     # Create diff table between TABLE_A and TABLE_B.
   dbx help [COMMAND]           # Describe available commands or one specific command
-  dbx import SRC               # import SRC CSV into table
-  dbx import_diff SRC_A SRC_B  # import then diff between SRC_A CSV and SRC_B CSV files.
-  dbx types SRC                # detect column types give a SRC CSV file
+  dbx import SRC               # Import SRC CSV into table
+  dbx import_diff SRC_A SRC_B  # Import then diff between SRC_A CSV and SRC_B CSV files.
+  dbx types SRC                # Detect column types give a SRC CSV file
 
 Options:
   [--db=Database URL: adapter://user:pass@host:port/db_name]
-  [--column-patterns=List of column patterns to override type info. Ex: phone:string external_ref:string ...]
-  [--sample=Number of rows to sample for type detection] # Default: 100
+  [--column-patterns=List of column patterns to override type info]
+  [--sample=Number of rows to sample for type detection]
+                                                                     # Default: 100
+  [--auto-index-pattern=Add index when column matches pattern]
+                                                                     # Default: ^(\w+_id|id)$
 ```
 
 ## Configuration
@@ -37,6 +87,9 @@ column_patterns:
 
 # Number of rows to sample for type detection
 sample: 100
+
+# Add index if column matches this pattern.
+auto_index_pattern: _id$
 ```
 
 ## Installation

data/dbx_sample.yml

@@ -11,3 +11,6 @@ column_patterns:
 
 # Number of rows to sample for type detection
 sample: 100
+
+# Add index if column matches this pattern.
+auto_index_pattern: _id$

data/exe/dbx

@@ -9,11 +9,12 @@ require 'pp'
 # #rubocop:disable all
 class CLI < Thor
   class_option :db, type: :string, banner: 'Database URL: adapter://user:pass@host:port/db_name'
-  class_option :column_patterns, type: :array, banner: 'List of column patterns to override type info. Ex: phone:string external_ref:string ...'
+  class_option :column_patterns, type: :array, banner: 'List of column patterns to override type info'
   class_option :sample, type: :numeric, banner: 'Number of rows to sample for type detection', default: 100
+  class_option :auto_index_pattern, type: :string, default: '^(\w+_id|id)$', banner: 'Add index when column matches pattern'
 
   # contents of the Thor class
-  desc 'types SRC', 'detect column types give a SRC CSV file'
+  desc 'types SRC', 'Detect column types give a SRC CSV file'
   def types(src)
     handle_global_options
     DBX.column_types(src, sample_rows: options[:sample]).each do |col, type|
@@ -21,7 +22,7 @@ class CLI < Thor
     end
   end
 
-  desc 'create SRC', 'create a table with types from SRC CSV file'
+  desc 'create SRC', 'Create a table with types from SRC CSV file'
   option :name
   option :force, type: :boolean
   def create(src)
@@ -33,7 +34,7 @@ class CLI < Thor
     )
   end
 
-  desc 'import SRC', 'import SRC CSV into table'
+  desc 'import SRC', 'Import SRC CSV into table'
   option :name
   option :force, type: :boolean
   def import(src)
@@ -45,7 +46,7 @@ class CLI < Thor
     )
   end
 
-  desc 'diff TABLE_A TABLE_B', 'create diff table between TABLE_A and TABLE_B.'
+  desc 'diff TABLE_A TABLE_B', 'Create diff table between TABLE_A and TABLE_B.'
   option :force, type: :boolean, banner: 'remove diff_ table if it exists'
   option :using, type: :array, banner: 'JOIN USING the columns list here. Ex: id'
   option :exclude_columns, type: :array, banner: 'Exclude columns from comparison and selection'
@@ -62,7 +63,7 @@ class CLI < Thor
     )
   end
 
-  desc 'import_diff SRC_A SRC_B', 'import then diff between SRC_A CSV and SRC_B CSV files.'
+  desc 'import_diff SRC_A SRC_B', 'Import then diff between SRC_A CSV and SRC_B CSV files.'
   option :force, type: :boolean, banner: 'remove diff_ table if it exists'
   option :using, type: :array, banner: 'JOIN USING the columns list here. Ex: id'
   option :exclude_columns, type: :array, banner: 'Exclude columns from comparison and selection'
@@ -97,6 +98,10 @@ class CLI < Thor
     if options[:db]
       DBX.config['db'] = options[:db]
     end
+
+    if options[:auto_index_pattern]
+      DBX.config['auto_index_pattern'] = options[:auto_index_pattern]
+    end
   end
 end
 CLI.start(ARGV)

data/lib/dbx.rb

@@ -34,6 +34,10 @@ module DBX
     config['sample_rows'] || 100
   end
 
+  def config_auto_index_pattern
+    config['auto_index_pattern']
+  end
+
   def config_db
     ENV['DATABASE_URL'] || config['db'] || raise('`db` not set as command line option or `dbx.yml`')
   end
@@ -94,14 +98,18 @@ module DBX
           pg.put_copy_data(line)
         end
       end
-      index_table(name)
+
+      unless config_auto_index_pattern.blank?
+        index_table(name, pattern: /#{config_auto_index_pattern}/)
+      end
     end
     name
   end
 
-  def index_table(table_name)
+  def index_table(table_name, pattern: nil)
     connection do |conn|
       conn.columns(table_name).each_with_index do |column, i|
+        next unless column.name =~ pattern
         conn.add_index(table_name, [column.name], name: "idx_#{table_name}_#{i.to_s.rjust(2,'0')}")
       end
     end

data/lib/dbx/version.rb

@@ -1,3 +1,3 @@
 module Dbx
-  VERSION = '0.1.2'.freeze
+  VERSION = '0.2.0'.freeze
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dbx
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Scott Pierce
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-08-23 00:00:00.000000000 Z
+date: 2018-08-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler