effective_developer 0.0.6 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 321b83d7d7cf7a803e2516db52bb1610b32dd8a0
-  data.tar.gz: 9f64632100cb5199d5b76bb4a6a57f8764ce64a3
+  metadata.gz: 6afcc37e01e11a752f1fa16ea168e739886571a5
+  data.tar.gz: 1f79a862bbaeaec604384015649860c8fc3ca53d
 SHA512:
-  metadata.gz: 1d8b4e802b1e9b8c8bbc7bc78dc9aa12cf8518a9d1a35a859a86289a1417852c1f14f95cec6ce609a57b544e2838b34e2b31f4c3329052ac396aa642903bb3a3
-  data.tar.gz: 7e7d7fba5828905f6b0663817424ba74a3532e857f84b8752e11cd37c1fb96bd62e5d0d218ab63bccbd883fde23dfbbba8995107b1f38206ecde74be2c9670c0
+  metadata.gz: 90ce51e217dab774391c2f2e697951a594383738b61b1c1473291804daadbb9dc88409932925faeb000e8abea7248a5a224e7e0bb8f543cfeea564c9e6be2deb
+  data.tar.gz: ce87d2c69e4844d62226ada1b014698269121d3db47265d0170e202ccba8ca168e2d77b4c18272357b01858bd3bdb773e89a180a9df0ba41f3ccc245b7ae60bd

data/README.md

@@ -18,7 +18,7 @@ Run the bundle command to install it:
 bundle install
 ```
 
-To use the included command line scripts in any directory, clone this repo:
+To use the included command line shell scripts in any directory, clone this repo:
 
 ```console
 git clone git@github.com:code-and-effect/effective_developer.git
@@ -70,7 +70,37 @@ A command line script that calls [BFG Repo-Cleaner](https://rtyley.github.io/bfg
 
 # Rake scripts
 
-## Reset Pk Sequence
+## csv:export
+
+Exports all database tables to individual .csv files.
+
+```ruby
+rake export:csv
+```
+
+## csv:import::foos
+
+Where table is the name of a model.  Dynamically created rake task when a `/lib/csv_importers/foos.rb` file is present.
+
+```ruby
+rake csv:import:foos
+```
+
+## csv:import::scaffold
+
+Scaffolds an `Effective::CSVImporter` file for each .csv file in `/lib/csv_importers/data/*.csv`
+
+```ruby
+rake csv:import:scaffold
+```
+
+or
+
+```ruby
+rake csv:import:scaffold[users]
+```
+
+## reset_pk_sequence
 
 If you ever run into the error `duplicate key violates unique constraint (id) error`, run this script:
 

data/app/models/effective/csv_importer.rb

@@ -2,22 +2,33 @@ require 'csv'
 
 module Effective
   class CSVImporter
-    attr_reader :current_row, :last_row
+    attr_reader :current_row, :last_row, :csv_file
 
     A=0;B=1;C=2;D=3;E=4;F=5;G=6;H=7;I=8;J=9;K=10;L=11;M=12;N=13;
     O=14;P=15;Q=16;R=17;S=18;T=19;U=20;V=21;W=22;X=23;Y=24;Z=25;
     AA=26;AB=27;AC=28;AD=29;AE=30;AF=31;AG=32;AH=33;AI=34;AJ=35;
     AK=36;AL=37;AM=38;AN=39;AO=40;AP=41;AQ=42;AR=43;AS=44;AT=45;
 
-    def initialize(csv_file, has_header_row = true)
-      @csv_file = csv_file.to_s
-      @has_header_row = has_header_row
+    def initialize(csv_file = default_csv_file(), header: true)
+      @has_header_row = header
+
+      @csv_file = csv_file
+      raise "#{@csv_file} does not exist" unless File.exists?(@csv_file)
     end
 
-    def csv_file
-      @csv_file
+    def columns
+      raise "Please define a method 'def columns' returning a Hash of {id: A, name: B}"
     end
 
+    def process_row
+      raise "Please define a method 'def process_row' to process your row"
+    end
+
+    # Override me if you need some before/after hooks
+    def before_import ; end
+    def after_import ; end
+
+    # This runs through each row and calls process_row() on it
     def import!
       log "Importing #{csv_file.split('/').last.sub('.csv', '')}"
 
@@ -30,37 +41,39 @@ module Effective
       log "Import complete (#{@errors_count} errors in #{@has_header_row ? @current_row_number-1 : @current_row_number} rows)"
     end
 
-    def with_each_row(&block)
-      @current_row_number = (@has_header_row ? 2 : 1)
+    # Returns an Array of Arrays, with each row run through normalize
+    def rows
+      @rows ||= [].tap do |rows|
+        CSV.foreach(csv_file, headers: @has_header_row) do |row|
+          rows << columns.map { |column, index| normalize(column, row[index].try(:strip).presence) }
+        end
+      end
+    end
 
-      CSV.foreach(csv_file, headers: @has_header_row) do |row|
-        @current_row = row
+    # UserStudentInfosImporter.new().where(id: 3, title: 'thing')
+    # Returns an Array of Hashes, representing any row that matches the selector
+    def where(attributes)
+      raise 'expected a Hash of attributes' unless attributes.kind_of?(Hash)
+      attributes.each { |column, _| raise "unknown column :#{column}" unless columns.key?(column) }
 
-        begin
-          yield
-          print colorize('.', :green)
-        rescue => e
-          error(e.message)
-          puts row
-          puts e.backtrace.first(3)
+      rows.map do |row|
+        if attributes.all? { |column, value| row[columns[column]] == value }
+          columns.inject({}) { |retval, (column, index)| retval[column] = row[index]; retval }
         end
-
-        @current_row_number += 1
-        @last_row = row
-      end
+      end.compact
     end
 
-    def columns
-      raise "Please define a method 'def columns' returning a Hash of {id: A, name: B}"
+    def where!(attributes)
+      where(attributes).presence || raise("csv row with #{attributes} not found")
     end
 
-    def process_row
-      raise "Please define a method 'def process_row' to process your row"
+    def find(attributes)
+      where(attributes).first
     end
 
-    # Override me if you need some before/after hooks
-    def before_import ; end
-    def after_import ; end
+    def find!(attributes)
+      find(attributes).presence || raise("csv row with #{attributes} not found")
+    end
 
     # Normalize the value based on column name
     def normalize(column, value)
@@ -73,11 +86,46 @@ module Effective
         parse_datetime(column, value)
       elsif column.ends_with?('_on')  # Date
         parse_datetime(column, value).beginning_of_day
+      elsif column.ends_with?('_to_i')
+        value.to_i
+      elsif column.ends_with?('_to_f')
+        value.to_f
+      elsif column.ends_with?('_to_s')
+        value.to_s
+      elsif column.ends_with?('_to_a')
+        if ['[]', '{}'].include?(value)
+          []
+        elsif value.starts_with?('{') && value.ends_with?('}')
+          YAML::load(value).keys.select { |str| str.to_s.present? }
+        else
+          YAML::load(value).to_a.select { |str| str.to_s.present? }
+        end
+      elsif column == 'id' || column.ends_with?('_id')
+        value.present? ? value.to_i : nil
       else
-        value.presence || ''.freeze
+        value.presence
       end
     end
 
+    # Takes an object and loops through all columns assigning the current row values
+    def assign_columns(obj, only: [], except: [])
+      assigns = (
+        if only.present?
+          only = Array(only)
+          columns.keep_if { |key, _| only.include?(key) }
+        elsif except.present?
+          except = Array(except)
+          columns.delete_if { |key, _| except.include?(key) }
+        end
+      )
+
+      (assigns || columns).each do |column, _|
+        obj.send("#{column}=", col(column)) if obj.respond_to?(column)
+      end
+
+      obj
+    end
+
     def log(message)
       puts "\n#{message}";
     end
@@ -93,6 +141,26 @@ module Effective
 
     protected
 
+    def with_each_row(&block)
+      @current_row_number = (@has_header_row ? 2 : 1)
+
+      CSV.foreach(csv_file, headers: @has_header_row) do |row|
+        @current_row = row
+
+        begin
+          yield
+          print colorize('.', :green)
+        rescue => e
+          error(e.message)
+          puts row
+          puts e.backtrace.first(3)
+        end
+
+        @current_row_number += 1
+        @last_row = row
+      end
+    end
+
     def col(column)
       raise "unknown column :#{column} passed to col()" unless columns.key?(column) || column.kind_of?(Integer)
 
@@ -147,5 +215,11 @@ module Effective
       end
     end
 
+    private
+
+    def default_csv_file
+      ("lib/csv_importers/data/#{self.class.name.gsub('CsvImporters::', '').underscore.gsub('_importer', '')}.csv")
+    end
+
   end
 end

data/lib/effective_developer/version.rb

@@ -1,3 +1,3 @@
 module EffectiveDeveloper
-  VERSION = '0.0.6'.freeze
+  VERSION = '0.0.7'.freeze
 end

data/lib/generators/effective_developer/csv_importer.rb.erb

@@ -0,0 +1,13 @@
+module CsvImporters
+  class <%= klass %>Importer < Effective::CSVImporter
+    def columns
+      {<% columns.each_with_index do |column, index| %>
+        <%= column %>: <%= (letters[index] || index) %><%= ',' unless (index+1) == columns.length %><% end %>
+      }
+    end
+
+    def process_row
+      raise 'todo'
+    end
+  end
+end

data/lib/tasks/effective_csv_importer.rake

@@ -1,6 +1,6 @@
 # effective_csv_importer 1.0
 
-# Creates one rake task per importer model, as well as a `rake import:all` task.
+# Creates one rake task per importer model, as well as a `rake csv:import:all` task.
 
 # Usage:
 
@@ -8,41 +8,92 @@
 # Put your csv data in /lib/csv_importers/data/posts.csv
 # Both filenames should be pluralized
 
-# rake import:posts (one task created per model)
-# rake import:all
+# rake csv:import:posts (one task created per model)
+# rake csv:import:all
+# rake csv:import:scaffold
+# rake csv:import:scaffold[users]
+# rake csv:export
 
-namespace :import do
-  Dir['lib/csv_importers/*.rb'].each do |file|
-    importer = file.sub('lib/csv_importers/', '').sub('_importer.rb', '')
-    csv_file = "lib/csv_importers/data/#{importer}.csv"
-    next unless File.exists?(csv_file)
+namespace :csv do
+  namespace :import do
+    # Create a rake task to import each csv file
+    Dir['lib/csv_importers/*.rb'].each do |file|
+      importer = file.sub('lib/csv_importers/', '').sub('_importer.rb', '')
+      csv_file = "lib/csv_importers/data/#{importer}.csv"
+      next unless File.exists?(csv_file)
 
-    # Create a rake task to import this file
-    desc "Import #{importer} from #{csv_file}"
+      # rake csv:import:foo
+      desc "Import #{importer} from #{csv_file}"
 
-    task importer => :environment do
-      require "#{Rails.application.root}/#{file}"
+      task importer => :environment do
+        require "#{Rails.application.root}/#{file}"
 
-      klass = "CsvImporters::#{importer.classify.pluralize}Importer".safe_constantize
-      raise "unable to constantize CsvImporters::#{importer.classify.pluralize}Importer for #{file}" unless klass
+        klass = "CsvImporters::#{importer.classify.pluralize}Importer".safe_constantize
+        raise "unable to constantize CsvImporters::#{importer.classify.pluralize}Importer for #{file}" unless klass
 
-      klass.new(csv_file).import!
+        klass.new().import!
+      end
+    end
+
+    # rake csv:import:all
+    desc 'Import all from /lib/csv_importers/*.rb'
+
+    task :all => :environment do
+      Dir['lib/csv_importers/*.rb'].each do |file|
+        importer = file.sub('lib/csv_importers/', '').sub('_importer.rb', '')
+        csv_file = "lib/csv_importers/data/#{importer}.csv"
+        next unless File.exists?(csv_file)
+
+        Rake::Task["csv:import:#{importer}"].invoke
+      end
+    end
+
+    # rake csv:scaffold
+    # rake csv:scaffold[users]
+    desc 'Scaffold an Effective::CSVImporter for each /lib/csv_importers/data/*.csv file'
+
+    task :scaffold, [:file_name] => :environment do |t, args|
+      args.with_defaults(file_name: 'all')
+
+      require 'csv'
+
+      generator = ERB.new(File.read(File.dirname(__FILE__) + '/../generators/effective_developer/csv_importer.rb.erb'))
+      letters = ('A'..'AT').to_a
+
+      Dir['lib/csv_importers/data/*.csv'].each do |file|
+        csv_file = file.split('/').last.gsub('.csv', '')
+
+        next if (Array(args.file_name) != ['all'] && Array(args.file_name).include?(csv_file) == false)
+
+        klass = csv_file.classify.pluralize
+        columns = CSV.open(file, 'r') { |csv| csv.first }
+
+        File.open("#{Rails.root}/lib/csv_importers/#{csv_file}_importer.rb", 'w') do |file|
+          file.write generator.result(binding)
+        end
+      end
     end
   end
-end
 
-# This task is kind of naive, because some imports could be order dependent. Use at your own risk.
-namespace :import do
-  desc "Import all from /lib/csv_importers/*.rb"
+  # rake csv:export
+  desc 'Export all database tables to /tmp/csv_exports/*.csv'
 
-  task :all => :environment do
-    Dir['lib/csv_importers/*.rb'].each do |file|
-      importer = file.sub('lib/csv_importers/', '').sub('_importer.rb', '')
-      csv_file = "lib/csv_importers/data/#{importer}.csv"
-      next unless File.exists?(csv_file)
+  task :export => :environment do
+    require 'csv'
 
-      Rake::Task["import:#{importer}"].invoke
+    path = Rails.root.to_s + '/tmp/csv_exports/'
+    FileUtils.mkdir_p(path) unless File.directory?(path)
+
+    (ActiveRecord::Base.connection.tables - ['schema_migrations']).each do |table|
+      records = ActiveRecord::Base.connection.exec_query("SELECT * FROM #{table} ORDER BY id")
+
+      CSV.open(path + "#{table}.csv", 'wb') do |csv|
+        csv << records.columns
+        records.rows.each { |row| csv << row }
+      end
     end
+
+    puts "Successfully csv exported #{ActiveRecord::Base.connection.tables.length} tables to #{path}"
   end
 
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: effective_developer
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.0.7
 platform: ruby
 authors:
 - Code and Effect
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-28 00:00:00.000000000 Z
+date: 2016-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rails
@@ -40,6 +40,7 @@ files:
 - lib/effective_developer.rb
 - lib/effective_developer/engine.rb
 - lib/effective_developer/version.rb
+- lib/generators/effective_developer/csv_importer.rb.erb
 - lib/generators/effective_developer/install_generator.rb
 - lib/tasks/effective_csv_importer.rake
 - lib/tasks/pg_pull.rake