datashift 0.13.0 → 0.14.0

data/lib/thor/export.thor

@@ -106,6 +106,71 @@ module Datashift
       end
 
     end
+    
+    desc "db", "Export every Active Record model" 
+
+    method_option :result, :aliases => '-r', :required => true, :desc => "Path in which to create excel files"
+    method_option :csv, :aliases => '-c', :desc => "Export to CSV instead - Excel is default."
+    method_option :prefix, :aliases => '-p', :desc => "For namespaced tables/models specify the table prefix e.g spree_"
+    method_option :module, :aliases => '-m', :desc => "For namespaced tables/models specify the Module name e.g Spree"
+    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
+    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
+ 
+    def db()
+     
+      require File.expand_path('config/environment.rb')
+      
+      require 'excel_exporter'
+      require 'csv_exporter'
+        
+      exporter = options[:csv] ?  DataShift::CsvExporter.new(nil) :  DataShift::ExcelExporter.new(nil)
+      
+      ext = options[:csv] ? '.csv' : '.xls'
+      
+      # Hmmm not many models appear - Rails uses autoload !
+      #ActiveRecord::Base.send(:subclasses).each do |model|
+       # puts model.name
+      #end
+       
+      parent = options[:module] ? Object.const_get(options[:module]) : Object
+      
+      ActiveRecord::Base.connection.tables.each do |table|
+            
+        table.sub!(options[:prefix],'') if(options[:prefix])
+
+        @result = File.join(options[:result], "#{table}#{ext}")
+        
+        begin      
+          @klass = parent.const_get(table.classify)    
+        rescue => e
+          puts e.inspect
+          puts "WARNING: Could not find an AR model for Table #{table}"
+          next
+        end
+
+        puts "Datashift: Start export to #{@result}"
+         
+        exporter.filename = @result
+        
+        raise "ERROR: No such Model [#{@klass}] found - check valid model supplied via -model <Class>" if(@klass.nil?)
+
+        begin
+          
+          if(options[:assoc])
+            opts = (options[:exclude]) ? {:exclude => options[:exclude]} : {}
+            logger.info("Datashift: Exporting with associations")
+            exporter.export_with_associations(@klass, @klass.all, opts)
+          else
+            exporter.export(@klass.all, :sheet_name => @klass.name)
+          end
+        rescue => e
+          puts e
+          puts e.backtrace
+          puts "Warning: Error during export, data may be incomplete"
+        end
+      end
+    end
+    
   end
 
 end

data/lib/thor/generate.thor

@@ -41,8 +41,6 @@ module Datashift
       # TODO - We're assuming run from a rails app/top level dir...
       # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
       require File.expand_path('config/environment.rb')
-   
-
 
       model = options[:model]
       result = options[:result]
@@ -63,7 +61,7 @@ module Datashift
         gen = DataShift::ExcelGenerator.new(result)
 
         opts =  { :remove => options[:remove],
-                  :remove_rails => options[:remove_rails]
+          :remove_rails => options[:remove_rails]
         }
           
         if(options[:assoc])
@@ -88,7 +86,7 @@ module Datashift
     method_option :model, :aliases => '-m', :required => true, :desc => "The active record model to export"
     method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
     method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
-    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
+    method_option :exclude, :aliases => '-x',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
     
     def csv()
      
@@ -130,6 +128,72 @@ module Datashift
       end
 
     end
+    
+    desc "db", "Generate a template for every Active Record model" 
+
+    method_option :result, :aliases => '-r', :required => true, :desc => "Path in which to create excel files"
+    method_option :csv, :aliases => '-c', :desc => "Export to CSV instead - Excel is default."
+    method_option :prefix, :aliases => '-p', :desc => "For namespaced tables/models specify the table prefix e.g spree_"
+    method_option :module, :aliases => '-m', :desc => "For namespaced tables/models specify the Module name e.g Spree"
+    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include all associations in the template"
+    method_option :exclude, :aliases => '-x',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
+    method_option :remove, :aliases => '-e',  :type => :array, :desc => "Don't generate the user supplied fields"
+    method_option :remove_rails, :type => :boolean, :desc => "Don't generate the standard Rails fields: #{DataShift::GeneratorBase::rails_columns.inspect}"
+     
+    def db()
+     
+      require File.expand_path('config/environment.rb')
+      
+      require 'excel_exporter'
+      require 'csv_exporter'
+        
+      exporter = options[:csv] ?  DataShift::CsvGenerator.new(nil) :  DataShift::ExcelGenerator.new(nil)
+      
+      ext = options[:csv] ? '.csv' : '.xls'
+       
+      parent = options[:module] ? Object.const_get(options[:module]) : Object
+      
+      ActiveRecord::Base.connection.tables.each do |table|
+            
+        table.sub!(options[:prefix],'') if(options[:prefix])
+
+        @result = File.join(options[:result], "#{table}#{ext}")
+        
+        begin      
+          @klass = parent.const_get(table.classify)    
+        rescue => e
+          puts e.inspect
+          puts "WARNING: Could not find an AR model for Table #{table}"
+          next
+        end
+
+        puts "Datashift: Start template generation to #{@result}" 
+        
+        raise "ERROR: No such Model [#{@klass}] found - check valid model supplied via -model <Class>" if(@klass.nil?)
+
+        begin
+          opts =  { :filename => @result,
+                    :remove => options[:remove],
+                    :remove_rails => options[:remove_rails],
+                    :sheet_name => @klass.name
+          }
+          
+          if(options[:assoc])   
+            opts[:exclude] = options[:exclude]
+            logger.info("Datashift: Generating with associations")
+            exporter.generate_with_associations(@klass, opts)
+          else
+            exporter.generate(@klass, opts)
+          end
+        rescue => e
+          puts e
+          puts e.backtrace
+          puts "Warning: Error during export, data may be incomplete"
+        end
+      end
+    end
+    
+    
   end
 
 end

data/spec/Gemfile

@@ -1,11 +1,22 @@
 source 'https://rubygems.org'
 
+       
+# DEFINE WHICH VERSIONS ON VITAL GEMS WE WANT TO TEST WITH
+
+gem 'rails', '4.0.4'
+
+gem 'spreadsheet'
+gem 'paperclip'
+
+
 gem 'rspec'  # Behavior Driven Development (BDD) for Ruby
 gem 'rspec-core'  # RSpec runner and example groups.
 gem 'rspec-expectations'  # RSpec matchers for should and should_not.
 gem 'rspec-mocks'  # RSpec test double framework with stubbing and mocking.
 gem 'rspec-rails'  # RSpec version 2.x for Rails version 3.x.
 
+gem 'sqlite3'
+
 # we need both, for JRuby testing of Excel and non JRuby csv
 platform :jruby do
   gem 'jruby-openssl'
@@ -13,13 +24,6 @@ platform :jruby do
 end
 
 platform :ruby do 
-  gem 'sqlite3'
+  #gem 'activerecord-sqlite3-adapter'
 end
-       
-# DEFINE WHICH VERSIONS WE WANT TO TEST WITH
-
-gem 'rails', '3.2.8'
-
-gem 'spreadsheet'
-gem 'paperclip'
 

data/spec/Gemfile.lock

@@ -1,115 +1,115 @@
 GEM
   remote: https://rubygems.org/
   specs:
-    actionmailer (3.2.8)
-      actionpack (= 3.2.8)
-      mail (~> 2.4.4)
-    actionpack (3.2.8)
-      activemodel (= 3.2.8)
-      activesupport (= 3.2.8)
-      builder (~> 3.0.0)
+    actionmailer (4.0.4)
+      actionpack (= 4.0.4)
+      mail (~> 2.5.4)
+    actionpack (4.0.4)
+      activesupport (= 4.0.4)
+      builder (~> 3.1.0)
       erubis (~> 2.7.0)
-      journey (~> 1.0.4)
-      rack (~> 1.4.0)
-      rack-cache (~> 1.2)
-      rack-test (~> 0.6.1)
-      sprockets (~> 2.1.3)
-    activemodel (3.2.8)
-      activesupport (= 3.2.8)
-      builder (~> 3.0.0)
-    activerecord (3.2.8)
-      activemodel (= 3.2.8)
-      activesupport (= 3.2.8)
-      arel (~> 3.0.2)
-      tzinfo (~> 0.3.29)
-    activerecord-jdbc-adapter (1.2.2)
-    activerecord-jdbcsqlite3-adapter (1.2.2)
-      activerecord-jdbc-adapter (~> 1.2.2)
+      rack (~> 1.5.2)
+      rack-test (~> 0.6.2)
+    activemodel (4.0.4)
+      activesupport (= 4.0.4)
+      builder (~> 3.1.0)
+    activerecord (4.0.4)
+      activemodel (= 4.0.4)
+      activerecord-deprecated_finders (~> 1.0.2)
+      activesupport (= 4.0.4)
+      arel (~> 4.0.0)
+    activerecord-deprecated_finders (1.0.3)
+    activerecord-jdbc-adapter (1.3.7)
+      activerecord (>= 2.2)
+    activerecord-jdbcsqlite3-adapter (1.3.7)
+      activerecord-jdbc-adapter (~> 1.3.7)
       jdbc-sqlite3 (~> 3.7.2)
-    activeresource (3.2.8)
-      activemodel (= 3.2.8)
-      activesupport (= 3.2.8)
-    activesupport (3.2.8)
-      i18n (~> 0.6)
-      multi_json (~> 1.0)
-    arel (3.0.2)
-    bouncy-castle-java (1.5.0146.1)
-    builder (3.0.3)
-    cocaine (0.3.1)
-    diff-lcs (1.1.3)
+    activesupport (4.0.4)
+      i18n (~> 0.6, >= 0.6.9)
+      minitest (~> 4.2)
+      multi_json (~> 1.3)
+      thread_safe (~> 0.1)
+      tzinfo (~> 0.3.37)
+    arel (4.0.2)
+    bouncy-castle-java (1.5.0147)
+    builder (3.1.4)
+    climate_control (0.0.3)
+      activesupport (>= 3.0)
+    cocaine (0.5.4)
+      climate_control (>= 0.0.3, < 1.0)
+    diff-lcs (1.2.5)
     erubis (2.7.0)
-    hike (1.2.1)
-    i18n (0.6.1)
-    jdbc-sqlite3 (3.7.2)
-    journey (1.0.4)
-    jruby-openssl (0.7.7)
-      bouncy-castle-java (>= 1.5.0146.1)
-    json (1.7.5)
-    json (1.7.5-java)
-    mail (2.4.4)
-      i18n (>= 0.4.0)
+    hike (1.2.3)
+    i18n (0.6.9)
+    jdbc-sqlite3 (3.7.2.1)
+    jruby-openssl (0.9.4)
+      bouncy-castle-java (>= 1.5.0147)
+    mail (2.5.4)
       mime-types (~> 1.16)
       treetop (~> 1.4.8)
-    mime-types (1.19)
-    multi_json (1.3.6)
-    paperclip (2.7.0)
-      activerecord (>= 2.3.0)
-      activesupport (>= 2.3.2)
-      cocaine (>= 0.0.2)
+    mime-types (1.25.1)
+    minitest (4.7.5)
+    multi_json (1.10.0)
+    paperclip (4.1.1)
+      activemodel (>= 3.0.0)
+      activesupport (>= 3.0.0)
+      cocaine (~> 0.5.3)
       mime-types
-    polyglot (0.3.3)
-    rack (1.4.1)
-    rack-cache (1.2)
-      rack (>= 0.4)
-    rack-ssl (1.3.2)
-      rack
-    rack-test (0.6.1)
+    polyglot (0.3.4)
+    rack (1.5.2)
+    rack-test (0.6.2)
       rack (>= 1.0)
-    rails (3.2.8)
-      actionmailer (= 3.2.8)
-      actionpack (= 3.2.8)
-      activerecord (= 3.2.8)
-      activeresource (= 3.2.8)
-      activesupport (= 3.2.8)
-      bundler (~> 1.0)
-      railties (= 3.2.8)
-    railties (3.2.8)
-      actionpack (= 3.2.8)
-      activesupport (= 3.2.8)
-      rack-ssl (~> 1.3.2)
+    rails (4.0.4)
+      actionmailer (= 4.0.4)
+      actionpack (= 4.0.4)
+      activerecord (= 4.0.4)
+      activesupport (= 4.0.4)
+      bundler (>= 1.3.0, < 2.0)
+      railties (= 4.0.4)
+      sprockets-rails (~> 2.0.0)
+    railties (4.0.4)
+      actionpack (= 4.0.4)
+      activesupport (= 4.0.4)
       rake (>= 0.8.7)
-      rdoc (~> 3.4)
-      thor (>= 0.14.6, < 2.0)
-    rake (0.9.2.2)
-    rdoc (3.12)
-      json (~> 1.4)
-    rspec (2.11.0)
-      rspec-core (~> 2.11.0)
-      rspec-expectations (~> 2.11.0)
-      rspec-mocks (~> 2.11.0)
-    rspec-core (2.11.1)
-    rspec-expectations (2.11.3)
-      diff-lcs (~> 1.1.3)
-    rspec-mocks (2.11.3)
-    rspec-rails (2.11.0)
+      thor (>= 0.18.1, < 2.0)
+    rake (10.3.2)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.8)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.6)
+    rspec-rails (2.14.2)
       actionpack (>= 3.0)
+      activemodel (>= 3.0)
       activesupport (>= 3.0)
       railties (>= 3.0)
-      rspec (~> 2.11.0)
-    ruby-ole (1.2.11.4)
-    spreadsheet (0.7.3)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    ruby-ole (1.2.11.7)
+    spreadsheet (0.9.7)
       ruby-ole (>= 1.0)
-    sprockets (2.1.3)
+    sprockets (2.12.1)
       hike (~> 1.2)
+      multi_json (~> 1.0)
       rack (~> 1.0)
       tilt (~> 1.1, != 1.3.0)
-    sqlite3 (1.3.6)
-    thor (0.16.0)
-    tilt (1.3.3)
-    treetop (1.4.10)
+    sprockets-rails (2.0.1)
+      actionpack (>= 3.0)
+      activesupport (>= 3.0)
+      sprockets (~> 2.8)
+    sqlite3 (1.3.9)
+    thor (0.19.1)
+    thread_safe (0.3.3)
+    thread_safe (0.3.3-java)
+    tilt (1.4.1)
+    treetop (1.4.15)
       polyglot
       polyglot (>= 0.3.1)
-    tzinfo (0.3.33)
+    tzinfo (0.3.39)
 
 PLATFORMS
   java
@@ -119,7 +119,7 @@ DEPENDENCIES
   activerecord-jdbcsqlite3-adapter
   jruby-openssl
   paperclip
-  rails (= 3.2.8)
+  rails (= 4.0.4)
   rspec
   rspec-core
   rspec-expectations

data/spec/csv_exporter_spec.rb

@@ -13,7 +13,7 @@ require 'csv_exporter'
 describe 'CSV Exporter' do
 
   before(:all) do
-    
+      
     # load our test model definitions - Project etc  
     require ifixture_file('test_model_defs')  
   
@@ -26,26 +26,41 @@ describe 'CSV Exporter' do
   end
   
   before(:each) do
-    MethodDictionary.clear
-    MethodDictionary.find_operators( Project )
+    DataShift::MethodDictionary.clear
+    DataShift::MethodDictionary.find_operators( Project )
     
     db_clear()    # todo read up about proper transactional fixtures
+    
+    Project.create( :value_as_string	=> 'Value as String', :value_as_boolean => true,	:value_as_double => 75.672)
+    Project.create( :value_as_string	=> 'Another Value as String', :value_as_boolean => false,	:value_as_double => 12)
+     
   end
   
   it "should be able to create a new CSV exporter" do
-    generator = CsvExporter.new( 'rspec_csv_empty.csv' )
+    exporter = DataShift::CsvExporter.new( 'rspec_csv_empty.csv' )
+      
+    exporter.should_not be_nil
+  end
+
+  it "should throw if not active record objects" do
+    exporter = DataShift::CsvExporter.new( 'rspec_csv_empty.csv' )
       
-    generator.should_not be_nil
+    expect{ exporter.export([123.45]) }.to raise_error(ArgumentError)
   end
   
-  it "should export a model to csv file" do
+
+  it "should export collection of model objects to .xls file" do
 
     expect = result_file('project_export_spec.csv')
 
-    exporter = CsvExporter.new( expect )
+    exporter = DataShift::CsvExporter.new( expect )
      
+    count = Project.count 
+    
     Project.create( :value_as_string	=> 'Value as String', :value_as_boolean => true,	:value_as_double => 75.672)
      
+    Project.count.should == count + 1
+    
     exporter.export(Project.all)
  
     File.exists?(expect).should be_true
@@ -57,14 +72,37 @@ describe 'CSV Exporter' do
     count.should == Project.count + 1
   end
 
+  it "should handle bad params to export" do
+
+    expect = result_file('project_first_export_spec.csv')
+
+    exporter = DataShift::CsvExporter.new( expect )
+    
+    expect{ exporter.export(nil) }.not_to raise_error
+
+    expect{ exporter.export([]) }.not_to raise_error
+   
+    puts "Can manually check file @ #{expect}"
+  end
+  
+  it "should export a model object to csv file" do
+
+    expect = result_file('project_first_export_spec.csv')
+
+    exporter = DataShift::CsvExporter.new( expect )
+    
+    exporter.export(Project.all[0])
+ 
+    File.exists?(expect).should be_true
+      
+    puts "Can manually check file @ #{expect}"
+  end
+
   it "should export a model and result of method calls on it to csv file" do
 
     expect = result_file('project_with_methods_export_spec.csv')
 
-    exporter = CsvExporter.new( expect )
-     
-    Project.create( :value_as_string	=> 'Value as String', :value_as_boolean => true,	:value_as_double => 75.672)
-    Project.create( :value_as_string	=> 'Another Value as String', :value_as_boolean => false,	:value_as_double => 12)
+    exporter = DataShift::CsvExporter.new( expect )
      
     exporter.export(Project.all, {:methods => [:multiply]})
  
@@ -85,7 +123,7 @@ describe 'CSV Exporter' do
     
     expect= result_file('project_plus_assoc_export_spec.csv')
 
-    gen = CsvExporter.new(expect)
+    gen = DataShift::CsvExporter.new(expect)
 
     gen.export_with_associations(Project, Project.all)