datashift 0.16.0 → 0.40.0

data/lib/thor/import.thor

@@ -1,110 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
-# Author ::   Tom Statter
-# Date ::     Mar 2012
-# License::   MIT.
-#
-# Usage::
-#
-#  To pull Datashift commands into your main application :
-#
-#     require 'datashift'
-#
-#     DataShift::load_commands
-#
-#  N.B Requires JRuby
-#
-# => bundle exec thor datashift:import:excel -m <active record class> -i <output_template.xls> -a
-#
-
-require 'datashift'
-  
-
-# Note, not DataShift, case sensitive, create namespace for command line : datashift
-module Datashift
-
-          
-  class Import < Thor     
-  
-    include DataShift::Logging
-      
-    desc "excel", "import .xls file for specifiec active record model" 
-    method_option :model, :aliases => '-m', :required => true, :desc => "The related active record model"
-    method_option :input, :aliases => '-i', :required => true, :desc => "The input .xls file"
-    method_option :config, :aliases => '-c', :desc => "YAML config file with defaults, over-rides etc"
-    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include any associations supplied in the input"
-    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
-    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
-     
-    def excel()
-     
-      # TODO - We're assuming run from a rails app/top level dir...
-      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
-      require File.expand_path('config/environment.rb')
-   
-      require 'excel_loader'
-
-      model = options[:model]
-
-      klass = DataShift::ModelMapper.class_from_string_or_raise( model )
-
-      loader_options = { :instance_methods => true }
-
-      if(options[:loader])
-        begin
-     
-          loader_klass = DataShift::ModelMapper::class_from_string(options[:loader])
-
-          loader = loader_klass.new(klass)
-
-          logger.info("INFO: Using loader : #{loader.class}")
-        rescue
-          logger.error("INFO: No specific #{model}Loader found  - using generic ExcelLoader")
-          loader = DataShift::ExcelLoader.new(klass, nil, loader_options)
-        end
-      else
-        logger.info("No Loader specified - using generic ExcelLoader")
-        loader = DataShift::ExcelLoader.new(klass, nil, loader_options)
-      end
-
-      #TOFIX - multi loggers to file + STDOUT
-      # loader.logger.verbose if(options['verbose'])
-      
-      loader.configure_from( options[:config] ) if(options[:config])
-
-      loader.perform_load(options[:input])
-    end
-    
-    desc "csv", "import CSV file for specified active record model" 
-    method_option :model, :aliases => '-m', :required => true, :desc => "The related active record model"
-    method_option :input, :aliases => '-i', :required => true, :desc => "The input .xls file"
-    method_option :config, :aliases => '-c', :desc => "YAML config file with defaults, over-rides etc"
-    method_option :assoc, :aliases => '-a', :type => :boolean, :desc => "Include any associations supplied in the input"
-    method_option :exclude, :aliases => '-e',  :type => :array, :desc => "Use with -a : Exclude association types. Any from #{DataShift::MethodDetail::supported_types_enum.to_a.inspect}"
-    method_option :verbose, :aliases => '-v', :type => :boolean, :desc => "Verbose logging"
-
-    def csv()
-     
-      # TODO - We're assuming run from a rails app/top level dir...
-      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
-      require File.expand_path('config/environment.rb')
-   
-      require 'csv_loader'
-
-      model = options[:model]
-
-      klass = DataShift::ModelMapper.class_from_string_or_raise( model )
-
-      loader = DataShift::CsvLoader.new(klass)
-
-      #TOFIX - multi loggers to file + STDOUT
-      # loader.logger.verbose if(options['verbose'])
-      
-      loader.configure_from( options[:config] ) if(options[:config])
-
-      loader.perform_load(options[:input])
-    end
-    
-  end
-
-end
-

data/lib/thor/mapping.thor

@@ -1,65 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2015
-# Author ::   Tom Statter
-# Date ::     Mar 2015
-# License::   MIT.
-#
-# Usage::
-#
-#  To pull Datashift commands into your main application :
-#
-#     require 'datashift'
-#
-#     DataShift::load_commands
-#
-require 'datashift'
-
-# Note, not DataShift, case sensitive, create namespace for command line : datashift
-module Datashift
-
-
-  class Mapping < Thor
-
-    include DataShift::Logging
-
-    desc "template", "Generate a simple mappings template\nInput is treated as the *source* unless otherwise directed"
-
-    method_option :model, :aliases => '-m', :desc => "The active record model to use for mappings"
-    method_option :model_as_dest, :aliases => '-d', :type=> :boolean, :desc => "Set model attributes as destination"
-
-    method_option :excel, :aliases => '-e', :desc => "The excel spreadsheet to use for mappings"
-    method_option :excel_as_dest, :aliases => '-a', :type=> :boolean, :desc => "Set excel headers as destination"
-
-    method_option :result, :aliases => '-r', :required => true, :desc => "Create template of model in supplied file"
-
-    def template()
-
-      # TODO - We're assuming run from a rails app/top level dir...
-      # ...can we make this more robust ? e.g what about when using active record but not in Rails app, 
-      require File.expand_path('config/environment.rb')
-
-      result = options[:result]
-
-      if(File.directory?(result))
-        result = File.join(result, "mapping_template.yaml")
-        puts "Output generated in #{result}"
-      end
-
-      logger.info "Datashift: Starting mapping template generation in #{result}"
-
-      mapper = DataShift::MappingGenerator.new(result)
-
-      model = options[:model]
-
-      mappings = String.new
-
-      mappings += mapper.generate(model, options) unless(model.nil? && options[:excel])
-
-      mappings += mapper.generate_from_excel(options[:excel], options) if(options[:excel])
-
-      File.open(result, 'w') { |f| f << mappings }
-
-    end
-
-  end
-
-end

data/lib/thor/tools.thor

@@ -1,84 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
-# Author ::   Tom Statter
-# Date ::     Sept 2012
-# License::   MIT.
-#
-# Usage::
-#
-#  To pull Datashift commands into your main application :
-#
-#     require 'datashift'
-#
-#     DataShift::load_commands
-#
-require 'datashift'
-  
-# Note, not DataShift, case sensitive, create namespace for command line : datashift
-module Datashift        
-  class Tools < Thor     
-  
-    include DataShift::Logging
-      
-    desc "zip", "Create zip of files" 
-
-    method_option :path, :aliases => '-p', :required => true, :desc => "The path to the digital files"
-    method_option :output, :aliases => '-o', :required => true, :desc => "The resulting zip file name"
- 
-    def zip()
-     
-      require 'zip/zip'
-      require 'zip/zipfilesystem'
-
-      output = options[:output]
-
-      Zip::ZipOutputStream.open(output) do |zos|
-        Dir[File.join(options[:path], '**', '*.*')].each do |p|
-          zos.put_next_entry(File.basename(p))
-          zos.print IO.read(p)
-        end
-      end
-    end
-    
-    desc "zip_matching", "Create zip of matching digital files e.g zip up pdf, jpg and png versions of a file" 
-
-    method_option :path, :aliases => '-p', :required => true, :desc => "The path to the digital files"
-    method_option :results, :aliases => '-r', :required => true, :desc => "The path to store resulting zip files"
- 
-    def zip_matching()
-     
-      require 'zip/zip'
-      require 'zip/zipfilesystem'
-
-      ready_to_zip = {}
-      Dir[File.join(options[:path], '**', '*.*')].each do |p|
-        next if File.directory? p
-        
-        basename = File.basename(p, '.*')
-        ready_to_zip[basename] ||= []       
-        ready_to_zip[basename] << p  
-      end 
-      
-      output = options[:results]
-      
-      FileUtils::mkdir_p(output) unless File.exists?(output)
-      
-      puts "Creating #{ready_to_zip.keys.size} new zips"
-      ready_to_zip.each do |basename, paths|
-      
-        z= File.join(output, basename + '.zip')
-        puts "zipping to #{z}"
-        
-        Zip::ZipOutputStream.open(z) do |zos|
-          paths.each do |file|
-            zos.put_next_entry(File.basename(file))
-            zos.print IO.read(file)
-          end
-        end
-      end
-      
-    end   
-    
-  end
-
-end
-

data/spec/Gemfile

@@ -1,31 +0,0 @@
-source 'https://rubygems.org'
-
-
-# DEFINE WHICH VERSIONS ON VITAL GEMS WE WANT TO TEST WITH
-
-gem 'rails', '4.1.8'
-
-gem 'spreadsheet', '1.0.0'
-gem 'paperclip'
-gem 'activesupport'
-
-gem 'database_cleaner'
-
-gem 'factory_girl_rails'
-
-gem 'rspec'
-gem 'rspec-core'
-gem 'rspec-expectations' # RSpec matchers for should and should_not.
-gem 'rspec-mocks'        # RSpec test double framework with stubbing and mocking.
-gem 'rspec-rails'       # RSpec version 2.x for Rails version 3.x.
-
-# we need both, for JRuby testing of Excel and non JRuby csv
-platform :jruby do
-  gem 'jruby-openssl'
-  gem 'activerecord-jdbcsqlite3-adapter'
-end
-
-platform :ruby do
-  gem 'sqlite3'
-end
-

data/spec/Gemfile.lock

@@ -1,134 +0,0 @@
-GEM
-  remote: https://rubygems.org/
-  specs:
-    actionmailer (4.1.8)
-      actionpack (= 4.1.8)
-      actionview (= 4.1.8)
-      mail (~> 2.5, >= 2.5.4)
-    actionpack (4.1.8)
-      actionview (= 4.1.8)
-      activesupport (= 4.1.8)
-      rack (~> 1.5.2)
-      rack-test (~> 0.6.2)
-    actionview (4.1.8)
-      activesupport (= 4.1.8)
-      builder (~> 3.1)
-      erubis (~> 2.7.0)
-    activemodel (4.1.8)
-      activesupport (= 4.1.8)
-      builder (~> 3.1)
-    activerecord (4.1.8)
-      activemodel (= 4.1.8)
-      activesupport (= 4.1.8)
-      arel (~> 5.0.0)
-    activesupport (4.1.8)
-      i18n (~> 0.6, >= 0.6.9)
-      json (~> 1.7, >= 1.7.7)
-      minitest (~> 5.1)
-      thread_safe (~> 0.1)
-      tzinfo (~> 1.1)
-    arel (5.0.1.20140414130214)
-    builder (3.2.2)
-    climate_control (0.0.3)
-      activesupport (>= 3.0)
-    cocaine (0.5.8)
-      climate_control (>= 0.0.3, < 1.0)
-    concurrent-ruby (1.0.1)
-    database_cleaner (1.5.1)
-    diff-lcs (1.2.5)
-    erubis (2.7.0)
-    factory_girl (4.5.0)
-      activesupport (>= 3.0.0)
-    factory_girl_rails (4.6.0)
-      factory_girl (~> 4.5.0)
-      railties (>= 3.0.0)
-    i18n (0.7.0)
-    json (1.8.3)
-    mail (2.6.3)
-      mime-types (>= 1.16, < 3)
-    mime-types (2.99.1)
-    mimemagic (0.3.0)
-    minitest (5.8.4)
-    paperclip (4.3.5)
-      activemodel (>= 3.2.0)
-      activesupport (>= 3.2.0)
-      cocaine (~> 0.5.5)
-      mime-types
-      mimemagic (= 0.3.0)
-    rack (1.5.5)
-    rack-test (0.6.3)
-      rack (>= 1.0)
-    rails (4.1.8)
-      actionmailer (= 4.1.8)
-      actionpack (= 4.1.8)
-      actionview (= 4.1.8)
-      activemodel (= 4.1.8)
-      activerecord (= 4.1.8)
-      activesupport (= 4.1.8)
-      bundler (>= 1.3.0, < 2.0)
-      railties (= 4.1.8)
-      sprockets-rails (~> 2.0)
-    railties (4.1.8)
-      actionpack (= 4.1.8)
-      activesupport (= 4.1.8)
-      rake (>= 0.8.7)
-      thor (>= 0.18.1, < 2.0)
-    rake (11.0.1)
-    rspec (3.4.0)
-      rspec-core (~> 3.4.0)
-      rspec-expectations (~> 3.4.0)
-      rspec-mocks (~> 3.4.0)
-    rspec-core (3.4.4)
-      rspec-support (~> 3.4.0)
-    rspec-expectations (3.4.0)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-mocks (3.4.1)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.4.0)
-    rspec-rails (3.4.2)
-      actionpack (>= 3.0, < 4.3)
-      activesupport (>= 3.0, < 4.3)
-      railties (>= 3.0, < 4.3)
-      rspec-core (~> 3.4.0)
-      rspec-expectations (~> 3.4.0)
-      rspec-mocks (~> 3.4.0)
-      rspec-support (~> 3.4.0)
-    rspec-support (3.4.1)
-    ruby-ole (1.2.12)
-    spreadsheet (1.0.0)
-      ruby-ole (>= 1.0)
-    sprockets (3.5.2)
-      concurrent-ruby (~> 1.0)
-      rack (> 1, < 3)
-    sprockets-rails (2.3.3)
-      actionpack (>= 3.0)
-      activesupport (>= 3.0)
-      sprockets (>= 2.8, < 4.0)
-    sqlite3 (1.3.11)
-    thor (0.19.1)
-    thread_safe (0.3.5)
-    tzinfo (1.2.2)
-      thread_safe (~> 0.1)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  activerecord-jdbcsqlite3-adapter
-  activesupport
-  database_cleaner
-  factory_girl_rails
-  jruby-openssl
-  paperclip
-  rails (= 4.1.8)
-  rspec
-  rspec-core
-  rspec-expectations
-  rspec-mocks
-  rspec-rails
-  spreadsheet (= 1.0.0)
-  sqlite3
-
-BUNDLED WITH
-   1.11.2

data/spec/csv_exporter_spec.rb

@@ -1,144 +0,0 @@
-# Copyright:: (c) Autotelik Media Ltd 2012
-# Author ::   Tom Statter
-# Date ::     Sept 2012
-# License::   MIT
-#
-# Details::   Specs for CSV aspect of export
-#
-require File.dirname(__FILE__) + '/spec_helper'
-
-require 'erb'
-require 'csv_exporter'
-
-describe 'CSV Exporter' do
-
-  before(:all) do
-    results_clear( "*.csv" )
-  end
-
-  before(:each) do
-    DataShift::MethodDictionary.clear
-    DataShift::MethodDictionary.find_operators( Project )
-
-    db_clear()    # todo read up about proper transactional fixtures
-  end
-
-  context 'simple project' do
-
-    before(:each) do
-      create( :project )
-    end
-
-    it "should be able to create a new CSV exporter" do
-      exporter = DataShift::CsvExporter.new( 'exp_rspec_csv_empty.csv' )
-
-      expect(exporter).not_to be_nil
-    end
-
-    it "should throw if not active record objects" do
-      exporter = DataShift::CsvExporter.new( 'exp_rspec_csv_empty.csv' )
-
-      expect{ exporter.export([123.45]) }.to raise_error(ArgumentError)
-    end
-
-
-    it "should export collection of model objects to .xls file" do
-
-      expected = result_file('exp_project_collection_spec.csv')
-
-      exporter = DataShift::CsvExporter.new( expected )
-
-      count = Project.count
-
-      Project.create( :value_as_string	=> 'Value as String', :value_as_boolean => true,	:value_as_double => 75.672)
-
-      Project.count.should == count + 1
-
-      exporter.export(Project.all)
-
-      expect(File.exists?(expected)).to eq true
-
-      puts "Can manually check file @ #{expected}"
-
-      File.foreach(expected) {}
-      count = $.
-      count.should == Project.count + 1
-    end
-
-    it "should handle bad params to export" do
-
-      expected = result_file('project_first_export_spec.csv')
-
-      exporter = DataShift::CsvExporter.new( expected )
-
-      expect{ exporter.export(nil) }.not_to raise_error
-
-      expect{ exporter.export([]) }.not_to raise_error
-
-      puts "Can manually check file @ #{expected}"
-    end
-
-    it "should export a model object to csv file" do
-
-      expected = result_file('project_first_export_spec.csv')
-
-      exporter = DataShift::CsvExporter.new( expected )
-
-      exporter.export(Project.all[0])
-
-      expect(File.exists?(expected)).to eq true
-
-      puts "Can manually check file @ #{expected}"
-    end
-
-    it "should export a model and result of method calls on it to csv file" do
-
-      expected = result_file('project_with_methods_export_spec.csv')
-
-      exporter = DataShift::CsvExporter.new( expected )
-
-      exporter.export(Project.all, {:methods => [:multiply]})
-
-      expect(File.exists?(expected)).to eq true
-
-      puts "Can manually check file @ #{expected}"
-
-      File.foreach(expected) {}
-      count = $.
-      count.should == Project.count + 1
-    end
-
-  end
-
-  it "should export a model and associations to csv" do
-
-    create( :project_user )
-    create_list(:project, 7)
-
-    expected = result_file('exp_project_plus_assoc_export_spec.csv')
-
-    gen = DataShift::CsvExporter.new(expected)
-
-    items = Project.all
-
-    gen.export_with_associations(Project, items)
-
-    File.foreach(expected) {}
-    count = $.
-    count.should == items.size + 1
-
-    expect(File.exists?(expected)).to eq true
-
-    csv = CSV.read(expected)
-
-    expect(csv[0]).to include 'owner'
-    expect(csv[0]).to include 'user'
-
-    user_inx = csv[0].index 'user'
-
-    expect(user_inx).to be > -1
-
-    expect( csv[1][user_inx] ).to include 'mr'
-  end
-
-end