data_miner 3.0.0.rc2 → 3.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: d3203cf46ab37ec16f8d9fdd4e51d379ce09f442
+  data.tar.gz: 426543897bff21b3fde15eaac09fb15ff0ae5666
+SHA512:
+  metadata.gz: 75e892e73e41479a63f042f6c3b5e1a8513d926b3ea80703ea1fd92e5c8c7097a820b6892d0ca90bffa154abcb2dfd484b2c226e29a95a682d1673e2b1eb12f8
+  data.tar.gz: 56adeddbd921d858b310f764712e9313a9d7537a8f1c511aa0d560fc94ca8b4ccc5d9b582744082f5ca3c5ae32820f2ec462f6952bd58500eed6e8c8e8263ae0

data/CHANGELOG

@@ -1,45 +1,29 @@
-3.0.0.rc2 / 2013-08-28
+3.0.0 / 2014-02-04
 
-* Enhancements
-
-  * (Re-)Start logging the beginning, erroring, and ending of steps
-  * Raise errors if a step fails
-
-3.0.0.rc1 / 2013-08-08
-
-* Enhancements
-
-  * :field_name can be passed as an Array like [ :first_name, :last_name ], which will join them with a space
-
-3.0.0.beta / 2013-07-26
-
-* Enhancements
-
-  * test steps with after: and every: options, referring to row counts in the previous step
-  * :limit option for import steps - oh yeah
-  * :random_skip option - goes nicely with :limit if you're debugging
-
-* Bug fixes
-
-  * Make sure to stringify data_miner(:append) option
-
-3.0.0.alpha / 2013-07-24
-
-* breaking changes
-
-  * :dictionary is now just a plain Hash-like object (responds to []) - and no longer attempts to refresh between runs  
-  * no more unit conversions
-  * always nullifies blank strings
-  * always overwrites columns whether there was a non-null value there before or not (no more :overwrite option)
-  * don't use synthesize option, just pass a block
-  * synthesized values get the same whitespace compression and stripping as other values
-  * not tested against the Earth library, which has particular requirements and makes testing too complicated
-  * doesn't keep DataMiner::Run or DataMiner::Run::ColumnStatistic records
-  * no more :matcher option
-
-* enhancements
+* Breaking changes
 
-  * if you have a postgres hstore column called "foo", you can do store 'foo.bar'
+  * (alpha) :dictionary is now just a plain Hash-like object (responds to []) - and no longer attempts to refresh between runs  
+  * (alpha) no more unit conversions
+  * (alpha) always nullifies blank strings
+  * (alpha) always overwrites columns whether there was a non-null value there before or not (no more :overwrite option)
+  * (alpha) don't use synthesize option, just pass a block
+  * (alpha) synthesized values get the same whitespace compression and stripping as other values
+  * (alpha) not tested against the Earth library, which has particular requirements and makes testing too complicated
+  * (alpha) doesn't keep DataMiner::Run or DataMiner::Run::ColumnStatistic records
+  * (alpha) no more :matcher option
+
+* Enhancements
+
+  * :date_format option takes format for Date.strptime
+  * :ignore_error options recovers from some value conversion errors
+  * (rc2) (Re-)Start logging the beginning, erroring, and ending of steps
+  * (rc2) Raise errors if a step fails
+  * (rc1) :field_name can be passed as an Array like [ :first_name, :last_name ], which will join them with a space
+  * (beta) test steps with after: and every: options, referring to row counts in the previous step
+  * (beta) :limit option for import steps - oh yeah
+  * (beta) :random_skip option - goes nicely with :limit if you're debugging
+  * (alpha) if you have a postgres hstore column called "foo", you can do store 'foo.bar'
+  * add MIT license
 
 2.5.2 / 2013-07-05
 

data/LICENSE

@@ -1,4 +1,4 @@
-Copyright (c) 2013 Seamus Abshere
+Copyright (c) 2014 Seamus Abshere
 
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the

data/data_miner.gemspec

@@ -10,6 +10,8 @@ Gem::Specification.new do |s|
   s.summary     = %{Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.}
   s.description = %q{Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models. Uses Upsert internally for speed.}
 
+  s.license = 'MIT'
+  
   s.rubyforge_project = "data_miner"
 
   s.files         = `git ls-files`.split("\n")

data/lib/data_miner.rb

@@ -1,5 +1,6 @@
 require 'singleton'
 require 'set'
+require 'date'
 require 'active_support'
 require 'active_support/version'
 if ::ActiveSupport::VERSION::MAJOR >= 3

data/lib/data_miner/attribute.rb

@@ -29,12 +29,15 @@ class DataMiner
       'upcase',
       'field_number',
       'chars',
+      'date_format',
+      'ignore_error',
     ]
 
     DEFAULT_SPLIT_PATTERN = /\s+/
     DEFAULT_SPLIT_KEEP = 0
     DEFAULT_DELIMITER = ', '
     DEFAULT_UPCASE = false
+    DEFAULT_IGNORE_ERROR = false
 
     # activerecord-3.2.6/lib/active_record/connection_adapters/column.rb
     TRUE_VALUES = [true, 1, '1', 't', 'T', 'true', 'TRUE', 'on', 'ON', 'yes', 'YES', 'y', 'Y']
@@ -91,6 +94,14 @@ class DataMiner
     # @return [TrueClass,FalseClass]
     attr_reader :upcase
 
+    # Date format to pass to Date.strptime
+    # @return [String]
+    attr_reader :date_format
+
+    # Ignore value conversion errors - value will be nil.
+    # @return [TrueClass, FalseClass]
+    attr_reader :ignore_error
+
     # Dictionary for translating.
     #
     # You pass a Hash or something that responds to []
@@ -108,9 +119,11 @@ class DataMiner
       @name = name.to_s
       @synthesize = blk if block_given?
       @dictionary = options['dictionary']
+      @ignore_error = options.fetch 'ignore_error', DEFAULT_IGNORE_ERROR
       if @static_boolean = options.has_key?('static')
         @static = options['static']
       end
+      @date_format = options['date_format']
       @field_number = options['field_number']
       @field_name_settings = options['field_name']
       @delimiter = options.fetch 'delimiter', DEFAULT_DELIMITER
@@ -240,10 +253,19 @@ class DataMiner
       if sprintf
         value = sprintf % value.to_f
       end
+      if date_format
+        value = Date.strptime value.to_s, date_format
+      end
       if dictionary
         value = dictionary[value]
       end
       value
+    rescue
+      if ignore_error
+        DataMiner.logger.debug { "Error in #{name}: #{$!.message}" }
+      else
+        raise $!
+      end
     end
 
     def hstore?

data/lib/data_miner/version.rb

@@ -1,3 +1,3 @@
 class DataMiner
-  VERSION = '3.0.0.rc2'
+  VERSION = '3.0.0'
 end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_miner
 version: !ruby/object:Gem::Version
-  version: 3.0.0.rc2
-  prerelease: 6
+  version: 3.0.0
 platform: ruby
 authors:
 - Seamus Abshere
@@ -13,342 +12,300 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-08-28 00:00:00.000000000 Z
+date: 2014-02-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: '3'
 - !ruby/object:Gem::Dependency
   name: errata
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.0.1
 - !ruby/object:Gem::Dependency
   name: remote_table
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.2
 - !ruby/object:Gem::Dependency
   name: upsert
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.3.1
 - !ruby/object:Gem::Dependency
   name: posix-spawn
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: unix_utils
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: roo
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.10.3
 - !ruby/object:Gem::Dependency
   name: rspec-expectations
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: active_record_inline_schema
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fuzzy_match
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: minitest-reporters
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rdiscount
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: sqlite3
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mysql2
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import
@@ -364,9 +321,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec
-- .yardopts
+- ".gitignore"
+- ".rspec"
+- ".yardopts"
 - CHANGELOG
 - Gemfile
 - LICENSE
@@ -402,28 +359,28 @@ files:
 - test/support/pets_funny.csv
 - test/test_data_miner.rb
 homepage: https://github.com/seamusabshere/data_miner
-licenses: []
+licenses:
+- MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>'
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: data_miner
-rubygems_version: 1.8.25
+rubygems_version: 2.2.1
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Download, pull out of a ZIP/TAR/GZ/BZ2 archive, parse, correct, and import
   XLS, ODS, XML, CSV, HTML, etc. into your ActiveRecord models.
 test_files:
@@ -445,4 +402,3 @@ test_files:
 - test/support/pets.csv
 - test/support/pets_funny.csv
 - test/test_data_miner.rb
-has_rdoc: