xlsx_importer 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b3d2b150f788f4b35e0c2fbe8705bb7dad67c425
+  data.tar.gz: 4827b0a0e3b365eeed33b429882d062a83022396
+SHA512:
+  metadata.gz: 30de840c9e84166184a78a1117d95d9d9f85448574afd58dc8f6a359de998090f98010be0c71fa3d45987dd4243416c2c3ce5a23855446fd1aeeb0b19ebc9ac8
+  data.tar.gz: 065786ed8e8d09b2d024108c66024997ec767fe93ef2a66e377254d5c636b617b384e1be34304c4a1dc6d15a2689965544853c390967873161e63c908bf5f9e5

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+/.idea/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in xlsx_importer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Martin Bianculli
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,74 @@
+# XlsxImporter
+
+I really like [Smarter CSV](https://github.com/tilo/smarter_csv) but as MS Excel cannot export CSV files in unicode,
+I need to import the spreadsheet directly to preserve the unicode characters.
+
+XlsxImporter is a port from Smarter CSV with a few specific options for spreadsheets import. It uses [SimpleXlsxReader](https://github.com/woahdae/simple_xlsx_reader)
+to read the MS Excel xlsx file. It only support xlsx files.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'xlsx_importer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install xlsx_importer
+
+## Usage
+
+```ruby
+    > XlsxImporter.process('/tmp/people.xlsx', options = {})
+     => [ {:first_name=>"Dan", :last_name=>"McAllister"},
+          {:first_name=>"Lucy", :last_name=>"Laweless"} ]
+```
+
+### Options
+
+A few options from Smarter CVS are supported, a few more were added.
+
+| Option                    | Default | Explanation
+|---------------------------|---------|-----------------------------------------------------------------------------
+| :key_mapping_hash         | nil     | a hash which maps headers from the spreadsheet to keys in the result hash
+| :remove_unmapped_keys     | false   | when using :key_mapping option, should non-mapped keys / columns be removed?
+| :sheet                    | 0       | the number of the sheet inside the workbook to parse
+| :strip_whitespace         | true    | remove whitespace before/after values and headers
+| :downcase_header          | true    | downcase all column headers
+| :strip_chars_from_headers | nil     | RegExp to remove extraneous characters from the header line
+| :remove_empty_hashes      | true    | remove / ignore any hashes which don't have any key/value pairs
+| :remove_empty_values      | true    | remove values which have nil or empty strings as values
+| :remove_zero_values       | false   | remove values which have a numeric value equal to zero / 0
+| :remove_values_matching   | nil     | removes key/value pairs if value matches given regular expressions
+| :header_row               | 1       | row number with the headers
+| :date_keys                | nil     | Array with the keys of date fields to perform DB compatible validation
+
+### Example
+
+```ruby
+  result = XlsxImporter.process('tmp/test.xlsx', {
+      :key_mapping_hash => {
+          :name => :full_name,
+          :date_of_creation => :created_at,
+          :date_of_termination => :deleted_at,
+      },
+      :date_keys => [:created_at, :deleted_at],
+      :remove_unmapped_keys => true,
+      :strip_chars_from_headers => /[\/\(\)]/,
+      :remove_values_matching => /^[\.\-]$/,
+      :sheet => 1,
+      :header_row => 3
+  })
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+
+desc "Open an irb session preloaded with this library"
+task :console do
+  sh "irb -rubygems -I lib -r xlsx_importer.rb"
+end

data/lib/extensions/hash.rb

@@ -0,0 +1,9 @@
+# the following extension for class Hash is needed (from Facets of Ruby library):
+
+class Hash
+  def self.zip(keys,values) # from Facets of Ruby library
+    h = {}
+    keys.size.times{ |i| h[ keys[i] ] = values[i] }
+    h
+  end
+end

data/lib/xlsx_importer.rb

@@ -0,0 +1,3 @@
+require "xlsx_importer/version"
+require "extensions/hash.rb"
+require "xlsx_importer/xlsx_importer"

data/lib/xlsx_importer/version.rb

@@ -0,0 +1,3 @@
+module XlsxImporter
+  VERSION = "0.0.1"
+end

data/lib/xlsx_importer/xlsx_importer.rb

@@ -0,0 +1,81 @@
+require 'simple_xlsx_reader'
+
+module XlsxImporter
+  def self.process(input, options={})
+
+    default_options = {
+      :key_mapping_hash => nil,
+      :remove_unmapped_keys => false,
+      :sheet => 0,
+      :strip_whitespace => true,
+      :downcase_header => true,
+      :strip_chars_from_headers => nil,
+      :remove_empty_hashes => true,
+      :remove_empty_values => true,
+      :remove_zero_values => false,
+      :remove_values_matching => nil,
+      :header_row => 1,
+      :date_keys => nil,
+    }
+    options = default_options.merge(options)
+
+    headers = []
+    result = []
+
+    doc = SimpleXlsxReader.open(input)
+    sheet = doc.sheets[options[:sheet]]
+
+    headers = sheet.rows[options[:header_row] - 1]
+    headers.map!{|x| x.gsub(options[:strip_chars_from_headers], '')} if options[:strip_chars_from_headers]
+    headers.map!{|x| x.respond_to?(:strip) ? x.strip : x}  if options[:strip_whitespace]
+    headers.map!{|x| x.gsub(/\s+/,'_')}
+    headers.map!{|x| x.respond_to?(:downcase) ? x.downcase : x } if options[:downcase_header]
+    headers.map!{|x| x.to_sym }
+
+    key_mapping_hash = options[:key_mapping_hash]
+    if !key_mapping_hash.nil? && key_mapping_hash.class == Hash && key_mapping_hash.keys.size > 0
+      headers.map!{|x| key_mapping_hash.has_key?(x) ? (key_mapping_hash[x].nil? ? nil : key_mapping_hash[x].to_sym) : (options[:remove_unmapped_keys] ? nil : x)}
+    end
+
+    line = 0
+    rows = sheet.rows.drop(options[:header_row])
+    begin
+      rows.each do |row|
+        line += 1
+
+        row.map!{|x| x.respond_to?(:strip) ? x.strip : x }  if options[:strip_whitespace]
+
+        hash = Hash.zip(headers,row)
+
+        hash.delete(nil); hash.delete('');
+        hash.delete_if {|k,v| v.nil? || v =~ /^\s*$/}  if options[:remove_empty_values]
+        hash.delete_if {|k,v| !v.nil? && v =~ /^(\d+|\d+\.\d+)$/ && v.to_f == 0} if options[:remove_zero_values]   # values are typically Strings!
+        hash.delete_if {|k,v| v =~ options[:remove_values_matching]} if options[:remove_values_matching]
+
+        options[:date_keys].map do |x|
+            hash[x] = nil if hash.has_key?(x) && self.date_validation(hash[x]).nil?
+        end if options[:date_keys].present? && options[:date_keys].class == Array && options[:date_keys].size > 0
+
+        next if hash.empty? if options[:remove_empty_hashes]
+
+        result << hash
+      end
+    rescue => e
+      puts "Error around line #{line} \n"
+      raise e
+    end
+
+    return result
+
+  end
+
+  def self.date_validation(date)
+    date = Date.parse(date) if date.class == 'String'
+    date.strftime('%Y-%m-%d') =~ /^([1-3][0-9]{3,3})-(0?[1-9]|1[0-2])-(0?[1-9]|[1-2][1-9]|3[0-1])$/
+  rescue
+    return nil
+  end
+
+
+
+end

data/xlsx_importer.gemspec

@@ -0,0 +1,25 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'xlsx_importer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "xlsx_importer"
+  spec.version       = XlsxImporter::VERSION
+  spec.authors       = ["Martin Bianculli"]
+  spec.email         = ["mbianculli@gmail.com"]
+  spec.description   = %q{Import a xlxs file}
+  spec.summary       = %q{Import a xlxs file}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+
+  spec.add_dependency "simple_xlsx_reader", "~> 1.0.1"
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: xlsx_importer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Martin Bianculli
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simple_xlsx_reader
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.1
+description: Import a xlxs file
+email:
+- mbianculli@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/extensions/hash.rb
+- lib/xlsx_importer.rb
+- lib/xlsx_importer/version.rb
+- lib/xlsx_importer/xlsx_importer.rb
+- xlsx_importer.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Import a xlxs file
+test_files: []