tin_opener 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e222a6323203b3affe9485fe42b365a009da3a53
-  data.tar.gz: e001229370d3121e99b23f7e5291ae3e270a76c1
+  metadata.gz: 9ebe2ed89555b70efb828ec91f80d77b69c0e7ff
+  data.tar.gz: ea1d9478b4d9f27ddd14a71beea052cd252edc97
 SHA512:
-  metadata.gz: b2be34285d30e29fe91863479e1313150cb1f453d98e22fe6da86b4ba8e9c1c668355c0611cd48509df4428a134c23b7f8b1bba65c2a6ca1456d060d2dc423a3
-  data.tar.gz: 4fd302a413208378608c6ca73e10add657a8c6dd4dbcb7ae3a08e4bf81db8e86f8b97d2ad93d2a30d3051a84b65c14b3fce73794d8f6475efc5b5e5beb39f319
+  metadata.gz: 464c8a01153d27a3edaade63dac993baf85e89cbc60c58e26826d610f73ffcced799b4aad954f78d34950ac1194ceb456dbb5dfb0f56b97d1686af0f5fcdd511
+  data.tar.gz: aae351b0ba80839a4f82356c89418f43a5712053320616f8409a40e37dc11639def9de02a5ce5a3e31988b2d656db6f06328bbadceb041480c93bfae2c487b41

data/README.md

@@ -1,6 +1,7 @@
 # tin_opener
 [![Build Status](https://travis-ci.org/dgilperez/tin_opener.svg?branch=master)](https://travis-ci.org/dgilperez/tin_opener)
 [![Code Climate](https://codeclimate.com/github/dgilperez/tin_opener/badges/gpa.svg)](https://codeclimate.com/github/dgilperez/tin_opener)
+[![Gem Version](https://badge.fury.io/rb/tin_opener.svg)](https://badge.fury.io/rb/tin_opener)
 
 Opening Data, one Tin at a time.
 
@@ -29,9 +30,13 @@ This gem provides the following models:
 Data input format support:
 
 * CSV
+* XLS (basic)
 
 ## TODO
 
 Support for more data formats:
 
-* Excel
+* XLST
+* XLS (test different scenarios)
+* XML
+* Custom Data Sources

data/app/controllers/tin_opener/data_sets_controller.rb

@@ -22,7 +22,8 @@ module TinOpener
       if @data_set.save
         redirect_to @data_set, notice: 'Data set was successfully created.'
       else
-        render :new
+        @data_sets = DataSet.all
+        render :index
       end
     end
 

data/app/models/tin_opener/data_set.rb

@@ -16,13 +16,14 @@ module TinOpener
     private
 
     def process_data_file
-      return unless data_file.is_a?(File)
+      return unless data_file.is_a?(File) || data_file.is_a?(Tempfile)
 
       data_file_processor = DataFileProcessor.new(file: data_file)
 
       self.headers = data_file_processor.headers
 
       import_records(data_file_processor.rows)
+      # TODO: benchmark and remove this allegedly slower alternative
       # data_file_processor.rows.each do |row|
       #   records.new(row_data: row)
       # end

data/app/views/tin_opener/data_sets/index.html.erb

@@ -24,6 +24,6 @@
 
 <br>
 
-New Data Set
+<h2>New Data Set</h2>
 
 <%= render 'form' %>

data/lib/tin_opener/data_file_processor.rb

@@ -1,22 +1,36 @@
 module TinOpener
+  class UnsupportedDataFileError < StandardError; end
+
   class DataFileProcessor
     def initialize(args = {})
-      @file      = args.fetch(:file)
-      @separator = args.fetch(:separator) { ';' }
+      @file = args.fetch(:file)
     end
 
-    def headers
-      @headers ||= rows.first.try do |row|
-        row.transform_values do |value|
-          value.class.name
+    delegate :headers, :rows, to: :file_type_processor
+
+    private
+
+    def file_type_processor
+      @file_type_processor ||= case mime_type
+        when 'text/csv', 'text/comma-separated-values', 'text/plain'
+          DataFileProcessors::CsvProcessor.new(file: @file)
+        when 'application/excel'
+          DataFileProcessors::XlsProcessor.new(file: @file)
+        else
+          fail UnsupportedDataFileError
         end
-      end
     end
 
-    def rows
-      @csv_data ||= CSV.parse(@file, col_sep: @separator, headers: true).map do |row|
-          row.to_hash.transform_keys{ |a| a.squeeze.strip.gsub(/\s/, '_').underscore.to_sym }
-        end
+    # TODO: Current implementation extracts MIME from file extension.
+    #       This will fail for files with bad extensions (a CSV renamed as a XLS),
+    #       or with files with no extension
+    #
+    #       Options to consider:
+    #       - Use ruby-filemagic gem
+    #       - Use or compare with output from (*nix only)
+    #           `file #{@file.path} --mime-type`.gsub("\n", '').split(':')[1].split('/')[1].gsub('x-', '').gsub(/jpeg/, 'jpg').gsub(/text/, 'txt').gsub(/x-/, '')
+    def mime_type
+      @mime_type ||= MIME::Types.type_for(@file.path).first.try(:simplified)
     end
   end
 end

data/lib/tin_opener/data_file_processors/csv_processor.rb

@@ -0,0 +1,24 @@
+module TinOpener
+  module DataFileProcessors
+    class CsvProcessor < TinOpener::DataFileProcessor
+      def initialize(args = {})
+        super
+        @separator = args.fetch(:separator) { ';' }
+      end
+
+      def headers
+        @headers ||= rows.first.try do |row|
+          row.transform_values do |value|
+            value ? value.class.name : 'String'
+          end
+        end
+      end
+
+      def rows
+        @csv_data ||= CSV.parse(@file, col_sep: @separator, headers: true).map do |row|
+            row.to_hash.transform_keys{ |a| a.gsub(/\s+/, ' ').strip.gsub(/\s/, '_').underscore.to_sym }
+          end
+      end
+    end
+  end
+end

data/lib/tin_opener/data_file_processors/xls_processor.rb

@@ -0,0 +1,36 @@
+module TinOpener
+  module DataFileProcessors
+    class XlsProcessor < TinOpener::DataFileProcessor
+      def initialize(args = {})
+        super
+      end
+
+      delegate :headers, :rows, to: :csv_processor
+
+      private
+
+      def raw_data
+        @raw_data ||= Roo::Spreadsheet.open(@file.path)
+      end
+
+      def csv_processor
+        @csv_processor ||= CsvProcessor.new(file: tempfile.tap { write_csv_data })
+      end
+
+      def tempfile
+        @tempfile ||= Tempfile.new(filename)
+      end
+
+      def filename
+        @file.path.split('/').last
+      end
+
+      # TODO: Roo has a built-in .to_yaml transform. Consider using that over CSV
+      #       whenever a YAML processor is implemented
+      def write_csv_data
+        # arguments: filename = nil, separator = ',', sheet = default_sheet
+        @csv_data ||= raw_data.to_csv(tempfile, ';')
+      end
+    end
+  end
+end

data/lib/tin_opener/version.rb

@@ -1,3 +1,3 @@
 module TinOpener
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/tin_opener.rb

@@ -1,6 +1,10 @@
 require "tin_opener/engine"
 require "tin_opener/data_file_processor"
+require "tin_opener/data_file_processors/csv_processor"
+require "tin_opener/data_file_processors/xls_processor"
 require "csv"
+require "roo"
+require "roo-xls"
 require "activerecord-import"
 require 'pry-byebug' unless Rails.env.production?