heydan 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1ee89c19c81cd1085881148d715aebb67e5feff2
-  data.tar.gz: d9b8fd855f3ad595247b2bbe0310ea1f94921cd9
+  metadata.gz: e22eb10825520b00a73152328eec8a81b25ae63f
+  data.tar.gz: 68e3ec6d5129df28b213239c6f319e2a0f740543
 SHA512:
-  metadata.gz: 86f5aab602eed69e4c1efb59950f54c07e25aa17ee3fadef6fc21ea5205ef3246cf029de5f9152d70700040140278b69352eb49cfdee60329b37b56b81f878e0
-  data.tar.gz: e78c0d64a5356e56dd5779c2d680ee2cc7211fca18dd9ffc66643aaaaad3dd64da9c2dc9526f17b50e3da61992083c19907f07e973aef7367713e3cd744e4205
+  metadata.gz: 4fef12494bc4c8def20878cfd4955990028df455fcee912761995a126785a887c5447391da8bd7cc1cade8b1224410564aeb819530083a15c303491942aa7963
+  data.tar.gz: a48552b34329556da69a746d3bfcb8bdadcb4ab33f4571a60cfcb16a781ec86a883d8b3af1a9b318c8d23400fd5fa29680b06cfee571b1be50991b108a6163ab

data/README.md

@@ -12,10 +12,14 @@ The goal is to create an easy versino controlled data import and processing tool
 
 ## Installation
 
-You will need git, ruby and elastic search
+You will need git & ruby for the core library
 
     $ gem install heydan
 
+If you would like to leverage the server & search functionality, you will need elasticsearch
+
+    http://elastic.co/downloads
+
 ## Command Line Usage
 
 HeyDan is meant to be used in a shell:
@@ -51,6 +55,7 @@ HeyDan is meant to be used in a shell:
     $ heydan upload
     #uploads contents of the datasets folder to aws as specified in the settings file.
 
+
 ## Adding Data
 
 You can contribute to HeyDan by creating a new source folder on github. Look at https://github.com/danmelton/heydan_sources as an example.

data/heydan.gemspec

@@ -37,6 +37,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency "sinatra","~>1.4"
   spec.add_dependency "sinatra-cross_origin","~>0.3"
   spec.add_dependency "fog-aws", "~>0.7.5"
+  spec.add_dependency "georuby", "~>2.5"
+  spec.add_dependency "spreadsheet", "~>1.0"
+  spec.add_dependency "rubyXL", "~> 3.3"
 
   #development/test depedencies
   spec.add_development_dependency "bundler", "~> 1.10"

data/lib/heydan/helper.rb

@@ -41,16 +41,24 @@ class HeyDan::Helper
       file = download(url)
       @data = case ext
         when 'csv'
-          get_csv_data(file)        
+          get_csv_data(file)
+        when 'xls'
+          get_excel_data(file)
+        when 'xlsx'
+          get_excel_data(file, 'xlsx')
         when 'zip'
           files = unzip(file)
+          return get_shapefile_data(files) if is_shapefile?(files)
           if files.size == 1
-            get_csv_data(files[0]) if is_csv?(files[0])
+            return get_csv_data(files[0]) if is_csv?(files[0])
+            return get_excel_data(files[0]) if is_excel?(files)
           else
             files.map { |f| get_csv_data(f) if is_csv?(f)} 
           end
         when 'txt'
           get_csv_data(file) if is_csv?(file)
+        when 'shp'
+          get_shapefile_data(file)
         else
           get_csv_data(file) if is_csv?(file)
         end
@@ -62,6 +70,53 @@ class HeyDan::Helper
       contents.match(/\b\t/).nil? || contents.match(/\b,/).nil? #not perfect
     end
 
+    def is_shapefile?(shapefile_array)
+      !get_shapefile(shapefile_array).nil?
+    end
+
+    def get_shapefile(shapefile_array)
+      shapefile_array.select { |file| file.to_s.include?('.shp')}[0]
+    end
+
+    def get_shapefile_data(shapefile_array)
+      file = get_shapefile(shapefile_array)
+      require 'geo_ruby'
+      require 'geo_ruby/shp'
+      
+      shp = GeoRuby::Shp4r::ShpFile.open(file)
+      data = [shp.fields.map(&:name) + ['geojson']]
+      shp.records.each do |record|
+        data << (record.data.attributes.values + [record.geometry.as_json])
+      end
+      data
+    end
+
+    def get_excel_file(files)
+      files.select { |file| file.to_s.include?('.xls') || file.to_s.include?('.xlsx')}[0]
+    end
+
+    def is_excel?(files)
+      !get_excel_file(files).nil?
+    end
+
+    def get_excel_data(file, type='xls')
+      if type == 'xls'
+        require 'spreadsheet'
+        book = Spreadsheet.open file
+        data = book.worksheets.map(&:rows)
+      else
+        require 'rubyXL'
+        book = RubyXL::Parser.parse(file)
+        data = book.worksheets.map do |w| 
+          w.sheet_data.rows.map { |row|
+            row.cells.map { |c| c.value } unless row.nil?
+          }
+        end
+      end
+      return data[0] if data.size == 1
+      data
+    end
+
     def get_csv_data(file)
       contents = File.read(file, :encoding => 'utf-8').encode("UTF-8", :invalid=>:replace, :replace=>"").gsub('"',"")
 

data/lib/heydan/version.rb

@@ -1,3 +1,3 @@
 module HeyDan
-  VERSION = "0.1.1"
+  VERSION = "0.1.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heydan
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Dan Melton
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-25 00:00:00.000000000 Z
+date: 2015-08-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
@@ -136,6 +136,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.7.5
+- !ruby/object:Gem::Dependency
+  name: georuby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.5'
+- !ruby/object:Gem::Dependency
+  name: spreadsheet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rubyXL
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.3'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement