heydan 0.1.1 → 0.1.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 1ee89c19c81cd1085881148d715aebb67e5feff2
4
- data.tar.gz: d9b8fd855f3ad595247b2bbe0310ea1f94921cd9
3
+ metadata.gz: e22eb10825520b00a73152328eec8a81b25ae63f
4
+ data.tar.gz: 68e3ec6d5129df28b213239c6f319e2a0f740543
5
5
  SHA512:
6
- metadata.gz: 86f5aab602eed69e4c1efb59950f54c07e25aa17ee3fadef6fc21ea5205ef3246cf029de5f9152d70700040140278b69352eb49cfdee60329b37b56b81f878e0
7
- data.tar.gz: e78c0d64a5356e56dd5779c2d680ee2cc7211fca18dd9ffc66643aaaaad3dd64da9c2dc9526f17b50e3da61992083c19907f07e973aef7367713e3cd744e4205
6
+ metadata.gz: 4fef12494bc4c8def20878cfd4955990028df455fcee912761995a126785a887c5447391da8bd7cc1cade8b1224410564aeb819530083a15c303491942aa7963
7
+ data.tar.gz: a48552b34329556da69a746d3bfcb8bdadcb4ab33f4571a60cfcb16a781ec86a883d8b3af1a9b318c8d23400fd5fa29680b06cfee571b1be50991b108a6163ab
data/README.md CHANGED
@@ -12,10 +12,14 @@ The goal is to create an easy versino controlled data import and processing tool
12
12
 
13
13
  ## Installation
14
14
 
15
- You will need git, ruby and elastic search
15
+ You will need git & ruby for the core library
16
16
 
17
17
  $ gem install heydan
18
18
 
19
+ If you would like to leverage the server & search functionality, you will need elasticsearch
20
+
21
+ http://elastic.co/downloads
22
+
19
23
  ## Command Line Usage
20
24
 
21
25
  HeyDan is meant to be used in a shell:
@@ -51,6 +55,7 @@ HeyDan is meant to be used in a shell:
51
55
  $ heydan upload
52
56
  #uploads contents of the datasets folder to aws as specified in the settings file.
53
57
 
58
+
54
59
  ## Adding Data
55
60
 
56
61
  You can contribute to HeyDan by creating a new source folder on github. Look at https://github.com/danmelton/heydan_sources as an example.
@@ -37,6 +37,9 @@ Gem::Specification.new do |spec|
37
37
  spec.add_dependency "sinatra","~>1.4"
38
38
  spec.add_dependency "sinatra-cross_origin","~>0.3"
39
39
  spec.add_dependency "fog-aws", "~>0.7.5"
40
+ spec.add_dependency "georuby", "~>2.5"
41
+ spec.add_dependency "spreadsheet", "~>1.0"
42
+ spec.add_dependency "rubyXL", "~> 3.3"
40
43
 
41
44
  #development/test depedencies
42
45
  spec.add_development_dependency "bundler", "~> 1.10"
@@ -41,16 +41,24 @@ class HeyDan::Helper
41
41
  file = download(url)
42
42
  @data = case ext
43
43
  when 'csv'
44
- get_csv_data(file)
44
+ get_csv_data(file)
45
+ when 'xls'
46
+ get_excel_data(file)
47
+ when 'xlsx'
48
+ get_excel_data(file, 'xlsx')
45
49
  when 'zip'
46
50
  files = unzip(file)
51
+ return get_shapefile_data(files) if is_shapefile?(files)
47
52
  if files.size == 1
48
- get_csv_data(files[0]) if is_csv?(files[0])
53
+ return get_csv_data(files[0]) if is_csv?(files[0])
54
+ return get_excel_data(files[0]) if is_excel?(files)
49
55
  else
50
56
  files.map { |f| get_csv_data(f) if is_csv?(f)}
51
57
  end
52
58
  when 'txt'
53
59
  get_csv_data(file) if is_csv?(file)
60
+ when 'shp'
61
+ get_shapefile_data(file)
54
62
  else
55
63
  get_csv_data(file) if is_csv?(file)
56
64
  end
@@ -62,6 +70,53 @@ class HeyDan::Helper
62
70
  contents.match(/\b\t/).nil? || contents.match(/\b,/).nil? #not perfect
63
71
  end
64
72
 
73
+ def is_shapefile?(shapefile_array)
74
+ !get_shapefile(shapefile_array).nil?
75
+ end
76
+
77
+ def get_shapefile(shapefile_array)
78
+ shapefile_array.select { |file| file.to_s.include?('.shp')}[0]
79
+ end
80
+
81
+ def get_shapefile_data(shapefile_array)
82
+ file = get_shapefile(shapefile_array)
83
+ require 'geo_ruby'
84
+ require 'geo_ruby/shp'
85
+
86
+ shp = GeoRuby::Shp4r::ShpFile.open(file)
87
+ data = [shp.fields.map(&:name) + ['geojson']]
88
+ shp.records.each do |record|
89
+ data << (record.data.attributes.values + [record.geometry.as_json])
90
+ end
91
+ data
92
+ end
93
+
94
+ def get_excel_file(files)
95
+ files.select { |file| file.to_s.include?('.xls') || file.to_s.include?('.xlsx')}[0]
96
+ end
97
+
98
+ def is_excel?(files)
99
+ !get_excel_file(files).nil?
100
+ end
101
+
102
+ def get_excel_data(file, type='xls')
103
+ if type == 'xls'
104
+ require 'spreadsheet'
105
+ book = Spreadsheet.open file
106
+ data = book.worksheets.map(&:rows)
107
+ else
108
+ require 'rubyXL'
109
+ book = RubyXL::Parser.parse(file)
110
+ data = book.worksheets.map do |w|
111
+ w.sheet_data.rows.map { |row|
112
+ row.cells.map { |c| c.value } unless row.nil?
113
+ }
114
+ end
115
+ end
116
+ return data[0] if data.size == 1
117
+ data
118
+ end
119
+
65
120
  def get_csv_data(file)
66
121
  contents = File.read(file, :encoding => 'utf-8').encode("UTF-8", :invalid=>:replace, :replace=>"").gsub('"',"")
67
122
 
@@ -1,3 +1,3 @@
1
1
  module HeyDan
2
- VERSION = "0.1.1"
2
+ VERSION = "0.1.3"
3
3
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: heydan
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.1
4
+ version: 0.1.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dan Melton
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2015-08-25 00:00:00.000000000 Z
11
+ date: 2015-08-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: thor
@@ -136,6 +136,48 @@ dependencies:
136
136
  - - "~>"
137
137
  - !ruby/object:Gem::Version
138
138
  version: 0.7.5
139
+ - !ruby/object:Gem::Dependency
140
+ name: georuby
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: '2.5'
146
+ type: :runtime
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: '2.5'
153
+ - !ruby/object:Gem::Dependency
154
+ name: spreadsheet
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '1.0'
160
+ type: :runtime
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '1.0'
167
+ - !ruby/object:Gem::Dependency
168
+ name: rubyXL
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '3.3'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '3.3'
139
181
  - !ruby/object:Gem::Dependency
140
182
  name: bundler
141
183
  requirement: !ruby/object:Gem::Requirement