heydan 0.1.1 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +6 -1
- data/heydan.gemspec +3 -0
- data/lib/heydan/helper.rb +57 -2
- data/lib/heydan/version.rb +1 -1
- metadata +44 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: e22eb10825520b00a73152328eec8a81b25ae63f
|
4
|
+
data.tar.gz: 68e3ec6d5129df28b213239c6f319e2a0f740543
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4fef12494bc4c8def20878cfd4955990028df455fcee912761995a126785a887c5447391da8bd7cc1cade8b1224410564aeb819530083a15c303491942aa7963
|
7
|
+
data.tar.gz: a48552b34329556da69a746d3bfcb8bdadcb4ab33f4571a60cfcb16a781ec86a883d8b3af1a9b318c8d23400fd5fa29680b06cfee571b1be50991b108a6163ab
|
data/README.md
CHANGED
@@ -12,10 +12,14 @@ The goal is to create an easy versino controlled data import and processing tool
|
|
12
12
|
|
13
13
|
## Installation
|
14
14
|
|
15
|
-
You will need git
|
15
|
+
You will need git & ruby for the core library
|
16
16
|
|
17
17
|
$ gem install heydan
|
18
18
|
|
19
|
+
If you would like to leverage the server & search functionality, you will need elasticsearch
|
20
|
+
|
21
|
+
http://elastic.co/downloads
|
22
|
+
|
19
23
|
## Command Line Usage
|
20
24
|
|
21
25
|
HeyDan is meant to be used in a shell:
|
@@ -51,6 +55,7 @@ HeyDan is meant to be used in a shell:
|
|
51
55
|
$ heydan upload
|
52
56
|
#uploads contents of the datasets folder to aws as specified in the settings file.
|
53
57
|
|
58
|
+
|
54
59
|
## Adding Data
|
55
60
|
|
56
61
|
You can contribute to HeyDan by creating a new source folder on github. Look at https://github.com/danmelton/heydan_sources as an example.
|
data/heydan.gemspec
CHANGED
@@ -37,6 +37,9 @@ Gem::Specification.new do |spec|
|
|
37
37
|
spec.add_dependency "sinatra","~>1.4"
|
38
38
|
spec.add_dependency "sinatra-cross_origin","~>0.3"
|
39
39
|
spec.add_dependency "fog-aws", "~>0.7.5"
|
40
|
+
spec.add_dependency "georuby", "~>2.5"
|
41
|
+
spec.add_dependency "spreadsheet", "~>1.0"
|
42
|
+
spec.add_dependency "rubyXL", "~> 3.3"
|
40
43
|
|
41
44
|
#development/test depedencies
|
42
45
|
spec.add_development_dependency "bundler", "~> 1.10"
|
data/lib/heydan/helper.rb
CHANGED
@@ -41,16 +41,24 @@ class HeyDan::Helper
|
|
41
41
|
file = download(url)
|
42
42
|
@data = case ext
|
43
43
|
when 'csv'
|
44
|
-
get_csv_data(file)
|
44
|
+
get_csv_data(file)
|
45
|
+
when 'xls'
|
46
|
+
get_excel_data(file)
|
47
|
+
when 'xlsx'
|
48
|
+
get_excel_data(file, 'xlsx')
|
45
49
|
when 'zip'
|
46
50
|
files = unzip(file)
|
51
|
+
return get_shapefile_data(files) if is_shapefile?(files)
|
47
52
|
if files.size == 1
|
48
|
-
get_csv_data(files[0]) if is_csv?(files[0])
|
53
|
+
return get_csv_data(files[0]) if is_csv?(files[0])
|
54
|
+
return get_excel_data(files[0]) if is_excel?(files)
|
49
55
|
else
|
50
56
|
files.map { |f| get_csv_data(f) if is_csv?(f)}
|
51
57
|
end
|
52
58
|
when 'txt'
|
53
59
|
get_csv_data(file) if is_csv?(file)
|
60
|
+
when 'shp'
|
61
|
+
get_shapefile_data(file)
|
54
62
|
else
|
55
63
|
get_csv_data(file) if is_csv?(file)
|
56
64
|
end
|
@@ -62,6 +70,53 @@ class HeyDan::Helper
|
|
62
70
|
contents.match(/\b\t/).nil? || contents.match(/\b,/).nil? #not perfect
|
63
71
|
end
|
64
72
|
|
73
|
+
def is_shapefile?(shapefile_array)
|
74
|
+
!get_shapefile(shapefile_array).nil?
|
75
|
+
end
|
76
|
+
|
77
|
+
def get_shapefile(shapefile_array)
|
78
|
+
shapefile_array.select { |file| file.to_s.include?('.shp')}[0]
|
79
|
+
end
|
80
|
+
|
81
|
+
def get_shapefile_data(shapefile_array)
|
82
|
+
file = get_shapefile(shapefile_array)
|
83
|
+
require 'geo_ruby'
|
84
|
+
require 'geo_ruby/shp'
|
85
|
+
|
86
|
+
shp = GeoRuby::Shp4r::ShpFile.open(file)
|
87
|
+
data = [shp.fields.map(&:name) + ['geojson']]
|
88
|
+
shp.records.each do |record|
|
89
|
+
data << (record.data.attributes.values + [record.geometry.as_json])
|
90
|
+
end
|
91
|
+
data
|
92
|
+
end
|
93
|
+
|
94
|
+
def get_excel_file(files)
|
95
|
+
files.select { |file| file.to_s.include?('.xls') || file.to_s.include?('.xlsx')}[0]
|
96
|
+
end
|
97
|
+
|
98
|
+
def is_excel?(files)
|
99
|
+
!get_excel_file(files).nil?
|
100
|
+
end
|
101
|
+
|
102
|
+
def get_excel_data(file, type='xls')
|
103
|
+
if type == 'xls'
|
104
|
+
require 'spreadsheet'
|
105
|
+
book = Spreadsheet.open file
|
106
|
+
data = book.worksheets.map(&:rows)
|
107
|
+
else
|
108
|
+
require 'rubyXL'
|
109
|
+
book = RubyXL::Parser.parse(file)
|
110
|
+
data = book.worksheets.map do |w|
|
111
|
+
w.sheet_data.rows.map { |row|
|
112
|
+
row.cells.map { |c| c.value } unless row.nil?
|
113
|
+
}
|
114
|
+
end
|
115
|
+
end
|
116
|
+
return data[0] if data.size == 1
|
117
|
+
data
|
118
|
+
end
|
119
|
+
|
65
120
|
def get_csv_data(file)
|
66
121
|
contents = File.read(file, :encoding => 'utf-8').encode("UTF-8", :invalid=>:replace, :replace=>"").gsub('"',"")
|
67
122
|
|
data/lib/heydan/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: heydan
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.
|
4
|
+
version: 0.1.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dan Melton
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-08-
|
11
|
+
date: 2015-08-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -136,6 +136,48 @@ dependencies:
|
|
136
136
|
- - "~>"
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: 0.7.5
|
139
|
+
- !ruby/object:Gem::Dependency
|
140
|
+
name: georuby
|
141
|
+
requirement: !ruby/object:Gem::Requirement
|
142
|
+
requirements:
|
143
|
+
- - "~>"
|
144
|
+
- !ruby/object:Gem::Version
|
145
|
+
version: '2.5'
|
146
|
+
type: :runtime
|
147
|
+
prerelease: false
|
148
|
+
version_requirements: !ruby/object:Gem::Requirement
|
149
|
+
requirements:
|
150
|
+
- - "~>"
|
151
|
+
- !ruby/object:Gem::Version
|
152
|
+
version: '2.5'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: spreadsheet
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - "~>"
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '1.0'
|
160
|
+
type: :runtime
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - "~>"
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '1.0'
|
167
|
+
- !ruby/object:Gem::Dependency
|
168
|
+
name: rubyXL
|
169
|
+
requirement: !ruby/object:Gem::Requirement
|
170
|
+
requirements:
|
171
|
+
- - "~>"
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '3.3'
|
174
|
+
type: :runtime
|
175
|
+
prerelease: false
|
176
|
+
version_requirements: !ruby/object:Gem::Requirement
|
177
|
+
requirements:
|
178
|
+
- - "~>"
|
179
|
+
- !ruby/object:Gem::Version
|
180
|
+
version: '3.3'
|
139
181
|
- !ruby/object:Gem::Dependency
|
140
182
|
name: bundler
|
141
183
|
requirement: !ruby/object:Gem::Requirement
|