msewage-importer 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +21 -0
- data/.rspec +2 -0
- data/Gemfile +15 -0
- data/Guardfile +18 -0
- data/LICENSE +22 -0
- data/README.md +55 -0
- data/Rakefile +21 -0
- data/WestBengal_2005.aspx.json +1 -0
- data/bin/msewage-importer +5 -0
- data/config/config-example.yml +4 -0
- data/lib/msewage-importer/api.rb +63 -0
- data/lib/msewage-importer/cli.rb +124 -0
- data/lib/msewage-importer/config.rb +71 -0
- data/lib/msewage-importer/geolocator.rb +29 -0
- data/lib/msewage-importer/importer.rb +95 -0
- data/lib/msewage-importer/importers/base.rb +51 -0
- data/lib/msewage-importer/importers/csv.rb +31 -0
- data/lib/msewage-importer/importers/json.rb +13 -0
- data/lib/msewage-importer/source_types.rb +54 -0
- data/lib/msewage-importer/version.rb +5 -0
- data/lib/msewage-importer.rb +16 -0
- data/msewage-importer.gemspec +30 -0
- data/spec/spec_helper.rb +22 -0
- metadata +247 -0
data/.gitignore
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
*.csv
|
2
|
+
*.gem
|
3
|
+
*.json
|
4
|
+
*.rbc
|
5
|
+
.bundle
|
6
|
+
.config
|
7
|
+
.yardoc
|
8
|
+
Gemfile.lock
|
9
|
+
InstalledFiles
|
10
|
+
_yardoc
|
11
|
+
config/config.yml
|
12
|
+
coverage
|
13
|
+
doc/
|
14
|
+
gems.tags
|
15
|
+
lib/bundler/man
|
16
|
+
pkg
|
17
|
+
rdoc
|
18
|
+
spec/reports
|
19
|
+
test/tmp
|
20
|
+
test/version_tmp
|
21
|
+
tmp
|
data/.rspec
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
source 'https://rubygems.org'
|
2
|
+
|
3
|
+
# Specify your gem's dependencies in msewage-importer.gemspec
|
4
|
+
gemspec
|
5
|
+
|
6
|
+
group :development do
|
7
|
+
gem "rb-fchange", :require => false
|
8
|
+
gem "rb-fsevent", :require => false
|
9
|
+
gem "rb-inotify", :require => false
|
10
|
+
end
|
11
|
+
|
12
|
+
group :test do
|
13
|
+
gem "rake"
|
14
|
+
gem "rspec"
|
15
|
+
end
|
data/Guardfile
ADDED
@@ -0,0 +1,18 @@
|
|
1
|
+
# A sample Guardfile
|
2
|
+
# More info at https://github.com/guard/guard#readme
|
3
|
+
|
4
|
+
guard 'bundler' do
|
5
|
+
watch('Gemfile')
|
6
|
+
watch(/^.+\.gemspec/)
|
7
|
+
end
|
8
|
+
|
9
|
+
guard 'ctags-bundler', src_path: ["lib", "spec/support"] do
|
10
|
+
watch(/^(lib|spec\/support)\/.*\.rb$/)
|
11
|
+
watch('Gemfile.lock')
|
12
|
+
end
|
13
|
+
|
14
|
+
guard 'rspec' do
|
15
|
+
watch(%r{^spec/.+_spec\.rb$})
|
16
|
+
watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" }
|
17
|
+
watch('spec/spec_helper.rb') { "spec" }
|
18
|
+
end
|
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
Copyright (c) 2012 Juan C. Müller
|
2
|
+
|
3
|
+
MIT License
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,55 @@
|
|
1
|
+
# Msewage::Importer
|
2
|
+
|
3
|
+
Command line tool to import a CSV file into mSewage (msewage.org). It is able to import one
|
4
|
+
kind of source at a time.
|
5
|
+
|
6
|
+
## Installation
|
7
|
+
|
8
|
+
$ gem install msewage-importer
|
9
|
+
|
10
|
+
## Usage
|
11
|
+
|
12
|
+
$ msewage-importer --type [source_type] --source [source.json]
|
13
|
+
|
14
|
+
Currently, the following source types are supported:
|
15
|
+
* open_defecation_site
|
16
|
+
* toilet
|
17
|
+
* latrine
|
18
|
+
* septic_tank
|
19
|
+
* sewage_outflow
|
20
|
+
* manure
|
21
|
+
* industrial_waste
|
22
|
+
* open_sewage
|
23
|
+
|
24
|
+
## Help
|
25
|
+
|
26
|
+
$ msewage-importer --help
|
27
|
+
|
28
|
+
## Type of data supported
|
29
|
+
|
30
|
+
At this moment, only JSON data sources are supported. These sources should conform to the
|
31
|
+
following format [mSewage API](http://data.mwater.co/msewage/apiv2#2)
|
32
|
+
|
33
|
+
```json
|
34
|
+
{
|
35
|
+
"sources": [
|
36
|
+
{
|
37
|
+
"name": "optional",
|
38
|
+
"description": "optional",
|
39
|
+
"location": "location string. Required."
|
40
|
+
},
|
41
|
+
...
|
42
|
+
]
|
43
|
+
}
|
44
|
+
|
45
|
+
```
|
46
|
+
|
47
|
+
The data might come from a script like [this one](https://github.com/tlevine/pri-toilet-hackers).
|
48
|
+
|
49
|
+
## Contributing
|
50
|
+
|
51
|
+
1. Fork it
|
52
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
53
|
+
3. Commit your changes (`git commit -am 'Added some feature'`)
|
54
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
55
|
+
5. Create new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
#!/usr/bin/env rake
|
2
|
+
|
3
|
+
$:.push File.expand_path("../lib", __FILE__)
|
4
|
+
|
5
|
+
require "rspec/core/rake_task"
|
6
|
+
require "msewage-importer/version"
|
7
|
+
require "bundler/gem_tasks"
|
8
|
+
|
9
|
+
RSpec::Core::RakeTask.new(:spec) do |spec|
|
10
|
+
spec.pattern = "spec/**/*_spec.rb"
|
11
|
+
end
|
12
|
+
|
13
|
+
desc "Clean backup and swap files, and artifacts"
|
14
|
+
task :clean do
|
15
|
+
require "fileutils"
|
16
|
+
Dir["{pkg/*,**/*~,**/.*.sw?,coverage/**,spec/reports/**}"].each do |file|
|
17
|
+
rm_rf file
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
task default: :spec
|
@@ -0,0 +1 @@
|
|
1
|
+
{"sources": [{"description": "Updated in 2005", "name": "NARUGRAM", "location": "NARUGRAM, RAINA-I, BARDHAMAN, WEST BENGAL, India"}, {"description": "Updated in 2005", "name": "HARINKHOLA-II", "location": "HARINKHOLA-II, ARAMBAGH, HOOGHLY, WEST BENGAL, India"}, {"description": "Updated in 2005", "name": "SALEPUR-II", "location": "SALEPUR-II, ARAMBAGH, HOOGHLY, WEST BENGAL, India"}, {"description": "Updated in 2005", "name": "JOYPUR", "location": "JOYPUR, AMTA-II, HOWRAH, WEST BENGAL, India"}, {"description": "Updated in 2005", "name": "CHAMRAIL", "location": "CHAMRAIL, BALLY-JAGACHA, HOWRAH, WEST BENGAL, India"}, {"description": "Updated in 2005", "name": "SABRA", "location": "SABRA, DANTAN-II, MEDINIPUR WEST, WEST BENGAL, India"}, {"description": "Updated in 2005", "name": "SHAWREE KOTBAR", "location": "SHAWREE KOTBAR, DANTAN-II, MEDINIPUR WEST, WEST BENGAL, India"}, {"description": "Updated in 2005", "name": "PANCHBERIYA", "location": "PANCHBERIYA, DASPUR-I, MEDINIPUR WEST, WEST BENGAL, India"}, {"description": "Updated in 2005", "name": "NARMA", "location": "NARMA, NARAYANGARH, MEDINIPUR WEST, WEST BENGAL, India"}, {"description": "Updated in 2005", "name": "PURBA KHILKAPUR", "location": "PURBA KHILKAPUR, BARASAT-I, NORTH 24 PARAGANAS, WEST BENGAL, India"}]}
|
@@ -0,0 +1,63 @@
|
|
1
|
+
require 'uuid'
|
2
|
+
require 'httparty'
|
3
|
+
|
4
|
+
module Msewage::Importer
|
5
|
+
class API
|
6
|
+
def initialize(config = Config.new)
|
7
|
+
@config = config
|
8
|
+
end
|
9
|
+
|
10
|
+
def insert(record)
|
11
|
+
res = HTTParty.put(new_source_url, body: new_source_body(record))
|
12
|
+
!res["code"].nil?
|
13
|
+
print "."
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
def new_source_url
|
19
|
+
"#{api_endpoint}/sources/#{source_uid}?clientuid=#{client_uid}"
|
20
|
+
end
|
21
|
+
|
22
|
+
def new_source_body(record)
|
23
|
+
{
|
24
|
+
created_by: user_name,
|
25
|
+
desc: record["desc"] || record["description"],
|
26
|
+
latitude: record["latitude"],
|
27
|
+
longitude: record["longitude"],
|
28
|
+
name: record["name"],
|
29
|
+
source_type: record["source_type"]
|
30
|
+
}
|
31
|
+
end
|
32
|
+
|
33
|
+
def client_uid
|
34
|
+
@client_uid ||= retrieve_client_uid
|
35
|
+
end
|
36
|
+
|
37
|
+
def retrieve_client_uid
|
38
|
+
body = { password: password }
|
39
|
+
path = "#{api_endpoint}/users/#{user_name}"
|
40
|
+
res = HTTParty.post(path, body: body)
|
41
|
+
res["clientuid"]
|
42
|
+
end
|
43
|
+
|
44
|
+
attr_reader :config
|
45
|
+
|
46
|
+
def source_uid
|
47
|
+
UUID.generate.gsub(/-/, '')
|
48
|
+
#"0" * 16
|
49
|
+
end
|
50
|
+
|
51
|
+
def user_name
|
52
|
+
config.msewage.username
|
53
|
+
end
|
54
|
+
|
55
|
+
def password
|
56
|
+
config.msewage.password
|
57
|
+
end
|
58
|
+
|
59
|
+
def api_endpoint
|
60
|
+
config.api_endpoint
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
@@ -0,0 +1,124 @@
|
|
1
|
+
require 'getoptlong'
|
2
|
+
require 'command_line_helper'
|
3
|
+
|
4
|
+
module Msewage::Importer
|
5
|
+
# Public: Entry point for command line interface
|
6
|
+
class CLI
|
7
|
+
include CommandLineHelper::HelpText
|
8
|
+
|
9
|
+
class << self
|
10
|
+
# Public: entry point for class
|
11
|
+
def run
|
12
|
+
self.new.setup_and_run
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
# Public: setup and run
|
17
|
+
def setup_and_run
|
18
|
+
setup
|
19
|
+
run
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
attr_reader :options
|
25
|
+
|
26
|
+
# Private: set up program name and process command line options
|
27
|
+
def setup
|
28
|
+
process_command_line_options
|
29
|
+
rescue GetoptLong::MissingArgument, GetoptLong::InvalidOption, GetoptLong::AmbiguousOption
|
30
|
+
puts
|
31
|
+
puts "Run #{program_name} --help to see the available options"
|
32
|
+
exit
|
33
|
+
end
|
34
|
+
|
35
|
+
# Private: do some more!
|
36
|
+
def run
|
37
|
+
Importer.new(options).import
|
38
|
+
end
|
39
|
+
|
40
|
+
# Private: Command line options
|
41
|
+
def options_possible
|
42
|
+
[
|
43
|
+
['--config', '-c', GetoptLong::REQUIRED_ARGUMENT, 'Override the configuration file'],
|
44
|
+
['--help', '-h', GetoptLong::NO_ARGUMENT, 'Show this text'],
|
45
|
+
['--password', '-p', GetoptLong::REQUIRED_ARGUMENT, 'mSewage.org password'],
|
46
|
+
['--source', '-s', GetoptLong::REQUIRED_ARGUMENT, 'Source file with data to import'],
|
47
|
+
['--type', '-t', GetoptLong::REQUIRED_ARGUMENT, 'Type of defecation site sources to import'],
|
48
|
+
['--types', '-T', GetoptLong::NO_ARGUMENT, 'Show type of defecation sites available'],
|
49
|
+
['--username', '-u', GetoptLong::REQUIRED_ARGUMENT, 'mSewage.org user name'],
|
50
|
+
['--verbose', '-v', GetoptLong::NO_ARGUMENT, 'Be extremely verbose'],
|
51
|
+
['--version', '-V', GetoptLong::NO_ARGUMENT, 'Show version info'],
|
52
|
+
]
|
53
|
+
end
|
54
|
+
|
55
|
+
# Private:
|
56
|
+
def process_command_line_options
|
57
|
+
@options = {}
|
58
|
+
|
59
|
+
cli_options.each do |opt, arg|
|
60
|
+
case opt
|
61
|
+
when '--help'
|
62
|
+
show_help_and_exit
|
63
|
+
when '--config'
|
64
|
+
options[:config_file_path] = arg
|
65
|
+
when '--verbose'
|
66
|
+
options[:verbose] = true
|
67
|
+
when '--version'
|
68
|
+
show_version_info_and_exit
|
69
|
+
when '--types'
|
70
|
+
show_types_and_exit
|
71
|
+
when '--type'
|
72
|
+
options[:type] = arg
|
73
|
+
when '--username'
|
74
|
+
options[:msewage] ||= {}
|
75
|
+
options[:msewage][:username] = arg
|
76
|
+
when '--password'
|
77
|
+
options[:msewage] ||= {}
|
78
|
+
options[:msewage][:password] = arg
|
79
|
+
when '--source'
|
80
|
+
options[:source] = arg
|
81
|
+
end
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
# Private:
|
86
|
+
def cli_options
|
87
|
+
@cli_options ||= GetoptLong.new(*options_possible.map{ |o| o.first(3) })
|
88
|
+
end
|
89
|
+
|
90
|
+
# Private:
|
91
|
+
def show_types_and_exit
|
92
|
+
puts " " << SourceTypes.types_supported * "\n "
|
93
|
+
exit
|
94
|
+
end
|
95
|
+
|
96
|
+
# Private:
|
97
|
+
def show_version_info_and_exit
|
98
|
+
puts version_info
|
99
|
+
exit
|
100
|
+
end
|
101
|
+
|
102
|
+
# Private:
|
103
|
+
def version_info
|
104
|
+
<<-EOV
|
105
|
+
#{program_name}, version #{version}
|
106
|
+
|
107
|
+
(c) Juan C. Muller, 2012
|
108
|
+
http://github.com/jcmuller/msewage-importer
|
109
|
+
EOV
|
110
|
+
end
|
111
|
+
|
112
|
+
# Private:
|
113
|
+
def version
|
114
|
+
VERSION
|
115
|
+
end
|
116
|
+
|
117
|
+
# Private:
|
118
|
+
def show_help_and_exit
|
119
|
+
STDOUT.puts help_info
|
120
|
+
exit
|
121
|
+
end
|
122
|
+
|
123
|
+
end
|
124
|
+
end
|
@@ -0,0 +1,71 @@
|
|
1
|
+
require "hashie"
|
2
|
+
require "yaml"
|
3
|
+
|
4
|
+
module Msewage::Importer
|
5
|
+
class Config
|
6
|
+
def initialize(options)
|
7
|
+
setup_config(options)
|
8
|
+
validate_config
|
9
|
+
end
|
10
|
+
|
11
|
+
def api_endpoint
|
12
|
+
API_ENDPOINT
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
attr_reader :config
|
18
|
+
|
19
|
+
API_ENDPOINT = "http://data.mwater.co/msewage/apiv2"
|
20
|
+
|
21
|
+
def setup_config(options)
|
22
|
+
options_from_config = config_file(config_file_path(options))
|
23
|
+
@config = Hashie::Mash.new
|
24
|
+
config.merge!(options_from_config) unless options_from_config.nil?
|
25
|
+
config.merge!(options)
|
26
|
+
end
|
27
|
+
|
28
|
+
def config_file_path(options)
|
29
|
+
options.delete(:config_file_path) || default_config_file_path
|
30
|
+
end
|
31
|
+
|
32
|
+
def validate_config
|
33
|
+
unless valid_keys?
|
34
|
+
STDERR.puts "Please create a config file in #{default_config_file_path}"
|
35
|
+
STDERR.puts "\nIt should look like:\n\n#{config_file_example}"
|
36
|
+
|
37
|
+
exit 1
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def valid_keys?
|
42
|
+
config[:msewage] && config.msewage[:username] && config.msewage[:password]
|
43
|
+
end
|
44
|
+
|
45
|
+
def default_config_file_path
|
46
|
+
"#{ENV['HOME']}/.config/msewage-importer/config.yml"
|
47
|
+
end
|
48
|
+
|
49
|
+
def config_file(config_file_path)
|
50
|
+
YAML.load_file(config_file_path)
|
51
|
+
rescue Errno::ENOENT
|
52
|
+
end
|
53
|
+
|
54
|
+
def config_file_example
|
55
|
+
File.read(config_file_example_path)
|
56
|
+
end
|
57
|
+
|
58
|
+
def config_file_example_path
|
59
|
+
File.expand_path('../../../config/config-example.yml', __FILE__)
|
60
|
+
end
|
61
|
+
|
62
|
+
def method_missing(meth, *args, &block)
|
63
|
+
return config[meth.to_s] if config.has_key?(meth.to_s)
|
64
|
+
super
|
65
|
+
end
|
66
|
+
|
67
|
+
def respond_to?(meth)
|
68
|
+
config.has_key?(meth.to_s) || super
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'geocoder'
|
2
|
+
|
3
|
+
module Msewage::Importer
|
4
|
+
class Geolocator
|
5
|
+
def initialize(config = Config.new)
|
6
|
+
#Geocoder.configure do |c|
|
7
|
+
# #c.lookup = :bing
|
8
|
+
# #c.api_key = config.geocoder.bing_key
|
9
|
+
# c.lookup = :google
|
10
|
+
# #c.lookup = :nominatim
|
11
|
+
# #c.lookup = :yandex
|
12
|
+
# #c.lookup = :mapquest
|
13
|
+
# #c.lookup = :freegeoip
|
14
|
+
#end
|
15
|
+
|
16
|
+
@config = config
|
17
|
+
end
|
18
|
+
|
19
|
+
def geolocate(location)
|
20
|
+
result = Geocoder.search(location)
|
21
|
+
result = result.first if result.is_a?(Array)
|
22
|
+
result.nil? ? nil : result.coordinates
|
23
|
+
end
|
24
|
+
|
25
|
+
private
|
26
|
+
|
27
|
+
attr_reader :config
|
28
|
+
end
|
29
|
+
end
|
@@ -0,0 +1,95 @@
|
|
1
|
+
require 'uuid'
|
2
|
+
|
3
|
+
module Msewage::Importer
|
4
|
+
class Importer
|
5
|
+
class NoTypeSpecified < RuntimeError
|
6
|
+
def initialize; super("No type of import specified"); end
|
7
|
+
end
|
8
|
+
|
9
|
+
class NoSourceSpecified < RuntimeError
|
10
|
+
def initialize; super("No source file to import specified"); end
|
11
|
+
end
|
12
|
+
|
13
|
+
class InvalidTypeSpecified < RuntimeError
|
14
|
+
def initialize; super("Invalid import type specified"); end
|
15
|
+
end
|
16
|
+
|
17
|
+
def initialize(options)
|
18
|
+
@config = Config.new(options)
|
19
|
+
puts options.inspect if config.verbose
|
20
|
+
end
|
21
|
+
|
22
|
+
def import
|
23
|
+
assert
|
24
|
+
importer = source_importer
|
25
|
+
sources = importer.import
|
26
|
+
import_sources(sources)
|
27
|
+
rescue NoTypeSpecified, NoSourceSpecified, InvalidTypeSpecified => e
|
28
|
+
STDERR.puts e
|
29
|
+
STDERR.puts "\nType #{$0} --help"
|
30
|
+
exit
|
31
|
+
end
|
32
|
+
|
33
|
+
private
|
34
|
+
|
35
|
+
attr_reader :config, :api
|
36
|
+
|
37
|
+
def geolocator
|
38
|
+
@geolocator ||= Geolocator.new(config)
|
39
|
+
end
|
40
|
+
|
41
|
+
def api
|
42
|
+
@api ||= API.new(config)
|
43
|
+
end
|
44
|
+
|
45
|
+
def ensure_coordinates(source)
|
46
|
+
unless source["latitude"] && source["longitude"]
|
47
|
+
coordinates = geolocator.geolocate(source["location"])
|
48
|
+
source["latitude"], source["longitude"] = coordinates
|
49
|
+
end
|
50
|
+
source
|
51
|
+
end
|
52
|
+
|
53
|
+
def add_source_type(source)
|
54
|
+
source["source_type"] = source_type
|
55
|
+
source
|
56
|
+
end
|
57
|
+
|
58
|
+
def source_importer
|
59
|
+
Importers::Base.factory(config.source)
|
60
|
+
end
|
61
|
+
|
62
|
+
def import_sources(sources)
|
63
|
+
sources.each do |source|
|
64
|
+
import_source(source)
|
65
|
+
end
|
66
|
+
puts
|
67
|
+
end
|
68
|
+
|
69
|
+
def import_source(source)
|
70
|
+
ensure_coordinates(source)
|
71
|
+
add_source_type(source)
|
72
|
+
unless api.insert(source)
|
73
|
+
STDERR.puts "Error inserting #{source.inspect}"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
77
|
+
def assert
|
78
|
+
raise NoTypeSpecified unless config_option(:type)
|
79
|
+
raise InvalidTypeSpecified unless valid_type?
|
80
|
+
raise NoSourceSpecified unless config_option(:source)
|
81
|
+
end
|
82
|
+
|
83
|
+
def config_option(option)
|
84
|
+
config.send(:config).send(:[], option)
|
85
|
+
end
|
86
|
+
|
87
|
+
def source_type
|
88
|
+
SourceTypes.given_type_to_apis(config.type)
|
89
|
+
end
|
90
|
+
|
91
|
+
def valid_type?
|
92
|
+
!!source_type
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
module Msewage::Importer
|
2
|
+
module Importers
|
3
|
+
class Base
|
4
|
+
class << self
|
5
|
+
def type(*file_types)
|
6
|
+
file_types.each do |file_type|
|
7
|
+
Importers::Base.importers[file_type] = self
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
def factory(file_name)
|
12
|
+
#load_importers
|
13
|
+
importers.each_pair do |extension, class_name|
|
14
|
+
if file_name =~ %r{#{extension}}
|
15
|
+
return class_name.new(file_name)
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def load_importers
|
21
|
+
#dirname = File.dirname(__FILE__)
|
22
|
+
#Dir[File.join(dirname, "*.rb")].each do |f|
|
23
|
+
# require f.gsub(%r{#{dirname}/lib/}, '').gsub(/.rb/, '') unless f =~ /base/
|
24
|
+
#end
|
25
|
+
end
|
26
|
+
|
27
|
+
def importers
|
28
|
+
@importers ||= {}
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def initialize(file_name)
|
33
|
+
@file_name = file_name
|
34
|
+
end
|
35
|
+
|
36
|
+
def import
|
37
|
+
raise "OVERRIDE ME"
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
|
42
|
+
attr_reader :file_name
|
43
|
+
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
dirname = File.dirname(__FILE__)
|
49
|
+
Dir[File.join(dirname, "*.rb")].each do |f|
|
50
|
+
require f.gsub(%r{#{dirname}/lib/}, '').gsub(/.rb/, '') unless f =~ /base/
|
51
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'csv'
|
2
|
+
|
3
|
+
module Msewage::Importer
|
4
|
+
module Importers
|
5
|
+
class CSV < Base
|
6
|
+
type :csv
|
7
|
+
|
8
|
+
def import
|
9
|
+
sources = read_file
|
10
|
+
headers = sources.shift
|
11
|
+
build_hash(headers, sources)
|
12
|
+
end
|
13
|
+
|
14
|
+
def read_file
|
15
|
+
::CSV.parse(File.read(file_name))
|
16
|
+
end
|
17
|
+
|
18
|
+
def build_hash(headers, sources)
|
19
|
+
[].tap do |array_with_sources|
|
20
|
+
sources.each do |line|
|
21
|
+
source = {}
|
22
|
+
line.each_with_index do |data, i|
|
23
|
+
source[headers[i]] = data
|
24
|
+
end
|
25
|
+
array_with_sources.push(source)
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,54 @@
|
|
1
|
+
module Msewage::Importer
|
2
|
+
class SourceTypes
|
3
|
+
class << self
|
4
|
+
def given_type_to_apis(type)
|
5
|
+
source_types[internal_types_to_msewages[type]]
|
6
|
+
end
|
7
|
+
|
8
|
+
def types_supported
|
9
|
+
%w{
|
10
|
+
animal_manure
|
11
|
+
combined_sewer_outflow
|
12
|
+
industrial_waste_outflow
|
13
|
+
latrine
|
14
|
+
open_defecation_site
|
15
|
+
open_sewage_canal_or_puddle
|
16
|
+
raw_sewage_outflow
|
17
|
+
septic_tank_cesspool
|
18
|
+
toilet
|
19
|
+
treatment_plant_outflow
|
20
|
+
}.sort
|
21
|
+
end
|
22
|
+
|
23
|
+
def internal_types_to_msewages
|
24
|
+
{
|
25
|
+
"open_defecation_site" => "Open defecation site",
|
26
|
+
"toilet" => "Toilet",
|
27
|
+
"latrine" => "Latrine",
|
28
|
+
"septic_tank_cesspool" => "Septic tank/cesspool",
|
29
|
+
"raw_sewage_outflow" => "Raw sewage outflow",
|
30
|
+
"treatment_plant_outflow" => "Treatment plant outflow",
|
31
|
+
"combined_sewer_outflow" => "Combined sewer outflow",
|
32
|
+
"animal_manure" => "Animal manure",
|
33
|
+
"industrial_waste_outflow" => "Industrial waste outflow",
|
34
|
+
"open_sewage_canal_or_puddle" => "Open sewage canal or puddle"
|
35
|
+
}
|
36
|
+
end
|
37
|
+
|
38
|
+
def source_types
|
39
|
+
{
|
40
|
+
"Open defecation site" => 0,
|
41
|
+
"Toilet" => 1,
|
42
|
+
"Latrine" => 2,
|
43
|
+
"Septic tank/cesspool" => 3,
|
44
|
+
"Raw sewage outflow" => 4,
|
45
|
+
"Treatment plant outflow" => 5,
|
46
|
+
"Combined sewer outflow" => 6,
|
47
|
+
"Animal manure" => 7,
|
48
|
+
"Industrial waste outflow" => 8,
|
49
|
+
"Open sewage canal or puddle" => 9
|
50
|
+
}
|
51
|
+
end
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require "msewage-importer/version"
|
2
|
+
|
3
|
+
module Msewage::Importer
|
4
|
+
autoload :API, 'msewage-importer/api'
|
5
|
+
autoload :CLI, 'msewage-importer/cli'
|
6
|
+
autoload :Config, 'msewage-importer/config'
|
7
|
+
autoload :Geolocator, 'msewage-importer/geolocator'
|
8
|
+
autoload :Importer, 'msewage-importer/importer'
|
9
|
+
autoload :SourceTypes, 'msewage-importer/source_types'
|
10
|
+
|
11
|
+
module Importers
|
12
|
+
autoload :Base, 'msewage-importer/importers/base'
|
13
|
+
autoload :JSON, 'msewage-importer/importers/json'
|
14
|
+
autoload :CSV, 'msewage-importer/importers/csv'
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,30 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require File.expand_path('../lib/msewage-importer/version', __FILE__)
|
3
|
+
|
4
|
+
Gem::Specification.new do |gem|
|
5
|
+
gem.authors = ["Juan C. Müller"]
|
6
|
+
gem.email = ["jcmuller@gmail.com"]
|
7
|
+
gem.description = %q{Import data from JSON or CSV files into mSewage (mSewage.org)}
|
8
|
+
gem.summary = %q{Help populate the mSewage database}
|
9
|
+
gem.homepage = "http://github.com/jcmuller/msewage-importer"
|
10
|
+
|
11
|
+
gem.files = `git ls-files`.split($\)
|
12
|
+
gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
|
13
|
+
gem.test_files = gem.files.grep(%r{^(test|spec|features)/})
|
14
|
+
gem.name = "msewage-importer"
|
15
|
+
gem.require_paths = ["lib"]
|
16
|
+
gem.version = Msewage::Importer::VERSION
|
17
|
+
|
18
|
+
gem.add_dependency("geocoder")
|
19
|
+
gem.add_dependency("hashie")
|
20
|
+
gem.add_dependency("command_line_helper")
|
21
|
+
gem.add_dependency("uuid")
|
22
|
+
|
23
|
+
gem.add_development_dependency("rake")
|
24
|
+
gem.add_development_dependency("guard")
|
25
|
+
gem.add_development_dependency("guard-bundler")
|
26
|
+
gem.add_development_dependency("guard-ctags-bundler")
|
27
|
+
gem.add_development_dependency("guard-rspec")
|
28
|
+
gem.add_development_dependency("terminal-notifier-guard")
|
29
|
+
gem.add_development_dependency("pry-debugger")
|
30
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
# This file was generated by the `rspec --init` command. Conventionally, all
|
2
|
+
# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
|
3
|
+
# Require this file using `require "spec_helper"` to ensure that it is only
|
4
|
+
# loaded once.
|
5
|
+
#
|
6
|
+
# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
|
7
|
+
require "rspec/core"
|
8
|
+
require "rspec/mocks"
|
9
|
+
|
10
|
+
require "msewage-importer"
|
11
|
+
|
12
|
+
RSpec.configure do |config|
|
13
|
+
config.treat_symbols_as_metadata_keys_with_true_values = true
|
14
|
+
config.run_all_when_everything_filtered = true
|
15
|
+
config.filter_run :focus
|
16
|
+
|
17
|
+
# Run specs in random order to surface order dependencies. If you find an
|
18
|
+
# order dependency and want to debug it, you can fix the order by providing
|
19
|
+
# the seed, which is printed after each run.
|
20
|
+
# --seed 1234
|
21
|
+
config.order = 'random'
|
22
|
+
end
|
metadata
ADDED
@@ -0,0 +1,247 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: msewage-importer
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
prerelease:
|
6
|
+
platform: ruby
|
7
|
+
authors:
|
8
|
+
- Juan C. Müller
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2012-12-03 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: geocoder
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
none: false
|
18
|
+
requirements:
|
19
|
+
- - ! '>='
|
20
|
+
- !ruby/object:Gem::Version
|
21
|
+
version: '0'
|
22
|
+
type: :runtime
|
23
|
+
prerelease: false
|
24
|
+
version_requirements: !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ! '>='
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
version: '0'
|
30
|
+
- !ruby/object:Gem::Dependency
|
31
|
+
name: hashie
|
32
|
+
requirement: !ruby/object:Gem::Requirement
|
33
|
+
none: false
|
34
|
+
requirements:
|
35
|
+
- - ! '>='
|
36
|
+
- !ruby/object:Gem::Version
|
37
|
+
version: '0'
|
38
|
+
type: :runtime
|
39
|
+
prerelease: false
|
40
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ! '>='
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
version: '0'
|
46
|
+
- !ruby/object:Gem::Dependency
|
47
|
+
name: command_line_helper
|
48
|
+
requirement: !ruby/object:Gem::Requirement
|
49
|
+
none: false
|
50
|
+
requirements:
|
51
|
+
- - ! '>='
|
52
|
+
- !ruby/object:Gem::Version
|
53
|
+
version: '0'
|
54
|
+
type: :runtime
|
55
|
+
prerelease: false
|
56
|
+
version_requirements: !ruby/object:Gem::Requirement
|
57
|
+
none: false
|
58
|
+
requirements:
|
59
|
+
- - ! '>='
|
60
|
+
- !ruby/object:Gem::Version
|
61
|
+
version: '0'
|
62
|
+
- !ruby/object:Gem::Dependency
|
63
|
+
name: uuid
|
64
|
+
requirement: !ruby/object:Gem::Requirement
|
65
|
+
none: false
|
66
|
+
requirements:
|
67
|
+
- - ! '>='
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
type: :runtime
|
71
|
+
prerelease: false
|
72
|
+
version_requirements: !ruby/object:Gem::Requirement
|
73
|
+
none: false
|
74
|
+
requirements:
|
75
|
+
- - ! '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: '0'
|
78
|
+
- !ruby/object:Gem::Dependency
|
79
|
+
name: rake
|
80
|
+
requirement: !ruby/object:Gem::Requirement
|
81
|
+
none: false
|
82
|
+
requirements:
|
83
|
+
- - ! '>='
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0'
|
86
|
+
type: :development
|
87
|
+
prerelease: false
|
88
|
+
version_requirements: !ruby/object:Gem::Requirement
|
89
|
+
none: false
|
90
|
+
requirements:
|
91
|
+
- - ! '>='
|
92
|
+
- !ruby/object:Gem::Version
|
93
|
+
version: '0'
|
94
|
+
- !ruby/object:Gem::Dependency
|
95
|
+
name: guard
|
96
|
+
requirement: !ruby/object:Gem::Requirement
|
97
|
+
none: false
|
98
|
+
requirements:
|
99
|
+
- - ! '>='
|
100
|
+
- !ruby/object:Gem::Version
|
101
|
+
version: '0'
|
102
|
+
type: :development
|
103
|
+
prerelease: false
|
104
|
+
version_requirements: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ! '>='
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
version: '0'
|
110
|
+
- !ruby/object:Gem::Dependency
|
111
|
+
name: guard-bundler
|
112
|
+
requirement: !ruby/object:Gem::Requirement
|
113
|
+
none: false
|
114
|
+
requirements:
|
115
|
+
- - ! '>='
|
116
|
+
- !ruby/object:Gem::Version
|
117
|
+
version: '0'
|
118
|
+
type: :development
|
119
|
+
prerelease: false
|
120
|
+
version_requirements: !ruby/object:Gem::Requirement
|
121
|
+
none: false
|
122
|
+
requirements:
|
123
|
+
- - ! '>='
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
- !ruby/object:Gem::Dependency
|
127
|
+
name: guard-ctags-bundler
|
128
|
+
requirement: !ruby/object:Gem::Requirement
|
129
|
+
none: false
|
130
|
+
requirements:
|
131
|
+
- - ! '>='
|
132
|
+
- !ruby/object:Gem::Version
|
133
|
+
version: '0'
|
134
|
+
type: :development
|
135
|
+
prerelease: false
|
136
|
+
version_requirements: !ruby/object:Gem::Requirement
|
137
|
+
none: false
|
138
|
+
requirements:
|
139
|
+
- - ! '>='
|
140
|
+
- !ruby/object:Gem::Version
|
141
|
+
version: '0'
|
142
|
+
- !ruby/object:Gem::Dependency
|
143
|
+
name: guard-rspec
|
144
|
+
requirement: !ruby/object:Gem::Requirement
|
145
|
+
none: false
|
146
|
+
requirements:
|
147
|
+
- - ! '>='
|
148
|
+
- !ruby/object:Gem::Version
|
149
|
+
version: '0'
|
150
|
+
type: :development
|
151
|
+
prerelease: false
|
152
|
+
version_requirements: !ruby/object:Gem::Requirement
|
153
|
+
none: false
|
154
|
+
requirements:
|
155
|
+
- - ! '>='
|
156
|
+
- !ruby/object:Gem::Version
|
157
|
+
version: '0'
|
158
|
+
- !ruby/object:Gem::Dependency
|
159
|
+
name: terminal-notifier-guard
|
160
|
+
requirement: !ruby/object:Gem::Requirement
|
161
|
+
none: false
|
162
|
+
requirements:
|
163
|
+
- - ! '>='
|
164
|
+
- !ruby/object:Gem::Version
|
165
|
+
version: '0'
|
166
|
+
type: :development
|
167
|
+
prerelease: false
|
168
|
+
version_requirements: !ruby/object:Gem::Requirement
|
169
|
+
none: false
|
170
|
+
requirements:
|
171
|
+
- - ! '>='
|
172
|
+
- !ruby/object:Gem::Version
|
173
|
+
version: '0'
|
174
|
+
- !ruby/object:Gem::Dependency
|
175
|
+
name: pry-debugger
|
176
|
+
requirement: !ruby/object:Gem::Requirement
|
177
|
+
none: false
|
178
|
+
requirements:
|
179
|
+
- - ! '>='
|
180
|
+
- !ruby/object:Gem::Version
|
181
|
+
version: '0'
|
182
|
+
type: :development
|
183
|
+
prerelease: false
|
184
|
+
version_requirements: !ruby/object:Gem::Requirement
|
185
|
+
none: false
|
186
|
+
requirements:
|
187
|
+
- - ! '>='
|
188
|
+
- !ruby/object:Gem::Version
|
189
|
+
version: '0'
|
190
|
+
description: Import data from JSON or CSV files into mSewage (mSewage.org)
|
191
|
+
email:
|
192
|
+
- jcmuller@gmail.com
|
193
|
+
executables:
|
194
|
+
- msewage-importer
|
195
|
+
extensions: []
|
196
|
+
extra_rdoc_files: []
|
197
|
+
files:
|
198
|
+
- .gitignore
|
199
|
+
- .rspec
|
200
|
+
- Gemfile
|
201
|
+
- Guardfile
|
202
|
+
- LICENSE
|
203
|
+
- README.md
|
204
|
+
- Rakefile
|
205
|
+
- WestBengal_2005.aspx.json
|
206
|
+
- bin/msewage-importer
|
207
|
+
- config/config-example.yml
|
208
|
+
- lib/msewage-importer.rb
|
209
|
+
- lib/msewage-importer/api.rb
|
210
|
+
- lib/msewage-importer/cli.rb
|
211
|
+
- lib/msewage-importer/config.rb
|
212
|
+
- lib/msewage-importer/geolocator.rb
|
213
|
+
- lib/msewage-importer/importer.rb
|
214
|
+
- lib/msewage-importer/importers/base.rb
|
215
|
+
- lib/msewage-importer/importers/csv.rb
|
216
|
+
- lib/msewage-importer/importers/json.rb
|
217
|
+
- lib/msewage-importer/source_types.rb
|
218
|
+
- lib/msewage-importer/version.rb
|
219
|
+
- msewage-importer.gemspec
|
220
|
+
- spec/spec_helper.rb
|
221
|
+
homepage: http://github.com/jcmuller/msewage-importer
|
222
|
+
licenses: []
|
223
|
+
post_install_message:
|
224
|
+
rdoc_options: []
|
225
|
+
require_paths:
|
226
|
+
- lib
|
227
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
228
|
+
none: false
|
229
|
+
requirements:
|
230
|
+
- - ! '>='
|
231
|
+
- !ruby/object:Gem::Version
|
232
|
+
version: '0'
|
233
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
234
|
+
none: false
|
235
|
+
requirements:
|
236
|
+
- - ! '>='
|
237
|
+
- !ruby/object:Gem::Version
|
238
|
+
version: '0'
|
239
|
+
requirements: []
|
240
|
+
rubyforge_project:
|
241
|
+
rubygems_version: 1.8.24
|
242
|
+
signing_key:
|
243
|
+
specification_version: 3
|
244
|
+
summary: Help populate the mSewage database
|
245
|
+
test_files:
|
246
|
+
- spec/spec_helper.rb
|
247
|
+
has_rdoc:
|