heydan 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +11 -0
- data/.rspec +2 -0
- data/.travis.yml +4 -0
- data/CODE_OF_CONDUCT.md +13 -0
- data/Gemfile +4 -0
- data/Guardfile +44 -0
- data/LICENSE.txt +11 -0
- data/README.md +84 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/heydan +11 -0
- data/bin/setup +7 -0
- data/heydan.gemspec +49 -0
- data/lib/heydan.rb +65 -0
- data/lib/heydan/base.rb +72 -0
- data/lib/heydan/cdn.rb +30 -0
- data/lib/heydan/cli.rb +80 -0
- data/lib/heydan/help_text.rb +83 -0
- data/lib/heydan/helper.rb +108 -0
- data/lib/heydan/import.rb +49 -0
- data/lib/heydan/jurisdiction_file.rb +119 -0
- data/lib/heydan/open_civic_identifiers.rb +51 -0
- data/lib/heydan/script.rb +179 -0
- data/lib/heydan/script_file.rb +44 -0
- data/lib/heydan/server.rb +55 -0
- data/lib/heydan/source_file.rb +79 -0
- data/lib/heydan/sources.rb +128 -0
- data/lib/heydan/version.rb +3 -0
- data/lib/templates/script.rb.erb +32 -0
- metadata +285 -0
@@ -0,0 +1,179 @@
|
|
1
|
+
require 'ruby-progressbar'
|
2
|
+
class HeyDan::Script
|
3
|
+
attr_accessor :jurisdiction_type
|
4
|
+
attr_accessor :fromsource
|
5
|
+
attr_accessor :data
|
6
|
+
attr_accessor :folder
|
7
|
+
attr_accessor :source
|
8
|
+
attr_accessor :variable
|
9
|
+
attr_accessor :id
|
10
|
+
attr_accessor :source_file
|
11
|
+
attr_accessor :identifiers #builds a hash with new_identifier => file_name
|
12
|
+
|
13
|
+
def initialize(opts)
|
14
|
+
@folder = opts[:folder]
|
15
|
+
@source = opts[:source]
|
16
|
+
@variable = opts[:variable]
|
17
|
+
@jurisdiction_type = HeyDan.options[:type]
|
18
|
+
@fromsource = HeyDan.options[:fromsource]
|
19
|
+
@identifiers = {}
|
20
|
+
@source_file = HeyDan::SourceFile.new(@folder, @source, @variable)
|
21
|
+
end
|
22
|
+
|
23
|
+
#overwritten by the developer
|
24
|
+
def build
|
25
|
+
end
|
26
|
+
|
27
|
+
#overwritten by the developer
|
28
|
+
def validate_build
|
29
|
+
end
|
30
|
+
|
31
|
+
#overwritten by the developer, can be dataset, attribute, or identifer
|
32
|
+
def type
|
33
|
+
'dataset'
|
34
|
+
end
|
35
|
+
|
36
|
+
#overwritten by developer
|
37
|
+
def version
|
38
|
+
1
|
39
|
+
end
|
40
|
+
|
41
|
+
def dataset_file_name
|
42
|
+
"#{@folder}_#{@source}_#{@variable}.csv"
|
43
|
+
end
|
44
|
+
|
45
|
+
def name
|
46
|
+
"#{@folder}_#{@source}_#{@variable}.csv"
|
47
|
+
end
|
48
|
+
|
49
|
+
def dataset_file_path
|
50
|
+
File.join(HeyDan.folders[:datasets], dataset_file_name)
|
51
|
+
end
|
52
|
+
|
53
|
+
#downloads from the cdn
|
54
|
+
def download
|
55
|
+
@data = HeyDan::Helper.get_data_from_url(HeyDan.cdn + '/' + dataset_file_name)
|
56
|
+
end
|
57
|
+
|
58
|
+
def process_from_source
|
59
|
+
build
|
60
|
+
validate_build
|
61
|
+
HeyDan::Helper.save_data(dataset_file_name, @data)
|
62
|
+
end
|
63
|
+
|
64
|
+
#runs through download, build and validate
|
65
|
+
def process
|
66
|
+
if @fromsource
|
67
|
+
puts "From Source is specified, processing from source for #{name}" if HeyDan.help?
|
68
|
+
process_from_source
|
69
|
+
end
|
70
|
+
|
71
|
+
begin
|
72
|
+
if HeyDan::Helper.dataset_exists?(dataset_file_name)
|
73
|
+
puts "Dataset for #{name} exists" if HeyDan.help?
|
74
|
+
else
|
75
|
+
download
|
76
|
+
end
|
77
|
+
rescue
|
78
|
+
puts "Had trouble downloading #{name}, processing from source instead" if HeyDan.help?
|
79
|
+
process_from_source
|
80
|
+
end
|
81
|
+
update_jurisdiction_files
|
82
|
+
end
|
83
|
+
|
84
|
+
def build_identifier_hash(identifier)
|
85
|
+
identifier_file = File.join(HeyDan.folders[:downloads], "identifiers_file_#{identifier}.json")
|
86
|
+
if File.exist?(identifier_file)
|
87
|
+
@identifiers = JSON.parse(File.read(identifier_file))
|
88
|
+
return @identifiers
|
89
|
+
end
|
90
|
+
HeyDan::HelpText.build_identifier(identifier)
|
91
|
+
get_identifiers_from_files(identifier)
|
92
|
+
File.open(identifier_file, 'w') do |file|
|
93
|
+
file.write(@identifiers.to_json)
|
94
|
+
end
|
95
|
+
@identifiers
|
96
|
+
end
|
97
|
+
|
98
|
+
def get_identifiers_from_files(identifier)
|
99
|
+
@identifiers = {}
|
100
|
+
Dir.glob(File.join(HeyDan.folders[:jurisdictions], '*.json')).each do |j|
|
101
|
+
jf = HeyDan::JurisdictionFile.new(name: j.gsub(HeyDan.folders[:jurisdictions] + '/', ''))
|
102
|
+
return if !jf.match_type?(@jurisdiction_type)
|
103
|
+
@identifiers["#{jf.get_identifier(identifier)}"] = j.gsub(HeyDan.folders[:jurisdictions] + '/', '')
|
104
|
+
end
|
105
|
+
@identifiers
|
106
|
+
end
|
107
|
+
|
108
|
+
def update_jurisdiction_files
|
109
|
+
get_data
|
110
|
+
get_identifiers
|
111
|
+
@progress = ProgressBar.create(:title => "Updating Files for #{@source} #{@variable} from #{@folder} for jurisdictions #{(' matching ' + @jurisdiction_type) if @jurisdiction_type}", :starting_at => 0, :total => @data[1..-1].size) if HeyDan.help?
|
112
|
+
self.send("add_#{type}s")
|
113
|
+
@progress.finish if HeyDan.help?
|
114
|
+
end
|
115
|
+
|
116
|
+
def add_datasets
|
117
|
+
metadata = @source_file.variable
|
118
|
+
metadata.keep_if { |k| ['id', 'name', 'short_description', 'tags'].include?(k)}
|
119
|
+
metadata["years"] = @data[0][1..-1]
|
120
|
+
id = metadata['id']
|
121
|
+
@data[1..-1].each_index do |i|
|
122
|
+
row = @data[i+1]
|
123
|
+
next if row[0].nil? || @identifiers[row[0]].nil?
|
124
|
+
jf = get_jurisdiction_filename(@identifiers[row[0]])
|
125
|
+
next if row[0].nil? || !jf.exists?
|
126
|
+
ds = jf.get_dataset(id)
|
127
|
+
if !ds.nil?
|
128
|
+
if ds['version'] >= version
|
129
|
+
@progress.progress = i if HeyDan.help?
|
130
|
+
next
|
131
|
+
end
|
132
|
+
end
|
133
|
+
jf.datasets.delete(ds)
|
134
|
+
metadata["version"] = version
|
135
|
+
metadata["data"] = row[1..-1]
|
136
|
+
jf.add_dataset(metadata)
|
137
|
+
jf.save
|
138
|
+
@progress.progress = i if HeyDan.help?
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def get_jurisdiction_filename(id)
|
143
|
+
HeyDan::JurisdictionFile.new(name: @identifiers[id] || id)
|
144
|
+
end
|
145
|
+
|
146
|
+
def add_identifiers
|
147
|
+
@data[1..-1].each_index do |i|
|
148
|
+
row = @data[i+1]
|
149
|
+
jf = get_jurisdiction_filename(row[0])
|
150
|
+
next if row[0].nil? || !jf.exists?
|
151
|
+
jf.add_identifier(@data[0][1], row[1])
|
152
|
+
jf.save
|
153
|
+
@progress.progress = i if HeyDan.help?
|
154
|
+
end
|
155
|
+
end
|
156
|
+
|
157
|
+
def add_attributes
|
158
|
+
@data[1..-1].each_index do |i|
|
159
|
+
row = @data[i+1]
|
160
|
+
jf = get_jurisdiction_filename(row[0])
|
161
|
+
next if row[0].nil? || !jf.exists?
|
162
|
+
jf.add_attribute(@data[0][1], row[1])
|
163
|
+
jf.save
|
164
|
+
@progress.progress = i if HeyDan.help?
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def get_identifiers
|
169
|
+
@id = @data[0][0]
|
170
|
+
@identifiers = build_identifier_hash(@id)
|
171
|
+
end
|
172
|
+
|
173
|
+
def get_data
|
174
|
+
if @data.nil?
|
175
|
+
@data = HeyDan::Helper.get_data(dataset_file_name)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
class HeyDan::ScriptFile
|
2
|
+
attr_accessor :folder
|
3
|
+
attr_accessor :source
|
4
|
+
attr_accessor :variable
|
5
|
+
attr_accessor :script_folder_path
|
6
|
+
attr_accessor :script_file_path
|
7
|
+
attr_accessor :class_name
|
8
|
+
attr_accessor :name
|
9
|
+
|
10
|
+
|
11
|
+
def initialize(folder, source, variable)
|
12
|
+
@folder, @source, @variable = folder, source, variable
|
13
|
+
@name = @folder + "_" + @source + "_" + @variable
|
14
|
+
create_class_name
|
15
|
+
@script_folder_path = File.join(HeyDan.folders[:sources], @folder, 'scripts')
|
16
|
+
@script_file_path = File.join(@script_folder_path, "#{@name}.rb")
|
17
|
+
end
|
18
|
+
|
19
|
+
def create_script_folder
|
20
|
+
FileUtils.mkdir_p @script_folder_path if !Dir.exist?(@script_folder_path)
|
21
|
+
end
|
22
|
+
|
23
|
+
def create_class_name
|
24
|
+
@class_name = @name.split('_').collect(&:capitalize).join
|
25
|
+
end
|
26
|
+
|
27
|
+
def template
|
28
|
+
template_path = File.join('lib', 'templates', 'script.rb.erb')
|
29
|
+
require 'erb'
|
30
|
+
ERB.new(File.read(template_path)).result binding
|
31
|
+
end
|
32
|
+
|
33
|
+
def save
|
34
|
+
create_script_folder
|
35
|
+
File.open(@script_file_path, 'w') do |f|
|
36
|
+
f.write(template)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
def eval_class
|
41
|
+
load script_file_path
|
42
|
+
return eval("#{@class_name}")
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'sinatra'
|
2
|
+
require 'json'
|
3
|
+
require 'sinatra/cross_origin'
|
4
|
+
require 'elasticsearch'
|
5
|
+
|
6
|
+
class HeyDan::Server < Sinatra::Base
|
7
|
+
|
8
|
+
configure do
|
9
|
+
enable :cross_origin
|
10
|
+
end
|
11
|
+
|
12
|
+
get '/entities/*.json' do
|
13
|
+
content_type 'application/json'
|
14
|
+
begin
|
15
|
+
file = File.join(HeyDan.folders[:jurisdictions], "#{params[:splat][0].gsub('/','::')}.json")
|
16
|
+
File.read(file)
|
17
|
+
rescue
|
18
|
+
halt 404, "{\"message\": \"I couldn\'t find #{params[:splat][0]}. Is the ID right?\"}"
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
get '/entities' do
|
23
|
+
content_type 'application/json'
|
24
|
+
begin
|
25
|
+
client = Elasticsearch::Client.new url: HeyDan.elasticsearch[:url], log: false
|
26
|
+
r = client.search index: 'jurisdictions',type: params[:type], :_source_include => ['id', 'entityType', 'name', 'entityUrl'],
|
27
|
+
body: {
|
28
|
+
from: params[:page] || 1,
|
29
|
+
size: params[:per_page] || 10
|
30
|
+
}
|
31
|
+
return {total: r["hits"]["total"], page: params[:page] || 1, entities: r["hits"]["hits"].map { |x| x["_source"]}}.to_json
|
32
|
+
rescue
|
33
|
+
halt 404, '{"message": "Hrmm, something is fishy here. Are you do something weird?"}'
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
get '/entities/*' do
|
38
|
+
content_type 'application/json'
|
39
|
+
begin
|
40
|
+
client = Elasticsearch::Client.new url: HeyDan.elasticsearch[:url], log: false
|
41
|
+
r = client.search index: 'jurisdictions', type: params[:type], :_source_include => ['id', 'entityType', 'name', 'entityUrl'],
|
42
|
+
body: {
|
43
|
+
query: {
|
44
|
+
simple_query_string:{query: "\"#{params[:splat][0]}*\"", fields: ['id']}
|
45
|
+
},
|
46
|
+
from: params[:page] || 1,
|
47
|
+
size: params[:per_page] || 10
|
48
|
+
}
|
49
|
+
return {total: r["hits"]["total"], page: params[:page] || 1, entities: r["hits"]["hits"].map { |x| x["_source"]}}.to_json
|
50
|
+
rescue
|
51
|
+
halt 404, '{"message": "Hrmm, something is fishy here. Are you do something weird?"}'
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
@@ -0,0 +1,79 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
class HeyDan::SourceFile
|
4
|
+
attr_accessor :json
|
5
|
+
attr_accessor :file_path
|
6
|
+
attr_accessor :folder_path
|
7
|
+
attr_accessor :folder
|
8
|
+
attr_accessor :name
|
9
|
+
attr_accessor :variable
|
10
|
+
|
11
|
+
def initialize(folder, name, variable=nil)
|
12
|
+
@folder = folder
|
13
|
+
@name = name
|
14
|
+
@variable = variable
|
15
|
+
@folder_path = File.join(HeyDan.folders[:sources],@folder)
|
16
|
+
@file_path = File.join(@folder_path, @name)
|
17
|
+
get_json
|
18
|
+
end
|
19
|
+
|
20
|
+
def get_json
|
21
|
+
if exist?
|
22
|
+
@json = JSON.parse(File.read(file_path))
|
23
|
+
else
|
24
|
+
@json = initial_json
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def create_folder
|
29
|
+
FileUtils.mkdir_p @folder_path if !Dir.exist?(@folder_path)
|
30
|
+
end
|
31
|
+
|
32
|
+
def exist?
|
33
|
+
File.exist?(@file_path)
|
34
|
+
end
|
35
|
+
|
36
|
+
def add_variable(variable_name)
|
37
|
+
@json['variables'][variable_name] = variable_json(variable_name) if variable(variable_name).nil?
|
38
|
+
variable(variable_name)
|
39
|
+
end
|
40
|
+
|
41
|
+
def create_variable_scripts
|
42
|
+
if @json['variables'].keys.size > 0
|
43
|
+
@json['variables'].keys.each do |variable|
|
44
|
+
create_script_file(variable)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def create_script_file(variable_name)
|
50
|
+
file = HeyDan::ScriptFile.new(@folder, @name, variable_name)
|
51
|
+
file.save
|
52
|
+
end
|
53
|
+
|
54
|
+
def initial_json
|
55
|
+
{'name' => @name, 'short_description' => 'A short description', 'long_description' => 'a longer description', 'notes' => nil, 'depends' => nil, 'sourceUrl' => 'the website of the source', 'variables' => {}}
|
56
|
+
end
|
57
|
+
|
58
|
+
def variable_json(variable_name)
|
59
|
+
{'id' => "#{@folder}_#{@name}_#{variable_name}",'name' => variable_name, 'short_description' => 'a short description', 'long_description' => 'a description of the variable', 'notes' => 'any notes about this variable', 'identifier' => 'open_civic_id or ansi_id or other', 'dates' => [2015], 'tags' => [], 'sourceUrl' => 'website for variables information if different than source', 'jurisdiction_types' => [], 'coverage' => {} }
|
60
|
+
end
|
61
|
+
|
62
|
+
def variable(variable_name=nil)
|
63
|
+
variable_name ||= @variable
|
64
|
+
@json['variables'][variable_name]
|
65
|
+
end
|
66
|
+
|
67
|
+
def variables
|
68
|
+
@json['variables'].keys
|
69
|
+
end
|
70
|
+
|
71
|
+
def save
|
72
|
+
create_folder
|
73
|
+
create_variable_scripts
|
74
|
+
File.open(@file_path, 'w') do |f|
|
75
|
+
f.write(JSON.pretty_generate(@json))
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
require 'git'
|
2
|
+
|
3
|
+
class HeyDan::Sources
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def source_exists?(name_or_link)
|
7
|
+
HeyDan.sources ||= {}
|
8
|
+
HeyDan.sources.keys.map(&:to_s).include?(name_or_link) || HeyDan.sources.values.include?(name_or_link)
|
9
|
+
end
|
10
|
+
|
11
|
+
def sync
|
12
|
+
keys = HeyDan.sources.keys
|
13
|
+
if keys
|
14
|
+
HeyDan.sources.keys.map(&:to_s).each { |source| update(source)}
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
def add(link)
|
19
|
+
raise 'Link must be a git link in the format of http(s)://url/*.git' if !correct_link?(link)
|
20
|
+
settings_file = HeyDan::Base.load_or_create_settings
|
21
|
+
name = extract_name(link)
|
22
|
+
if !source_exists?(link)
|
23
|
+
HeyDan.sources ||= {}
|
24
|
+
HeyDan.sources.merge!({"#{name}" => link})
|
25
|
+
HeyDan::Base.save_settings
|
26
|
+
end
|
27
|
+
update(name)
|
28
|
+
end
|
29
|
+
|
30
|
+
def correct_link?(link)
|
31
|
+
!link.match(/(http|https):\/\/\w+\.\w+(\/\w+)+\.git/).nil?
|
32
|
+
end
|
33
|
+
|
34
|
+
def update(name)
|
35
|
+
if directory_exist?(name)
|
36
|
+
HeyDan::HelpText.git_update(name)
|
37
|
+
g = Git.open(source_folder(name))
|
38
|
+
g.pull
|
39
|
+
else
|
40
|
+
HeyDan::HelpText.git_clone(name)
|
41
|
+
g = Git.clone(HeyDan.sources[name.to_sym], name, {:path => HeyDan.folders[:sources]})
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def source_folder(name)
|
46
|
+
File.join(HeyDan.folders[:sources],name)
|
47
|
+
end
|
48
|
+
|
49
|
+
def directory_exist?(name)
|
50
|
+
Dir.exists?(source_folder(name))
|
51
|
+
end
|
52
|
+
|
53
|
+
def extract_name(git_link)
|
54
|
+
git_link.match(/(\w+)\.git$/i)[1]
|
55
|
+
end
|
56
|
+
|
57
|
+
def create_folder(name)
|
58
|
+
FileUtils.mkdir_p source_folder(name)
|
59
|
+
end
|
60
|
+
|
61
|
+
def create_source(folder, name)
|
62
|
+
file = HeyDan::SourceFile.new(folder, name)
|
63
|
+
file.save
|
64
|
+
end
|
65
|
+
|
66
|
+
def create_variable(folder, name, variable)
|
67
|
+
file = HeyDan::SourceFile.new(folder, name)
|
68
|
+
file.add_variable(variable)
|
69
|
+
file.save
|
70
|
+
end
|
71
|
+
|
72
|
+
def source_exist?(folder, source_name)
|
73
|
+
file = HeyDan::SourceFile.new(folder, source_name)
|
74
|
+
file.exist?
|
75
|
+
end
|
76
|
+
|
77
|
+
def variable_exist?(folder, source_name, variable)
|
78
|
+
file = HeyDan::SourceFile.new(folder, source_name)
|
79
|
+
return file.exist? if !file.exist?
|
80
|
+
!file.variable(variable).nil?
|
81
|
+
end
|
82
|
+
|
83
|
+
def only_letters_and_underscores?(text)
|
84
|
+
end
|
85
|
+
|
86
|
+
def create(folder, source_name, variable=nil)
|
87
|
+
create_folder(folder) if !directory_exist?(folder)
|
88
|
+
create_source(folder, source_name) if !source_exist?(folder, source_name)
|
89
|
+
create_variable(folder, source_name, variable) if !variable_exist?(folder, source_name, variable)
|
90
|
+
end
|
91
|
+
|
92
|
+
def build(folder=nil, name=nil, variable=nil)
|
93
|
+
if variable && name && folder
|
94
|
+
build_variable(folder, name, variable)
|
95
|
+
elsif name && folder
|
96
|
+
build_source(folder, name)
|
97
|
+
elsif folder
|
98
|
+
build_folder(folder)
|
99
|
+
else
|
100
|
+
HeyDan.sources.keys.each { |source| build_folder(source.to_s)}
|
101
|
+
end
|
102
|
+
end
|
103
|
+
|
104
|
+
def sources(folder)
|
105
|
+
Dir.glob(source_folder(folder) + '/*').select { |x| !x.include?('/scripts') && !x.include?('.')}.map { |x| x.split('/')[-1]}
|
106
|
+
end
|
107
|
+
|
108
|
+
def build_folder(folder)
|
109
|
+
sources(folder).each do |source|
|
110
|
+
build_source(folder, source)
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def build_source(folder, source)
|
115
|
+
source_file = HeyDan::SourceFile.new(folder, source)
|
116
|
+
source_file.variables.each do |var|
|
117
|
+
build_variable(folder, source, var)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
def build_variable(folder, source, variable)
|
122
|
+
script = HeyDan::ScriptFile.new(folder, source, variable)
|
123
|
+
var = script.eval_class.send(:new, {folder: folder, source: source, variable: variable}.merge(HeyDan.options))
|
124
|
+
var.process
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
end
|