heydan 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,179 @@
1
+ require 'ruby-progressbar'
2
+ class HeyDan::Script
3
+ attr_accessor :jurisdiction_type
4
+ attr_accessor :fromsource
5
+ attr_accessor :data
6
+ attr_accessor :folder
7
+ attr_accessor :source
8
+ attr_accessor :variable
9
+ attr_accessor :id
10
+ attr_accessor :source_file
11
+ attr_accessor :identifiers #builds a hash with new_identifier => file_name
12
+
13
+ def initialize(opts)
14
+ @folder = opts[:folder]
15
+ @source = opts[:source]
16
+ @variable = opts[:variable]
17
+ @jurisdiction_type = HeyDan.options[:type]
18
+ @fromsource = HeyDan.options[:fromsource]
19
+ @identifiers = {}
20
+ @source_file = HeyDan::SourceFile.new(@folder, @source, @variable)
21
+ end
22
+
23
+ #overwritten by the developer
24
+ def build
25
+ end
26
+
27
+ #overwritten by the developer
28
+ def validate_build
29
+ end
30
+
31
+ #overwritten by the developer, can be dataset, attribute, or identifer
32
+ def type
33
+ 'dataset'
34
+ end
35
+
36
+ #overwritten by developer
37
+ def version
38
+ 1
39
+ end
40
+
41
+ def dataset_file_name
42
+ "#{@folder}_#{@source}_#{@variable}.csv"
43
+ end
44
+
45
+ def name
46
+ "#{@folder}_#{@source}_#{@variable}.csv"
47
+ end
48
+
49
+ def dataset_file_path
50
+ File.join(HeyDan.folders[:datasets], dataset_file_name)
51
+ end
52
+
53
+ #downloads from the cdn
54
+ def download
55
+ @data = HeyDan::Helper.get_data_from_url(HeyDan.cdn + '/' + dataset_file_name)
56
+ end
57
+
58
+ def process_from_source
59
+ build
60
+ validate_build
61
+ HeyDan::Helper.save_data(dataset_file_name, @data)
62
+ end
63
+
64
+ #runs through download, build and validate
65
+ def process
66
+ if @fromsource
67
+ puts "From Source is specified, processing from source for #{name}" if HeyDan.help?
68
+ process_from_source
69
+ end
70
+
71
+ begin
72
+ if HeyDan::Helper.dataset_exists?(dataset_file_name)
73
+ puts "Dataset for #{name} exists" if HeyDan.help?
74
+ else
75
+ download
76
+ end
77
+ rescue
78
+ puts "Had trouble downloading #{name}, processing from source instead" if HeyDan.help?
79
+ process_from_source
80
+ end
81
+ update_jurisdiction_files
82
+ end
83
+
84
+ def build_identifier_hash(identifier)
85
+ identifier_file = File.join(HeyDan.folders[:downloads], "identifiers_file_#{identifier}.json")
86
+ if File.exist?(identifier_file)
87
+ @identifiers = JSON.parse(File.read(identifier_file))
88
+ return @identifiers
89
+ end
90
+ HeyDan::HelpText.build_identifier(identifier)
91
+ get_identifiers_from_files(identifier)
92
+ File.open(identifier_file, 'w') do |file|
93
+ file.write(@identifiers.to_json)
94
+ end
95
+ @identifiers
96
+ end
97
+
98
+ def get_identifiers_from_files(identifier)
99
+ @identifiers = {}
100
+ Dir.glob(File.join(HeyDan.folders[:jurisdictions], '*.json')).each do |j|
101
+ jf = HeyDan::JurisdictionFile.new(name: j.gsub(HeyDan.folders[:jurisdictions] + '/', ''))
102
+ return if !jf.match_type?(@jurisdiction_type)
103
+ @identifiers["#{jf.get_identifier(identifier)}"] = j.gsub(HeyDan.folders[:jurisdictions] + '/', '')
104
+ end
105
+ @identifiers
106
+ end
107
+
108
+ def update_jurisdiction_files
109
+ get_data
110
+ get_identifiers
111
+ @progress = ProgressBar.create(:title => "Updating Files for #{@source} #{@variable} from #{@folder} for jurisdictions #{(' matching ' + @jurisdiction_type) if @jurisdiction_type}", :starting_at => 0, :total => @data[1..-1].size) if HeyDan.help?
112
+ self.send("add_#{type}s")
113
+ @progress.finish if HeyDan.help?
114
+ end
115
+
116
+ def add_datasets
117
+ metadata = @source_file.variable
118
+ metadata.keep_if { |k| ['id', 'name', 'short_description', 'tags'].include?(k)}
119
+ metadata["years"] = @data[0][1..-1]
120
+ id = metadata['id']
121
+ @data[1..-1].each_index do |i|
122
+ row = @data[i+1]
123
+ next if row[0].nil? || @identifiers[row[0]].nil?
124
+ jf = get_jurisdiction_filename(@identifiers[row[0]])
125
+ next if row[0].nil? || !jf.exists?
126
+ ds = jf.get_dataset(id)
127
+ if !ds.nil?
128
+ if ds['version'] >= version
129
+ @progress.progress = i if HeyDan.help?
130
+ next
131
+ end
132
+ end
133
+ jf.datasets.delete(ds)
134
+ metadata["version"] = version
135
+ metadata["data"] = row[1..-1]
136
+ jf.add_dataset(metadata)
137
+ jf.save
138
+ @progress.progress = i if HeyDan.help?
139
+ end
140
+ end
141
+
142
+ def get_jurisdiction_filename(id)
143
+ HeyDan::JurisdictionFile.new(name: @identifiers[id] || id)
144
+ end
145
+
146
+ def add_identifiers
147
+ @data[1..-1].each_index do |i|
148
+ row = @data[i+1]
149
+ jf = get_jurisdiction_filename(row[0])
150
+ next if row[0].nil? || !jf.exists?
151
+ jf.add_identifier(@data[0][1], row[1])
152
+ jf.save
153
+ @progress.progress = i if HeyDan.help?
154
+ end
155
+ end
156
+
157
+ def add_attributes
158
+ @data[1..-1].each_index do |i|
159
+ row = @data[i+1]
160
+ jf = get_jurisdiction_filename(row[0])
161
+ next if row[0].nil? || !jf.exists?
162
+ jf.add_attribute(@data[0][1], row[1])
163
+ jf.save
164
+ @progress.progress = i if HeyDan.help?
165
+ end
166
+ end
167
+
168
+ def get_identifiers
169
+ @id = @data[0][0]
170
+ @identifiers = build_identifier_hash(@id)
171
+ end
172
+
173
+ def get_data
174
+ if @data.nil?
175
+ @data = HeyDan::Helper.get_data(dataset_file_name)
176
+ end
177
+ end
178
+
179
+ end
@@ -0,0 +1,44 @@
1
+ class HeyDan::ScriptFile
2
+ attr_accessor :folder
3
+ attr_accessor :source
4
+ attr_accessor :variable
5
+ attr_accessor :script_folder_path
6
+ attr_accessor :script_file_path
7
+ attr_accessor :class_name
8
+ attr_accessor :name
9
+
10
+
11
+ def initialize(folder, source, variable)
12
+ @folder, @source, @variable = folder, source, variable
13
+ @name = @folder + "_" + @source + "_" + @variable
14
+ create_class_name
15
+ @script_folder_path = File.join(HeyDan.folders[:sources], @folder, 'scripts')
16
+ @script_file_path = File.join(@script_folder_path, "#{@name}.rb")
17
+ end
18
+
19
+ def create_script_folder
20
+ FileUtils.mkdir_p @script_folder_path if !Dir.exist?(@script_folder_path)
21
+ end
22
+
23
+ def create_class_name
24
+ @class_name = @name.split('_').collect(&:capitalize).join
25
+ end
26
+
27
+ def template
28
+ template_path = File.join('lib', 'templates', 'script.rb.erb')
29
+ require 'erb'
30
+ ERB.new(File.read(template_path)).result binding
31
+ end
32
+
33
+ def save
34
+ create_script_folder
35
+ File.open(@script_file_path, 'w') do |f|
36
+ f.write(template)
37
+ end
38
+ end
39
+
40
+ def eval_class
41
+ load script_file_path
42
+ return eval("#{@class_name}")
43
+ end
44
+ end
@@ -0,0 +1,55 @@
1
+ require 'sinatra'
2
+ require 'json'
3
+ require 'sinatra/cross_origin'
4
+ require 'elasticsearch'
5
+
6
+ class HeyDan::Server < Sinatra::Base
7
+
8
+ configure do
9
+ enable :cross_origin
10
+ end
11
+
12
+ get '/entities/*.json' do
13
+ content_type 'application/json'
14
+ begin
15
+ file = File.join(HeyDan.folders[:jurisdictions], "#{params[:splat][0].gsub('/','::')}.json")
16
+ File.read(file)
17
+ rescue
18
+ halt 404, "{\"message\": \"I couldn\'t find #{params[:splat][0]}. Is the ID right?\"}"
19
+ end
20
+ end
21
+
22
+ get '/entities' do
23
+ content_type 'application/json'
24
+ begin
25
+ client = Elasticsearch::Client.new url: HeyDan.elasticsearch[:url], log: false
26
+ r = client.search index: 'jurisdictions',type: params[:type], :_source_include => ['id', 'entityType', 'name', 'entityUrl'],
27
+ body: {
28
+ from: params[:page] || 1,
29
+ size: params[:per_page] || 10
30
+ }
31
+ return {total: r["hits"]["total"], page: params[:page] || 1, entities: r["hits"]["hits"].map { |x| x["_source"]}}.to_json
32
+ rescue
33
+ halt 404, '{"message": "Hrmm, something is fishy here. Are you do something weird?"}'
34
+ end
35
+ end
36
+
37
+ get '/entities/*' do
38
+ content_type 'application/json'
39
+ begin
40
+ client = Elasticsearch::Client.new url: HeyDan.elasticsearch[:url], log: false
41
+ r = client.search index: 'jurisdictions', type: params[:type], :_source_include => ['id', 'entityType', 'name', 'entityUrl'],
42
+ body: {
43
+ query: {
44
+ simple_query_string:{query: "\"#{params[:splat][0]}*\"", fields: ['id']}
45
+ },
46
+ from: params[:page] || 1,
47
+ size: params[:per_page] || 10
48
+ }
49
+ return {total: r["hits"]["total"], page: params[:page] || 1, entities: r["hits"]["hits"].map { |x| x["_source"]}}.to_json
50
+ rescue
51
+ halt 404, '{"message": "Hrmm, something is fishy here. Are you do something weird?"}'
52
+ end
53
+ end
54
+
55
+ end
@@ -0,0 +1,79 @@
1
+ require 'json'
2
+
3
+ class HeyDan::SourceFile
4
+ attr_accessor :json
5
+ attr_accessor :file_path
6
+ attr_accessor :folder_path
7
+ attr_accessor :folder
8
+ attr_accessor :name
9
+ attr_accessor :variable
10
+
11
+ def initialize(folder, name, variable=nil)
12
+ @folder = folder
13
+ @name = name
14
+ @variable = variable
15
+ @folder_path = File.join(HeyDan.folders[:sources],@folder)
16
+ @file_path = File.join(@folder_path, @name)
17
+ get_json
18
+ end
19
+
20
+ def get_json
21
+ if exist?
22
+ @json = JSON.parse(File.read(file_path))
23
+ else
24
+ @json = initial_json
25
+ end
26
+ end
27
+
28
+ def create_folder
29
+ FileUtils.mkdir_p @folder_path if !Dir.exist?(@folder_path)
30
+ end
31
+
32
+ def exist?
33
+ File.exist?(@file_path)
34
+ end
35
+
36
+ def add_variable(variable_name)
37
+ @json['variables'][variable_name] = variable_json(variable_name) if variable(variable_name).nil?
38
+ variable(variable_name)
39
+ end
40
+
41
+ def create_variable_scripts
42
+ if @json['variables'].keys.size > 0
43
+ @json['variables'].keys.each do |variable|
44
+ create_script_file(variable)
45
+ end
46
+ end
47
+ end
48
+
49
+ def create_script_file(variable_name)
50
+ file = HeyDan::ScriptFile.new(@folder, @name, variable_name)
51
+ file.save
52
+ end
53
+
54
+ def initial_json
55
+ {'name' => @name, 'short_description' => 'A short description', 'long_description' => 'a longer description', 'notes' => nil, 'depends' => nil, 'sourceUrl' => 'the website of the source', 'variables' => {}}
56
+ end
57
+
58
+ def variable_json(variable_name)
59
+ {'id' => "#{@folder}_#{@name}_#{variable_name}",'name' => variable_name, 'short_description' => 'a short description', 'long_description' => 'a description of the variable', 'notes' => 'any notes about this variable', 'identifier' => 'open_civic_id or ansi_id or other', 'dates' => [2015], 'tags' => [], 'sourceUrl' => 'website for variables information if different than source', 'jurisdiction_types' => [], 'coverage' => {} }
60
+ end
61
+
62
+ def variable(variable_name=nil)
63
+ variable_name ||= @variable
64
+ @json['variables'][variable_name]
65
+ end
66
+
67
+ def variables
68
+ @json['variables'].keys
69
+ end
70
+
71
+ def save
72
+ create_folder
73
+ create_variable_scripts
74
+ File.open(@file_path, 'w') do |f|
75
+ f.write(JSON.pretty_generate(@json))
76
+ end
77
+ end
78
+
79
+ end
@@ -0,0 +1,128 @@
1
+ require 'git'
2
+
3
+ class HeyDan::Sources
4
+ class << self
5
+
6
+ def source_exists?(name_or_link)
7
+ HeyDan.sources ||= {}
8
+ HeyDan.sources.keys.map(&:to_s).include?(name_or_link) || HeyDan.sources.values.include?(name_or_link)
9
+ end
10
+
11
+ def sync
12
+ keys = HeyDan.sources.keys
13
+ if keys
14
+ HeyDan.sources.keys.map(&:to_s).each { |source| update(source)}
15
+ end
16
+ end
17
+
18
+ def add(link)
19
+ raise 'Link must be a git link in the format of http(s)://url/*.git' if !correct_link?(link)
20
+ settings_file = HeyDan::Base.load_or_create_settings
21
+ name = extract_name(link)
22
+ if !source_exists?(link)
23
+ HeyDan.sources ||= {}
24
+ HeyDan.sources.merge!({"#{name}" => link})
25
+ HeyDan::Base.save_settings
26
+ end
27
+ update(name)
28
+ end
29
+
30
+ def correct_link?(link)
31
+ !link.match(/(http|https):\/\/\w+\.\w+(\/\w+)+\.git/).nil?
32
+ end
33
+
34
+ def update(name)
35
+ if directory_exist?(name)
36
+ HeyDan::HelpText.git_update(name)
37
+ g = Git.open(source_folder(name))
38
+ g.pull
39
+ else
40
+ HeyDan::HelpText.git_clone(name)
41
+ g = Git.clone(HeyDan.sources[name.to_sym], name, {:path => HeyDan.folders[:sources]})
42
+ end
43
+ end
44
+
45
+ def source_folder(name)
46
+ File.join(HeyDan.folders[:sources],name)
47
+ end
48
+
49
+ def directory_exist?(name)
50
+ Dir.exists?(source_folder(name))
51
+ end
52
+
53
+ def extract_name(git_link)
54
+ git_link.match(/(\w+)\.git$/i)[1]
55
+ end
56
+
57
+ def create_folder(name)
58
+ FileUtils.mkdir_p source_folder(name)
59
+ end
60
+
61
+ def create_source(folder, name)
62
+ file = HeyDan::SourceFile.new(folder, name)
63
+ file.save
64
+ end
65
+
66
+ def create_variable(folder, name, variable)
67
+ file = HeyDan::SourceFile.new(folder, name)
68
+ file.add_variable(variable)
69
+ file.save
70
+ end
71
+
72
+ def source_exist?(folder, source_name)
73
+ file = HeyDan::SourceFile.new(folder, source_name)
74
+ file.exist?
75
+ end
76
+
77
+ def variable_exist?(folder, source_name, variable)
78
+ file = HeyDan::SourceFile.new(folder, source_name)
79
+ return file.exist? if !file.exist?
80
+ !file.variable(variable).nil?
81
+ end
82
+
83
+ def only_letters_and_underscores?(text)
84
+ end
85
+
86
+ def create(folder, source_name, variable=nil)
87
+ create_folder(folder) if !directory_exist?(folder)
88
+ create_source(folder, source_name) if !source_exist?(folder, source_name)
89
+ create_variable(folder, source_name, variable) if !variable_exist?(folder, source_name, variable)
90
+ end
91
+
92
+ def build(folder=nil, name=nil, variable=nil)
93
+ if variable && name && folder
94
+ build_variable(folder, name, variable)
95
+ elsif name && folder
96
+ build_source(folder, name)
97
+ elsif folder
98
+ build_folder(folder)
99
+ else
100
+ HeyDan.sources.keys.each { |source| build_folder(source.to_s)}
101
+ end
102
+ end
103
+
104
+ def sources(folder)
105
+ Dir.glob(source_folder(folder) + '/*').select { |x| !x.include?('/scripts') && !x.include?('.')}.map { |x| x.split('/')[-1]}
106
+ end
107
+
108
+ def build_folder(folder)
109
+ sources(folder).each do |source|
110
+ build_source(folder, source)
111
+ end
112
+ end
113
+
114
+ def build_source(folder, source)
115
+ source_file = HeyDan::SourceFile.new(folder, source)
116
+ source_file.variables.each do |var|
117
+ build_variable(folder, source, var)
118
+ end
119
+ end
120
+
121
+ def build_variable(folder, source, variable)
122
+ script = HeyDan::ScriptFile.new(folder, source, variable)
123
+ var = script.eval_class.send(:new, {folder: folder, source: source, variable: variable}.merge(HeyDan.options))
124
+ var.process
125
+ end
126
+
127
+ end
128
+ end