tax_generator 0.5.2 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.travis.yml +15 -0
- data/README.md +1 -1
- data/lib/tax_generator/all.rb +3 -2
- data/lib/tax_generator/classes/file_creator.rb +8 -6
- data/lib/tax_generator/classes/processor.rb +255 -250
- data/lib/tax_generator/classes/taxonomy_tree.rb +6 -6
- data/lib/tax_generator/helpers/application_helper.rb +2 -17
- data/lib/tax_generator/helpers/methodic_actor.rb +6 -0
- data/lib/tax_generator/version.rb +2 -2
- data/tax_generator.gemspec +4 -4
- metadata +22 -40
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c747d61bb720bf5072dc9e9ec0870d11974e31c3
|
4
|
+
data.tar.gz: 971979df05fb6b2d3632f24b93c62ed318745ad4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d0463ac2a1ce210da8b8e5ae4ea3348695c3d50e3e4a2bd526c7a05d8abe161cbdb26e02cd9bdceea1e7eed70928d563b5558bfa26d886899ce97be379916dc8
|
7
|
+
data.tar.gz: 94489169e099c7bf427d3693571c31ee7e508f680f1a078e8ef73c903f7fba861e82feae9f9db37341199ac80de6e440c9cfb16d26b18e11b983c3cb5014f49f
|
data/.travis.yml
ADDED
@@ -0,0 +1,15 @@
|
|
1
|
+
sudo: false
|
2
|
+
cache: bundler
|
3
|
+
language: ruby
|
4
|
+
before_install:
|
5
|
+
- "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc"
|
6
|
+
- gem install bundler
|
7
|
+
rvm:
|
8
|
+
- 2.0.0
|
9
|
+
- 2.1.5
|
10
|
+
- 2.2.2
|
11
|
+
- 2.2.3
|
12
|
+
env:
|
13
|
+
- RAILS_ENV=test RACK_ENV=test
|
14
|
+
notifications:
|
15
|
+
email: false
|
data/README.md
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
tax_generator
|
2
2
|
=============
|
3
3
|
|
4
|
-
[](http://badge.fury.io/rb/tax_generator) [](https://github.com/bogdanRada/tax_generator)
|
4
|
+
[](http://badge.fury.io/rb/tax_generator) [](https://github.com/bogdanRada/tax_generator) [](https://github.com/bogdanRada/tax_generator)
|
5
5
|
|
6
6
|
Description
|
7
7
|
-----------
|
data/lib/tax_generator/all.rb
CHANGED
@@ -10,8 +10,8 @@ require 'active_support/core_ext/module/delegation'
|
|
10
10
|
require 'slop'
|
11
11
|
require 'nokogiri'
|
12
12
|
require 'tree'
|
13
|
-
require '
|
14
|
-
require '
|
13
|
+
require 'concurrent'
|
14
|
+
require 'concurrent-edge'
|
15
15
|
|
16
16
|
require 'logger'
|
17
17
|
require 'fileutils'
|
@@ -20,6 +20,7 @@ require 'erb'
|
|
20
20
|
require 'tilt'
|
21
21
|
require 'tilt/erb'
|
22
22
|
require 'thread'
|
23
|
+
require 'securerandom'
|
23
24
|
|
24
25
|
%w(classes helpers).each do |folder_name|
|
25
26
|
Gem.find_files("tax_generator/#{folder_name}/**/*.rb").each { |path| require path }
|
@@ -1,4 +1,5 @@
|
|
1
1
|
require_relative '../helpers/application_helper'
|
2
|
+
require_relative '../helpers/methodic_actor'
|
2
3
|
module TaxGenerator
|
3
4
|
# class used to create the files
|
4
5
|
#
|
@@ -14,9 +15,9 @@ module TaxGenerator
|
|
14
15
|
# @return [TaxGenerator::TaxonomyTree] the taxonomy tree holding all the nodes from the taxonomy xml document
|
15
16
|
# @!attribute output_folder
|
16
17
|
# @return [String] the output folder where the new files will be created
|
17
|
-
class FileCreator
|
18
|
-
include
|
19
|
-
include
|
18
|
+
class FileCreator < Concurrent::Actor::RestartingContext
|
19
|
+
include Concurrent::Async
|
20
|
+
include MethodicActor
|
20
21
|
include TaxGenerator::ApplicationHelper
|
21
22
|
|
22
23
|
attr_reader :processor, :job, :job_id, :destination, :taxonomy, :output_folder
|
@@ -40,12 +41,13 @@ module TaxGenerator
|
|
40
41
|
# @return [void]
|
41
42
|
#
|
42
43
|
# @api public
|
43
|
-
def
|
44
|
+
def initialize(*args)
|
45
|
+
job = args[0]
|
46
|
+
@processor = args[1]
|
44
47
|
job = job.stringify_keys
|
45
48
|
@job = job
|
46
|
-
@processor = manager
|
47
49
|
process_job(job)
|
48
|
-
|
50
|
+
processor.register_worker_for_job(job, self)
|
49
51
|
end
|
50
52
|
|
51
53
|
# processes the job information by retrieving keys from the hash
|
@@ -18,14 +18,22 @@ module TaxGenerator
|
|
18
18
|
# @!attribute worker_to_job
|
19
19
|
# @return [Hash] each key from the list is the workers mailbox address, and the value is the job being handled by the worker
|
20
20
|
class Processor
|
21
|
-
include
|
22
|
-
include Celluloid::Logger
|
23
|
-
include Celluloid::Notifications
|
21
|
+
include Concurrent::Async
|
24
22
|
include TaxGenerator::ApplicationHelper
|
25
23
|
|
26
|
-
|
24
|
+
QUEUES = Concurrent::Map.new do |hash, queue_name| #:nodoc:
|
25
|
+
hash.compute_if_absent(queue_name) {
|
26
|
+
Concurrent::ThreadPoolExecutor.new(
|
27
|
+
min_threads: 10, # create 10 threads at startup
|
28
|
+
max_threads: 50, # create at most 50 threads
|
29
|
+
max_queue: 0, # unbounded queue of work waiting for an available thread
|
30
|
+
)
|
31
|
+
}
|
32
|
+
end
|
33
|
+
|
34
|
+
attr_reader :options, :taxonomy, :jobs, :job_to_worker, :worker_to_job
|
27
35
|
|
28
|
-
trap_exit :worker_died
|
36
|
+
#trap_exit :worker_died
|
29
37
|
|
30
38
|
# receives a list of options that are used to determine the input files and output and input folders
|
31
39
|
#
|
@@ -41,11 +49,7 @@ module TaxGenerator
|
|
41
49
|
#
|
42
50
|
# @api public
|
43
51
|
def initialize(options = {})
|
44
|
-
Celluloid.boot
|
45
52
|
@options = options.is_a?(Hash) ? options.symbolize_keys : {}
|
46
|
-
@worker_supervisor = Celluloid::SupervisionGroup.run!
|
47
|
-
@workers = @worker_supervisor.pool(TaxGenerator::FileCreator, as: :workers, size: 50)
|
48
|
-
Actor.current.link @workers
|
49
53
|
@jobs_mutex = Mutex.new
|
50
54
|
@job_to_worker_mutex = Mutex.new
|
51
55
|
@worker_to_job_mutex = Mutex.new
|
@@ -54,279 +58,280 @@ module TaxGenerator
|
|
54
58
|
@worker_to_job = {}
|
55
59
|
end
|
56
60
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
# @api public
|
63
|
-
def input_folder
|
64
|
-
@options.fetch(:input_dir, "#{root}/data/input")
|
65
|
-
end
|
61
|
+
def enqueue(job_data, queue: 'default') #:nodoc:
|
62
|
+
QUEUES[queue].post(job_data) { |job|
|
63
|
+
TaxGenerator::FileCreator.spawn(name: "worker_#{job['atlas_id']}", args: [job, self])
|
64
|
+
}
|
65
|
+
end
|
66
66
|
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
67
|
+
# otherwise the default path
|
68
|
+
#
|
69
|
+
# @return [String]
|
70
|
+
#
|
71
|
+
# @api public
|
72
|
+
# returns the input folder from the options list
|
73
|
+
def input_folder
|
74
|
+
@options.fetch(:input_dir, "#{root}/data/input")
|
75
|
+
end
|
76
76
|
|
77
|
-
|
78
|
-
|
79
|
-
|
80
|
-
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
77
|
+
# returns the taxonomy filename from the option list
|
78
|
+
# otherwise the default filename
|
79
|
+
#
|
80
|
+
# @return [String]
|
81
|
+
#
|
82
|
+
# @api public
|
83
|
+
def taxonomy_file_name
|
84
|
+
@options.fetch(:taxonomy_filename, 'taxonomy.xml')
|
85
|
+
end
|
86
86
|
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
|
94
|
-
|
95
|
-
|
87
|
+
# returns the destinations filename from the option list
|
88
|
+
# otherwise the default filename
|
89
|
+
#
|
90
|
+
# @return [String]
|
91
|
+
#
|
92
|
+
# @api public
|
93
|
+
def destinations_file_name
|
94
|
+
@options.fetch(:destinations_filename, 'destinations.xml')
|
95
|
+
end
|
96
96
|
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
97
|
+
# returns the output folder path from the option list
|
98
|
+
# otherwise the default path
|
99
|
+
#
|
100
|
+
# @return [String]
|
101
|
+
#
|
102
|
+
# @api public
|
103
|
+
def output_folder
|
104
|
+
@options.fetch(:output_dir, "#{root}/data/output")
|
105
|
+
end
|
105
106
|
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
107
|
+
# returns the full path to the taxonomy file
|
108
|
+
#
|
109
|
+
# @return [String]
|
110
|
+
#
|
111
|
+
# @api public
|
112
|
+
def taxonomy_file_path
|
113
|
+
File.join(input_folder, taxonomy_file_name)
|
114
|
+
end
|
114
115
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
|
116
|
+
# returns the full path to the destinations file
|
117
|
+
#
|
118
|
+
# @return [String]
|
119
|
+
#
|
120
|
+
# @api public
|
121
|
+
def destinations_file_path
|
122
|
+
File.join(input_folder, destinations_file_name)
|
123
|
+
end
|
123
124
|
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
FileUtils.cp_r(Dir["#{File.join(root, 'templates', 'static')}/*"], static_output_dir)
|
133
|
-
end
|
125
|
+
# returns the full path to the static folder
|
126
|
+
#
|
127
|
+
# @return [String]
|
128
|
+
#
|
129
|
+
# @api public
|
130
|
+
def static_output_dir
|
131
|
+
File.join(output_folder, 'static')
|
132
|
+
end
|
134
133
|
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
134
|
+
# cleans the output folder and re-creates it and the static folder
|
135
|
+
#
|
136
|
+
# @return [void]
|
137
|
+
#
|
138
|
+
# @api public
|
139
|
+
def prepare_output_dirs
|
140
|
+
FileUtils.rm_rf Dir["#{output_folder}/**/*"]
|
141
|
+
create_directories(output_folder, static_output_dir)
|
142
|
+
FileUtils.cp_r(Dir["#{File.join(root, 'templates', 'static')}/*"], static_output_dir)
|
143
143
|
end
|
144
|
-
end
|
145
144
|
|
146
|
-
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
# @api public
|
153
|
-
def register_jobs(*jobs)
|
154
|
-
jobs.pmap do |job|
|
155
|
-
job = job.stringify_keys
|
145
|
+
# checks if all workers finished and returns true or false
|
146
|
+
#
|
147
|
+
# @return [Boolean]
|
148
|
+
#
|
149
|
+
# @api public
|
150
|
+
def all_workers_finished?
|
156
151
|
@jobs_mutex.synchronize do
|
157
|
-
@jobs
|
152
|
+
@jobs.all? { |_job_id, job| job['status'] == 'finished' }
|
158
153
|
end
|
159
154
|
end
|
160
|
-
end
|
161
155
|
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
@jobs_mutex.synchronize do
|
176
|
-
@jobs.pmap do |_job_id, job|
|
177
|
-
@workers.async.work(job, current_actor) if @workers.alive?
|
156
|
+
# registers all the jobs so that the managers can have access to them at any time
|
157
|
+
#
|
158
|
+
# @param [Array] jobs the jobs that will be registered
|
159
|
+
#
|
160
|
+
# @return [void]
|
161
|
+
#
|
162
|
+
# @api public
|
163
|
+
def register_jobs(*jobs)
|
164
|
+
jobs.each do |job|
|
165
|
+
job = job.stringify_keys
|
166
|
+
@jobs_mutex.synchronize do
|
167
|
+
@jobs[job['atlas_id']] = job
|
168
|
+
end
|
178
169
|
end
|
179
170
|
end
|
180
|
-
end
|
181
171
|
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
172
|
+
# registers all the jobs, and then delegates them to workers
|
173
|
+
# @see #register_jobs
|
174
|
+
# @see TaxGenerator::FileCreator#work
|
175
|
+
#
|
176
|
+
# @param [Array] jobs the jobs that will be delegated to the workers
|
177
|
+
#
|
178
|
+
# @return [void]
|
179
|
+
#
|
180
|
+
# @api public
|
181
|
+
def delegate_job(*jobs)
|
182
|
+
# jobs need to be added into the manager before starting task to avoid adding new key while iterating
|
183
|
+
register_jobs(*jobs)
|
184
|
+
@jobs_mutex.synchronize do
|
185
|
+
@jobs.each do |_job_id, job|
|
186
|
+
enqueue(job)
|
187
|
+
end
|
188
|
+
end
|
195
189
|
end
|
196
|
-
|
197
|
-
|
198
|
-
|
190
|
+
|
191
|
+
# parses the destinations xml document, gets each destination and adds a new job for that
|
192
|
+
# destination in the job list and then returns it
|
193
|
+
# @see #nokogiri_xml
|
194
|
+
#
|
195
|
+
# @return [Array<Hash>]
|
196
|
+
#
|
197
|
+
# @api public
|
198
|
+
def fetch_file_jobs
|
199
|
+
jobs = []
|
200
|
+
@taxonomy.taxonomies.each do |taxonomy_node|
|
201
|
+
jobs << { atlas_id: taxonomy_node.name, taxonomy: @taxonomy, destination: nil, output_folder: output_folder }
|
202
|
+
end
|
203
|
+
nokogiri_xml(destinations_file_path).xpath('//destination').each do |destination|
|
204
|
+
atlas_id = destination.attributes['atlas_id']
|
205
|
+
jobs << { atlas_id: atlas_id.value, taxonomy: @taxonomy, destination: destination, output_folder: output_folder }
|
206
|
+
end
|
207
|
+
jobs
|
199
208
|
end
|
200
|
-
jobs
|
201
|
-
end
|
202
209
|
|
203
|
-
|
204
|
-
|
205
|
-
|
206
|
-
|
207
|
-
|
208
|
-
|
209
|
-
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
215
|
-
|
210
|
+
# fetches the jobs for file generation, then delegates the jobs to workers and waits untill workers finish
|
211
|
+
# @see #fetch_file_jobs
|
212
|
+
# @see #delegate_job
|
213
|
+
# @see #wait_jobs_termination
|
214
|
+
#
|
215
|
+
# @return [void]
|
216
|
+
#
|
217
|
+
# @api public
|
218
|
+
def generate_files
|
219
|
+
jobs = fetch_file_jobs
|
220
|
+
delegate_job(*jobs)
|
221
|
+
wait_jobs_termination
|
222
|
+
end
|
216
223
|
|
217
|
-
|
218
|
-
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
|
224
|
-
|
225
|
-
|
226
|
-
end
|
224
|
+
# retrieves the information about the node from the tree and generates for each destination a new File
|
225
|
+
# @see #create_file
|
226
|
+
#
|
227
|
+
# @return [void]
|
228
|
+
#
|
229
|
+
# @api public
|
230
|
+
def wait_jobs_termination
|
231
|
+
sleep(0.1) until all_workers_finished?
|
232
|
+
end
|
227
233
|
|
228
|
-
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
234
|
+
# registeres a worker inside the job_to_worker storage using the 'atlas_id' key from the job hash
|
235
|
+
# @param [TaxGenerator::FileCreator] worker the worker that needs to be registered
|
236
|
+
# @param [Hash] job the job that will be used to register the worker
|
237
|
+
#
|
238
|
+
# @return [void]
|
239
|
+
#
|
240
|
+
# @api public
|
241
|
+
def register_job_to_worker(job, worker)
|
242
|
+
@job_to_worker_mutex.synchronize do
|
243
|
+
@job_to_worker[job['atlas_id']] = worker
|
244
|
+
end
|
238
245
|
end
|
239
|
-
end
|
240
246
|
|
241
|
-
|
242
|
-
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
247
|
+
# registeres a job to a worker, using the mailbox address of the worker (which is unique)
|
248
|
+
# @param [TaxGenerator::FileCreator] worker the worker that will be used to registerr the job
|
249
|
+
# @param [Hash] job the job that willbe registered to a worker
|
250
|
+
#
|
251
|
+
# @return [void]
|
252
|
+
#
|
253
|
+
# @api public
|
254
|
+
def register_worker_to_job(job, worker)
|
255
|
+
@worker_to_job_mutex.synchronize do
|
256
|
+
@worker_to_job[worker.name] = job
|
257
|
+
end
|
251
258
|
end
|
252
|
-
end
|
253
259
|
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
|
267
|
-
|
268
|
-
|
269
|
-
|
270
|
-
|
271
|
-
end
|
260
|
+
# registers the worker so that the current actor has access to it at any given time and starts the worker
|
261
|
+
# @see TaxGenerator::FileCreator#start_work
|
262
|
+
# @see #register_job_to_worker
|
263
|
+
# @see #register_worker_to_job
|
264
|
+
#
|
265
|
+
# @param [Hash] job the job that the worker will work
|
266
|
+
# @param [TaxGenerator::FileCreator] worker the worker that will create the file
|
267
|
+
#
|
268
|
+
# @return [void]
|
269
|
+
#
|
270
|
+
# @api public
|
271
|
+
def register_worker_for_job(job, worker)
|
272
|
+
register_job_to_worker(job, worker)
|
273
|
+
register_worker_to_job(job, worker)
|
274
|
+
log_message("worker #{worker.job_id} registed into manager")
|
275
|
+
worker.tell(:start_work)
|
276
|
+
end
|
272
277
|
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
279
|
-
|
280
|
-
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
287
|
-
|
288
|
-
|
278
|
+
# generates the taxonomy tree , prints it and generates the files
|
279
|
+
# @see TaxGenerator::TaxonomyTree#new
|
280
|
+
# @see Tree::TreeNode#print_tree
|
281
|
+
# @see #generate_files
|
282
|
+
#
|
283
|
+
# @return [void]
|
284
|
+
#
|
285
|
+
# @api public
|
286
|
+
def work
|
287
|
+
prepare_output_dirs
|
288
|
+
if File.directory?(input_folder) && File.file?(taxonomy_file_path) && File.file?(destinations_file_path)
|
289
|
+
@taxonomy = TaxGenerator::TaxonomyTree.new(taxonomy_file_path)
|
290
|
+
@taxonomy.print_tree
|
291
|
+
generate_files
|
292
|
+
else
|
293
|
+
log_message('Please provide valid options', log_method: 'fatal')
|
294
|
+
end
|
289
295
|
end
|
290
|
-
end
|
291
296
|
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
297
|
+
# fetches the job from a worker
|
298
|
+
# @param [TaxGenerator::FileCreator] worker the worker that died
|
299
|
+
#
|
300
|
+
# @return [void]
|
301
|
+
#
|
302
|
+
# @api public
|
303
|
+
def get_job_from_worker(worker)
|
304
|
+
@worker_to_job_mutex.synchronize do
|
305
|
+
@worker_to_job[worker.name]
|
306
|
+
end
|
301
307
|
end
|
302
|
-
end
|
303
308
|
|
304
|
-
|
305
|
-
|
306
|
-
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
309
|
+
# deletes the worker from the worker_to_job storage
|
310
|
+
# @param [TaxGenerator::FileCreator] worker the worker that died
|
311
|
+
#
|
312
|
+
# @return [void]
|
313
|
+
#
|
314
|
+
# @api public
|
315
|
+
def delete_from_worker_to_job(worker)
|
316
|
+
name = worker.name
|
317
|
+
@worker_to_job_mutex.synchronize do
|
318
|
+
@worker_to_job.delete(name)
|
319
|
+
end
|
320
|
+
name
|
314
321
|
end
|
315
|
-
mailbox
|
316
|
-
end
|
317
322
|
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
328
|
-
|
329
|
-
|
323
|
+
# logs the message about working being dead if a worker crashes
|
324
|
+
# @param [TaxGenerator::FileCreator] worker the worker that died
|
325
|
+
# @param [String] reason the reason for which the worker died
|
326
|
+
#
|
327
|
+
# @return [void]
|
328
|
+
#
|
329
|
+
# @api public
|
330
|
+
def worker_died(worker, reason)
|
331
|
+
job = get_job_from_worker(worker)
|
332
|
+
worker_name = delete_from_worker_to_job(worker)
|
333
|
+
return if reason.blank? || job.blank?
|
334
|
+
log_message("worker job #{job['atlas_id']} with name #{worker_name.inspect} died for reason: #{log_error(reason)}", log_method: 'fatal')
|
335
|
+
end
|
330
336
|
end
|
331
337
|
end
|
332
|
-
end
|
@@ -9,7 +9,7 @@ module TaxGenerator
|
|
9
9
|
# @return [Nokogiri::XML] the xml document used to build the tree
|
10
10
|
class TaxonomyTree
|
11
11
|
include TaxGenerator::ApplicationHelper
|
12
|
-
attr_reader :root_node, :document
|
12
|
+
attr_reader :root_node, :document, :taxonomies
|
13
13
|
|
14
14
|
# receives a file path that will be parsed and used to build the tree
|
15
15
|
# @see Tree::TreeNode#new
|
@@ -23,6 +23,7 @@ module TaxGenerator
|
|
23
23
|
def initialize(file_path)
|
24
24
|
@document = nokogiri_xml(file_path)
|
25
25
|
@root_node = TaxGenerator::TaxonomyNode.new('ROOT', 'ROOT')
|
26
|
+
@taxonomies = []
|
26
27
|
find_taxonomies
|
27
28
|
end
|
28
29
|
|
@@ -35,11 +36,10 @@ module TaxGenerator
|
|
35
36
|
#
|
36
37
|
# @api public
|
37
38
|
def find_taxonomies
|
38
|
-
|
39
|
-
@document.xpath('.//taxonomy').pmap do |taxonomy_node|
|
40
|
-
count += 1
|
39
|
+
@document.xpath('.//taxonomy').each do |taxonomy_node|
|
41
40
|
taxonomy_name = taxonomy_node.at_xpath('.//taxonomy_name')
|
42
|
-
tax_node = insert_node(
|
41
|
+
tax_node = insert_node(SecureRandom.uuid, taxonomy_name.content, @root_node)
|
42
|
+
@taxonomies << tax_node
|
43
43
|
add_node(taxonomy_node, tax_node, skip_add: true)
|
44
44
|
end
|
45
45
|
end
|
@@ -110,7 +110,7 @@ module TaxGenerator
|
|
110
110
|
def add_node(taxonomy_node, node, options = {})
|
111
111
|
tax_node = options[:skip_add].present? ? node : add_taxonomy_node(taxonomy_node, node)
|
112
112
|
return unless taxonomy_node.children.any?
|
113
|
-
taxonomy_node.xpath('./node').
|
113
|
+
taxonomy_node.xpath('./node').each do |child_node|
|
114
114
|
add_node(child_node, tax_node) if tax_node.present?
|
115
115
|
end
|
116
116
|
end
|
@@ -49,26 +49,12 @@ module TaxGenerator
|
|
49
49
|
#
|
50
50
|
# @api public
|
51
51
|
def create_directories(*args)
|
52
|
-
args.
|
52
|
+
args.each do |argument|
|
53
53
|
FileUtils.mkdir_p(argument) unless File.directory?(argument)
|
54
54
|
end
|
55
55
|
end
|
56
56
|
|
57
|
-
|
58
|
-
#
|
59
|
-
#
|
60
|
-
# @return [void]
|
61
|
-
#
|
62
|
-
# @api public
|
63
|
-
def set_celluloid_exception_handling
|
64
|
-
Celluloid.logger = app_logger
|
65
|
-
Celluloid.task_class = Celluloid::TaskThread
|
66
|
-
Celluloid.exception_handler do |ex|
|
67
|
-
unless ex.is_a?(Interrupt)
|
68
|
-
log_error(ex)
|
69
|
-
end
|
70
|
-
end
|
71
|
-
end
|
57
|
+
|
72
58
|
|
73
59
|
# Reads a file and interpretes it as ERB
|
74
60
|
#
|
@@ -133,7 +119,6 @@ module TaxGenerator
|
|
133
119
|
#
|
134
120
|
# @api public
|
135
121
|
def execute_with_rescue
|
136
|
-
set_celluloid_exception_handling
|
137
122
|
yield if block_given?
|
138
123
|
rescue Interrupt
|
139
124
|
rescue_interrupt
|
data/tax_generator.gemspec
CHANGED
@@ -18,8 +18,8 @@ Gem::Specification.new do |s|
|
|
18
18
|
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
19
19
|
|
20
20
|
s.add_runtime_dependency 'nokogiri', '~> 1.6', '>= 1.6.7'
|
21
|
-
s.add_runtime_dependency '
|
22
|
-
s.add_runtime_dependency '
|
21
|
+
s.add_runtime_dependency 'concurrent-ruby', '~> 1.0', '>= 1.0'
|
22
|
+
s.add_runtime_dependency 'concurrent-ruby-edge', '~> 0.2' , '>= 0.2'
|
23
23
|
s.add_runtime_dependency 'slop', '~> 4.2', '>= 4.2.1'
|
24
24
|
s.add_runtime_dependency 'activesupport', '~> 4.2', '>= 4.2.5'
|
25
25
|
s.add_runtime_dependency 'rubytree', '~> 0.9', '>= 0.9.6'
|
@@ -31,8 +31,8 @@ Gem::Specification.new do |s|
|
|
31
31
|
s.add_development_dependency 'simplecov-summary', '~> 0.0.4', '>= 0.0.4'
|
32
32
|
s.add_development_dependency 'mocha', '~> 1.1', '>= 1.1'
|
33
33
|
|
34
|
-
s.add_development_dependency '
|
35
|
-
|
34
|
+
s.add_development_dependency 'concurrent-ruby-ext', '~> 1.0' , '>= 1.0'
|
35
|
+
|
36
36
|
s.add_development_dependency 'yard', '~> 0.8', '>= 0.8.7'
|
37
37
|
s.add_development_dependency 'yard-rspec', '~> 0.1', '>= 0.1'
|
38
38
|
s.add_development_dependency 'redcarpet', '~> 3.3', '>= 3.3'
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: tax_generator
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.6.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- bogdanRada
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-27 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: nokogiri
|
@@ -31,35 +31,35 @@ dependencies:
|
|
31
31
|
- !ruby/object:Gem::Version
|
32
32
|
version: 1.6.7
|
33
33
|
- !ruby/object:Gem::Dependency
|
34
|
-
name:
|
34
|
+
name: concurrent-ruby
|
35
35
|
requirement: !ruby/object:Gem::Requirement
|
36
36
|
requirements:
|
37
37
|
- - "~>"
|
38
38
|
- !ruby/object:Gem::Version
|
39
|
-
version: '0
|
40
|
-
- - "
|
39
|
+
version: '1.0'
|
40
|
+
- - ">="
|
41
41
|
- !ruby/object:Gem::Version
|
42
|
-
version:
|
42
|
+
version: '1.0'
|
43
43
|
type: :runtime
|
44
44
|
prerelease: false
|
45
45
|
version_requirements: !ruby/object:Gem::Requirement
|
46
46
|
requirements:
|
47
47
|
- - "~>"
|
48
48
|
- !ruby/object:Gem::Version
|
49
|
-
version: '0
|
50
|
-
- - "
|
49
|
+
version: '1.0'
|
50
|
+
- - ">="
|
51
51
|
- !ruby/object:Gem::Version
|
52
|
-
version:
|
52
|
+
version: '1.0'
|
53
53
|
- !ruby/object:Gem::Dependency
|
54
|
-
name:
|
54
|
+
name: concurrent-ruby-edge
|
55
55
|
requirement: !ruby/object:Gem::Requirement
|
56
56
|
requirements:
|
57
57
|
- - "~>"
|
58
58
|
- !ruby/object:Gem::Version
|
59
59
|
version: '0.2'
|
60
|
-
- - "
|
60
|
+
- - ">="
|
61
61
|
- !ruby/object:Gem::Version
|
62
|
-
version: 0.2
|
62
|
+
version: '0.2'
|
63
63
|
type: :runtime
|
64
64
|
prerelease: false
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
@@ -67,9 +67,9 @@ dependencies:
|
|
67
67
|
- - "~>"
|
68
68
|
- !ruby/object:Gem::Version
|
69
69
|
version: '0.2'
|
70
|
-
- - "
|
70
|
+
- - ">="
|
71
71
|
- !ruby/object:Gem::Version
|
72
|
-
version: 0.2
|
72
|
+
version: '0.2'
|
73
73
|
- !ruby/object:Gem::Dependency
|
74
74
|
name: slop
|
75
75
|
requirement: !ruby/object:Gem::Requirement
|
@@ -251,45 +251,25 @@ dependencies:
|
|
251
251
|
- !ruby/object:Gem::Version
|
252
252
|
version: '1.1'
|
253
253
|
- !ruby/object:Gem::Dependency
|
254
|
-
name:
|
255
|
-
requirement: !ruby/object:Gem::Requirement
|
256
|
-
requirements:
|
257
|
-
- - "~>"
|
258
|
-
- !ruby/object:Gem::Version
|
259
|
-
version: '0.33'
|
260
|
-
- - ">="
|
261
|
-
- !ruby/object:Gem::Version
|
262
|
-
version: '0.33'
|
263
|
-
type: :development
|
264
|
-
prerelease: false
|
265
|
-
version_requirements: !ruby/object:Gem::Requirement
|
266
|
-
requirements:
|
267
|
-
- - "~>"
|
268
|
-
- !ruby/object:Gem::Version
|
269
|
-
version: '0.33'
|
270
|
-
- - ">="
|
271
|
-
- !ruby/object:Gem::Version
|
272
|
-
version: '0.33'
|
273
|
-
- !ruby/object:Gem::Dependency
|
274
|
-
name: reek
|
254
|
+
name: concurrent-ruby-ext
|
275
255
|
requirement: !ruby/object:Gem::Requirement
|
276
256
|
requirements:
|
277
257
|
- - "~>"
|
278
258
|
- !ruby/object:Gem::Version
|
279
|
-
version: '
|
259
|
+
version: '1.0'
|
280
260
|
- - ">="
|
281
261
|
- !ruby/object:Gem::Version
|
282
|
-
version: '
|
262
|
+
version: '1.0'
|
283
263
|
type: :development
|
284
264
|
prerelease: false
|
285
265
|
version_requirements: !ruby/object:Gem::Requirement
|
286
266
|
requirements:
|
287
267
|
- - "~>"
|
288
268
|
- !ruby/object:Gem::Version
|
289
|
-
version: '
|
269
|
+
version: '1.0'
|
290
270
|
- - ">="
|
291
271
|
- !ruby/object:Gem::Version
|
292
|
-
version: '
|
272
|
+
version: '1.0'
|
293
273
|
- !ruby/object:Gem::Dependency
|
294
274
|
name: yard
|
295
275
|
requirement: !ruby/object:Gem::Requirement
|
@@ -404,6 +384,7 @@ files:
|
|
404
384
|
- ".reek"
|
405
385
|
- ".rspec"
|
406
386
|
- ".rubocop.yml"
|
387
|
+
- ".travis.yml"
|
407
388
|
- Gemfile
|
408
389
|
- LICENSE
|
409
390
|
- README.md
|
@@ -424,6 +405,7 @@ files:
|
|
424
405
|
- lib/tax_generator/classes/taxonomy_tree.rb
|
425
406
|
- lib/tax_generator/cli.rb
|
426
407
|
- lib/tax_generator/helpers/application_helper.rb
|
408
|
+
- lib/tax_generator/helpers/methodic_actor.rb
|
427
409
|
- lib/tax_generator/version.rb
|
428
410
|
- spec/lib/tax_generator/application_spec.rb
|
429
411
|
- spec/lib/tax_generator/classes/destination_spec.rb
|
@@ -456,7 +438,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
456
438
|
version: '0'
|
457
439
|
requirements: []
|
458
440
|
rubyforge_project:
|
459
|
-
rubygems_version: 2.
|
441
|
+
rubygems_version: 2.5.1
|
460
442
|
signing_key:
|
461
443
|
specification_version: 4
|
462
444
|
summary: Tax generator is a simple XML processor and generator of HTMl files and generates
|