picsolve_rabbitmq_uploader 0.0.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.travis.yml +3 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +21 -0
- data/README.md +34 -0
- data/Rakefile +24 -0
- data/bin/picsolve_rabbitmq_uploader +13 -0
- data/lib/picsolve_rabbitmq_uploader.rb +171 -0
- data/lib/picsolve_rabbitmq_uploader/version.rb +4 -0
- data/picsolve_rabbitmq_uploader.gemspec +39 -0
- data/upload.dist.xml +10 -0
- metadata +144 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 360cc18137fa90503565ddfa0b8d5faba63476ff
|
4
|
+
data.tar.gz: 7b712b1cbca765073f6ab7f170fddab3751b73df
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 7a5301aa9312c76c121eef5b8149db8c334e3447ac7cadbab8557e3c1aa772046176d292c8e104338bf7d692678e5d64a53b273cc31574dda7d7dfe80a40b6b7
|
7
|
+
data.tar.gz: 190b9652fe2546264c79e36f1a276e0a4e59bb2811a2492d09ac6177c3c63fcfca1bff7378f685cf6864b6f8db33a0499dc4bd01689df8ecab793a892e244045
|
data/.gitignore
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2015 Christian Simon
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
# PicsolveRabbitmqUploader
|
2
|
+
|
3
|
+
Script to retry sending web upload messages to rabbitmq. It takes full path and
|
4
|
+
file name of the psi error log. Typically this script should be run with all
|
5
|
+
the error logs generated while rabbitmq service was down.
|
6
|
+
|
7
|
+
## Usage
|
8
|
+
|
9
|
+
```sh
|
10
|
+
# Install gem
|
11
|
+
gem install picsolve_rabbitmq_uploader-0.0.1.gem
|
12
|
+
|
13
|
+
## Create a upload.xml config file in CWD (cf. upload.dist.xml)
|
14
|
+
|
15
|
+
# Send failed uploads to rabbitmq
|
16
|
+
picsolve_rabbitmq_uploader C:/logfile.txt
|
17
|
+
```
|
18
|
+
|
19
|
+
## Development
|
20
|
+
|
21
|
+
```sh
|
22
|
+
# Clone repo
|
23
|
+
|
24
|
+
# Install all ruby dependencies
|
25
|
+
bundle install
|
26
|
+
|
27
|
+
## Do development
|
28
|
+
|
29
|
+
# Run tests
|
30
|
+
bundle exec rake test
|
31
|
+
|
32
|
+
# Build package
|
33
|
+
bundle exec rake build
|
34
|
+
```
|
data/Rakefile
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
begin
|
2
|
+
require 'bundler/gem_tasks'
|
3
|
+
rescue LoadError
|
4
|
+
task :gem
|
5
|
+
end
|
6
|
+
|
7
|
+
begin
|
8
|
+
require 'rspec/core/rake_task'
|
9
|
+
RSpec::Core::RakeTask.new(:spec)
|
10
|
+
rescue LoadError
|
11
|
+
task :spec
|
12
|
+
end
|
13
|
+
|
14
|
+
begin
|
15
|
+
require 'rubocop/rake_task'
|
16
|
+
RuboCop::RakeTask.new
|
17
|
+
rescue LoadError
|
18
|
+
task :rubocop
|
19
|
+
end
|
20
|
+
|
21
|
+
desc 'Run all tests'
|
22
|
+
task test: [:spec, :rubocop]
|
23
|
+
|
24
|
+
task default: :test
|
@@ -0,0 +1,13 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'picsolve_rabbitmq_uploader'
|
4
|
+
|
5
|
+
begin
|
6
|
+
PicsolveRabbitmqUploader::RabbitUploader.new.run
|
7
|
+
rescue => e
|
8
|
+
PicsolveRabbitmqUploader.logger.fatal "uncatched exception: #{e}"
|
9
|
+
e.backtrace.each do |line|
|
10
|
+
PicsolveRabbitmqUploader.logger.debug line
|
11
|
+
end
|
12
|
+
exit 1
|
13
|
+
end
|
@@ -0,0 +1,171 @@
|
|
1
|
+
require 'picsolve_rabbitmq_uploader/version'
|
2
|
+
require 'nokogiri'
|
3
|
+
require 'bunny'
|
4
|
+
|
5
|
+
# Parse logs and upload failed messaged to rabbitmq
|
6
|
+
module PicsolveRabbitmqUploader
|
7
|
+
def self.logger
|
8
|
+
@logger ||= create_logger
|
9
|
+
end
|
10
|
+
|
11
|
+
def self.create_logger
|
12
|
+
l = Logger.new(STDOUT)
|
13
|
+
l.level = Logger::DEBUG
|
14
|
+
l
|
15
|
+
end
|
16
|
+
|
17
|
+
# reads a psi config xml
|
18
|
+
class RabbitSettings
|
19
|
+
attr_reader :config
|
20
|
+
|
21
|
+
def log
|
22
|
+
PicsolveRabbitmqUploader.logger
|
23
|
+
end
|
24
|
+
|
25
|
+
def initialize
|
26
|
+
c = {}
|
27
|
+
read.xpath('//rabbitmq').children.each do |child|
|
28
|
+
next if child.name == 'text'
|
29
|
+
c[child.name] = child.text
|
30
|
+
end
|
31
|
+
@config = c
|
32
|
+
end
|
33
|
+
|
34
|
+
def read
|
35
|
+
log.info "Parsing config file '#{config_file}'"
|
36
|
+
f = File.open(config_file)
|
37
|
+
doc = Nokogiri::HTML(f)
|
38
|
+
f.close
|
39
|
+
doc
|
40
|
+
end
|
41
|
+
|
42
|
+
def config_file
|
43
|
+
@config_file ||= create_config_file
|
44
|
+
end
|
45
|
+
|
46
|
+
def create_config_file
|
47
|
+
paths = [
|
48
|
+
'upload.xml',
|
49
|
+
'PsiConfig.xml'
|
50
|
+
]
|
51
|
+
paths.each do |f|
|
52
|
+
log.debug "Testing for config file '#{f}'"
|
53
|
+
return f if File.exist? f
|
54
|
+
end
|
55
|
+
|
56
|
+
fail "No config file found: #{paths}"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
|
60
|
+
# uploads missing uploads to rabbitmq
|
61
|
+
class RabbitUploader
|
62
|
+
def log
|
63
|
+
PicsolveRabbitmqUploader.logger
|
64
|
+
end
|
65
|
+
|
66
|
+
def run(argv = ARGV)
|
67
|
+
@argv = argv
|
68
|
+
|
69
|
+
@config = RabbitSettings.new.config
|
70
|
+
|
71
|
+
msgs = messages
|
72
|
+
|
73
|
+
# stop here if no messages to upload
|
74
|
+
return unless msgs.length > 0
|
75
|
+
|
76
|
+
upload_rmq(msgs)
|
77
|
+
end
|
78
|
+
|
79
|
+
def publish_msgs_rmq(msgs)
|
80
|
+
# send all messages to exchange
|
81
|
+
msgs.each do |msg|
|
82
|
+
log.info "Publish on exchange=#{@config['exchange']} " \
|
83
|
+
"to to queue=#{@config['queueupload']} data=#{msg}"
|
84
|
+
@exchange.publish(
|
85
|
+
msg,
|
86
|
+
mandatory: true,
|
87
|
+
routing_key: @config['queueupload']
|
88
|
+
)
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
def ensure_queue
|
93
|
+
return if conn_rmq.queue_exists?(@config['queueupload'])
|
94
|
+
|
95
|
+
# create a queue
|
96
|
+
@channel.queue(
|
97
|
+
@config['queueupload'],
|
98
|
+
durable: true
|
99
|
+
)
|
100
|
+
end
|
101
|
+
|
102
|
+
def upload_rmq(msgs)
|
103
|
+
log.info 'connect to RabbitMQ'
|
104
|
+
|
105
|
+
conn_rmq.start
|
106
|
+
|
107
|
+
@channel = conn_rmq.create_channel
|
108
|
+
|
109
|
+
ensure_queue
|
110
|
+
|
111
|
+
# get exchange
|
112
|
+
@exchange = @channel.topic(@config['exchange'], durable: true)
|
113
|
+
|
114
|
+
publish_msgs_rmq msgs
|
115
|
+
|
116
|
+
log.info 'disconnect from RabbitMQ'
|
117
|
+
conn_rmq.stop
|
118
|
+
end
|
119
|
+
|
120
|
+
def conn_rmq
|
121
|
+
@conn_rmq ||= create_conn_rmq
|
122
|
+
end
|
123
|
+
|
124
|
+
def create_conn_rmq
|
125
|
+
Bunny.new(
|
126
|
+
host: @config['host'],
|
127
|
+
port: @config['port'],
|
128
|
+
user: @config['username'],
|
129
|
+
password: @config['password']
|
130
|
+
)
|
131
|
+
end
|
132
|
+
|
133
|
+
def filter_line(line)
|
134
|
+
m = line.match(
|
135
|
+
/Failed to send upload message to RabbitMQ psi\.uploads:(.*)$/
|
136
|
+
)
|
137
|
+
|
138
|
+
# return nil if regex fails
|
139
|
+
return nil unless m
|
140
|
+
|
141
|
+
m[1]
|
142
|
+
end
|
143
|
+
|
144
|
+
def messages
|
145
|
+
m = log_files.map do |file|
|
146
|
+
File.open(file).map do |line|
|
147
|
+
filter_line line
|
148
|
+
end
|
149
|
+
end
|
150
|
+
|
151
|
+
m.flatten.select do |d|
|
152
|
+
!d.nil?
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
def log_files
|
157
|
+
@log_files ||= existing_log_files
|
158
|
+
end
|
159
|
+
|
160
|
+
def existing_log_files
|
161
|
+
paths = @argv.select do |f|
|
162
|
+
log.debug "Testing for existence and readability of log file '#{f}'"
|
163
|
+
File.exist?(f) && File.readable?(f)
|
164
|
+
end
|
165
|
+
|
166
|
+
return paths if paths.length > 0
|
167
|
+
|
168
|
+
fail "No log file found, please specify it as parameter: #{paths}"
|
169
|
+
end
|
170
|
+
end
|
171
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'picsolve_rabbitmq_uploader/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = 'picsolve_rabbitmq_uploader'
|
8
|
+
spec.version = PicsolveRabbitmqUploader::VERSION
|
9
|
+
spec.authors = ['Christian Simon', 'Juan Morales']
|
10
|
+
spec.email = ['christian.simon@picsolve.com','juan.morales@picsolve.com']
|
11
|
+
|
12
|
+
spec.summary = 'Script to retry sending web upload messages to rabbitmq'
|
13
|
+
|
14
|
+
spec.description = '
|
15
|
+
Script to retry sending web upload messages to rabbitmq
|
16
|
+
This script depends on python3-pika package.
|
17
|
+
It takes full path and file name of the psi error log. Typically this script
|
18
|
+
should be run with all the error logs generated while rabbitmq service
|
19
|
+
was down
|
20
|
+
'
|
21
|
+
|
22
|
+
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
23
|
+
f.match(%r{^(test|spec|features)/})
|
24
|
+
end
|
25
|
+
spec.bindir = 'bin'
|
26
|
+
spec.executables = spec.files.grep(%r{^bin/}) do |f|
|
27
|
+
File.basename(f)
|
28
|
+
end
|
29
|
+
spec.require_paths = ['lib']
|
30
|
+
|
31
|
+
spec.add_development_dependency 'bundler', '~> 1.9'
|
32
|
+
spec.add_development_dependency 'rake', '~> 10.0'
|
33
|
+
spec.add_development_dependency 'rspec'
|
34
|
+
spec.add_development_dependency 'rubocop'
|
35
|
+
spec.add_dependency 'nokogiri'
|
36
|
+
# depeding on the ruby version installed, we need one or another version of bunny
|
37
|
+
spec.add_dependency 'bunny', '<= 1.7.0' if RUBY_VERSION[0].to_i < 2
|
38
|
+
spec.add_dependency 'bunny', '~> 2.0' if RUBY_VERSION[0].to_i >= 2
|
39
|
+
end
|
data/upload.dist.xml
ADDED
metadata
ADDED
@@ -0,0 +1,144 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: picsolve_rabbitmq_uploader
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.5
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Christian Simon
|
8
|
+
- Juan Morales
|
9
|
+
autorequire:
|
10
|
+
bindir: bin
|
11
|
+
cert_chain: []
|
12
|
+
date: 2015-09-30 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - "~>"
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '1.9'
|
21
|
+
type: :development
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - "~>"
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '1.9'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: rake
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - "~>"
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '10.0'
|
35
|
+
type: :development
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - "~>"
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '10.0'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: rspec
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '0'
|
49
|
+
type: :development
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '0'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: rubocop
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '0'
|
63
|
+
type: :development
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '0'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: nokogiri
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '0'
|
77
|
+
type: :runtime
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '0'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: bunny
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - "~>"
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '2.0'
|
91
|
+
type: :runtime
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - "~>"
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '2.0'
|
98
|
+
description: "\nScript to retry sending web upload messages to rabbitmq\nThis script
|
99
|
+
depends on python3-pika package.\nIt takes full path and file name of the psi error
|
100
|
+
log. Typically this script\nshould be run with all the error logs generated while
|
101
|
+
rabbitmq service\nwas down\n "
|
102
|
+
email:
|
103
|
+
- christian.simon@picsolve.com
|
104
|
+
- juan.morales@picsolve.com
|
105
|
+
executables:
|
106
|
+
- picsolve_rabbitmq_uploader
|
107
|
+
extensions: []
|
108
|
+
extra_rdoc_files: []
|
109
|
+
files:
|
110
|
+
- ".gitignore"
|
111
|
+
- ".travis.yml"
|
112
|
+
- Gemfile
|
113
|
+
- LICENSE.txt
|
114
|
+
- README.md
|
115
|
+
- Rakefile
|
116
|
+
- bin/picsolve_rabbitmq_uploader
|
117
|
+
- lib/picsolve_rabbitmq_uploader.rb
|
118
|
+
- lib/picsolve_rabbitmq_uploader/version.rb
|
119
|
+
- picsolve_rabbitmq_uploader.gemspec
|
120
|
+
- upload.dist.xml
|
121
|
+
homepage:
|
122
|
+
licenses: []
|
123
|
+
metadata: {}
|
124
|
+
post_install_message:
|
125
|
+
rdoc_options: []
|
126
|
+
require_paths:
|
127
|
+
- lib
|
128
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
129
|
+
requirements:
|
130
|
+
- - ">="
|
131
|
+
- !ruby/object:Gem::Version
|
132
|
+
version: '0'
|
133
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
134
|
+
requirements:
|
135
|
+
- - ">="
|
136
|
+
- !ruby/object:Gem::Version
|
137
|
+
version: '0'
|
138
|
+
requirements: []
|
139
|
+
rubyforge_project:
|
140
|
+
rubygems_version: 2.2.2
|
141
|
+
signing_key:
|
142
|
+
specification_version: 4
|
143
|
+
summary: Script to retry sending web upload messages to rabbitmq
|
144
|
+
test_files: []
|