picsolve_rabbitmq_uploader 0.0.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 360cc18137fa90503565ddfa0b8d5faba63476ff
4
+ data.tar.gz: 7b712b1cbca765073f6ab7f170fddab3751b73df
5
+ SHA512:
6
+ metadata.gz: 7a5301aa9312c76c121eef5b8149db8c334e3447ac7cadbab8557e3c1aa772046176d292c8e104338bf7d692678e5d64a53b273cc31574dda7d7dfe80a40b6b7
7
+ data.tar.gz: 190b9652fe2546264c79e36f1a276e0a4e59bb2811a2492d09ac6177c3c63fcfca1bff7378f685cf6864b6f8db33a0499dc4bd01689df8ecab793a892e244045
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.travis.yml ADDED
@@ -0,0 +1,3 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.1.5
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in picsolve_rabbitmq_uploader.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2015 Christian Simon
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,34 @@
1
+ # PicsolveRabbitmqUploader
2
+
3
+ Script to retry sending web upload messages to rabbitmq. It takes full path and
4
+ file name of the psi error log. Typically this script should be run with all
5
+ the error logs generated while rabbitmq service was down.
6
+
7
+ ## Usage
8
+
9
+ ```sh
10
+ # Install gem
11
+ gem install picsolve_rabbitmq_uploader-0.0.1.gem
12
+
13
+ ## Create a upload.xml config file in CWD (cf. upload.dist.xml)
14
+
15
+ # Send failed uploads to rabbitmq
16
+ picsolve_rabbitmq_uploader C:/logfile.txt
17
+ ```
18
+
19
+ ## Development
20
+
21
+ ```sh
22
+ # Clone repo
23
+
24
+ # Install all ruby dependencies
25
+ bundle install
26
+
27
+ ## Do development
28
+
29
+ # Run tests
30
+ bundle exec rake test
31
+
32
+ # Build package
33
+ bundle exec rake build
34
+ ```
data/Rakefile ADDED
@@ -0,0 +1,24 @@
1
+ begin
2
+ require 'bundler/gem_tasks'
3
+ rescue LoadError
4
+ task :gem
5
+ end
6
+
7
+ begin
8
+ require 'rspec/core/rake_task'
9
+ RSpec::Core::RakeTask.new(:spec)
10
+ rescue LoadError
11
+ task :spec
12
+ end
13
+
14
+ begin
15
+ require 'rubocop/rake_task'
16
+ RuboCop::RakeTask.new
17
+ rescue LoadError
18
+ task :rubocop
19
+ end
20
+
21
+ desc 'Run all tests'
22
+ task test: [:spec, :rubocop]
23
+
24
+ task default: :test
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'picsolve_rabbitmq_uploader'
4
+
5
+ begin
6
+ PicsolveRabbitmqUploader::RabbitUploader.new.run
7
+ rescue => e
8
+ PicsolveRabbitmqUploader.logger.fatal "uncatched exception: #{e}"
9
+ e.backtrace.each do |line|
10
+ PicsolveRabbitmqUploader.logger.debug line
11
+ end
12
+ exit 1
13
+ end
@@ -0,0 +1,171 @@
1
+ require 'picsolve_rabbitmq_uploader/version'
2
+ require 'nokogiri'
3
+ require 'bunny'
4
+
5
+ # Parse logs and upload failed messaged to rabbitmq
6
+ module PicsolveRabbitmqUploader
7
+ def self.logger
8
+ @logger ||= create_logger
9
+ end
10
+
11
+ def self.create_logger
12
+ l = Logger.new(STDOUT)
13
+ l.level = Logger::DEBUG
14
+ l
15
+ end
16
+
17
+ # reads a psi config xml
18
+ class RabbitSettings
19
+ attr_reader :config
20
+
21
+ def log
22
+ PicsolveRabbitmqUploader.logger
23
+ end
24
+
25
+ def initialize
26
+ c = {}
27
+ read.xpath('//rabbitmq').children.each do |child|
28
+ next if child.name == 'text'
29
+ c[child.name] = child.text
30
+ end
31
+ @config = c
32
+ end
33
+
34
+ def read
35
+ log.info "Parsing config file '#{config_file}'"
36
+ f = File.open(config_file)
37
+ doc = Nokogiri::HTML(f)
38
+ f.close
39
+ doc
40
+ end
41
+
42
+ def config_file
43
+ @config_file ||= create_config_file
44
+ end
45
+
46
+ def create_config_file
47
+ paths = [
48
+ 'upload.xml',
49
+ 'PsiConfig.xml'
50
+ ]
51
+ paths.each do |f|
52
+ log.debug "Testing for config file '#{f}'"
53
+ return f if File.exist? f
54
+ end
55
+
56
+ fail "No config file found: #{paths}"
57
+ end
58
+ end
59
+
60
+ # uploads missing uploads to rabbitmq
61
+ class RabbitUploader
62
+ def log
63
+ PicsolveRabbitmqUploader.logger
64
+ end
65
+
66
+ def run(argv = ARGV)
67
+ @argv = argv
68
+
69
+ @config = RabbitSettings.new.config
70
+
71
+ msgs = messages
72
+
73
+ # stop here if no messages to upload
74
+ return unless msgs.length > 0
75
+
76
+ upload_rmq(msgs)
77
+ end
78
+
79
+ def publish_msgs_rmq(msgs)
80
+ # send all messages to exchange
81
+ msgs.each do |msg|
82
+ log.info "Publish on exchange=#{@config['exchange']} " \
83
+ "to to queue=#{@config['queueupload']} data=#{msg}"
84
+ @exchange.publish(
85
+ msg,
86
+ mandatory: true,
87
+ routing_key: @config['queueupload']
88
+ )
89
+ end
90
+ end
91
+
92
+ def ensure_queue
93
+ return if conn_rmq.queue_exists?(@config['queueupload'])
94
+
95
+ # create a queue
96
+ @channel.queue(
97
+ @config['queueupload'],
98
+ durable: true
99
+ )
100
+ end
101
+
102
+ def upload_rmq(msgs)
103
+ log.info 'connect to RabbitMQ'
104
+
105
+ conn_rmq.start
106
+
107
+ @channel = conn_rmq.create_channel
108
+
109
+ ensure_queue
110
+
111
+ # get exchange
112
+ @exchange = @channel.topic(@config['exchange'], durable: true)
113
+
114
+ publish_msgs_rmq msgs
115
+
116
+ log.info 'disconnect from RabbitMQ'
117
+ conn_rmq.stop
118
+ end
119
+
120
+ def conn_rmq
121
+ @conn_rmq ||= create_conn_rmq
122
+ end
123
+
124
+ def create_conn_rmq
125
+ Bunny.new(
126
+ host: @config['host'],
127
+ port: @config['port'],
128
+ user: @config['username'],
129
+ password: @config['password']
130
+ )
131
+ end
132
+
133
+ def filter_line(line)
134
+ m = line.match(
135
+ /Failed to send upload message to RabbitMQ psi\.uploads:(.*)$/
136
+ )
137
+
138
+ # return nil if regex fails
139
+ return nil unless m
140
+
141
+ m[1]
142
+ end
143
+
144
+ def messages
145
+ m = log_files.map do |file|
146
+ File.open(file).map do |line|
147
+ filter_line line
148
+ end
149
+ end
150
+
151
+ m.flatten.select do |d|
152
+ !d.nil?
153
+ end
154
+ end
155
+
156
+ def log_files
157
+ @log_files ||= existing_log_files
158
+ end
159
+
160
+ def existing_log_files
161
+ paths = @argv.select do |f|
162
+ log.debug "Testing for existence and readability of log file '#{f}'"
163
+ File.exist?(f) && File.readable?(f)
164
+ end
165
+
166
+ return paths if paths.length > 0
167
+
168
+ fail "No log file found, please specify it as parameter: #{paths}"
169
+ end
170
+ end
171
+ end
@@ -0,0 +1,4 @@
1
+ # verison number
2
+ module PicsolveRabbitmqUploader
3
+ VERSION = '0.0.5'
4
+ end
@@ -0,0 +1,39 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'picsolve_rabbitmq_uploader/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'picsolve_rabbitmq_uploader'
8
+ spec.version = PicsolveRabbitmqUploader::VERSION
9
+ spec.authors = ['Christian Simon', 'Juan Morales']
10
+ spec.email = ['christian.simon@picsolve.com','juan.morales@picsolve.com']
11
+
12
+ spec.summary = 'Script to retry sending web upload messages to rabbitmq'
13
+
14
+ spec.description = '
15
+ Script to retry sending web upload messages to rabbitmq
16
+ This script depends on python3-pika package.
17
+ It takes full path and file name of the psi error log. Typically this script
18
+ should be run with all the error logs generated while rabbitmq service
19
+ was down
20
+ '
21
+
22
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
23
+ f.match(%r{^(test|spec|features)/})
24
+ end
25
+ spec.bindir = 'bin'
26
+ spec.executables = spec.files.grep(%r{^bin/}) do |f|
27
+ File.basename(f)
28
+ end
29
+ spec.require_paths = ['lib']
30
+
31
+ spec.add_development_dependency 'bundler', '~> 1.9'
32
+ spec.add_development_dependency 'rake', '~> 10.0'
33
+ spec.add_development_dependency 'rspec'
34
+ spec.add_development_dependency 'rubocop'
35
+ spec.add_dependency 'nokogiri'
36
+ # depeding on the ruby version installed, we need one or another version of bunny
37
+ spec.add_dependency 'bunny', '<= 1.7.0' if RUBY_VERSION[0].to_i < 2
38
+ spec.add_dependency 'bunny', '~> 2.0' if RUBY_VERSION[0].to_i >= 2
39
+ end
data/upload.dist.xml ADDED
@@ -0,0 +1,10 @@
1
+ <?xml version="1.0" encoding="ISO-8859-1"?>
2
+
3
+ <RabbitMQ>
4
+ <host>host1</host>
5
+ <port>port1</port>
6
+ <exchange>exchange1</exchange>
7
+ <username>username1</username>
8
+ <password>password1</password>
9
+ <queueUpload>queueUpload1</queueUpload>
10
+ </RabbitMQ>
metadata ADDED
@@ -0,0 +1,144 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: picsolve_rabbitmq_uploader
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.5
5
+ platform: ruby
6
+ authors:
7
+ - Christian Simon
8
+ - Juan Morales
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2015-09-30 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - "~>"
19
+ - !ruby/object:Gem::Version
20
+ version: '1.9'
21
+ type: :development
22
+ prerelease: false
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - "~>"
26
+ - !ruby/object:Gem::Version
27
+ version: '1.9'
28
+ - !ruby/object:Gem::Dependency
29
+ name: rake
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - "~>"
33
+ - !ruby/object:Gem::Version
34
+ version: '10.0'
35
+ type: :development
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - "~>"
40
+ - !ruby/object:Gem::Version
41
+ version: '10.0'
42
+ - !ruby/object:Gem::Dependency
43
+ name: rspec
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
49
+ type: :development
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - ">="
54
+ - !ruby/object:Gem::Version
55
+ version: '0'
56
+ - !ruby/object:Gem::Dependency
57
+ name: rubocop
58
+ requirement: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: '0'
63
+ type: :development
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ - !ruby/object:Gem::Dependency
71
+ name: nokogiri
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - ">="
75
+ - !ruby/object:Gem::Version
76
+ version: '0'
77
+ type: :runtime
78
+ prerelease: false
79
+ version_requirements: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - ">="
82
+ - !ruby/object:Gem::Version
83
+ version: '0'
84
+ - !ruby/object:Gem::Dependency
85
+ name: bunny
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - "~>"
89
+ - !ruby/object:Gem::Version
90
+ version: '2.0'
91
+ type: :runtime
92
+ prerelease: false
93
+ version_requirements: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - "~>"
96
+ - !ruby/object:Gem::Version
97
+ version: '2.0'
98
+ description: "\nScript to retry sending web upload messages to rabbitmq\nThis script
99
+ depends on python3-pika package.\nIt takes full path and file name of the psi error
100
+ log. Typically this script\nshould be run with all the error logs generated while
101
+ rabbitmq service\nwas down\n "
102
+ email:
103
+ - christian.simon@picsolve.com
104
+ - juan.morales@picsolve.com
105
+ executables:
106
+ - picsolve_rabbitmq_uploader
107
+ extensions: []
108
+ extra_rdoc_files: []
109
+ files:
110
+ - ".gitignore"
111
+ - ".travis.yml"
112
+ - Gemfile
113
+ - LICENSE.txt
114
+ - README.md
115
+ - Rakefile
116
+ - bin/picsolve_rabbitmq_uploader
117
+ - lib/picsolve_rabbitmq_uploader.rb
118
+ - lib/picsolve_rabbitmq_uploader/version.rb
119
+ - picsolve_rabbitmq_uploader.gemspec
120
+ - upload.dist.xml
121
+ homepage:
122
+ licenses: []
123
+ metadata: {}
124
+ post_install_message:
125
+ rdoc_options: []
126
+ require_paths:
127
+ - lib
128
+ required_ruby_version: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
133
+ required_rubygems_version: !ruby/object:Gem::Requirement
134
+ requirements:
135
+ - - ">="
136
+ - !ruby/object:Gem::Version
137
+ version: '0'
138
+ requirements: []
139
+ rubyforge_project:
140
+ rubygems_version: 2.2.2
141
+ signing_key:
142
+ specification_version: 4
143
+ summary: Script to retry sending web upload messages to rabbitmq
144
+ test_files: []