picsolve_rabbitmq_uploader 0.0.5

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 360cc18137fa90503565ddfa0b8d5faba63476ff
4
+ data.tar.gz: 7b712b1cbca765073f6ab7f170fddab3751b73df
5
+ SHA512:
6
+ metadata.gz: 7a5301aa9312c76c121eef5b8149db8c334e3447ac7cadbab8557e3c1aa772046176d292c8e104338bf7d692678e5d64a53b273cc31574dda7d7dfe80a40b6b7
7
+ data.tar.gz: 190b9652fe2546264c79e36f1a276e0a4e59bb2811a2492d09ac6177c3c63fcfca1bff7378f685cf6864b6f8db33a0499dc4bd01689df8ecab793a892e244045
data/.gitignore ADDED
@@ -0,0 +1,9 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
data/.travis.yml ADDED
@@ -0,0 +1,3 @@
1
+ language: ruby
2
+ rvm:
3
+ - 2.1.5
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in picsolve_rabbitmq_uploader.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,21 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2015 Christian Simon
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in
13
+ all copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
+ THE SOFTWARE.
data/README.md ADDED
@@ -0,0 +1,34 @@
1
+ # PicsolveRabbitmqUploader
2
+
3
+ Script to retry sending web upload messages to rabbitmq. It takes full path and
4
+ file name of the psi error log. Typically this script should be run with all
5
+ the error logs generated while rabbitmq service was down.
6
+
7
+ ## Usage
8
+
9
+ ```sh
10
+ # Install gem
11
+ gem install picsolve_rabbitmq_uploader-0.0.1.gem
12
+
13
+ ## Create a upload.xml config file in CWD (cf. upload.dist.xml)
14
+
15
+ # Send failed uploads to rabbitmq
16
+ picsolve_rabbitmq_uploader C:/logfile.txt
17
+ ```
18
+
19
+ ## Development
20
+
21
+ ```sh
22
+ # Clone repo
23
+
24
+ # Install all ruby dependencies
25
+ bundle install
26
+
27
+ ## Do development
28
+
29
+ # Run tests
30
+ bundle exec rake test
31
+
32
+ # Build package
33
+ bundle exec rake build
34
+ ```
data/Rakefile ADDED
@@ -0,0 +1,24 @@
1
+ begin
2
+ require 'bundler/gem_tasks'
3
+ rescue LoadError
4
+ task :gem
5
+ end
6
+
7
+ begin
8
+ require 'rspec/core/rake_task'
9
+ RSpec::Core::RakeTask.new(:spec)
10
+ rescue LoadError
11
+ task :spec
12
+ end
13
+
14
+ begin
15
+ require 'rubocop/rake_task'
16
+ RuboCop::RakeTask.new
17
+ rescue LoadError
18
+ task :rubocop
19
+ end
20
+
21
+ desc 'Run all tests'
22
+ task test: [:spec, :rubocop]
23
+
24
+ task default: :test
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'picsolve_rabbitmq_uploader'
4
+
5
+ begin
6
+ PicsolveRabbitmqUploader::RabbitUploader.new.run
7
+ rescue => e
8
+ PicsolveRabbitmqUploader.logger.fatal "uncatched exception: #{e}"
9
+ e.backtrace.each do |line|
10
+ PicsolveRabbitmqUploader.logger.debug line
11
+ end
12
+ exit 1
13
+ end
@@ -0,0 +1,171 @@
1
+ require 'picsolve_rabbitmq_uploader/version'
2
+ require 'nokogiri'
3
+ require 'bunny'
4
+
5
+ # Parse logs and upload failed messaged to rabbitmq
6
+ module PicsolveRabbitmqUploader
7
+ def self.logger
8
+ @logger ||= create_logger
9
+ end
10
+
11
+ def self.create_logger
12
+ l = Logger.new(STDOUT)
13
+ l.level = Logger::DEBUG
14
+ l
15
+ end
16
+
17
+ # reads a psi config xml
18
+ class RabbitSettings
19
+ attr_reader :config
20
+
21
+ def log
22
+ PicsolveRabbitmqUploader.logger
23
+ end
24
+
25
+ def initialize
26
+ c = {}
27
+ read.xpath('//rabbitmq').children.each do |child|
28
+ next if child.name == 'text'
29
+ c[child.name] = child.text
30
+ end
31
+ @config = c
32
+ end
33
+
34
+ def read
35
+ log.info "Parsing config file '#{config_file}'"
36
+ f = File.open(config_file)
37
+ doc = Nokogiri::HTML(f)
38
+ f.close
39
+ doc
40
+ end
41
+
42
+ def config_file
43
+ @config_file ||= create_config_file
44
+ end
45
+
46
+ def create_config_file
47
+ paths = [
48
+ 'upload.xml',
49
+ 'PsiConfig.xml'
50
+ ]
51
+ paths.each do |f|
52
+ log.debug "Testing for config file '#{f}'"
53
+ return f if File.exist? f
54
+ end
55
+
56
+ fail "No config file found: #{paths}"
57
+ end
58
+ end
59
+
60
+ # uploads missing uploads to rabbitmq
61
+ class RabbitUploader
62
+ def log
63
+ PicsolveRabbitmqUploader.logger
64
+ end
65
+
66
+ def run(argv = ARGV)
67
+ @argv = argv
68
+
69
+ @config = RabbitSettings.new.config
70
+
71
+ msgs = messages
72
+
73
+ # stop here if no messages to upload
74
+ return unless msgs.length > 0
75
+
76
+ upload_rmq(msgs)
77
+ end
78
+
79
+ def publish_msgs_rmq(msgs)
80
+ # send all messages to exchange
81
+ msgs.each do |msg|
82
+ log.info "Publish on exchange=#{@config['exchange']} " \
83
+ "to to queue=#{@config['queueupload']} data=#{msg}"
84
+ @exchange.publish(
85
+ msg,
86
+ mandatory: true,
87
+ routing_key: @config['queueupload']
88
+ )
89
+ end
90
+ end
91
+
92
+ def ensure_queue
93
+ return if conn_rmq.queue_exists?(@config['queueupload'])
94
+
95
+ # create a queue
96
+ @channel.queue(
97
+ @config['queueupload'],
98
+ durable: true
99
+ )
100
+ end
101
+
102
+ def upload_rmq(msgs)
103
+ log.info 'connect to RabbitMQ'
104
+
105
+ conn_rmq.start
106
+
107
+ @channel = conn_rmq.create_channel
108
+
109
+ ensure_queue
110
+
111
+ # get exchange
112
+ @exchange = @channel.topic(@config['exchange'], durable: true)
113
+
114
+ publish_msgs_rmq msgs
115
+
116
+ log.info 'disconnect from RabbitMQ'
117
+ conn_rmq.stop
118
+ end
119
+
120
+ def conn_rmq
121
+ @conn_rmq ||= create_conn_rmq
122
+ end
123
+
124
+ def create_conn_rmq
125
+ Bunny.new(
126
+ host: @config['host'],
127
+ port: @config['port'],
128
+ user: @config['username'],
129
+ password: @config['password']
130
+ )
131
+ end
132
+
133
+ def filter_line(line)
134
+ m = line.match(
135
+ /Failed to send upload message to RabbitMQ psi\.uploads:(.*)$/
136
+ )
137
+
138
+ # return nil if regex fails
139
+ return nil unless m
140
+
141
+ m[1]
142
+ end
143
+
144
+ def messages
145
+ m = log_files.map do |file|
146
+ File.open(file).map do |line|
147
+ filter_line line
148
+ end
149
+ end
150
+
151
+ m.flatten.select do |d|
152
+ !d.nil?
153
+ end
154
+ end
155
+
156
+ def log_files
157
+ @log_files ||= existing_log_files
158
+ end
159
+
160
+ def existing_log_files
161
+ paths = @argv.select do |f|
162
+ log.debug "Testing for existence and readability of log file '#{f}'"
163
+ File.exist?(f) && File.readable?(f)
164
+ end
165
+
166
+ return paths if paths.length > 0
167
+
168
+ fail "No log file found, please specify it as parameter: #{paths}"
169
+ end
170
+ end
171
+ end
@@ -0,0 +1,4 @@
1
+ # verison number
2
+ module PicsolveRabbitmqUploader
3
+ VERSION = '0.0.5'
4
+ end
@@ -0,0 +1,39 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'picsolve_rabbitmq_uploader/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = 'picsolve_rabbitmq_uploader'
8
+ spec.version = PicsolveRabbitmqUploader::VERSION
9
+ spec.authors = ['Christian Simon', 'Juan Morales']
10
+ spec.email = ['christian.simon@picsolve.com','juan.morales@picsolve.com']
11
+
12
+ spec.summary = 'Script to retry sending web upload messages to rabbitmq'
13
+
14
+ spec.description = '
15
+ Script to retry sending web upload messages to rabbitmq
16
+ This script depends on python3-pika package.
17
+ It takes full path and file name of the psi error log. Typically this script
18
+ should be run with all the error logs generated while rabbitmq service
19
+ was down
20
+ '
21
+
22
+ spec.files = `git ls-files -z`.split("\x0").reject do |f|
23
+ f.match(%r{^(test|spec|features)/})
24
+ end
25
+ spec.bindir = 'bin'
26
+ spec.executables = spec.files.grep(%r{^bin/}) do |f|
27
+ File.basename(f)
28
+ end
29
+ spec.require_paths = ['lib']
30
+
31
+ spec.add_development_dependency 'bundler', '~> 1.9'
32
+ spec.add_development_dependency 'rake', '~> 10.0'
33
+ spec.add_development_dependency 'rspec'
34
+ spec.add_development_dependency 'rubocop'
35
+ spec.add_dependency 'nokogiri'
36
+ # depeding on the ruby version installed, we need one or another version of bunny
37
+ spec.add_dependency 'bunny', '<= 1.7.0' if RUBY_VERSION[0].to_i < 2
38
+ spec.add_dependency 'bunny', '~> 2.0' if RUBY_VERSION[0].to_i >= 2
39
+ end
data/upload.dist.xml ADDED
@@ -0,0 +1,10 @@
1
+ <?xml version="1.0" encoding="ISO-8859-1"?>
2
+
3
+ <RabbitMQ>
4
+ <host>host1</host>
5
+ <port>port1</port>
6
+ <exchange>exchange1</exchange>
7
+ <username>username1</username>
8
+ <password>password1</password>
9
+ <queueUpload>queueUpload1</queueUpload>
10
+ </RabbitMQ>
metadata ADDED
@@ -0,0 +1,144 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: picsolve_rabbitmq_uploader
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.5
5
+ platform: ruby
6
+ authors:
7
+ - Christian Simon
8
+ - Juan Morales
9
+ autorequire:
10
+ bindir: bin
11
+ cert_chain: []
12
+ date: 2015-09-30 00:00:00.000000000 Z
13
+ dependencies:
14
+ - !ruby/object:Gem::Dependency
15
+ name: bundler
16
+ requirement: !ruby/object:Gem::Requirement
17
+ requirements:
18
+ - - "~>"
19
+ - !ruby/object:Gem::Version
20
+ version: '1.9'
21
+ type: :development
22
+ prerelease: false
23
+ version_requirements: !ruby/object:Gem::Requirement
24
+ requirements:
25
+ - - "~>"
26
+ - !ruby/object:Gem::Version
27
+ version: '1.9'
28
+ - !ruby/object:Gem::Dependency
29
+ name: rake
30
+ requirement: !ruby/object:Gem::Requirement
31
+ requirements:
32
+ - - "~>"
33
+ - !ruby/object:Gem::Version
34
+ version: '10.0'
35
+ type: :development
36
+ prerelease: false
37
+ version_requirements: !ruby/object:Gem::Requirement
38
+ requirements:
39
+ - - "~>"
40
+ - !ruby/object:Gem::Version
41
+ version: '10.0'
42
+ - !ruby/object:Gem::Dependency
43
+ name: rspec
44
+ requirement: !ruby/object:Gem::Requirement
45
+ requirements:
46
+ - - ">="
47
+ - !ruby/object:Gem::Version
48
+ version: '0'
49
+ type: :development
50
+ prerelease: false
51
+ version_requirements: !ruby/object:Gem::Requirement
52
+ requirements:
53
+ - - ">="
54
+ - !ruby/object:Gem::Version
55
+ version: '0'
56
+ - !ruby/object:Gem::Dependency
57
+ name: rubocop
58
+ requirement: !ruby/object:Gem::Requirement
59
+ requirements:
60
+ - - ">="
61
+ - !ruby/object:Gem::Version
62
+ version: '0'
63
+ type: :development
64
+ prerelease: false
65
+ version_requirements: !ruby/object:Gem::Requirement
66
+ requirements:
67
+ - - ">="
68
+ - !ruby/object:Gem::Version
69
+ version: '0'
70
+ - !ruby/object:Gem::Dependency
71
+ name: nokogiri
72
+ requirement: !ruby/object:Gem::Requirement
73
+ requirements:
74
+ - - ">="
75
+ - !ruby/object:Gem::Version
76
+ version: '0'
77
+ type: :runtime
78
+ prerelease: false
79
+ version_requirements: !ruby/object:Gem::Requirement
80
+ requirements:
81
+ - - ">="
82
+ - !ruby/object:Gem::Version
83
+ version: '0'
84
+ - !ruby/object:Gem::Dependency
85
+ name: bunny
86
+ requirement: !ruby/object:Gem::Requirement
87
+ requirements:
88
+ - - "~>"
89
+ - !ruby/object:Gem::Version
90
+ version: '2.0'
91
+ type: :runtime
92
+ prerelease: false
93
+ version_requirements: !ruby/object:Gem::Requirement
94
+ requirements:
95
+ - - "~>"
96
+ - !ruby/object:Gem::Version
97
+ version: '2.0'
98
+ description: "\nScript to retry sending web upload messages to rabbitmq\nThis script
99
+ depends on python3-pika package.\nIt takes full path and file name of the psi error
100
+ log. Typically this script\nshould be run with all the error logs generated while
101
+ rabbitmq service\nwas down\n "
102
+ email:
103
+ - christian.simon@picsolve.com
104
+ - juan.morales@picsolve.com
105
+ executables:
106
+ - picsolve_rabbitmq_uploader
107
+ extensions: []
108
+ extra_rdoc_files: []
109
+ files:
110
+ - ".gitignore"
111
+ - ".travis.yml"
112
+ - Gemfile
113
+ - LICENSE.txt
114
+ - README.md
115
+ - Rakefile
116
+ - bin/picsolve_rabbitmq_uploader
117
+ - lib/picsolve_rabbitmq_uploader.rb
118
+ - lib/picsolve_rabbitmq_uploader/version.rb
119
+ - picsolve_rabbitmq_uploader.gemspec
120
+ - upload.dist.xml
121
+ homepage:
122
+ licenses: []
123
+ metadata: {}
124
+ post_install_message:
125
+ rdoc_options: []
126
+ require_paths:
127
+ - lib
128
+ required_ruby_version: !ruby/object:Gem::Requirement
129
+ requirements:
130
+ - - ">="
131
+ - !ruby/object:Gem::Version
132
+ version: '0'
133
+ required_rubygems_version: !ruby/object:Gem::Requirement
134
+ requirements:
135
+ - - ">="
136
+ - !ruby/object:Gem::Version
137
+ version: '0'
138
+ requirements: []
139
+ rubyforge_project:
140
+ rubygems_version: 2.2.2
141
+ signing_key:
142
+ specification_version: 4
143
+ summary: Script to retry sending web upload messages to rabbitmq
144
+ test_files: []