xcflushd 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.dockerignore +2 -0
- data/.rspec +1 -0
- data/.ruby-version +1 -1
- data/.travis.yml +3 -1
- data/CHANGELOG.md +26 -0
- data/Dockerfile +6 -3
- data/Gemfile.lock +33 -27
- data/Makefile +1 -1
- data/README.md +15 -7
- data/exe/xcflushd +4 -2
- data/lib/xcflushd/flusher.rb +23 -16
- data/lib/xcflushd/priority_auth_renewer.rb +9 -18
- data/lib/xcflushd/runner.rb +17 -5
- data/lib/xcflushd/storage.rb +7 -1
- data/lib/xcflushd/threading.rb +2 -4
- data/lib/xcflushd/version.rb +1 -1
- data/lib/xcflushd.rb +1 -0
- data/script/launch +4 -0
- data/script/test +6 -0
- data/xcflushd.gemspec +12 -12
- metadata +66 -63
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 8ce36d38201d9c3738a277cddbbae9b1f7a79a05
|
4
|
+
data.tar.gz: bddd1d1f27f34ded85d5d8cebeeced9b292ac7f0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: d4f02d26ef2fb8cdf559b5741f6c514df6a63b5a071618cc7048138ee43003d5717de46cc57c60a93d0337b46066038a58d0c3521328a890470d219609b50a3c
|
7
|
+
data.tar.gz: 8f01cfd539c1e763288f9149efa16755b05edaeb22c5d29bdae7baac6df13e1a0310e20b11cc99ce831ad3fa4b81c44da5680ea398fcb2a54b028c2090765f09
|
data/.dockerignore
ADDED
data/.rspec
CHANGED
data/.ruby-version
CHANGED
@@ -1 +1 @@
|
|
1
|
-
|
1
|
+
jruby-9.1.8.0
|
data/.travis.yml
CHANGED
data/CHANGELOG.md
ADDED
@@ -0,0 +1,26 @@
|
|
1
|
+
## [1.1.0] - 2017-05-02
|
2
|
+
- Switched from MRI to JRuby. our tests show that JRuby performs better than
|
3
|
+
MRI when there is a high number of calls to be made to 3scale's backend in a
|
4
|
+
flushing cycle. One of the reasons is that JRuby allows us to make several
|
5
|
+
calls to 3scale's backend in parallel as well as parse their XML responses.
|
6
|
+
- Updated dependencies: 3scale_client (2.11.0), gli (2.16.0), redis(3.3.3),
|
7
|
+
concurrent-ruby(1.0.5), simplecov (0.14.1), rubocop (0.48.1).
|
8
|
+
- Changed the timezone in the Dockerfile to UTC. This way, timestamps can be
|
9
|
+
easily translated from log files or other sources to whatever is needed.
|
10
|
+
- Increased the number of messages that the priority auth renewer can process
|
11
|
+
at the same time. We accomplished that by reducing the waiting times between
|
12
|
+
publishing attempts. Those waiting times were unnecessarily long.
|
13
|
+
- Fixed a bug that affected the way thread pools were being used. We were using
|
14
|
+
concurrent-ruby's ThreadPoolExecutor without specifying a max size for the
|
15
|
+
queue, and that type of pool only spawns a new thread when the queue is full.
|
16
|
+
That means that in practice, we specified a 'min:max' number of threads, but
|
17
|
+
only min were spawned. Now we are using FixedThreadPools and always take
|
18
|
+
the max number of threads for the input, ignoring the min.
|
19
|
+
- Added logging that shows the run time of each of the 5 phases of a
|
20
|
+
flushing cycle: 1- getting the reports from Redis, 2- reporting to 3scale,
|
21
|
+
3- waiting to leave some time to the reports to take effect, 4- getting the
|
22
|
+
auths from 3scale, 5- renewing the auths in Redis. This is useful for
|
23
|
+
debugging purposes.
|
24
|
+
- Minor changes in the Dockerfile to make it run on CentOS / RHEL.
|
25
|
+
- Improved performance of the flushing phase that takes care of renewing
|
26
|
+
authorizations in Redis by using the hmset command.
|
data/Dockerfile
CHANGED
@@ -7,7 +7,7 @@ ENV DEBIAN_FRONTEND=noninteractive
|
|
7
7
|
ARG USER_NAME=user
|
8
8
|
ARG USER_HOME="/home/${USER_NAME}"
|
9
9
|
# Timezone
|
10
|
-
ARG TZ="
|
10
|
+
ARG TZ="UTC"
|
11
11
|
|
12
12
|
RUN echo "Acquire::http {No-Cache=True;};" > /etc/apt/apt.conf.d/no-cache \
|
13
13
|
&& echo 'APT {Install-Recommends="false";Install-Suggests="false";};' > /etc/apt/apt.conf.d/no-recommends \
|
@@ -15,7 +15,7 @@ RUN echo "Acquire::http {No-Cache=True;};" > /etc/apt/apt.conf.d/no-cache \
|
|
15
15
|
&& rm -f /etc/cron.daily/apt \
|
16
16
|
&& rm -f /etc/cron.daily/dpkg \
|
17
17
|
&& rm -f /etc/cron.daily/passwd \
|
18
|
-
&& apt-get update -y -q && apt-get install -y -q apt-utils \
|
18
|
+
&& apt-get update -y -q && apt-get install -y -q apt-utils tzdata \
|
19
19
|
&& apt-get dist-upgrade -y -q \
|
20
20
|
&& apt-get install -y -q sudo cron logrotate wget curl unzip ca-certificates \
|
21
21
|
iputils-arping inetutils-ping inetutils-telnet net-tools nmap iotop tmux vim \
|
@@ -33,6 +33,8 @@ RUN echo "Acquire::http {No-Cache=True;};" > /etc/apt/apt.conf.d/no-cache \
|
|
33
33
|
&& chown -R ${USER_NAME}: ${USER_HOME} \
|
34
34
|
&& apt-get -y -q clean
|
35
35
|
|
36
|
+
RUN apt-get install -y -q openjdk-8-jre
|
37
|
+
|
36
38
|
USER ${USER_NAME}
|
37
39
|
|
38
40
|
ARG RBENV_ROOT="${USER_HOME}/.rbenv"
|
@@ -96,4 +98,5 @@ RUN chown -R ${USER_NAME}: ${APP_HOME}
|
|
96
98
|
USER ${USER_NAME}
|
97
99
|
RUN bundle install
|
98
100
|
|
99
|
-
|
101
|
+
ARG JRUBY_EXEC="jruby -Xcompile.invokedynamic=true -J-XX:ReservedCodeCacheSize=256M -J-XX:+UseCodeCacheFlushing -J-Xmn512m -J-Xms2048m -J-Xmx2048m -J-server -J-Djruby.objectspace.enabled=false -J-Djruby.thread.pool.enabled=true -J-Djruby.thread.pool.ttl=600 -J-Djruby.compile.mode=FORCE --server --headless -S"
|
102
|
+
CMD ${JRUBY_EXEC} bundle exec exe/xcflushd run
|
data/Gemfile.lock
CHANGED
@@ -1,40 +1,45 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
xcflushd (1.
|
5
|
-
3scale_client (~> 2.
|
6
|
-
concurrent-ruby (
|
7
|
-
daemons (
|
8
|
-
gli (
|
9
|
-
hiredis (
|
10
|
-
net-http-persistent (
|
11
|
-
redis (
|
4
|
+
xcflushd (1.1.0)
|
5
|
+
3scale_client (~> 2.11)
|
6
|
+
concurrent-ruby (~> 1.0.5)
|
7
|
+
daemons (~> 1.2.4)
|
8
|
+
gli (~> 2.16.0)
|
9
|
+
hiredis (~> 0.6.1)
|
10
|
+
net-http-persistent (~> 2.9.4)
|
11
|
+
redis (~> 3.3.3)
|
12
12
|
|
13
13
|
GEM
|
14
14
|
remote: https://rubygems.org/
|
15
15
|
specs:
|
16
|
-
3scale_client (2.
|
16
|
+
3scale_client (2.11.0)
|
17
17
|
nokogiri
|
18
18
|
ast (2.3.0)
|
19
|
-
concurrent-ruby (1.0.
|
19
|
+
concurrent-ruby (1.0.5)
|
20
|
+
concurrent-ruby (1.0.5-java)
|
20
21
|
daemons (1.2.4)
|
21
|
-
diff-lcs (1.
|
22
|
+
diff-lcs (1.3)
|
22
23
|
docile (1.1.5)
|
23
24
|
fakeredis (0.6.0)
|
24
25
|
redis (~> 3.2)
|
25
|
-
gli (2.
|
26
|
+
gli (2.16.0)
|
26
27
|
hiredis (0.6.1)
|
27
|
-
|
28
|
+
hiredis (0.6.1-java)
|
29
|
+
json (2.1.0)
|
30
|
+
json (2.1.0-java)
|
28
31
|
mini_portile2 (2.1.0)
|
29
32
|
net-http-persistent (2.9.4)
|
30
|
-
nokogiri (1.
|
33
|
+
nokogiri (1.7.1)
|
31
34
|
mini_portile2 (~> 2.1.0)
|
32
|
-
|
35
|
+
nokogiri (1.7.1-java)
|
36
|
+
parser (2.4.0.0)
|
33
37
|
ast (~> 2.2)
|
34
38
|
powerpack (0.1.1)
|
35
|
-
rainbow (2.
|
39
|
+
rainbow (2.2.2)
|
40
|
+
rake
|
36
41
|
rake (11.3.0)
|
37
|
-
redis (3.3.
|
42
|
+
redis (3.3.3)
|
38
43
|
rspec (3.5.0)
|
39
44
|
rspec-core (~> 3.5.0)
|
40
45
|
rspec-expectations (~> 3.5.0)
|
@@ -48,31 +53,32 @@ GEM
|
|
48
53
|
diff-lcs (>= 1.2.0, < 2.0)
|
49
54
|
rspec-support (~> 3.5.0)
|
50
55
|
rspec-support (3.5.0)
|
51
|
-
rubocop (0.
|
52
|
-
parser (>= 2.3.
|
56
|
+
rubocop (0.48.1)
|
57
|
+
parser (>= 2.3.3.1, < 3.0)
|
53
58
|
powerpack (~> 0.1)
|
54
59
|
rainbow (>= 1.99.1, < 3.0)
|
55
60
|
ruby-progressbar (~> 1.7)
|
56
61
|
unicode-display_width (~> 1.0, >= 1.0.1)
|
57
62
|
ruby-progressbar (1.8.1)
|
58
|
-
simplecov (0.
|
63
|
+
simplecov (0.14.1)
|
59
64
|
docile (~> 1.1.0)
|
60
65
|
json (>= 1.8, < 3)
|
61
66
|
simplecov-html (~> 0.10.0)
|
62
67
|
simplecov-html (0.10.0)
|
63
|
-
unicode-display_width (1.1
|
68
|
+
unicode-display_width (1.2.1)
|
64
69
|
|
65
70
|
PLATFORMS
|
71
|
+
java
|
66
72
|
ruby
|
67
73
|
|
68
74
|
DEPENDENCIES
|
69
|
-
bundler (~> 1.
|
70
|
-
fakeredis (~> 0.6
|
75
|
+
bundler (~> 1.14)
|
76
|
+
fakeredis (~> 0.6)
|
71
77
|
rake (~> 11.0)
|
72
|
-
rspec (~> 3.
|
73
|
-
rubocop (~> 0.
|
74
|
-
simplecov (~> 0.
|
78
|
+
rspec (~> 3.5)
|
79
|
+
rubocop (~> 0.48)
|
80
|
+
simplecov (~> 0.14)
|
75
81
|
xcflushd!
|
76
82
|
|
77
83
|
BUNDLED WITH
|
78
|
-
1.
|
84
|
+
1.14.6
|
data/Makefile
CHANGED
data/README.md
CHANGED
@@ -63,27 +63,35 @@ $ make build
|
|
63
63
|
|
64
64
|
Run:
|
65
65
|
```
|
66
|
-
$ docker run --rm xcflushd
|
66
|
+
$ docker run --rm -it xcflushd script/launch help
|
67
67
|
```
|
68
68
|
|
69
|
-
You can send the options as params
|
69
|
+
You can send the options as params for `script/launch`:
|
70
70
|
```
|
71
|
-
$ docker run --rm -it xcflushd
|
71
|
+
$ docker run --rm -it xcflushd script/launch run --auth-ttl 900 --provider-key my_provider_key --redis 127.0.0.1:6379 --frequency 300 --backend https://su1.3scale.net:443
|
72
72
|
```
|
73
73
|
|
74
74
|
Please note that the help command will also show you abbreviated flags you can
|
75
|
-
use at your convenience.
|
75
|
+
use at your convenience. Also, `script/launch` sets all the JRuby flags
|
76
|
+
recommended for a production environment. If you'd like to set different ones,
|
77
|
+
you can run:
|
78
|
+
```
|
79
|
+
$ docker run --rm -it xcflushd JRUBY_OPTS="..." jruby -S bundle exec xcflushd help
|
80
|
+
```
|
76
81
|
|
77
82
|
### Locally
|
78
83
|
|
84
|
+
This instructions are for JRuby, the Ruby implementation that we recommend for
|
85
|
+
running xcflushd.
|
86
|
+
|
79
87
|
Install the dependencies:
|
80
88
|
```
|
81
|
-
$ bundle install
|
89
|
+
$ jruby -S bundle install
|
82
90
|
```
|
83
91
|
|
84
|
-
Run the program:
|
92
|
+
Run the program with the recommended flags for production:
|
85
93
|
```
|
86
|
-
$
|
94
|
+
$ script/launch help
|
87
95
|
```
|
88
96
|
|
89
97
|
### Openshift
|
data/exe/xcflushd
CHANGED
@@ -31,10 +31,12 @@ command :run do |c|
|
|
31
31
|
c.desc 'Reporting frequency (in seconds)'
|
32
32
|
c.flag [:f, :frequency], required: true, type: Integer, must_match: POSITIVE_N_RE
|
33
33
|
|
34
|
-
c.desc 'Number of threads for the main thread pool (min:max)'
|
34
|
+
c.desc 'Number of threads for the main thread pool (min:max). '\
|
35
|
+
'min is not used, it is there just for backwards compatibility.'
|
35
36
|
c.flag [:t, :threads], default_value: 'auto', type: PositiveMinMaxInt
|
36
37
|
|
37
|
-
c.desc 'Number of threads for the priority auth renewer (min:max)'
|
38
|
+
c.desc 'Number of threads for the priority auth renewer (min:max). '\
|
39
|
+
'min is not used, it is there just for backwards compatibility.'
|
38
40
|
c.flag [:p, :'prio-threads'], default_value: 'auto', type: PositiveMinMaxInt
|
39
41
|
|
40
42
|
c.desc 'Run this program as a background service'
|
data/lib/xcflushd/flusher.rb
CHANGED
@@ -1,5 +1,4 @@
|
|
1
1
|
require 'concurrent'
|
2
|
-
require 'xcflushd/threading'
|
3
2
|
|
4
3
|
module Xcflushd
|
5
4
|
class Flusher
|
@@ -9,21 +8,14 @@ module Xcflushd
|
|
9
8
|
|
10
9
|
XcflushdError = Class.new(StandardError)
|
11
10
|
|
12
|
-
def initialize(reporter, authorizer, storage, auth_ttl, error_handler, threads)
|
11
|
+
def initialize(reporter, authorizer, storage, auth_ttl, error_handler, logger, threads)
|
13
12
|
@reporter = reporter
|
14
13
|
@authorizer = authorizer
|
15
14
|
@storage = storage
|
16
15
|
@auth_ttl = auth_ttl
|
17
16
|
@error_handler = error_handler
|
18
|
-
|
19
|
-
|
20
|
-
[threads.min, threads.max]
|
21
|
-
else
|
22
|
-
Threading.default_threads_value
|
23
|
-
end
|
24
|
-
|
25
|
-
@thread_pool = Concurrent::ThreadPoolExecutor.new(
|
26
|
-
min_threads: min_threads, max_threads: max_threads)
|
17
|
+
@logger = logger
|
18
|
+
@thread_pool = Concurrent::FixedThreadPool.new(threads)
|
27
19
|
end
|
28
20
|
|
29
21
|
def shutdown
|
@@ -40,8 +32,11 @@ module Xcflushd
|
|
40
32
|
|
41
33
|
# TODO: decide if we want to renew the authorizations every time.
|
42
34
|
def flush
|
43
|
-
reports_to_flush = reports
|
44
|
-
|
35
|
+
reports_to_flush = run_and_log_time('Getting the reports from Redis') do
|
36
|
+
reports
|
37
|
+
end
|
38
|
+
|
39
|
+
run_and_log_time('Reporting to 3scale') { report(reports_to_flush) }
|
45
40
|
|
46
41
|
# Ideally, we would like to ensure that once we start checking
|
47
42
|
# authorizations, they have taken into account the reports that we just
|
@@ -50,15 +45,21 @@ module Xcflushd
|
|
50
45
|
# processed.
|
51
46
|
# For now, let's just wait a few seconds. This will greatly mitigate the
|
52
47
|
# problem.
|
53
|
-
|
48
|
+
run_and_log_time('Giving reports some time to be processed') do
|
49
|
+
sleep(WAIT_TIME_REPORT_AUTH)
|
50
|
+
end
|
51
|
+
|
52
|
+
auths = run_and_log_time('Getting the auths from 3scale') do
|
53
|
+
authorizations(reports_to_flush)
|
54
|
+
end
|
54
55
|
|
55
|
-
renew(
|
56
|
+
run_and_log_time('Renewing the auths in Redis') { renew(auths) }
|
56
57
|
end
|
57
58
|
|
58
59
|
private
|
59
60
|
|
60
61
|
attr_reader :reporter, :authorizer, :storage, :auth_ttl,
|
61
|
-
:error_handler, :thread_pool
|
62
|
+
:error_handler, :logger, :thread_pool
|
62
63
|
|
63
64
|
def reports
|
64
65
|
storage.reports_to_flush
|
@@ -142,5 +143,11 @@ module Xcflushd
|
|
142
143
|
end.to_h
|
143
144
|
end
|
144
145
|
|
146
|
+
def run_and_log_time(action, &blk)
|
147
|
+
t = Time.now
|
148
|
+
res = blk.call
|
149
|
+
logger.debug("#{action} took #{(Time.now - t).round(3)} seconds")
|
150
|
+
res
|
151
|
+
end
|
145
152
|
end
|
146
153
|
end
|
@@ -1,5 +1,3 @@
|
|
1
|
-
require 'xcflushd/threading'
|
2
|
-
|
3
1
|
module Xcflushd
|
4
2
|
# Apart from flushing all the cached reports and renewing the authorizations
|
5
3
|
# periodically, we need to provide a mechanism to renew a specific auth at
|
@@ -22,9 +20,8 @@ module Xcflushd
|
|
22
20
|
# status once it gets it from 3scale.
|
23
21
|
class PriorityAuthRenewer
|
24
22
|
|
25
|
-
|
26
|
-
|
27
|
-
private_constant :TIMES_TO_PUBLISH
|
23
|
+
PUBLISH_WAIT_TIMES = [0.002, 0.005, 0.005, 0.01, 0.015].freeze
|
24
|
+
private_constant :PUBLISH_WAIT_TIMES
|
28
25
|
|
29
26
|
# We need two separate Redis clients: one for subscribing to a channel and
|
30
27
|
# the other one to publish to different channels. It is specified in the
|
@@ -47,15 +44,7 @@ module Xcflushd
|
|
47
44
|
# ensure thread-safety.
|
48
45
|
@current_auths = Concurrent::Map.new
|
49
46
|
|
50
|
-
|
51
|
-
[threads.min, threads.max]
|
52
|
-
else
|
53
|
-
Threading.default_threads_value
|
54
|
-
end
|
55
|
-
|
56
|
-
@thread_pool = Concurrent::ThreadPoolExecutor.new(
|
57
|
-
min_threads: min_threads,
|
58
|
-
max_threads: max_threads)
|
47
|
+
@thread_pool = Concurrent::FixedThreadPool.new(threads)
|
59
48
|
end
|
60
49
|
|
61
50
|
def shutdown
|
@@ -201,16 +190,18 @@ module Xcflushd
|
|
201
190
|
# between the moment the requests performs the publish and the
|
202
191
|
# subscribe actions. To mitigate the problem we can publish several
|
203
192
|
# times during some ms. We will see if this is good enough.
|
204
|
-
# Trade-off:
|
205
|
-
#
|
193
|
+
# Trade-off: Waiting long times between publishing attempts reduces the
|
194
|
+
# probability of triggering the problem described. However, it also makes
|
195
|
+
# incoming requests slow because tasks accumulate in the thread pool
|
196
|
+
# waiting.
|
206
197
|
publish_failures = 0
|
207
|
-
|
198
|
+
PUBLISH_WAIT_TIMES.each do |wait_time|
|
208
199
|
begin
|
209
200
|
publish_auth(combination, authorization)
|
210
201
|
rescue
|
211
202
|
publish_failures += 1
|
212
203
|
end
|
213
|
-
sleep(
|
204
|
+
sleep(wait_time)
|
214
205
|
end
|
215
206
|
|
216
207
|
if publish_failures > 0
|
data/lib/xcflushd/runner.rb
CHANGED
@@ -2,6 +2,7 @@ require 'xcflushd'
|
|
2
2
|
require 'redis'
|
3
3
|
require '3scale_client'
|
4
4
|
require 'xcflushd/3scale_client_ext'
|
5
|
+
require 'xcflushd/threading'
|
5
6
|
|
6
7
|
module Xcflushd
|
7
8
|
class Runner
|
@@ -27,7 +28,9 @@ module Xcflushd
|
|
27
28
|
|
28
29
|
redis_host = opts[:redis].host
|
29
30
|
redis_port = opts[:redis].port
|
30
|
-
|
31
|
+
redis_driver = RUBY_ENGINE == 'jruby' ? :ruby : :hiredis
|
32
|
+
|
33
|
+
redis = Redis.new(host: redis_host, port: redis_port, driver: redis_driver)
|
31
34
|
storage = Storage.new(redis, @logger, StorageKeys)
|
32
35
|
|
33
36
|
threescale = ThreeScale::Client.new(provider_key: opts[:provider_key],
|
@@ -39,19 +42,20 @@ module Xcflushd
|
|
39
42
|
reporter = Reporter.new(threescale)
|
40
43
|
authorizer = Authorizer.new(threescale)
|
41
44
|
|
42
|
-
redis_pub = Redis.new(host: redis_host, port: redis_port, driver:
|
43
|
-
redis_sub = Redis.new(host: redis_host, port: redis_port, driver:
|
45
|
+
redis_pub = Redis.new(host: redis_host, port: redis_port, driver: redis_driver)
|
46
|
+
redis_sub = Redis.new(host: redis_host, port: redis_port, driver: redis_driver)
|
44
47
|
|
45
48
|
auth_ttl = opts[:auth_ttl]
|
46
49
|
|
47
50
|
error_handler = FlusherErrorHandler.new(@logger, storage)
|
48
51
|
@flusher = Flusher.new(reporter, authorizer, storage,
|
49
|
-
auth_ttl, error_handler,
|
52
|
+
auth_ttl, error_handler, @logger,
|
53
|
+
flusher_threads(opts[:threads]))
|
50
54
|
|
51
55
|
@prio_auth_renewer = PriorityAuthRenewer.new(authorizer, storage,
|
52
56
|
redis_pub, redis_sub,
|
53
57
|
auth_ttl, @logger,
|
54
|
-
opts[:prio_threads])
|
58
|
+
prio_threads(opts[:prio_threads]))
|
55
59
|
|
56
60
|
@prio_auth_renewer_thread = start_priority_auth_renewer
|
57
61
|
|
@@ -60,6 +64,14 @@ module Xcflushd
|
|
60
64
|
|
61
65
|
private
|
62
66
|
|
67
|
+
def flusher_threads(opts_threads)
|
68
|
+
opts_threads ? opts_threads.max : Threading.default_threads
|
69
|
+
end
|
70
|
+
|
71
|
+
def prio_threads(opts_prio_threads)
|
72
|
+
opts_prio_threads ? opts_prio_threads.max : Threading.default_threads
|
73
|
+
end
|
74
|
+
|
63
75
|
def start_priority_auth_renewer
|
64
76
|
Thread.new do
|
65
77
|
loop do
|
data/lib/xcflushd/storage.rb
CHANGED
@@ -74,9 +74,15 @@ module Xcflushd
|
|
74
74
|
hash_key = hash_key(:auth, service_id, credentials)
|
75
75
|
|
76
76
|
authorizations.each_slice(REDIS_BATCH_KEYS) do |authorizations_slice|
|
77
|
+
# Array with the hash key and all the sorted key-values
|
78
|
+
hmset_args = [hash_key]
|
79
|
+
|
77
80
|
authorizations_slice.each do |metric, auth|
|
78
|
-
|
81
|
+
hmset_args << metric
|
82
|
+
hmset_args << auth_value(auth)
|
79
83
|
end
|
84
|
+
|
85
|
+
storage.hmset(*hmset_args)
|
80
86
|
end
|
81
87
|
|
82
88
|
set_auth_validity(service_id, credentials, auth_ttl)
|
data/lib/xcflushd/threading.rb
CHANGED
@@ -3,10 +3,8 @@ require 'concurrent'
|
|
3
3
|
|
4
4
|
module Xcflushd
|
5
5
|
module Threading
|
6
|
-
def self.
|
7
|
-
|
8
|
-
# default thread pool minimum is zero
|
9
|
-
return 0, cpus * 4
|
6
|
+
def self.default_threads
|
7
|
+
Concurrent.processor_count * 4
|
10
8
|
end
|
11
9
|
end
|
12
10
|
end
|
data/lib/xcflushd/version.rb
CHANGED
data/lib/xcflushd.rb
CHANGED
data/script/launch
ADDED
@@ -0,0 +1,4 @@
|
|
1
|
+
#!/bin/bash
|
2
|
+
|
3
|
+
export JRUBY_OPTS="-Xcompile.invokedynamic=true -J-XX:ReservedCodeCacheSize=256M -J-XX:+UseCodeCacheFlushing -J-Xmn512m -J-Xms2048m -J-Xmx2048m -J-server -J-Djruby.objectspace.enabled=false -J-Djruby.thread.pool.enabled=true -J-Djruby.thread.pool.ttl=600 -J-Djruby.compile.mode=FORCE --server --headless"
|
4
|
+
jruby -S bundle exec xcflushd $@
|
data/script/test
CHANGED
@@ -4,6 +4,12 @@ SCRIPT_DIR=$(dirname "$(readlink -f $0)")
|
|
4
4
|
|
5
5
|
pushd ${SCRIPT_DIR} > /dev/null
|
6
6
|
export TEST_COVERAGE=1
|
7
|
+
|
8
|
+
# 'debug' is needed for test coverage, 'dev' makes the JVM boot fast at the
|
9
|
+
# expense of long-running code performance among other things recommended for
|
10
|
+
# development: https://github.com/jruby/jruby/wiki/Improving-startup-time
|
11
|
+
export JRUBY_OPTS="--debug --dev"
|
12
|
+
|
7
13
|
bundle exec rake spec
|
8
14
|
STATUS=$?
|
9
15
|
popd > /dev/null
|
data/xcflushd.gemspec
CHANGED
@@ -26,18 +26,18 @@ Gem::Specification.new do |spec|
|
|
26
26
|
|
27
27
|
spec.required_ruby_version = '>= 2.1.0'
|
28
28
|
|
29
|
-
spec.add_runtime_dependency "3scale_client", "~> 2.
|
30
|
-
spec.add_runtime_dependency "gli", "
|
31
|
-
spec.add_runtime_dependency "redis", "
|
32
|
-
spec.add_runtime_dependency "hiredis", "
|
33
|
-
spec.add_runtime_dependency "concurrent-ruby", "1.0.
|
34
|
-
spec.add_runtime_dependency "net-http-persistent", "2.9.4"
|
35
|
-
spec.add_runtime_dependency "daemons", "
|
29
|
+
spec.add_runtime_dependency "3scale_client", "~> 2.11"
|
30
|
+
spec.add_runtime_dependency "gli", "~> 2.16.0"
|
31
|
+
spec.add_runtime_dependency "redis", "~> 3.3.3"
|
32
|
+
spec.add_runtime_dependency "hiredis", "~> 0.6.1"
|
33
|
+
spec.add_runtime_dependency "concurrent-ruby", "~> 1.0.5"
|
34
|
+
spec.add_runtime_dependency "net-http-persistent", "~> 2.9.4"
|
35
|
+
spec.add_runtime_dependency "daemons", "~> 1.2.4"
|
36
36
|
|
37
|
-
spec.add_development_dependency "bundler", "~> 1.
|
37
|
+
spec.add_development_dependency "bundler", "~> 1.14"
|
38
38
|
spec.add_development_dependency "rake", "~> 11.0"
|
39
|
-
spec.add_development_dependency "rspec", "~> 3.
|
40
|
-
spec.add_development_dependency "fakeredis", "~> 0.6
|
41
|
-
spec.add_development_dependency "simplecov", "~> 0.
|
42
|
-
spec.add_development_dependency "rubocop", "~> 0.
|
39
|
+
spec.add_development_dependency "rspec", "~> 3.5"
|
40
|
+
spec.add_development_dependency "fakeredis", "~> 0.6"
|
41
|
+
spec.add_development_dependency "simplecov", "~> 0.14"
|
42
|
+
spec.add_development_dependency "rubocop", "~> 0.48"
|
43
43
|
end
|
metadata
CHANGED
@@ -1,198 +1,198 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: xcflushd
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alejandro Martinez Ruiz
|
8
8
|
- David Ortiz Lopez
|
9
|
-
autorequire:
|
9
|
+
autorequire:
|
10
10
|
bindir: exe
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-
|
12
|
+
date: 2017-05-02 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
|
-
name: 3scale_client
|
16
15
|
requirement: !ruby/object:Gem::Requirement
|
17
16
|
requirements:
|
18
17
|
- - "~>"
|
19
18
|
- !ruby/object:Gem::Version
|
20
|
-
version: '2.
|
21
|
-
|
19
|
+
version: '2.11'
|
20
|
+
name: 3scale_client
|
22
21
|
prerelease: false
|
22
|
+
type: :runtime
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - "~>"
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: '2.
|
27
|
+
version: '2.11'
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
|
-
name: gli
|
30
29
|
requirement: !ruby/object:Gem::Requirement
|
31
30
|
requirements:
|
32
|
-
- -
|
31
|
+
- - "~>"
|
33
32
|
- !ruby/object:Gem::Version
|
34
|
-
version: 2.
|
35
|
-
|
33
|
+
version: 2.16.0
|
34
|
+
name: gli
|
36
35
|
prerelease: false
|
36
|
+
type: :runtime
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
|
-
- -
|
39
|
+
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: 2.
|
41
|
+
version: 2.16.0
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
|
-
name: redis
|
44
43
|
requirement: !ruby/object:Gem::Requirement
|
45
44
|
requirements:
|
46
|
-
- -
|
45
|
+
- - "~>"
|
47
46
|
- !ruby/object:Gem::Version
|
48
|
-
version: 3.3.
|
49
|
-
|
47
|
+
version: 3.3.3
|
48
|
+
name: redis
|
50
49
|
prerelease: false
|
50
|
+
type: :runtime
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
52
52
|
requirements:
|
53
|
-
- -
|
53
|
+
- - "~>"
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version: 3.3.
|
55
|
+
version: 3.3.3
|
56
56
|
- !ruby/object:Gem::Dependency
|
57
|
-
name: hiredis
|
58
57
|
requirement: !ruby/object:Gem::Requirement
|
59
58
|
requirements:
|
60
|
-
- -
|
59
|
+
- - "~>"
|
61
60
|
- !ruby/object:Gem::Version
|
62
61
|
version: 0.6.1
|
63
|
-
|
62
|
+
name: hiredis
|
64
63
|
prerelease: false
|
64
|
+
type: :runtime
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
66
66
|
requirements:
|
67
|
-
- -
|
67
|
+
- - "~>"
|
68
68
|
- !ruby/object:Gem::Version
|
69
69
|
version: 0.6.1
|
70
70
|
- !ruby/object:Gem::Dependency
|
71
|
-
name: concurrent-ruby
|
72
71
|
requirement: !ruby/object:Gem::Requirement
|
73
72
|
requirements:
|
74
|
-
- -
|
73
|
+
- - "~>"
|
75
74
|
- !ruby/object:Gem::Version
|
76
|
-
version: 1.0.
|
77
|
-
|
75
|
+
version: 1.0.5
|
76
|
+
name: concurrent-ruby
|
78
77
|
prerelease: false
|
78
|
+
type: :runtime
|
79
79
|
version_requirements: !ruby/object:Gem::Requirement
|
80
80
|
requirements:
|
81
|
-
- -
|
81
|
+
- - "~>"
|
82
82
|
- !ruby/object:Gem::Version
|
83
|
-
version: 1.0.
|
83
|
+
version: 1.0.5
|
84
84
|
- !ruby/object:Gem::Dependency
|
85
|
-
name: net-http-persistent
|
86
85
|
requirement: !ruby/object:Gem::Requirement
|
87
86
|
requirements:
|
88
|
-
- -
|
87
|
+
- - "~>"
|
89
88
|
- !ruby/object:Gem::Version
|
90
89
|
version: 2.9.4
|
91
|
-
|
90
|
+
name: net-http-persistent
|
92
91
|
prerelease: false
|
92
|
+
type: :runtime
|
93
93
|
version_requirements: !ruby/object:Gem::Requirement
|
94
94
|
requirements:
|
95
|
-
- -
|
95
|
+
- - "~>"
|
96
96
|
- !ruby/object:Gem::Version
|
97
97
|
version: 2.9.4
|
98
98
|
- !ruby/object:Gem::Dependency
|
99
|
-
name: daemons
|
100
99
|
requirement: !ruby/object:Gem::Requirement
|
101
100
|
requirements:
|
102
|
-
- -
|
101
|
+
- - "~>"
|
103
102
|
- !ruby/object:Gem::Version
|
104
103
|
version: 1.2.4
|
105
|
-
|
104
|
+
name: daemons
|
106
105
|
prerelease: false
|
106
|
+
type: :runtime
|
107
107
|
version_requirements: !ruby/object:Gem::Requirement
|
108
108
|
requirements:
|
109
|
-
- -
|
109
|
+
- - "~>"
|
110
110
|
- !ruby/object:Gem::Version
|
111
111
|
version: 1.2.4
|
112
112
|
- !ruby/object:Gem::Dependency
|
113
|
-
name: bundler
|
114
113
|
requirement: !ruby/object:Gem::Requirement
|
115
114
|
requirements:
|
116
115
|
- - "~>"
|
117
116
|
- !ruby/object:Gem::Version
|
118
|
-
version: '1.
|
119
|
-
|
117
|
+
version: '1.14'
|
118
|
+
name: bundler
|
120
119
|
prerelease: false
|
120
|
+
type: :development
|
121
121
|
version_requirements: !ruby/object:Gem::Requirement
|
122
122
|
requirements:
|
123
123
|
- - "~>"
|
124
124
|
- !ruby/object:Gem::Version
|
125
|
-
version: '1.
|
125
|
+
version: '1.14'
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
|
-
name: rake
|
128
127
|
requirement: !ruby/object:Gem::Requirement
|
129
128
|
requirements:
|
130
129
|
- - "~>"
|
131
130
|
- !ruby/object:Gem::Version
|
132
131
|
version: '11.0'
|
133
|
-
|
132
|
+
name: rake
|
134
133
|
prerelease: false
|
134
|
+
type: :development
|
135
135
|
version_requirements: !ruby/object:Gem::Requirement
|
136
136
|
requirements:
|
137
137
|
- - "~>"
|
138
138
|
- !ruby/object:Gem::Version
|
139
139
|
version: '11.0'
|
140
140
|
- !ruby/object:Gem::Dependency
|
141
|
-
name: rspec
|
142
141
|
requirement: !ruby/object:Gem::Requirement
|
143
142
|
requirements:
|
144
143
|
- - "~>"
|
145
144
|
- !ruby/object:Gem::Version
|
146
|
-
version: '3.
|
147
|
-
|
145
|
+
version: '3.5'
|
146
|
+
name: rspec
|
148
147
|
prerelease: false
|
148
|
+
type: :development
|
149
149
|
version_requirements: !ruby/object:Gem::Requirement
|
150
150
|
requirements:
|
151
151
|
- - "~>"
|
152
152
|
- !ruby/object:Gem::Version
|
153
|
-
version: '3.
|
153
|
+
version: '3.5'
|
154
154
|
- !ruby/object:Gem::Dependency
|
155
|
-
name: fakeredis
|
156
155
|
requirement: !ruby/object:Gem::Requirement
|
157
156
|
requirements:
|
158
157
|
- - "~>"
|
159
158
|
- !ruby/object:Gem::Version
|
160
|
-
version: 0.6
|
161
|
-
|
159
|
+
version: '0.6'
|
160
|
+
name: fakeredis
|
162
161
|
prerelease: false
|
162
|
+
type: :development
|
163
163
|
version_requirements: !ruby/object:Gem::Requirement
|
164
164
|
requirements:
|
165
165
|
- - "~>"
|
166
166
|
- !ruby/object:Gem::Version
|
167
|
-
version: 0.6
|
167
|
+
version: '0.6'
|
168
168
|
- !ruby/object:Gem::Dependency
|
169
|
-
name: simplecov
|
170
169
|
requirement: !ruby/object:Gem::Requirement
|
171
170
|
requirements:
|
172
171
|
- - "~>"
|
173
172
|
- !ruby/object:Gem::Version
|
174
|
-
version: 0.
|
175
|
-
|
173
|
+
version: '0.14'
|
174
|
+
name: simplecov
|
176
175
|
prerelease: false
|
176
|
+
type: :development
|
177
177
|
version_requirements: !ruby/object:Gem::Requirement
|
178
178
|
requirements:
|
179
179
|
- - "~>"
|
180
180
|
- !ruby/object:Gem::Version
|
181
|
-
version: 0.
|
181
|
+
version: '0.14'
|
182
182
|
- !ruby/object:Gem::Dependency
|
183
|
-
name: rubocop
|
184
183
|
requirement: !ruby/object:Gem::Requirement
|
185
184
|
requirements:
|
186
185
|
- - "~>"
|
187
186
|
- !ruby/object:Gem::Version
|
188
|
-
version: 0.
|
189
|
-
|
187
|
+
version: '0.48'
|
188
|
+
name: rubocop
|
190
189
|
prerelease: false
|
190
|
+
type: :development
|
191
191
|
version_requirements: !ruby/object:Gem::Requirement
|
192
192
|
requirements:
|
193
193
|
- - "~>"
|
194
194
|
- !ruby/object:Gem::Version
|
195
|
-
version: 0.
|
195
|
+
version: '0.48'
|
196
196
|
description: xcflushd is a daemon that connects to a Redis database containing 3scale's
|
197
197
|
XC API Management data and flushes it to the 3scale service for cached reporting
|
198
198
|
and authorizations. Check https://github.com/3scale/apicast-xc for an implementation
|
@@ -205,6 +205,7 @@ extensions: []
|
|
205
205
|
extra_rdoc_files: []
|
206
206
|
files:
|
207
207
|
- ".codeclimate.yml"
|
208
|
+
- ".dockerignore"
|
208
209
|
- ".gitignore"
|
209
210
|
- ".rspec"
|
210
211
|
- ".rubocop.yml"
|
@@ -212,6 +213,7 @@ files:
|
|
212
213
|
- ".ruby-version"
|
213
214
|
- ".simplecov"
|
214
215
|
- ".travis.yml"
|
216
|
+
- CHANGELOG.md
|
215
217
|
- Dockerfile
|
216
218
|
- Gemfile
|
217
219
|
- Gemfile.lock
|
@@ -240,13 +242,14 @@ files:
|
|
240
242
|
- lib/xcflushd/storage_keys.rb
|
241
243
|
- lib/xcflushd/threading.rb
|
242
244
|
- lib/xcflushd/version.rb
|
245
|
+
- script/launch
|
243
246
|
- script/test
|
244
247
|
- xcflushd.gemspec
|
245
248
|
homepage: https://github.com/3scale/xcflushd
|
246
249
|
licenses:
|
247
250
|
- Apache-2.0
|
248
251
|
metadata: {}
|
249
|
-
post_install_message:
|
252
|
+
post_install_message:
|
250
253
|
rdoc_options: []
|
251
254
|
require_paths:
|
252
255
|
- lib
|
@@ -261,9 +264,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
261
264
|
- !ruby/object:Gem::Version
|
262
265
|
version: '0'
|
263
266
|
requirements: []
|
264
|
-
rubyforge_project:
|
265
|
-
rubygems_version: 2.6.
|
266
|
-
signing_key:
|
267
|
+
rubyforge_project:
|
268
|
+
rubygems_version: 2.6.12
|
269
|
+
signing_key:
|
267
270
|
specification_version: 4
|
268
271
|
summary: Daemon for flushing XC reports to 3scale.
|
269
272
|
test_files: []
|