pseudo_cleaner 0.0.34 → 0.0.35
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/.gitignore +1 -0
- data/.rspec +2 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +2 -1
- data/lib/pseudo_cleaner/configuration.rb +6 -0
- data/lib/pseudo_cleaner/cucumber.rb +45 -2
- data/lib/pseudo_cleaner/master_cleaner.rb +149 -34
- data/lib/pseudo_cleaner/redis_cleaner.rb +665 -0
- data/lib/pseudo_cleaner/rspec.rb +29 -2
- data/lib/pseudo_cleaner/spinach.rb +57 -11
- data/lib/pseudo_cleaner/table_cleaner.rb +201 -45
- data/lib/pseudo_cleaner/version.rb +1 -1
- data/lib/pseudo_cleaner.rb +1 -0
- data/pseudo_cleaner.gemspec +4 -0
- data/spec/fixtures/resque.yml +4 -0
- data/spec/rails_helper.rb +10 -0
- data/spec/redis_cleaner_spec.rb +162 -0
- data/spec/spec_helper.rb +87 -0
- metadata +72 -5
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YzliYTc5NmFlMGQyM2E2OTZiNmNkZmI1NWQzOGViMWZmZjRiNWFiZA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
Nzk4NTNhODc2ZmQ0NGU5YWIyZjVkNmI4YWYyMjBiZmFlYWE3OWQ1ZQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MGZkODUwODZiOGFjMzhjYTY0MjI4M2FmYmFhOWVhNDM5MjM1YjRmODE3N2Y5
|
10
|
+
N2M1ZjNhNDk3YWZjYzBjYmQ5NTU0MGJhMzQ0ODYyNzI5ZDQ0NmJkZjE2YTc5
|
11
|
+
NmM0MTU1YzI3YzI2MWU5ZjVhZTljOGQ5NzczZDA0ZGZmNDA3ODU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NWNmMTQxNzFkZWZhOGI2MGNjZjU4YmFmMmU0NzJhOGI4ZTBiNmFkNDBlMTZk
|
14
|
+
Y2NjNjU4NTk5NjI2OWIyY2EyMGIxY2UwZGNlOTYwYzhjZjJlZTY1OWRlNGIy
|
15
|
+
OTNiYTEzMDc4YzUzMzI4YjY1NzQzODgxMTM1MzYyN2U2MTg2MDI=
|
data/.gitignore
CHANGED
data/.rspec
ADDED
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
pseudo_cleaner
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby-1.9.3-p327
|
data/Gemfile
CHANGED
@@ -15,6 +15,9 @@ module PseudoCleaner
|
|
15
15
|
attr_accessor :single_cleaner_set
|
16
16
|
attr_accessor :post_transaction_analysis
|
17
17
|
attr_accessor :db_connections
|
18
|
+
attr_accessor :peek_data_on_error
|
19
|
+
attr_accessor :peek_data_not_on_error
|
20
|
+
attr_accessor :enable_full_data_dump_tag
|
18
21
|
|
19
22
|
def self.current_instance
|
20
23
|
self.instance
|
@@ -27,6 +30,9 @@ module PseudoCleaner
|
|
27
30
|
@single_cleaner_set = true # true because I hope it will improve performance
|
28
31
|
@post_transaction_analysis = false # should only be set true if you are searching for a problem
|
29
32
|
@db_connections = {}
|
33
|
+
@peek_data_on_error = true
|
34
|
+
@peek_data_not_on_error = false
|
35
|
+
@enable_full_data_dump_tag = true
|
30
36
|
end
|
31
37
|
|
32
38
|
# Backwards comaptibility...
|
@@ -33,11 +33,20 @@ class CucumberHook
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def start_test(scenario, strategy)
|
36
|
-
PseudoCleaner::MasterCleaner.start_example(scenario, strategy)
|
36
|
+
PseudoCleaner::MasterCleaner.start_example(scenario, strategy, "PseudoCleaner::start_test - #{report_name(scenario)}")
|
37
37
|
end
|
38
38
|
|
39
39
|
def end_test(scenario)
|
40
|
-
PseudoCleaner::MasterCleaner.end_example(scenario)
|
40
|
+
PseudoCleaner::MasterCleaner.end_example(scenario, "PseudoCleaner::end_test - #{report_name(scenario)}")
|
41
|
+
end
|
42
|
+
|
43
|
+
def report_name(scenario)
|
44
|
+
report_name = "Unknown"
|
45
|
+
if scenario.respond_to?(:feature)
|
46
|
+
report_name = "#{scenario.feature.title}:#{scenario.title}"
|
47
|
+
elsif scenario.respond_to?(:line)
|
48
|
+
report_name = "Line - #{scenario.line}"
|
49
|
+
end
|
41
50
|
end
|
42
51
|
|
43
52
|
def run_test(scenario, strategy, block)
|
@@ -49,6 +58,14 @@ class CucumberHook
|
|
49
58
|
end_test(scenario)
|
50
59
|
end
|
51
60
|
end
|
61
|
+
|
62
|
+
def peek_data_inline(scenario)
|
63
|
+
PseudoCleaner::MasterCleaner.peek_data_inline("PseudoCleaner::peek_data - #{report_name(scenario)}")
|
64
|
+
end
|
65
|
+
|
66
|
+
def peek_data_new_test(scenario)
|
67
|
+
PseudoCleaner::MasterCleaner.peek_data_new_test("PseudoCleaner::peek_data - #{report_name(scenario)}")
|
68
|
+
end
|
52
69
|
end
|
53
70
|
|
54
71
|
##
|
@@ -116,6 +133,32 @@ else
|
|
116
133
|
end
|
117
134
|
end
|
118
135
|
|
136
|
+
After("~@full_data_dump") do |scenario|
|
137
|
+
if scenario.failed?
|
138
|
+
if PseudoCleaner::Configuration.instance.peek_data_on_error
|
139
|
+
CucumberHook.instance.peek_data_inline(scenario)
|
140
|
+
end
|
141
|
+
else
|
142
|
+
if PseudoCleaner::Configuration.instance.peek_data_not_on_error
|
143
|
+
CucumberHook.instance.peek_data_new_test(scenario)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
After("@full_data_dump") do |scenario|
|
149
|
+
if scenario.failed?
|
150
|
+
if PseudoCleaner::Configuration.instance.enable_full_data_dump_tag ||
|
151
|
+
PseudoCleaner::Configuration.instance.peek_data_on_error
|
152
|
+
CucumberHook.instance.peek_data_inline(scenario)
|
153
|
+
end
|
154
|
+
else
|
155
|
+
if PseudoCleaner::Configuration.instance.enable_full_data_dump_tag ||
|
156
|
+
PseudoCleaner::Configuration.instance.peek_data_not_on_error
|
157
|
+
CucumberHook.instance.peek_data_new_test(scenario)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
119
162
|
at_exit do
|
120
163
|
# We end suite in case a custom cleaner wants/needs to.
|
121
164
|
PseudoCleaner::MasterCleaner.end_suite
|
@@ -4,7 +4,10 @@ module PseudoCleaner
|
|
4
4
|
class MasterCleaner
|
5
5
|
@@suite_cleaner = nil
|
6
6
|
@@cleaner_classes = nil
|
7
|
+
@@redis_classes = nil
|
7
8
|
@@cleaner_classes_sorted = false
|
9
|
+
@@report_table = nil
|
10
|
+
@@report_error = false
|
8
11
|
|
9
12
|
CLEANING_STRATEGIES = [:transaction, :truncation, :deletion, :pseudo_delete, :none]
|
10
13
|
DB_CLEANER_CLEANING_STRATEGIES =
|
@@ -21,13 +24,57 @@ module PseudoCleaner
|
|
21
24
|
VALID_TEST_METHODS = VALID_START_METHODS + VALID_END_METHODS
|
22
25
|
|
23
26
|
class << self
|
24
|
-
def
|
25
|
-
|
26
|
-
|
27
|
+
def report_table
|
28
|
+
@@report_table
|
29
|
+
end
|
30
|
+
|
31
|
+
def report_error
|
32
|
+
@@report_error = true
|
33
|
+
end
|
34
|
+
|
35
|
+
def within_report_block(description, &block)
|
36
|
+
@@report_error = false
|
37
|
+
@@report_table = nil
|
38
|
+
|
39
|
+
if Object.const_defined?("Cornucopia", false) &&
|
40
|
+
Cornucopia.const_defined?("Util", false) &&
|
41
|
+
Cornucopia::Util.const_defined?("ReportBuilder", false)
|
42
|
+
Cornucopia::Util::ReportBuilder.current_report.within_test(description) do
|
43
|
+
Cornucopia::Util::ReportBuilder.current_report.within_section(description) do |report|
|
44
|
+
report.within_table do |report_table|
|
45
|
+
@@report_table = report_table
|
46
|
+
|
47
|
+
block.yield
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
unless @@report_error
|
52
|
+
Cornucopia::Util::ReportBuilder.current_report.test_succeeded
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
@@report_table = nil
|
57
|
+
else
|
58
|
+
block.yield
|
27
59
|
end
|
28
|
-
|
29
|
-
|
30
|
-
|
60
|
+
end
|
61
|
+
|
62
|
+
def start_suite(description = nil)
|
63
|
+
description ||= "PseudoCleaner::MasterCleaner.start_suite"
|
64
|
+
|
65
|
+
within_report_block(description) do
|
66
|
+
if @@suite_cleaner
|
67
|
+
@@suite_cleaner.reset_suite
|
68
|
+
end
|
69
|
+
@@suite_cleaner = PseudoCleaner::MasterCleaner.new(:suite)
|
70
|
+
@@suite_cleaner.start :pseudo_delete
|
71
|
+
@@suite_cleaner
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def clean_redis(redis)
|
76
|
+
@@redis_classes ||= []
|
77
|
+
@@redis_classes << redis
|
31
78
|
end
|
32
79
|
|
33
80
|
def database_cleaner
|
@@ -38,43 +85,55 @@ module PseudoCleaner
|
|
38
85
|
end
|
39
86
|
end
|
40
87
|
|
41
|
-
def start_example(example_class, strategy)
|
42
|
-
|
43
|
-
|
88
|
+
def start_example(example_class, strategy, description = nil)
|
89
|
+
description ||= "PseudoCleaner::MasterCleaner.start_example"
|
90
|
+
|
91
|
+
within_report_block(description) do
|
92
|
+
pseudo_cleaner_data = {}
|
93
|
+
pseudo_cleaner_data[:test_strategy] = strategy
|
44
94
|
|
45
|
-
|
46
|
-
|
95
|
+
unless strategy == :none
|
96
|
+
raise "invalid strategy" unless PseudoCleaner::MasterCleaner::DB_CLEANER_CLEANING_STRATEGIES.has_key? strategy
|
47
97
|
|
48
|
-
|
49
|
-
|
50
|
-
|
98
|
+
DatabaseCleaner.strategy = PseudoCleaner::MasterCleaner::DB_CLEANER_CLEANING_STRATEGIES[strategy]
|
99
|
+
unless [:pseudo_delete].include? strategy
|
100
|
+
PseudoCleaner::MasterCleaner.database_cleaner.start
|
101
|
+
end
|
102
|
+
|
103
|
+
pseudo_cleaner_data[:pseudo_state] = PseudoCleaner::MasterCleaner.start_test strategy
|
51
104
|
end
|
52
105
|
|
53
|
-
pseudo_cleaner_data
|
106
|
+
example_class.instance_variable_set(:@pseudo_cleaner_data, pseudo_cleaner_data)
|
54
107
|
end
|
55
|
-
|
56
|
-
example_class.instance_variable_set(:@pseudo_cleaner_data, pseudo_cleaner_data)
|
57
108
|
end
|
58
109
|
|
59
|
-
def end_example(example_class)
|
60
|
-
|
110
|
+
def end_example(example_class, description = nil)
|
111
|
+
description ||= "PseudoCleaner::MasterCleaner.end_example"
|
61
112
|
|
62
|
-
|
63
|
-
|
64
|
-
PseudoCleaner::MasterCleaner.database_cleaner.clean
|
65
|
-
end
|
113
|
+
within_report_block(description) do
|
114
|
+
pseudo_cleaner_data = example_class.instance_variable_get(:@pseudo_cleaner_data)
|
66
115
|
|
67
|
-
|
68
|
-
|
69
|
-
PseudoCleaner::MasterCleaner.
|
70
|
-
|
116
|
+
unless pseudo_cleaner_data[:test_strategy] == :none
|
117
|
+
unless [:pseudo_delete].include? pseudo_cleaner_data[:test_strategy]
|
118
|
+
PseudoCleaner::MasterCleaner.database_cleaner.clean
|
119
|
+
end
|
71
120
|
|
72
|
-
|
121
|
+
case pseudo_cleaner_data[:test_strategy]
|
122
|
+
when :deletion, :truncation
|
123
|
+
PseudoCleaner::MasterCleaner.database_reset
|
124
|
+
end
|
125
|
+
|
126
|
+
pseudo_cleaner_data[:pseudo_state].end test_type: :test, test_strategy: pseudo_cleaner_data[:test_strategy]
|
127
|
+
end
|
73
128
|
end
|
74
129
|
end
|
75
130
|
|
76
|
-
def end_suite
|
77
|
-
|
131
|
+
def end_suite(description = nil)
|
132
|
+
description ||= "PseudoCleaner::MasterCleaner.end_suite"
|
133
|
+
|
134
|
+
within_report_block(description) do
|
135
|
+
@@suite_cleaner.end test_strategy: :pseudo_delete if @@suite_cleaner
|
136
|
+
end
|
78
137
|
end
|
79
138
|
|
80
139
|
def start_test test_strategy
|
@@ -109,10 +168,14 @@ module PseudoCleaner
|
|
109
168
|
raise body_error if body_error
|
110
169
|
end
|
111
170
|
|
112
|
-
def reset_database
|
113
|
-
PseudoCleaner::MasterCleaner.
|
171
|
+
def reset_database(description = nil)
|
172
|
+
description ||= "PseudoCleaner::MasterCleaner.reset_database"
|
114
173
|
|
115
|
-
|
174
|
+
within_report_block(description) do
|
175
|
+
PseudoCleaner::MasterCleaner.database_cleaner.clean_with(:truncation)
|
176
|
+
|
177
|
+
PseudoCleaner::MasterCleaner.database_reset
|
178
|
+
end
|
116
179
|
end
|
117
180
|
|
118
181
|
def database_reset
|
@@ -189,6 +252,7 @@ module PseudoCleaner
|
|
189
252
|
|
190
253
|
PseudoCleaner::MasterCleaner.create_table_cleaners
|
191
254
|
PseudoCleaner::MasterCleaner.create_custom_cleaners
|
255
|
+
PseudoCleaner::MasterCleaner.create_redis_cleaners
|
192
256
|
end
|
193
257
|
|
194
258
|
@@cleaner_classes
|
@@ -245,6 +309,14 @@ module PseudoCleaner
|
|
245
309
|
end
|
246
310
|
end
|
247
311
|
|
312
|
+
def create_redis_cleaners
|
313
|
+
if @@redis_classes
|
314
|
+
@@redis_classes.each do |redis|
|
315
|
+
PseudoCleaner::MasterCleaner.cleaner_classes << [redis, nil, PseudoCleaner::RedisCleaner]
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
248
320
|
def find_file_class(seeder_file, seeder_root)
|
249
321
|
check_class = Object
|
250
322
|
full_module_name = []
|
@@ -265,7 +337,46 @@ module PseudoCleaner
|
|
265
337
|
end
|
266
338
|
|
267
339
|
def review_rows(&block)
|
268
|
-
@@suite_cleaner.review_rows
|
340
|
+
@@suite_cleaner.review_rows(&block)
|
341
|
+
end
|
342
|
+
|
343
|
+
def peek_data_inline(description = nil)
|
344
|
+
@@report_error = false
|
345
|
+
@@report_table = nil
|
346
|
+
|
347
|
+
if Object.const_defined?("Cornucopia", false) &&
|
348
|
+
Cornucopia.const_defined?("Util", false) &&
|
349
|
+
Cornucopia::Util.const_defined?("ReportBuilder", false)
|
350
|
+
Cornucopia::Util::ReportBuilder.current_report.within_section(description) do |report|
|
351
|
+
report.within_hidden_table do |outer_report_table|
|
352
|
+
Cornucopia::Util::ReportTable.new(
|
353
|
+
nested_table: outer_report_table,
|
354
|
+
suppress_blank_table: true) do |report_table|
|
355
|
+
@@report_table = report_table
|
356
|
+
|
357
|
+
peek_values
|
358
|
+
end
|
359
|
+
end
|
360
|
+
end
|
361
|
+
|
362
|
+
@@report_table = nil
|
363
|
+
else
|
364
|
+
PseudoCleaner::Logger.write(description)
|
365
|
+
|
366
|
+
peek_values
|
367
|
+
end
|
368
|
+
end
|
369
|
+
|
370
|
+
def peek_data_new_test(description = nil)
|
371
|
+
description ||= "PseudoCleaner::MasterCleaner.peek_data"
|
372
|
+
|
373
|
+
within_report_block(description) do
|
374
|
+
peek_values
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
378
|
+
def peek_values
|
379
|
+
@@suite_cleaner.peek_values
|
269
380
|
end
|
270
381
|
end
|
271
382
|
|
@@ -374,5 +485,9 @@ module PseudoCleaner
|
|
374
485
|
def review_rows(&block)
|
375
486
|
run_all_cleaners(:review_rows, @cleaners, &block)
|
376
487
|
end
|
488
|
+
|
489
|
+
def peek_values
|
490
|
+
run_all_cleaners(:peek_values, @cleaners)
|
491
|
+
end
|
377
492
|
end
|
378
493
|
end
|
@@ -0,0 +1,665 @@
|
|
1
|
+
require "redis"
|
2
|
+
require "redis-namespace"
|
3
|
+
require "pseudo_cleaner/master_cleaner"
|
4
|
+
require "pseudo_cleaner/configuration"
|
5
|
+
require "pseudo_cleaner/logger"
|
6
|
+
require "colorize"
|
7
|
+
|
8
|
+
module PseudoCleaner
|
9
|
+
##
|
10
|
+
# This class "cleans" a single redis connection.
|
11
|
+
#
|
12
|
+
# The cleaning is done by opening a monitor connection on redis and monitoring it for any actions that change values
|
13
|
+
# in redis.
|
14
|
+
#
|
15
|
+
# Long term, I was thinking about keeping some stats on the redis calls, but for now, I'm not.
|
16
|
+
#
|
17
|
+
# The cleaner roughly works as follows:
|
18
|
+
# * Get a list of all existing keys in the database as a level starting point.
|
19
|
+
# * Start a monitor
|
20
|
+
# * The monitor records any key that is updated or changed
|
21
|
+
# * When a test ends
|
22
|
+
# * Ask the monitor for a list of all changed keys.
|
23
|
+
# * The monitor resets the list of all changed keys.
|
24
|
+
# * When the suite ends
|
25
|
+
# * Get a list of all of the keys
|
26
|
+
# * Compare that list to the starting level
|
27
|
+
#
|
28
|
+
# Again, like the TableCleaner, if items are updated, this won't be able to "fix" that, but it will report on it.
|
29
|
+
#
|
30
|
+
# At this time, my code only pays attention to one database. It ignores other databases. We could extend things
|
31
|
+
# and have the monitor watch multiple databases. I'll have to do that later if I find a need.
|
32
|
+
|
33
|
+
# NOTE: Like the database cleaner, if the test is interrupted and test_end isn't called, the redis database may be
|
34
|
+
# left in an uncertain state.
|
35
|
+
|
36
|
+
# I'm not a huge fan of sleeps. In the non-rails world, I used to be able to do a sleep(0) to signal the system to
|
37
|
+
# check if somebody else needed to do some work. Testing with Rails, I find I have to actually sleep, so I do a
|
38
|
+
# very short time like 0.01.
|
39
|
+
class RedisCleaner
|
40
|
+
# SUITE_KEY = "PseudoDelete::RedisCleaner:initial_redis_state"
|
41
|
+
|
42
|
+
FLUSH_COMMANDS =
|
43
|
+
[
|
44
|
+
"flushall",
|
45
|
+
"flushdb"
|
46
|
+
]
|
47
|
+
WRITE_COMMANDS =
|
48
|
+
[
|
49
|
+
"append",
|
50
|
+
"bitop",
|
51
|
+
"blpop",
|
52
|
+
"brpop",
|
53
|
+
"brpoplpush",
|
54
|
+
"decr",
|
55
|
+
"decrby",
|
56
|
+
"del",
|
57
|
+
"expire",
|
58
|
+
"expireat",
|
59
|
+
"getset",
|
60
|
+
"hset",
|
61
|
+
"hsetnx",
|
62
|
+
"hincrby",
|
63
|
+
"hincrbyfloat",
|
64
|
+
"hmset",
|
65
|
+
"hdel",
|
66
|
+
"incr",
|
67
|
+
"incrby",
|
68
|
+
"incrbyfloat",
|
69
|
+
"linsert",
|
70
|
+
"lpop",
|
71
|
+
"lpush",
|
72
|
+
"lpushx",
|
73
|
+
"lrem",
|
74
|
+
"lset",
|
75
|
+
"ltrim",
|
76
|
+
"mapped_hmset",
|
77
|
+
"mapped_mset",
|
78
|
+
"mapped_msetnx",
|
79
|
+
"move",
|
80
|
+
"mset",
|
81
|
+
"msetnx",
|
82
|
+
"persist",
|
83
|
+
"pexpire",
|
84
|
+
"pexpireat",
|
85
|
+
"psetex",
|
86
|
+
"rename",
|
87
|
+
"renamenx",
|
88
|
+
"restore",
|
89
|
+
"rpop",
|
90
|
+
"rpoplpush",
|
91
|
+
"rpush",
|
92
|
+
"rpushx",
|
93
|
+
"sadd",
|
94
|
+
"sdiffstore",
|
95
|
+
"set",
|
96
|
+
"setbit",
|
97
|
+
"setex",
|
98
|
+
"setnx",
|
99
|
+
"setrange",
|
100
|
+
"sinterstore",
|
101
|
+
"smove",
|
102
|
+
"sort",
|
103
|
+
"spop",
|
104
|
+
"srem",
|
105
|
+
"sunionstore",
|
106
|
+
"zadd",
|
107
|
+
"zincrby",
|
108
|
+
"zinterstore",
|
109
|
+
"zrem",
|
110
|
+
"zremrangebyrank",
|
111
|
+
"zremrangebyscore",
|
112
|
+
]
|
113
|
+
READ_COMMANDS =
|
114
|
+
[
|
115
|
+
"bitcount",
|
116
|
+
"bitop",
|
117
|
+
"dump",
|
118
|
+
"exists",
|
119
|
+
"get",
|
120
|
+
"getbit",
|
121
|
+
"getrange",
|
122
|
+
"hget",
|
123
|
+
"hmget",
|
124
|
+
"hexists",
|
125
|
+
"hlen",
|
126
|
+
"hkeys",
|
127
|
+
"hscan",
|
128
|
+
"hscan_each",
|
129
|
+
"hvals",
|
130
|
+
"hgetall",
|
131
|
+
"lindex",
|
132
|
+
"llen",
|
133
|
+
"lrange",
|
134
|
+
"mapped_hmget",
|
135
|
+
"mapped_mget",
|
136
|
+
"mget",
|
137
|
+
"persist",
|
138
|
+
"scard",
|
139
|
+
"scan",
|
140
|
+
"scan_each",
|
141
|
+
"sdiff",
|
142
|
+
"sismember",
|
143
|
+
"smembers",
|
144
|
+
"srandmember",
|
145
|
+
"sscan",
|
146
|
+
"sscan_each",
|
147
|
+
"strlen",
|
148
|
+
"sunion",
|
149
|
+
"type",
|
150
|
+
"zcard",
|
151
|
+
"zcount",
|
152
|
+
"zrange",
|
153
|
+
"zrangebyscore",
|
154
|
+
"zrank",
|
155
|
+
"zrevrange",
|
156
|
+
"zrevrangebyscore",
|
157
|
+
"zrevrank",
|
158
|
+
"zscan",
|
159
|
+
"zscan_each",
|
160
|
+
"zscore",
|
161
|
+
]
|
162
|
+
|
163
|
+
attr_reader :monitor_thread
|
164
|
+
attr_reader :initial_keys
|
165
|
+
attr_accessor :options
|
166
|
+
|
167
|
+
class RedisMessage
|
168
|
+
attr_reader :message
|
169
|
+
attr_reader :time_stamp
|
170
|
+
attr_reader :db
|
171
|
+
attr_reader :host
|
172
|
+
attr_reader :port
|
173
|
+
attr_reader :command
|
174
|
+
attr_reader :cur_pos
|
175
|
+
|
176
|
+
def initialize(message_string)
|
177
|
+
@message = message_string
|
178
|
+
|
179
|
+
parse_message
|
180
|
+
end
|
181
|
+
|
182
|
+
def parse_message
|
183
|
+
if @message =~ /[0-9]+\.[0-9]+ \[[0-9]+ [^:]+:[^\]]+\] \"[^\"]+\"/
|
184
|
+
end_pos = @message.index(" ")
|
185
|
+
@time_stamp = @message[0..end_pos - 1]
|
186
|
+
|
187
|
+
@cur_pos = end_pos + 2 # " ["
|
188
|
+
end_pos = @message.index(" ", @cur_pos)
|
189
|
+
@db = @message[@cur_pos..end_pos - 1].to_i
|
190
|
+
|
191
|
+
@cur_pos = end_pos + 1
|
192
|
+
end_pos = @message.index(":", @cur_pos)
|
193
|
+
@host = @message[@cur_pos..end_pos - 1]
|
194
|
+
|
195
|
+
@cur_pos = end_pos + 1
|
196
|
+
end_pos = @message.index("]", @cur_pos)
|
197
|
+
@port = @message[@cur_pos..end_pos - 1].to_i
|
198
|
+
|
199
|
+
@cur_pos = end_pos + 2 # "] "
|
200
|
+
@command = next_value.downcase
|
201
|
+
else
|
202
|
+
@command = @message
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
def next_value
|
207
|
+
in_quote = (@message[@cur_pos] == '"')
|
208
|
+
if in_quote
|
209
|
+
@cur_pos += 1
|
210
|
+
end_pos = @cur_pos
|
211
|
+
while (end_pos && end_pos < @message.length)
|
212
|
+
end_pos = @message.index("\"", end_pos)
|
213
|
+
|
214
|
+
num_backslashes = 0
|
215
|
+
back_pos = end_pos
|
216
|
+
|
217
|
+
while @message[back_pos - 1] == "\\"
|
218
|
+
num_backslashes += 1
|
219
|
+
back_pos -= 1
|
220
|
+
end
|
221
|
+
|
222
|
+
break if (num_backslashes % 2) == 0
|
223
|
+
end_pos += 1
|
224
|
+
end
|
225
|
+
else
|
226
|
+
end_pos = @message.index(" ", @cur_pos)
|
227
|
+
end
|
228
|
+
the_value = @message[@cur_pos..end_pos - 1]
|
229
|
+
end_pos += 1 if in_quote
|
230
|
+
|
231
|
+
@cur_pos = end_pos + 1
|
232
|
+
|
233
|
+
the_value.gsub("\\\\", "\\").gsub("\\\"", "\"")
|
234
|
+
end
|
235
|
+
|
236
|
+
def keys
|
237
|
+
unless defined?(@message_keys)
|
238
|
+
@message_keys = []
|
239
|
+
|
240
|
+
if Redis::Namespace::COMMANDS.include? command
|
241
|
+
handling = Redis::Namespace::COMMANDS[command.to_s.downcase]
|
242
|
+
|
243
|
+
(before, after) = handling
|
244
|
+
|
245
|
+
case before
|
246
|
+
when :first
|
247
|
+
@message_keys << next_value
|
248
|
+
|
249
|
+
when :all
|
250
|
+
while @cur_pos < @message.length
|
251
|
+
@message_keys << next_value
|
252
|
+
end
|
253
|
+
|
254
|
+
when :exclude_first
|
255
|
+
next_value
|
256
|
+
while @cur_pos < @message.length
|
257
|
+
@message_keys << next_value
|
258
|
+
end
|
259
|
+
|
260
|
+
when :exclude_last
|
261
|
+
while @cur_pos < @message.length
|
262
|
+
@message_keys << next_value
|
263
|
+
end
|
264
|
+
@message_keys.delete_at(@message_keys.length - 1)
|
265
|
+
|
266
|
+
when :exclude_options
|
267
|
+
options = ["weights", "aggregate", "sum", "min", "max"]
|
268
|
+
while @cur_pos < @message.length
|
269
|
+
@message_keys << next_value
|
270
|
+
if options.include?(@message_keys[-1].downcase)
|
271
|
+
@message_keys.delete_at(@message_keys.length - 1)
|
272
|
+
break
|
273
|
+
end
|
274
|
+
end
|
275
|
+
|
276
|
+
when :alternate
|
277
|
+
while @cur_pos < @message.length
|
278
|
+
@message_keys << next_value
|
279
|
+
next_value
|
280
|
+
end
|
281
|
+
|
282
|
+
when :sort
|
283
|
+
next_value
|
284
|
+
|
285
|
+
while @cur_pos < @message.length
|
286
|
+
a_value = next_value
|
287
|
+
if a_value.downcase == "store"
|
288
|
+
@message_keys[0] = next_value
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
# when :eval_style
|
293
|
+
#
|
294
|
+
# when :scan_style
|
295
|
+
end
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
@message_keys
|
300
|
+
end
|
301
|
+
|
302
|
+
def to_s
|
303
|
+
{
|
304
|
+
time_stamp: time_stamp,
|
305
|
+
db: db,
|
306
|
+
host: host,
|
307
|
+
port: port,
|
308
|
+
command: command,
|
309
|
+
message: message,
|
310
|
+
cur_pos: cur_pos
|
311
|
+
}.to_s
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
def initialize(start_method, end_method, table, options)
|
316
|
+
@initial_keys = SortedSet.new
|
317
|
+
@monitor_thread = nil
|
318
|
+
@redis_name = nil
|
319
|
+
@suite_altered_keys = SortedSet.new
|
320
|
+
|
321
|
+
unless PseudoCleaner::MasterCleaner::VALID_START_METHODS.include?(start_method)
|
322
|
+
raise "You must specify a valid start function from: #{PseudoCleaner::MasterCleaner::VALID_START_METHODS}."
|
323
|
+
end
|
324
|
+
unless PseudoCleaner::MasterCleaner::VALID_END_METHODS.include?(end_method)
|
325
|
+
raise "You must specify a valid end function from: #{PseudoCleaner::MasterCleaner::VALID_END_METHODS}."
|
326
|
+
end
|
327
|
+
|
328
|
+
@options = options
|
329
|
+
|
330
|
+
@options[:table_start_method] ||= start_method
|
331
|
+
@options[:table_end_method] ||= end_method
|
332
|
+
@options[:output_diagnostics] ||= PseudoCleaner::Configuration.current_instance.output_diagnostics ||
|
333
|
+
PseudoCleaner::Configuration.current_instance.post_transaction_analysis
|
334
|
+
|
335
|
+
@redis = table
|
336
|
+
end
|
337
|
+
|
338
|
+
def <=>(right_object)
|
339
|
+
if (right_object.is_a?(PseudoCleaner::RedisCleaner))
|
340
|
+
return 0
|
341
|
+
elsif (right_object.is_a?(PseudoCleaner::TableCleaner))
|
342
|
+
return 1
|
343
|
+
else
|
344
|
+
if right_object.respond_to?(:<=>)
|
345
|
+
comparison = (right_object <=> self)
|
346
|
+
if comparison
|
347
|
+
return -1 * comparison
|
348
|
+
end
|
349
|
+
end
|
350
|
+
end
|
351
|
+
|
352
|
+
return 1
|
353
|
+
end
|
354
|
+
|
355
|
+
def redis
|
356
|
+
@redis ||= Redis.current
|
357
|
+
end
|
358
|
+
|
359
|
+
def suite_start test_strategy
|
360
|
+
@test_strategy ||= test_strategy
|
361
|
+
|
362
|
+
# if redis.type(PseudoCleaner::RedisCleaner::SUITE_KEY) == "set"
|
363
|
+
# @initial_keys = SortedSet.new(redis.smembers(PseudoCleaner::RedisCleaner::SUITE_KEY))
|
364
|
+
# report_end_of_suite_state "before suite start"
|
365
|
+
# end
|
366
|
+
# redis.del PseudoCleaner::RedisCleaner::SUITE_KEY
|
367
|
+
|
368
|
+
start_monitor
|
369
|
+
end
|
370
|
+
|
371
|
+
def test_start test_strategy
|
372
|
+
@test_strategy ||= test_strategy
|
373
|
+
|
374
|
+
synchronize_test_values do |test_values|
|
375
|
+
if test_values && !test_values.empty?
|
376
|
+
report_dirty_values "values altered before the test started", test_values
|
377
|
+
|
378
|
+
test_values.each do |value|
|
379
|
+
redis.del value unless initial_keys.include?(value)
|
380
|
+
end
|
381
|
+
end
|
382
|
+
end
|
383
|
+
end
|
384
|
+
|
385
|
+
def test_end test_strategy
|
386
|
+
synchronize_test_values do |updated_values|
|
387
|
+
if updated_values && !updated_values.empty?
|
388
|
+
report_keys = []
|
389
|
+
|
390
|
+
if @options[:output_diagnostics]
|
391
|
+
report_dirty_values "updated values", updated_values
|
392
|
+
end
|
393
|
+
|
394
|
+
updated_values.each do |value|
|
395
|
+
if initial_keys.include?(value)
|
396
|
+
report_keys << value
|
397
|
+
@suite_altered_keys << value
|
398
|
+
else
|
399
|
+
redis.del(value)
|
400
|
+
end
|
401
|
+
end
|
402
|
+
|
403
|
+
report_dirty_values "initial values altered by test", report_keys
|
404
|
+
end
|
405
|
+
end
|
406
|
+
end
|
407
|
+
|
408
|
+
def suite_end test_strategy
|
409
|
+
report_end_of_suite_state "suite end"
|
410
|
+
|
411
|
+
if monitor_thread
|
412
|
+
monitor_thread.kill
|
413
|
+
@monitor_thread = nil
|
414
|
+
end
|
415
|
+
end
|
416
|
+
|
417
|
+
def reset_suite
|
418
|
+
report_end_of_suite_state "reset suite"
|
419
|
+
|
420
|
+
if monitor_thread
|
421
|
+
monitor_thread.kill
|
422
|
+
@monitor_thread = nil
|
423
|
+
start_monitor
|
424
|
+
end
|
425
|
+
end
|
426
|
+
|
427
|
+
def ignore_regexes
|
428
|
+
[]
|
429
|
+
end
|
430
|
+
|
431
|
+
def ignore_key(key)
|
432
|
+
ignore_regexes.detect { |ignore_regex| key =~ ignore_regex }
|
433
|
+
end
|
434
|
+
|
435
|
+
def redis_name
|
436
|
+
unless @redis_name
|
437
|
+
redis_options = redis.client.options.with_indifferent_access
|
438
|
+
@redis_name = "#{redis_options[:host]}:#{redis_options[:port]}/#{redis_options[:db]}"
|
439
|
+
end
|
440
|
+
|
441
|
+
@redis_name
|
442
|
+
end
|
443
|
+
|
444
|
+
def review_rows(&block)
|
445
|
+
synchronize_test_values do |updated_values|
|
446
|
+
if updated_values && !updated_values.empty?
|
447
|
+
updated_values.each do |updated_value|
|
448
|
+
unless ignore_key(updated_value)
|
449
|
+
block.yield redis_name, report_record(updated_value)
|
450
|
+
end
|
451
|
+
end
|
452
|
+
end
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
456
|
+
def peek_values
|
457
|
+
synchronize_test_values do |updated_values|
|
458
|
+
if updated_values && !updated_values.empty?
|
459
|
+
output_values = false
|
460
|
+
|
461
|
+
if PseudoCleaner::MasterCleaner.report_table
|
462
|
+
Cornucopia::Util::ReportTable.new(nested_table: PseudoCleaner::MasterCleaner.report_table,
|
463
|
+
nested_table_label: redis_name,
|
464
|
+
suppress_blank_table: true) do |report_table|
|
465
|
+
updated_values.each do |updated_value|
|
466
|
+
unless ignore_key(updated_value)
|
467
|
+
output_values = true
|
468
|
+
report_table.write_stats updated_value, report_record(updated_value)
|
469
|
+
end
|
470
|
+
end
|
471
|
+
end
|
472
|
+
else
|
473
|
+
PseudoCleaner::Logger.write(" #{redis_name}")
|
474
|
+
|
475
|
+
updated_values.each do |updated_value|
|
476
|
+
unless ignore_key(updated_value)
|
477
|
+
output_values = true
|
478
|
+
PseudoCleaner::Logger.write(" #{updated_value}: #{report_record(updated_value)}")
|
479
|
+
end
|
480
|
+
end
|
481
|
+
end
|
482
|
+
|
483
|
+
PseudoCleaner::MasterCleaner.report_error if output_values
|
484
|
+
end
|
485
|
+
end
|
486
|
+
end
|
487
|
+
|
488
|
+
def synchronize_key
|
489
|
+
@synchronize_key ||= "redis_cleaner::synchronization_key_#{rand(1..1_000_000_000_000_000_000)}_#{rand(1..1_000_000_000_000_000_000)}"
|
490
|
+
end
|
491
|
+
|
492
|
+
def synchronize_end_key
|
493
|
+
@synchronize_end_key ||= "redis_cleaner::synchronization_end_key_#{rand(1..1_000_000_000_000_000_000)}_#{rand(1..1_000_000_000_000_000_000)}"
|
494
|
+
end
|
495
|
+
|
496
|
+
def report_end_of_suite_state report_reason
|
497
|
+
current_keys = SortedSet.new(redis.keys)
|
498
|
+
|
499
|
+
deleted_keys = initial_keys - current_keys
|
500
|
+
new_keys = current_keys - initial_keys
|
501
|
+
|
502
|
+
# filter out values we inserted that will go away on their own.
|
503
|
+
new_keys = new_keys.select { |key| (key =~ /redis_cleaner::synchronization_(?:end_)?key_[0-9]+_[0-9]+/).nil? }
|
504
|
+
|
505
|
+
report_dirty_values "new values as of #{report_reason}", new_keys
|
506
|
+
report_dirty_values "values deleted before #{report_reason}", deleted_keys
|
507
|
+
report_dirty_values "initial values changed during suite run", @suite_altered_keys
|
508
|
+
|
509
|
+
@suite_altered_keys = SortedSet.new
|
510
|
+
|
511
|
+
new_keys.each do |key_value|
|
512
|
+
redis.del key_value
|
513
|
+
end
|
514
|
+
end
|
515
|
+
|
516
|
+
def synchronize_test_values(&block)
|
517
|
+
updated_values = nil
|
518
|
+
|
519
|
+
if monitor_thread
|
520
|
+
redis.setex(synchronize_key, 1, true)
|
521
|
+
updated_values = queue.pop
|
522
|
+
end
|
523
|
+
|
524
|
+
block.yield updated_values
|
525
|
+
|
526
|
+
redis.setex(synchronize_end_key, 1, true)
|
527
|
+
end
|
528
|
+
|
529
|
+
def queue
|
530
|
+
@queue ||= Queue.new
|
531
|
+
end
|
532
|
+
|
533
|
+
def start_monitor
|
534
|
+
cleaner_class = self
|
535
|
+
|
536
|
+
@initial_keys = SortedSet.new(redis.keys)
|
537
|
+
# @initial_keys.add(PseudoCleaner::RedisCleaner::SUITE_KEY)
|
538
|
+
# @initial_keys.each do |key_value|
|
539
|
+
# redis.sadd(PseudoCleaner::RedisCleaner::SUITE_KEY, key_value)
|
540
|
+
# end
|
541
|
+
if @options[:output_diagnostics]
|
542
|
+
if PseudoCleaner::MasterCleaner.report_table
|
543
|
+
Cornucopia::Util::ReportTable.new(nested_table: PseudoCleaner::MasterCleaner.report_table,
|
544
|
+
nested_table_label: redis_name,
|
545
|
+
suppress_blank_table: true) do |report_table|
|
546
|
+
report_table.write_stats "initial keys count", @initial_keys.count
|
547
|
+
end
|
548
|
+
else
|
549
|
+
PseudoCleaner::Logger.write("#{redis_name}")
|
550
|
+
PseudoCleaner::Logger.write(" Initial keys count - #{@initial_keys.count}")
|
551
|
+
end
|
552
|
+
end
|
553
|
+
|
554
|
+
unless @monitor_thread
|
555
|
+
@monitor_thread = Thread.new do
|
556
|
+
in_redis_cleanup = false
|
557
|
+
updated_keys = SortedSet.new
|
558
|
+
|
559
|
+
monitor_redis = Redis.new(cleaner_class.redis.client.options)
|
560
|
+
redis_options = monitor_redis.client.options.with_indifferent_access
|
561
|
+
cleaner_class_db = redis_options[:db]
|
562
|
+
|
563
|
+
monitor_redis.monitor do |message|
|
564
|
+
redis_message = RedisMessage.new message
|
565
|
+
|
566
|
+
if redis_message.db == cleaner_class_db
|
567
|
+
process_command = true
|
568
|
+
|
569
|
+
if redis_message.command == "setex"
|
570
|
+
if redis_message.keys[0] == cleaner_class.synchronize_key
|
571
|
+
process_command = false
|
572
|
+
|
573
|
+
in_redis_cleanup = true
|
574
|
+
return_values = updated_keys
|
575
|
+
updated_keys = SortedSet.new
|
576
|
+
cleaner_class.queue << return_values
|
577
|
+
elsif redis_message.keys[0] == cleaner_class.synchronize_end_key
|
578
|
+
in_redis_cleanup = false
|
579
|
+
cleaner_class.monitor_thread[:updated] = nil
|
580
|
+
process_command = false
|
581
|
+
end
|
582
|
+
elsif redis_message.command == "del"
|
583
|
+
if in_redis_cleanup
|
584
|
+
process_command = false
|
585
|
+
end
|
586
|
+
end
|
587
|
+
|
588
|
+
if process_command
|
589
|
+
# flush...
|
590
|
+
if PseudoCleaner::RedisCleaner::WRITE_COMMANDS.include? redis_message.command
|
591
|
+
updated_keys.merge(redis_message.keys)
|
592
|
+
elsif PseudoCleaner::RedisCleaner::FLUSH_COMMANDS.include? redis_message.command
|
593
|
+
# Not sure I can get the keys at this point...
|
594
|
+
# updated_keys.merge(cleaner_class.redis.keys)
|
595
|
+
end
|
596
|
+
end
|
597
|
+
elsif "flushall" == redis_message.command
|
598
|
+
# Not sure I can get the keys at this point...
|
599
|
+
# updated_keys.merge(cleaner_class.redis.keys)
|
600
|
+
end
|
601
|
+
end
|
602
|
+
end
|
603
|
+
|
604
|
+
sleep(0.01)
|
605
|
+
redis.get(synchronize_key)
|
606
|
+
end
|
607
|
+
end
|
608
|
+
|
609
|
+
def report_record(key_name)
|
610
|
+
key_hash = { key: key_name, type: redis.type(key_name), ttl: redis.ttl(key_name) }
|
611
|
+
case key_hash[:type]
|
612
|
+
when "string"
|
613
|
+
key_hash[:value] = redis.get(key_name)
|
614
|
+
when "list"
|
615
|
+
key_hash[:list] = { len: redis.llen(key_name), values: redis.lrange(key_name, 0, -1) }
|
616
|
+
when "set"
|
617
|
+
key_hash[:set] = redis.smembers(key_name)
|
618
|
+
when "zset"
|
619
|
+
key_hash[:sorted_set] = redis.smembers(key_name)
|
620
|
+
when "hash"
|
621
|
+
key_hash[:list] = { len: redis.hlen(key_name), values: redis.hgetall(key_name) }
|
622
|
+
end
|
623
|
+
|
624
|
+
if key_hash[:value].nil? &&
|
625
|
+
key_hash[:list].nil? &&
|
626
|
+
key_hash[:set].nil? &&
|
627
|
+
key_hash[:sorted_set].nil? &&
|
628
|
+
key_hash[:hash].nil?
|
629
|
+
key_hash[:value] = "[[DELETED]]"
|
630
|
+
end
|
631
|
+
|
632
|
+
key_hash
|
633
|
+
end
|
634
|
+
|
635
|
+
def report_dirty_values message, test_values
|
636
|
+
if test_values && !test_values.empty?
|
637
|
+
output_values = false
|
638
|
+
|
639
|
+
if PseudoCleaner::MasterCleaner.report_table
|
640
|
+
Cornucopia::Util::ReportTable.new(nested_table: PseudoCleaner::MasterCleaner.report_table,
|
641
|
+
nested_table_label: redis_name,
|
642
|
+
suppress_blank_table: true) do |report_table|
|
643
|
+
report_table.write_stats "action", message
|
644
|
+
test_values.each_with_index do |key_name, index|
|
645
|
+
unless ignore_key(key_name)
|
646
|
+
output_values = true
|
647
|
+
report_table.write_stats index, report_record(key_name)
|
648
|
+
end
|
649
|
+
end
|
650
|
+
end
|
651
|
+
else
|
652
|
+
PseudoCleaner::Logger.write("********* RedisCleaner - #{message}".red.on_light_white)
|
653
|
+
test_values.each do |key_name|
|
654
|
+
unless ignore_key(key_name)
|
655
|
+
output_values = true
|
656
|
+
PseudoCleaner::Logger.write(" #{key_name}: #{report_record(key_name)}".red.on_light_white)
|
657
|
+
end
|
658
|
+
end
|
659
|
+
end
|
660
|
+
|
661
|
+
PseudoCleaner::MasterCleaner.report_error if output_values
|
662
|
+
end
|
663
|
+
end
|
664
|
+
end
|
665
|
+
end
|