pseudo_cleaner 0.0.34 → 0.0.35
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/.gitignore +1 -0
- data/.rspec +2 -0
- data/.ruby-gemset +1 -0
- data/.ruby-version +1 -0
- data/Gemfile +2 -1
- data/lib/pseudo_cleaner/configuration.rb +6 -0
- data/lib/pseudo_cleaner/cucumber.rb +45 -2
- data/lib/pseudo_cleaner/master_cleaner.rb +149 -34
- data/lib/pseudo_cleaner/redis_cleaner.rb +665 -0
- data/lib/pseudo_cleaner/rspec.rb +29 -2
- data/lib/pseudo_cleaner/spinach.rb +57 -11
- data/lib/pseudo_cleaner/table_cleaner.rb +201 -45
- data/lib/pseudo_cleaner/version.rb +1 -1
- data/lib/pseudo_cleaner.rb +1 -0
- data/pseudo_cleaner.gemspec +4 -0
- data/spec/fixtures/resque.yml +4 -0
- data/spec/rails_helper.rb +10 -0
- data/spec/redis_cleaner_spec.rb +162 -0
- data/spec/spec_helper.rb +87 -0
- metadata +72 -5
checksums.yaml
CHANGED
@@ -1,15 +1,15 @@
|
|
1
1
|
---
|
2
2
|
!binary "U0hBMQ==":
|
3
3
|
metadata.gz: !binary |-
|
4
|
-
|
4
|
+
YzliYTc5NmFlMGQyM2E2OTZiNmNkZmI1NWQzOGViMWZmZjRiNWFiZA==
|
5
5
|
data.tar.gz: !binary |-
|
6
|
-
|
6
|
+
Nzk4NTNhODc2ZmQ0NGU5YWIyZjVkNmI4YWYyMjBiZmFlYWE3OWQ1ZQ==
|
7
7
|
SHA512:
|
8
8
|
metadata.gz: !binary |-
|
9
|
-
|
10
|
-
|
11
|
-
|
9
|
+
MGZkODUwODZiOGFjMzhjYTY0MjI4M2FmYmFhOWVhNDM5MjM1YjRmODE3N2Y5
|
10
|
+
N2M1ZjNhNDk3YWZjYzBjYmQ5NTU0MGJhMzQ0ODYyNzI5ZDQ0NmJkZjE2YTc5
|
11
|
+
NmM0MTU1YzI3YzI2MWU5ZjVhZTljOGQ5NzczZDA0ZGZmNDA3ODU=
|
12
12
|
data.tar.gz: !binary |-
|
13
|
-
|
14
|
-
|
15
|
-
|
13
|
+
NWNmMTQxNzFkZWZhOGI2MGNjZjU4YmFmMmU0NzJhOGI4ZTBiNmFkNDBlMTZk
|
14
|
+
Y2NjNjU4NTk5NjI2OWIyY2EyMGIxY2UwZGNlOTYwYzhjZjJlZTY1OWRlNGIy
|
15
|
+
OTNiYTEzMDc4YzUzMzI4YjY1NzQzODgxMTM1MzYyN2U2MTg2MDI=
|
data/.gitignore
CHANGED
data/.rspec
ADDED
data/.ruby-gemset
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
pseudo_cleaner
|
data/.ruby-version
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
ruby-1.9.3-p327
|
data/Gemfile
CHANGED
@@ -15,6 +15,9 @@ module PseudoCleaner
|
|
15
15
|
attr_accessor :single_cleaner_set
|
16
16
|
attr_accessor :post_transaction_analysis
|
17
17
|
attr_accessor :db_connections
|
18
|
+
attr_accessor :peek_data_on_error
|
19
|
+
attr_accessor :peek_data_not_on_error
|
20
|
+
attr_accessor :enable_full_data_dump_tag
|
18
21
|
|
19
22
|
def self.current_instance
|
20
23
|
self.instance
|
@@ -27,6 +30,9 @@ module PseudoCleaner
|
|
27
30
|
@single_cleaner_set = true # true because I hope it will improve performance
|
28
31
|
@post_transaction_analysis = false # should only be set true if you are searching for a problem
|
29
32
|
@db_connections = {}
|
33
|
+
@peek_data_on_error = true
|
34
|
+
@peek_data_not_on_error = false
|
35
|
+
@enable_full_data_dump_tag = true
|
30
36
|
end
|
31
37
|
|
32
38
|
# Backwards comaptibility...
|
@@ -33,11 +33,20 @@ class CucumberHook
|
|
33
33
|
end
|
34
34
|
|
35
35
|
def start_test(scenario, strategy)
|
36
|
-
PseudoCleaner::MasterCleaner.start_example(scenario, strategy)
|
36
|
+
PseudoCleaner::MasterCleaner.start_example(scenario, strategy, "PseudoCleaner::start_test - #{report_name(scenario)}")
|
37
37
|
end
|
38
38
|
|
39
39
|
def end_test(scenario)
|
40
|
-
PseudoCleaner::MasterCleaner.end_example(scenario)
|
40
|
+
PseudoCleaner::MasterCleaner.end_example(scenario, "PseudoCleaner::end_test - #{report_name(scenario)}")
|
41
|
+
end
|
42
|
+
|
43
|
+
def report_name(scenario)
|
44
|
+
report_name = "Unknown"
|
45
|
+
if scenario.respond_to?(:feature)
|
46
|
+
report_name = "#{scenario.feature.title}:#{scenario.title}"
|
47
|
+
elsif scenario.respond_to?(:line)
|
48
|
+
report_name = "Line - #{scenario.line}"
|
49
|
+
end
|
41
50
|
end
|
42
51
|
|
43
52
|
def run_test(scenario, strategy, block)
|
@@ -49,6 +58,14 @@ class CucumberHook
|
|
49
58
|
end_test(scenario)
|
50
59
|
end
|
51
60
|
end
|
61
|
+
|
62
|
+
def peek_data_inline(scenario)
|
63
|
+
PseudoCleaner::MasterCleaner.peek_data_inline("PseudoCleaner::peek_data - #{report_name(scenario)}")
|
64
|
+
end
|
65
|
+
|
66
|
+
def peek_data_new_test(scenario)
|
67
|
+
PseudoCleaner::MasterCleaner.peek_data_new_test("PseudoCleaner::peek_data - #{report_name(scenario)}")
|
68
|
+
end
|
52
69
|
end
|
53
70
|
|
54
71
|
##
|
@@ -116,6 +133,32 @@ else
|
|
116
133
|
end
|
117
134
|
end
|
118
135
|
|
136
|
+
After("~@full_data_dump") do |scenario|
|
137
|
+
if scenario.failed?
|
138
|
+
if PseudoCleaner::Configuration.instance.peek_data_on_error
|
139
|
+
CucumberHook.instance.peek_data_inline(scenario)
|
140
|
+
end
|
141
|
+
else
|
142
|
+
if PseudoCleaner::Configuration.instance.peek_data_not_on_error
|
143
|
+
CucumberHook.instance.peek_data_new_test(scenario)
|
144
|
+
end
|
145
|
+
end
|
146
|
+
end
|
147
|
+
|
148
|
+
After("@full_data_dump") do |scenario|
|
149
|
+
if scenario.failed?
|
150
|
+
if PseudoCleaner::Configuration.instance.enable_full_data_dump_tag ||
|
151
|
+
PseudoCleaner::Configuration.instance.peek_data_on_error
|
152
|
+
CucumberHook.instance.peek_data_inline(scenario)
|
153
|
+
end
|
154
|
+
else
|
155
|
+
if PseudoCleaner::Configuration.instance.enable_full_data_dump_tag ||
|
156
|
+
PseudoCleaner::Configuration.instance.peek_data_not_on_error
|
157
|
+
CucumberHook.instance.peek_data_new_test(scenario)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
119
162
|
at_exit do
|
120
163
|
# We end suite in case a custom cleaner wants/needs to.
|
121
164
|
PseudoCleaner::MasterCleaner.end_suite
|
@@ -4,7 +4,10 @@ module PseudoCleaner
|
|
4
4
|
class MasterCleaner
|
5
5
|
@@suite_cleaner = nil
|
6
6
|
@@cleaner_classes = nil
|
7
|
+
@@redis_classes = nil
|
7
8
|
@@cleaner_classes_sorted = false
|
9
|
+
@@report_table = nil
|
10
|
+
@@report_error = false
|
8
11
|
|
9
12
|
CLEANING_STRATEGIES = [:transaction, :truncation, :deletion, :pseudo_delete, :none]
|
10
13
|
DB_CLEANER_CLEANING_STRATEGIES =
|
@@ -21,13 +24,57 @@ module PseudoCleaner
|
|
21
24
|
VALID_TEST_METHODS = VALID_START_METHODS + VALID_END_METHODS
|
22
25
|
|
23
26
|
class << self
|
24
|
-
def
|
25
|
-
|
26
|
-
|
27
|
+
def report_table
|
28
|
+
@@report_table
|
29
|
+
end
|
30
|
+
|
31
|
+
def report_error
|
32
|
+
@@report_error = true
|
33
|
+
end
|
34
|
+
|
35
|
+
def within_report_block(description, &block)
|
36
|
+
@@report_error = false
|
37
|
+
@@report_table = nil
|
38
|
+
|
39
|
+
if Object.const_defined?("Cornucopia", false) &&
|
40
|
+
Cornucopia.const_defined?("Util", false) &&
|
41
|
+
Cornucopia::Util.const_defined?("ReportBuilder", false)
|
42
|
+
Cornucopia::Util::ReportBuilder.current_report.within_test(description) do
|
43
|
+
Cornucopia::Util::ReportBuilder.current_report.within_section(description) do |report|
|
44
|
+
report.within_table do |report_table|
|
45
|
+
@@report_table = report_table
|
46
|
+
|
47
|
+
block.yield
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
unless @@report_error
|
52
|
+
Cornucopia::Util::ReportBuilder.current_report.test_succeeded
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
@@report_table = nil
|
57
|
+
else
|
58
|
+
block.yield
|
27
59
|
end
|
28
|
-
|
29
|
-
|
30
|
-
|
60
|
+
end
|
61
|
+
|
62
|
+
def start_suite(description = nil)
|
63
|
+
description ||= "PseudoCleaner::MasterCleaner.start_suite"
|
64
|
+
|
65
|
+
within_report_block(description) do
|
66
|
+
if @@suite_cleaner
|
67
|
+
@@suite_cleaner.reset_suite
|
68
|
+
end
|
69
|
+
@@suite_cleaner = PseudoCleaner::MasterCleaner.new(:suite)
|
70
|
+
@@suite_cleaner.start :pseudo_delete
|
71
|
+
@@suite_cleaner
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def clean_redis(redis)
|
76
|
+
@@redis_classes ||= []
|
77
|
+
@@redis_classes << redis
|
31
78
|
end
|
32
79
|
|
33
80
|
def database_cleaner
|
@@ -38,43 +85,55 @@ module PseudoCleaner
|
|
38
85
|
end
|
39
86
|
end
|
40
87
|
|
41
|
-
def start_example(example_class, strategy)
|
42
|
-
|
43
|
-
|
88
|
+
def start_example(example_class, strategy, description = nil)
|
89
|
+
description ||= "PseudoCleaner::MasterCleaner.start_example"
|
90
|
+
|
91
|
+
within_report_block(description) do
|
92
|
+
pseudo_cleaner_data = {}
|
93
|
+
pseudo_cleaner_data[:test_strategy] = strategy
|
44
94
|
|
45
|
-
|
46
|
-
|
95
|
+
unless strategy == :none
|
96
|
+
raise "invalid strategy" unless PseudoCleaner::MasterCleaner::DB_CLEANER_CLEANING_STRATEGIES.has_key? strategy
|
47
97
|
|
48
|
-
|
49
|
-
|
50
|
-
|
98
|
+
DatabaseCleaner.strategy = PseudoCleaner::MasterCleaner::DB_CLEANER_CLEANING_STRATEGIES[strategy]
|
99
|
+
unless [:pseudo_delete].include? strategy
|
100
|
+
PseudoCleaner::MasterCleaner.database_cleaner.start
|
101
|
+
end
|
102
|
+
|
103
|
+
pseudo_cleaner_data[:pseudo_state] = PseudoCleaner::MasterCleaner.start_test strategy
|
51
104
|
end
|
52
105
|
|
53
|
-
pseudo_cleaner_data
|
106
|
+
example_class.instance_variable_set(:@pseudo_cleaner_data, pseudo_cleaner_data)
|
54
107
|
end
|
55
|
-
|
56
|
-
example_class.instance_variable_set(:@pseudo_cleaner_data, pseudo_cleaner_data)
|
57
108
|
end
|
58
109
|
|
59
|
-
def end_example(example_class)
|
60
|
-
|
110
|
+
def end_example(example_class, description = nil)
|
111
|
+
description ||= "PseudoCleaner::MasterCleaner.end_example"
|
61
112
|
|
62
|
-
|
63
|
-
|
64
|
-
PseudoCleaner::MasterCleaner.database_cleaner.clean
|
65
|
-
end
|
113
|
+
within_report_block(description) do
|
114
|
+
pseudo_cleaner_data = example_class.instance_variable_get(:@pseudo_cleaner_data)
|
66
115
|
|
67
|
-
|
68
|
-
|
69
|
-
PseudoCleaner::MasterCleaner.
|
70
|
-
|
116
|
+
unless pseudo_cleaner_data[:test_strategy] == :none
|
117
|
+
unless [:pseudo_delete].include? pseudo_cleaner_data[:test_strategy]
|
118
|
+
PseudoCleaner::MasterCleaner.database_cleaner.clean
|
119
|
+
end
|
71
120
|
|
72
|
-
|
121
|
+
case pseudo_cleaner_data[:test_strategy]
|
122
|
+
when :deletion, :truncation
|
123
|
+
PseudoCleaner::MasterCleaner.database_reset
|
124
|
+
end
|
125
|
+
|
126
|
+
pseudo_cleaner_data[:pseudo_state].end test_type: :test, test_strategy: pseudo_cleaner_data[:test_strategy]
|
127
|
+
end
|
73
128
|
end
|
74
129
|
end
|
75
130
|
|
76
|
-
def end_suite
|
77
|
-
|
131
|
+
def end_suite(description = nil)
|
132
|
+
description ||= "PseudoCleaner::MasterCleaner.end_suite"
|
133
|
+
|
134
|
+
within_report_block(description) do
|
135
|
+
@@suite_cleaner.end test_strategy: :pseudo_delete if @@suite_cleaner
|
136
|
+
end
|
78
137
|
end
|
79
138
|
|
80
139
|
def start_test test_strategy
|
@@ -109,10 +168,14 @@ module PseudoCleaner
|
|
109
168
|
raise body_error if body_error
|
110
169
|
end
|
111
170
|
|
112
|
-
def reset_database
|
113
|
-
PseudoCleaner::MasterCleaner.
|
171
|
+
def reset_database(description = nil)
|
172
|
+
description ||= "PseudoCleaner::MasterCleaner.reset_database"
|
114
173
|
|
115
|
-
|
174
|
+
within_report_block(description) do
|
175
|
+
PseudoCleaner::MasterCleaner.database_cleaner.clean_with(:truncation)
|
176
|
+
|
177
|
+
PseudoCleaner::MasterCleaner.database_reset
|
178
|
+
end
|
116
179
|
end
|
117
180
|
|
118
181
|
def database_reset
|
@@ -189,6 +252,7 @@ module PseudoCleaner
|
|
189
252
|
|
190
253
|
PseudoCleaner::MasterCleaner.create_table_cleaners
|
191
254
|
PseudoCleaner::MasterCleaner.create_custom_cleaners
|
255
|
+
PseudoCleaner::MasterCleaner.create_redis_cleaners
|
192
256
|
end
|
193
257
|
|
194
258
|
@@cleaner_classes
|
@@ -245,6 +309,14 @@ module PseudoCleaner
|
|
245
309
|
end
|
246
310
|
end
|
247
311
|
|
312
|
+
def create_redis_cleaners
|
313
|
+
if @@redis_classes
|
314
|
+
@@redis_classes.each do |redis|
|
315
|
+
PseudoCleaner::MasterCleaner.cleaner_classes << [redis, nil, PseudoCleaner::RedisCleaner]
|
316
|
+
end
|
317
|
+
end
|
318
|
+
end
|
319
|
+
|
248
320
|
def find_file_class(seeder_file, seeder_root)
|
249
321
|
check_class = Object
|
250
322
|
full_module_name = []
|
@@ -265,7 +337,46 @@ module PseudoCleaner
|
|
265
337
|
end
|
266
338
|
|
267
339
|
def review_rows(&block)
|
268
|
-
@@suite_cleaner.review_rows
|
340
|
+
@@suite_cleaner.review_rows(&block)
|
341
|
+
end
|
342
|
+
|
343
|
+
def peek_data_inline(description = nil)
|
344
|
+
@@report_error = false
|
345
|
+
@@report_table = nil
|
346
|
+
|
347
|
+
if Object.const_defined?("Cornucopia", false) &&
|
348
|
+
Cornucopia.const_defined?("Util", false) &&
|
349
|
+
Cornucopia::Util.const_defined?("ReportBuilder", false)
|
350
|
+
Cornucopia::Util::ReportBuilder.current_report.within_section(description) do |report|
|
351
|
+
report.within_hidden_table do |outer_report_table|
|
352
|
+
Cornucopia::Util::ReportTable.new(
|
353
|
+
nested_table: outer_report_table,
|
354
|
+
suppress_blank_table: true) do |report_table|
|
355
|
+
@@report_table = report_table
|
356
|
+
|
357
|
+
peek_values
|
358
|
+
end
|
359
|
+
end
|
360
|
+
end
|
361
|
+
|
362
|
+
@@report_table = nil
|
363
|
+
else
|
364
|
+
PseudoCleaner::Logger.write(description)
|
365
|
+
|
366
|
+
peek_values
|
367
|
+
end
|
368
|
+
end
|
369
|
+
|
370
|
+
def peek_data_new_test(description = nil)
|
371
|
+
description ||= "PseudoCleaner::MasterCleaner.peek_data"
|
372
|
+
|
373
|
+
within_report_block(description) do
|
374
|
+
peek_values
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
378
|
+
def peek_values
|
379
|
+
@@suite_cleaner.peek_values
|
269
380
|
end
|
270
381
|
end
|
271
382
|
|
@@ -374,5 +485,9 @@ module PseudoCleaner
|
|
374
485
|
def review_rows(&block)
|
375
486
|
run_all_cleaners(:review_rows, @cleaners, &block)
|
376
487
|
end
|
488
|
+
|
489
|
+
def peek_values
|
490
|
+
run_all_cleaners(:peek_values, @cleaners)
|
491
|
+
end
|
377
492
|
end
|
378
493
|
end
|
@@ -0,0 +1,665 @@
|
|
1
|
+
require "redis"
|
2
|
+
require "redis-namespace"
|
3
|
+
require "pseudo_cleaner/master_cleaner"
|
4
|
+
require "pseudo_cleaner/configuration"
|
5
|
+
require "pseudo_cleaner/logger"
|
6
|
+
require "colorize"
|
7
|
+
|
8
|
+
module PseudoCleaner
|
9
|
+
##
|
10
|
+
# This class "cleans" a single redis connection.
|
11
|
+
#
|
12
|
+
# The cleaning is done by opening a monitor connection on redis and monitoring it for any actions that change values
|
13
|
+
# in redis.
|
14
|
+
#
|
15
|
+
# Long term, I was thinking about keeping some stats on the redis calls, but for now, I'm not.
|
16
|
+
#
|
17
|
+
# The cleaner roughly works as follows:
|
18
|
+
# * Get a list of all existing keys in the database as a level starting point.
|
19
|
+
# * Start a monitor
|
20
|
+
# * The monitor records any key that is updated or changed
|
21
|
+
# * When a test ends
|
22
|
+
# * Ask the monitor for a list of all changed keys.
|
23
|
+
# * The monitor resets the list of all changed keys.
|
24
|
+
# * When the suite ends
|
25
|
+
# * Get a list of all of the keys
|
26
|
+
# * Compare that list to the starting level
|
27
|
+
#
|
28
|
+
# Again, like the TableCleaner, if items are updated, this won't be able to "fix" that, but it will report on it.
|
29
|
+
#
|
30
|
+
# At this time, my code only pays attention to one database. It ignores other databases. We could extend things
|
31
|
+
# and have the monitor watch multiple databases. I'll have to do that later if I find a need.
|
32
|
+
|
33
|
+
# NOTE: Like the database cleaner, if the test is interrupted and test_end isn't called, the redis database may be
|
34
|
+
# left in an uncertain state.
|
35
|
+
|
36
|
+
# I'm not a huge fan of sleeps. In the non-rails world, I used to be able to do a sleep(0) to signal the system to
|
37
|
+
# check if somebody else needed to do some work. Testing with Rails, I find I have to actually sleep, so I do a
|
38
|
+
# very short time like 0.01.
|
39
|
+
class RedisCleaner
|
40
|
+
# SUITE_KEY = "PseudoDelete::RedisCleaner:initial_redis_state"
|
41
|
+
|
42
|
+
FLUSH_COMMANDS =
|
43
|
+
[
|
44
|
+
"flushall",
|
45
|
+
"flushdb"
|
46
|
+
]
|
47
|
+
WRITE_COMMANDS =
|
48
|
+
[
|
49
|
+
"append",
|
50
|
+
"bitop",
|
51
|
+
"blpop",
|
52
|
+
"brpop",
|
53
|
+
"brpoplpush",
|
54
|
+
"decr",
|
55
|
+
"decrby",
|
56
|
+
"del",
|
57
|
+
"expire",
|
58
|
+
"expireat",
|
59
|
+
"getset",
|
60
|
+
"hset",
|
61
|
+
"hsetnx",
|
62
|
+
"hincrby",
|
63
|
+
"hincrbyfloat",
|
64
|
+
"hmset",
|
65
|
+
"hdel",
|
66
|
+
"incr",
|
67
|
+
"incrby",
|
68
|
+
"incrbyfloat",
|
69
|
+
"linsert",
|
70
|
+
"lpop",
|
71
|
+
"lpush",
|
72
|
+
"lpushx",
|
73
|
+
"lrem",
|
74
|
+
"lset",
|
75
|
+
"ltrim",
|
76
|
+
"mapped_hmset",
|
77
|
+
"mapped_mset",
|
78
|
+
"mapped_msetnx",
|
79
|
+
"move",
|
80
|
+
"mset",
|
81
|
+
"msetnx",
|
82
|
+
"persist",
|
83
|
+
"pexpire",
|
84
|
+
"pexpireat",
|
85
|
+
"psetex",
|
86
|
+
"rename",
|
87
|
+
"renamenx",
|
88
|
+
"restore",
|
89
|
+
"rpop",
|
90
|
+
"rpoplpush",
|
91
|
+
"rpush",
|
92
|
+
"rpushx",
|
93
|
+
"sadd",
|
94
|
+
"sdiffstore",
|
95
|
+
"set",
|
96
|
+
"setbit",
|
97
|
+
"setex",
|
98
|
+
"setnx",
|
99
|
+
"setrange",
|
100
|
+
"sinterstore",
|
101
|
+
"smove",
|
102
|
+
"sort",
|
103
|
+
"spop",
|
104
|
+
"srem",
|
105
|
+
"sunionstore",
|
106
|
+
"zadd",
|
107
|
+
"zincrby",
|
108
|
+
"zinterstore",
|
109
|
+
"zrem",
|
110
|
+
"zremrangebyrank",
|
111
|
+
"zremrangebyscore",
|
112
|
+
]
|
113
|
+
READ_COMMANDS =
|
114
|
+
[
|
115
|
+
"bitcount",
|
116
|
+
"bitop",
|
117
|
+
"dump",
|
118
|
+
"exists",
|
119
|
+
"get",
|
120
|
+
"getbit",
|
121
|
+
"getrange",
|
122
|
+
"hget",
|
123
|
+
"hmget",
|
124
|
+
"hexists",
|
125
|
+
"hlen",
|
126
|
+
"hkeys",
|
127
|
+
"hscan",
|
128
|
+
"hscan_each",
|
129
|
+
"hvals",
|
130
|
+
"hgetall",
|
131
|
+
"lindex",
|
132
|
+
"llen",
|
133
|
+
"lrange",
|
134
|
+
"mapped_hmget",
|
135
|
+
"mapped_mget",
|
136
|
+
"mget",
|
137
|
+
"persist",
|
138
|
+
"scard",
|
139
|
+
"scan",
|
140
|
+
"scan_each",
|
141
|
+
"sdiff",
|
142
|
+
"sismember",
|
143
|
+
"smembers",
|
144
|
+
"srandmember",
|
145
|
+
"sscan",
|
146
|
+
"sscan_each",
|
147
|
+
"strlen",
|
148
|
+
"sunion",
|
149
|
+
"type",
|
150
|
+
"zcard",
|
151
|
+
"zcount",
|
152
|
+
"zrange",
|
153
|
+
"zrangebyscore",
|
154
|
+
"zrank",
|
155
|
+
"zrevrange",
|
156
|
+
"zrevrangebyscore",
|
157
|
+
"zrevrank",
|
158
|
+
"zscan",
|
159
|
+
"zscan_each",
|
160
|
+
"zscore",
|
161
|
+
]
|
162
|
+
|
163
|
+
attr_reader :monitor_thread
|
164
|
+
attr_reader :initial_keys
|
165
|
+
attr_accessor :options
|
166
|
+
|
167
|
+
class RedisMessage
|
168
|
+
attr_reader :message
|
169
|
+
attr_reader :time_stamp
|
170
|
+
attr_reader :db
|
171
|
+
attr_reader :host
|
172
|
+
attr_reader :port
|
173
|
+
attr_reader :command
|
174
|
+
attr_reader :cur_pos
|
175
|
+
|
176
|
+
def initialize(message_string)
|
177
|
+
@message = message_string
|
178
|
+
|
179
|
+
parse_message
|
180
|
+
end
|
181
|
+
|
182
|
+
def parse_message
|
183
|
+
if @message =~ /[0-9]+\.[0-9]+ \[[0-9]+ [^:]+:[^\]]+\] \"[^\"]+\"/
|
184
|
+
end_pos = @message.index(" ")
|
185
|
+
@time_stamp = @message[0..end_pos - 1]
|
186
|
+
|
187
|
+
@cur_pos = end_pos + 2 # " ["
|
188
|
+
end_pos = @message.index(" ", @cur_pos)
|
189
|
+
@db = @message[@cur_pos..end_pos - 1].to_i
|
190
|
+
|
191
|
+
@cur_pos = end_pos + 1
|
192
|
+
end_pos = @message.index(":", @cur_pos)
|
193
|
+
@host = @message[@cur_pos..end_pos - 1]
|
194
|
+
|
195
|
+
@cur_pos = end_pos + 1
|
196
|
+
end_pos = @message.index("]", @cur_pos)
|
197
|
+
@port = @message[@cur_pos..end_pos - 1].to_i
|
198
|
+
|
199
|
+
@cur_pos = end_pos + 2 # "] "
|
200
|
+
@command = next_value.downcase
|
201
|
+
else
|
202
|
+
@command = @message
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
def next_value
|
207
|
+
in_quote = (@message[@cur_pos] == '"')
|
208
|
+
if in_quote
|
209
|
+
@cur_pos += 1
|
210
|
+
end_pos = @cur_pos
|
211
|
+
while (end_pos && end_pos < @message.length)
|
212
|
+
end_pos = @message.index("\"", end_pos)
|
213
|
+
|
214
|
+
num_backslashes = 0
|
215
|
+
back_pos = end_pos
|
216
|
+
|
217
|
+
while @message[back_pos - 1] == "\\"
|
218
|
+
num_backslashes += 1
|
219
|
+
back_pos -= 1
|
220
|
+
end
|
221
|
+
|
222
|
+
break if (num_backslashes % 2) == 0
|
223
|
+
end_pos += 1
|
224
|
+
end
|
225
|
+
else
|
226
|
+
end_pos = @message.index(" ", @cur_pos)
|
227
|
+
end
|
228
|
+
the_value = @message[@cur_pos..end_pos - 1]
|
229
|
+
end_pos += 1 if in_quote
|
230
|
+
|
231
|
+
@cur_pos = end_pos + 1
|
232
|
+
|
233
|
+
the_value.gsub("\\\\", "\\").gsub("\\\"", "\"")
|
234
|
+
end
|
235
|
+
|
236
|
+
def keys
|
237
|
+
unless defined?(@message_keys)
|
238
|
+
@message_keys = []
|
239
|
+
|
240
|
+
if Redis::Namespace::COMMANDS.include? command
|
241
|
+
handling = Redis::Namespace::COMMANDS[command.to_s.downcase]
|
242
|
+
|
243
|
+
(before, after) = handling
|
244
|
+
|
245
|
+
case before
|
246
|
+
when :first
|
247
|
+
@message_keys << next_value
|
248
|
+
|
249
|
+
when :all
|
250
|
+
while @cur_pos < @message.length
|
251
|
+
@message_keys << next_value
|
252
|
+
end
|
253
|
+
|
254
|
+
when :exclude_first
|
255
|
+
next_value
|
256
|
+
while @cur_pos < @message.length
|
257
|
+
@message_keys << next_value
|
258
|
+
end
|
259
|
+
|
260
|
+
when :exclude_last
|
261
|
+
while @cur_pos < @message.length
|
262
|
+
@message_keys << next_value
|
263
|
+
end
|
264
|
+
@message_keys.delete_at(@message_keys.length - 1)
|
265
|
+
|
266
|
+
when :exclude_options
|
267
|
+
options = ["weights", "aggregate", "sum", "min", "max"]
|
268
|
+
while @cur_pos < @message.length
|
269
|
+
@message_keys << next_value
|
270
|
+
if options.include?(@message_keys[-1].downcase)
|
271
|
+
@message_keys.delete_at(@message_keys.length - 1)
|
272
|
+
break
|
273
|
+
end
|
274
|
+
end
|
275
|
+
|
276
|
+
when :alternate
|
277
|
+
while @cur_pos < @message.length
|
278
|
+
@message_keys << next_value
|
279
|
+
next_value
|
280
|
+
end
|
281
|
+
|
282
|
+
when :sort
|
283
|
+
next_value
|
284
|
+
|
285
|
+
while @cur_pos < @message.length
|
286
|
+
a_value = next_value
|
287
|
+
if a_value.downcase == "store"
|
288
|
+
@message_keys[0] = next_value
|
289
|
+
end
|
290
|
+
end
|
291
|
+
|
292
|
+
# when :eval_style
|
293
|
+
#
|
294
|
+
# when :scan_style
|
295
|
+
end
|
296
|
+
end
|
297
|
+
end
|
298
|
+
|
299
|
+
@message_keys
|
300
|
+
end
|
301
|
+
|
302
|
+
def to_s
|
303
|
+
{
|
304
|
+
time_stamp: time_stamp,
|
305
|
+
db: db,
|
306
|
+
host: host,
|
307
|
+
port: port,
|
308
|
+
command: command,
|
309
|
+
message: message,
|
310
|
+
cur_pos: cur_pos
|
311
|
+
}.to_s
|
312
|
+
end
|
313
|
+
end
|
314
|
+
|
315
|
+
def initialize(start_method, end_method, table, options)
|
316
|
+
@initial_keys = SortedSet.new
|
317
|
+
@monitor_thread = nil
|
318
|
+
@redis_name = nil
|
319
|
+
@suite_altered_keys = SortedSet.new
|
320
|
+
|
321
|
+
unless PseudoCleaner::MasterCleaner::VALID_START_METHODS.include?(start_method)
|
322
|
+
raise "You must specify a valid start function from: #{PseudoCleaner::MasterCleaner::VALID_START_METHODS}."
|
323
|
+
end
|
324
|
+
unless PseudoCleaner::MasterCleaner::VALID_END_METHODS.include?(end_method)
|
325
|
+
raise "You must specify a valid end function from: #{PseudoCleaner::MasterCleaner::VALID_END_METHODS}."
|
326
|
+
end
|
327
|
+
|
328
|
+
@options = options
|
329
|
+
|
330
|
+
@options[:table_start_method] ||= start_method
|
331
|
+
@options[:table_end_method] ||= end_method
|
332
|
+
@options[:output_diagnostics] ||= PseudoCleaner::Configuration.current_instance.output_diagnostics ||
|
333
|
+
PseudoCleaner::Configuration.current_instance.post_transaction_analysis
|
334
|
+
|
335
|
+
@redis = table
|
336
|
+
end
|
337
|
+
|
338
|
+
def <=>(right_object)
|
339
|
+
if (right_object.is_a?(PseudoCleaner::RedisCleaner))
|
340
|
+
return 0
|
341
|
+
elsif (right_object.is_a?(PseudoCleaner::TableCleaner))
|
342
|
+
return 1
|
343
|
+
else
|
344
|
+
if right_object.respond_to?(:<=>)
|
345
|
+
comparison = (right_object <=> self)
|
346
|
+
if comparison
|
347
|
+
return -1 * comparison
|
348
|
+
end
|
349
|
+
end
|
350
|
+
end
|
351
|
+
|
352
|
+
return 1
|
353
|
+
end
|
354
|
+
|
355
|
+
def redis
|
356
|
+
@redis ||= Redis.current
|
357
|
+
end
|
358
|
+
|
359
|
+
def suite_start test_strategy
|
360
|
+
@test_strategy ||= test_strategy
|
361
|
+
|
362
|
+
# if redis.type(PseudoCleaner::RedisCleaner::SUITE_KEY) == "set"
|
363
|
+
# @initial_keys = SortedSet.new(redis.smembers(PseudoCleaner::RedisCleaner::SUITE_KEY))
|
364
|
+
# report_end_of_suite_state "before suite start"
|
365
|
+
# end
|
366
|
+
# redis.del PseudoCleaner::RedisCleaner::SUITE_KEY
|
367
|
+
|
368
|
+
start_monitor
|
369
|
+
end
|
370
|
+
|
371
|
+
def test_start test_strategy
|
372
|
+
@test_strategy ||= test_strategy
|
373
|
+
|
374
|
+
synchronize_test_values do |test_values|
|
375
|
+
if test_values && !test_values.empty?
|
376
|
+
report_dirty_values "values altered before the test started", test_values
|
377
|
+
|
378
|
+
test_values.each do |value|
|
379
|
+
redis.del value unless initial_keys.include?(value)
|
380
|
+
end
|
381
|
+
end
|
382
|
+
end
|
383
|
+
end
|
384
|
+
|
385
|
+
def test_end test_strategy
|
386
|
+
synchronize_test_values do |updated_values|
|
387
|
+
if updated_values && !updated_values.empty?
|
388
|
+
report_keys = []
|
389
|
+
|
390
|
+
if @options[:output_diagnostics]
|
391
|
+
report_dirty_values "updated values", updated_values
|
392
|
+
end
|
393
|
+
|
394
|
+
updated_values.each do |value|
|
395
|
+
if initial_keys.include?(value)
|
396
|
+
report_keys << value
|
397
|
+
@suite_altered_keys << value
|
398
|
+
else
|
399
|
+
redis.del(value)
|
400
|
+
end
|
401
|
+
end
|
402
|
+
|
403
|
+
report_dirty_values "initial values altered by test", report_keys
|
404
|
+
end
|
405
|
+
end
|
406
|
+
end
|
407
|
+
|
408
|
+
def suite_end test_strategy
|
409
|
+
report_end_of_suite_state "suite end"
|
410
|
+
|
411
|
+
if monitor_thread
|
412
|
+
monitor_thread.kill
|
413
|
+
@monitor_thread = nil
|
414
|
+
end
|
415
|
+
end
|
416
|
+
|
417
|
+
def reset_suite
|
418
|
+
report_end_of_suite_state "reset suite"
|
419
|
+
|
420
|
+
if monitor_thread
|
421
|
+
monitor_thread.kill
|
422
|
+
@monitor_thread = nil
|
423
|
+
start_monitor
|
424
|
+
end
|
425
|
+
end
|
426
|
+
|
427
|
+
def ignore_regexes
|
428
|
+
[]
|
429
|
+
end
|
430
|
+
|
431
|
+
def ignore_key(key)
|
432
|
+
ignore_regexes.detect { |ignore_regex| key =~ ignore_regex }
|
433
|
+
end
|
434
|
+
|
435
|
+
def redis_name
|
436
|
+
unless @redis_name
|
437
|
+
redis_options = redis.client.options.with_indifferent_access
|
438
|
+
@redis_name = "#{redis_options[:host]}:#{redis_options[:port]}/#{redis_options[:db]}"
|
439
|
+
end
|
440
|
+
|
441
|
+
@redis_name
|
442
|
+
end
|
443
|
+
|
444
|
+
def review_rows(&block)
|
445
|
+
synchronize_test_values do |updated_values|
|
446
|
+
if updated_values && !updated_values.empty?
|
447
|
+
updated_values.each do |updated_value|
|
448
|
+
unless ignore_key(updated_value)
|
449
|
+
block.yield redis_name, report_record(updated_value)
|
450
|
+
end
|
451
|
+
end
|
452
|
+
end
|
453
|
+
end
|
454
|
+
end
|
455
|
+
|
456
|
+
def peek_values
|
457
|
+
synchronize_test_values do |updated_values|
|
458
|
+
if updated_values && !updated_values.empty?
|
459
|
+
output_values = false
|
460
|
+
|
461
|
+
if PseudoCleaner::MasterCleaner.report_table
|
462
|
+
Cornucopia::Util::ReportTable.new(nested_table: PseudoCleaner::MasterCleaner.report_table,
|
463
|
+
nested_table_label: redis_name,
|
464
|
+
suppress_blank_table: true) do |report_table|
|
465
|
+
updated_values.each do |updated_value|
|
466
|
+
unless ignore_key(updated_value)
|
467
|
+
output_values = true
|
468
|
+
report_table.write_stats updated_value, report_record(updated_value)
|
469
|
+
end
|
470
|
+
end
|
471
|
+
end
|
472
|
+
else
|
473
|
+
PseudoCleaner::Logger.write(" #{redis_name}")
|
474
|
+
|
475
|
+
updated_values.each do |updated_value|
|
476
|
+
unless ignore_key(updated_value)
|
477
|
+
output_values = true
|
478
|
+
PseudoCleaner::Logger.write(" #{updated_value}: #{report_record(updated_value)}")
|
479
|
+
end
|
480
|
+
end
|
481
|
+
end
|
482
|
+
|
483
|
+
PseudoCleaner::MasterCleaner.report_error if output_values
|
484
|
+
end
|
485
|
+
end
|
486
|
+
end
|
487
|
+
|
488
|
+
def synchronize_key
|
489
|
+
@synchronize_key ||= "redis_cleaner::synchronization_key_#{rand(1..1_000_000_000_000_000_000)}_#{rand(1..1_000_000_000_000_000_000)}"
|
490
|
+
end
|
491
|
+
|
492
|
+
def synchronize_end_key
|
493
|
+
@synchronize_end_key ||= "redis_cleaner::synchronization_end_key_#{rand(1..1_000_000_000_000_000_000)}_#{rand(1..1_000_000_000_000_000_000)}"
|
494
|
+
end
|
495
|
+
|
496
|
+
def report_end_of_suite_state report_reason
|
497
|
+
current_keys = SortedSet.new(redis.keys)
|
498
|
+
|
499
|
+
deleted_keys = initial_keys - current_keys
|
500
|
+
new_keys = current_keys - initial_keys
|
501
|
+
|
502
|
+
# filter out values we inserted that will go away on their own.
|
503
|
+
new_keys = new_keys.select { |key| (key =~ /redis_cleaner::synchronization_(?:end_)?key_[0-9]+_[0-9]+/).nil? }
|
504
|
+
|
505
|
+
report_dirty_values "new values as of #{report_reason}", new_keys
|
506
|
+
report_dirty_values "values deleted before #{report_reason}", deleted_keys
|
507
|
+
report_dirty_values "initial values changed during suite run", @suite_altered_keys
|
508
|
+
|
509
|
+
@suite_altered_keys = SortedSet.new
|
510
|
+
|
511
|
+
new_keys.each do |key_value|
|
512
|
+
redis.del key_value
|
513
|
+
end
|
514
|
+
end
|
515
|
+
|
516
|
+
def synchronize_test_values(&block)
|
517
|
+
updated_values = nil
|
518
|
+
|
519
|
+
if monitor_thread
|
520
|
+
redis.setex(synchronize_key, 1, true)
|
521
|
+
updated_values = queue.pop
|
522
|
+
end
|
523
|
+
|
524
|
+
block.yield updated_values
|
525
|
+
|
526
|
+
redis.setex(synchronize_end_key, 1, true)
|
527
|
+
end
|
528
|
+
|
529
|
+
def queue
|
530
|
+
@queue ||= Queue.new
|
531
|
+
end
|
532
|
+
|
533
|
+
def start_monitor
|
534
|
+
cleaner_class = self
|
535
|
+
|
536
|
+
@initial_keys = SortedSet.new(redis.keys)
|
537
|
+
# @initial_keys.add(PseudoCleaner::RedisCleaner::SUITE_KEY)
|
538
|
+
# @initial_keys.each do |key_value|
|
539
|
+
# redis.sadd(PseudoCleaner::RedisCleaner::SUITE_KEY, key_value)
|
540
|
+
# end
|
541
|
+
if @options[:output_diagnostics]
|
542
|
+
if PseudoCleaner::MasterCleaner.report_table
|
543
|
+
Cornucopia::Util::ReportTable.new(nested_table: PseudoCleaner::MasterCleaner.report_table,
|
544
|
+
nested_table_label: redis_name,
|
545
|
+
suppress_blank_table: true) do |report_table|
|
546
|
+
report_table.write_stats "initial keys count", @initial_keys.count
|
547
|
+
end
|
548
|
+
else
|
549
|
+
PseudoCleaner::Logger.write("#{redis_name}")
|
550
|
+
PseudoCleaner::Logger.write(" Initial keys count - #{@initial_keys.count}")
|
551
|
+
end
|
552
|
+
end
|
553
|
+
|
554
|
+
unless @monitor_thread
|
555
|
+
@monitor_thread = Thread.new do
|
556
|
+
in_redis_cleanup = false
|
557
|
+
updated_keys = SortedSet.new
|
558
|
+
|
559
|
+
monitor_redis = Redis.new(cleaner_class.redis.client.options)
|
560
|
+
redis_options = monitor_redis.client.options.with_indifferent_access
|
561
|
+
cleaner_class_db = redis_options[:db]
|
562
|
+
|
563
|
+
monitor_redis.monitor do |message|
|
564
|
+
redis_message = RedisMessage.new message
|
565
|
+
|
566
|
+
if redis_message.db == cleaner_class_db
|
567
|
+
process_command = true
|
568
|
+
|
569
|
+
if redis_message.command == "setex"
|
570
|
+
if redis_message.keys[0] == cleaner_class.synchronize_key
|
571
|
+
process_command = false
|
572
|
+
|
573
|
+
in_redis_cleanup = true
|
574
|
+
return_values = updated_keys
|
575
|
+
updated_keys = SortedSet.new
|
576
|
+
cleaner_class.queue << return_values
|
577
|
+
elsif redis_message.keys[0] == cleaner_class.synchronize_end_key
|
578
|
+
in_redis_cleanup = false
|
579
|
+
cleaner_class.monitor_thread[:updated] = nil
|
580
|
+
process_command = false
|
581
|
+
end
|
582
|
+
elsif redis_message.command == "del"
|
583
|
+
if in_redis_cleanup
|
584
|
+
process_command = false
|
585
|
+
end
|
586
|
+
end
|
587
|
+
|
588
|
+
if process_command
|
589
|
+
# flush...
|
590
|
+
if PseudoCleaner::RedisCleaner::WRITE_COMMANDS.include? redis_message.command
|
591
|
+
updated_keys.merge(redis_message.keys)
|
592
|
+
elsif PseudoCleaner::RedisCleaner::FLUSH_COMMANDS.include? redis_message.command
|
593
|
+
# Not sure I can get the keys at this point...
|
594
|
+
# updated_keys.merge(cleaner_class.redis.keys)
|
595
|
+
end
|
596
|
+
end
|
597
|
+
elsif "flushall" == redis_message.command
|
598
|
+
# Not sure I can get the keys at this point...
|
599
|
+
# updated_keys.merge(cleaner_class.redis.keys)
|
600
|
+
end
|
601
|
+
end
|
602
|
+
end
|
603
|
+
|
604
|
+
sleep(0.01)
|
605
|
+
redis.get(synchronize_key)
|
606
|
+
end
|
607
|
+
end
|
608
|
+
|
609
|
+
def report_record(key_name)
|
610
|
+
key_hash = { key: key_name, type: redis.type(key_name), ttl: redis.ttl(key_name) }
|
611
|
+
case key_hash[:type]
|
612
|
+
when "string"
|
613
|
+
key_hash[:value] = redis.get(key_name)
|
614
|
+
when "list"
|
615
|
+
key_hash[:list] = { len: redis.llen(key_name), values: redis.lrange(key_name, 0, -1) }
|
616
|
+
when "set"
|
617
|
+
key_hash[:set] = redis.smembers(key_name)
|
618
|
+
when "zset"
|
619
|
+
key_hash[:sorted_set] = redis.smembers(key_name)
|
620
|
+
when "hash"
|
621
|
+
key_hash[:list] = { len: redis.hlen(key_name), values: redis.hgetall(key_name) }
|
622
|
+
end
|
623
|
+
|
624
|
+
if key_hash[:value].nil? &&
|
625
|
+
key_hash[:list].nil? &&
|
626
|
+
key_hash[:set].nil? &&
|
627
|
+
key_hash[:sorted_set].nil? &&
|
628
|
+
key_hash[:hash].nil?
|
629
|
+
key_hash[:value] = "[[DELETED]]"
|
630
|
+
end
|
631
|
+
|
632
|
+
key_hash
|
633
|
+
end
|
634
|
+
|
635
|
+
def report_dirty_values message, test_values
|
636
|
+
if test_values && !test_values.empty?
|
637
|
+
output_values = false
|
638
|
+
|
639
|
+
if PseudoCleaner::MasterCleaner.report_table
|
640
|
+
Cornucopia::Util::ReportTable.new(nested_table: PseudoCleaner::MasterCleaner.report_table,
|
641
|
+
nested_table_label: redis_name,
|
642
|
+
suppress_blank_table: true) do |report_table|
|
643
|
+
report_table.write_stats "action", message
|
644
|
+
test_values.each_with_index do |key_name, index|
|
645
|
+
unless ignore_key(key_name)
|
646
|
+
output_values = true
|
647
|
+
report_table.write_stats index, report_record(key_name)
|
648
|
+
end
|
649
|
+
end
|
650
|
+
end
|
651
|
+
else
|
652
|
+
PseudoCleaner::Logger.write("********* RedisCleaner - #{message}".red.on_light_white)
|
653
|
+
test_values.each do |key_name|
|
654
|
+
unless ignore_key(key_name)
|
655
|
+
output_values = true
|
656
|
+
PseudoCleaner::Logger.write(" #{key_name}: #{report_record(key_name)}".red.on_light_white)
|
657
|
+
end
|
658
|
+
end
|
659
|
+
end
|
660
|
+
|
661
|
+
PseudoCleaner::MasterCleaner.report_error if output_values
|
662
|
+
end
|
663
|
+
end
|
664
|
+
end
|
665
|
+
end
|