experimental 0.1.0 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +6 -14
- data/README.markdown +194 -200
- data/app/models/experimental/experiment.rb +29 -26
- data/app/models/experimental/subject.rb +2 -2
- data/lib/experimental/loader.rb +34 -0
- data/lib/experimental/overrides.rb +28 -0
- data/lib/experimental/railtie.rb +13 -0
- data/lib/experimental/source/active_record.rb +13 -0
- data/lib/experimental/source/base.rb +15 -0
- data/lib/experimental/source/cache.rb +45 -0
- data/lib/experimental/source/configuration.rb +21 -0
- data/lib/experimental/source.rb +8 -0
- data/lib/experimental/test.rb +21 -0
- data/lib/experimental/version.rb +1 -1
- data/lib/experimental.rb +45 -2
- data/lib/tasks/{experiment.rake → experimental.rake} +3 -2
- metadata +46 -25
- data/app/models/experimental/cache.rb +0 -69
- data/app/models/experimental/loader.rb +0 -64
checksums.yaml
CHANGED
@@ -1,15 +1,7 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
metadata.gz: !binary |-
|
9
|
-
ZGVmMGFjYWNkZWExMGJmNWRiNmVmMjRiZWQzMTUyNGFhZGRmNThiODI2MTg3
|
10
|
-
NTQzMjI4NzBmZGJkNmFlNDI5YTM2MThmMmJkY2JiZWVkODBhZTcyZmM0OWVm
|
11
|
-
NzQzNGUwOGMwMGVjZjhiM2QzM2NhNmQxYTAyMDJhNTI3NzIyNjE=
|
12
|
-
data.tar.gz: !binary |-
|
13
|
-
N2Q3ZjM1OWRjN2NiMzA4ZWExYWU5ODY0ZDcxMjA5NTgzOWI4OWE2OWIzNmJk
|
14
|
-
MjM1Njk1NTk4Mjc1NWRmNzM4N2YwM2M1YjViZmRlODFlMDIxOWRkM2I1NmU3
|
15
|
-
MWNiODNhMmEzOGZhMDBkM2JlMDY1MjRkN2Q5MTNjYWEyZDQ1ZmU=
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 1c78c9b7d6c365503d9c865f63b3e24f7abbdec5
|
4
|
+
data.tar.gz: adc2a47c070863a3a87432d4eca968614155d3a9
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 9eee13366381e3d96a44abc8da8b003c405850a140f96091ec7d8410dad321d3501b4ac3577976217d5a46b279b6cdbc7704cd219dedf6ab1633a05431b480b8
|
7
|
+
data.tar.gz: 755c66a9193290985db2a6852a68c01edd3780fc94672d9691743fc6b02b3407051242ef30c8882b6753e98184e1f9a94850913976b878ba7a3b6cd76541b36d
|
data/README.markdown
CHANGED
@@ -12,135 +12,134 @@ be removed make the site explode
|
|
12
12
|
|
13
13
|
## Installation
|
14
14
|
|
15
|
-
|
15
|
+
`rails g experimental`
|
16
16
|
|
17
17
|
### Routes
|
18
18
|
|
19
|
-
|
20
|
-
|
19
|
+
```ruby
|
20
|
+
resources :experiments, only: [:index, :new, :create] do
|
21
|
+
collection do
|
22
|
+
get :inactive
|
23
|
+
post :set_winner
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
namespace :singles_admin do
|
28
|
+
resources :experiments, only: [:index, :new, :create] do
|
21
29
|
collection do
|
22
30
|
get :inactive
|
23
31
|
post :set_winner
|
24
32
|
end
|
25
33
|
end
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
namespace :singles_admin do
|
30
|
-
resources :experiments, :only => [:index, :new, :create] do
|
31
|
-
collection do
|
32
|
-
get :inactive
|
33
|
-
post :set_winner
|
34
|
-
end
|
35
|
-
end
|
36
|
-
end
|
37
|
-
````
|
34
|
+
end
|
35
|
+
end
|
36
|
+
```
|
38
37
|
|
39
38
|
### Admin Frontend
|
40
39
|
|
41
40
|
#### Create your own admin controller:
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
end
|
56
|
-
end
|
57
|
-
|
58
|
-
def base_resource_name
|
59
|
-
"singles_admin_experiment"
|
60
|
-
end
|
61
|
-
|
41
|
+
```ruby
|
42
|
+
class Admin::ExperimentsController < ApplicationController
|
43
|
+
include Experimental::ControllerActions
|
44
|
+
|
45
|
+
alias_method :index, :experiments_index
|
46
|
+
alias_method :new, :experiments_new
|
47
|
+
alias_method :set_winner, :experiments_set_winner
|
48
|
+
|
49
|
+
def create
|
50
|
+
if experiments_create
|
51
|
+
redirect_to admin_experiments_path
|
52
|
+
else
|
53
|
+
render :new
|
62
54
|
end
|
63
|
-
|
64
|
-
|
65
|
-
#### Using ActiveAdmin:
|
66
|
-
|
67
|
-
``` rails g active_admin:resource Experiment```
|
68
|
-
|
69
|
-
````
|
70
|
-
require 'experimental/controller_actions'
|
55
|
+
end
|
71
56
|
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
class_eval do
|
78
|
-
include Experimental::ControllerActions
|
79
|
-
end
|
80
|
-
|
81
|
-
def base_resource_name
|
82
|
-
"admin_experiment"
|
83
|
-
end
|
57
|
+
def base_resource_name
|
58
|
+
"singles_admin_experiment"
|
59
|
+
end
|
60
|
+
end
|
61
|
+
```
|
84
62
|
|
85
|
-
|
86
|
-
if experiments_create
|
87
|
-
redirect_to admin_experiments_path
|
88
|
-
else
|
89
|
-
render :new
|
90
|
-
end
|
91
|
-
end
|
63
|
+
#### Using ActiveAdmin:
|
92
64
|
|
93
|
-
|
94
|
-
experiments_new
|
95
|
-
end
|
96
|
-
end
|
65
|
+
`rails g active_admin:resource Experiment`
|
97
66
|
|
98
|
-
|
99
|
-
|
100
|
-
experiments_set_winner
|
101
|
-
end
|
67
|
+
```ruby
|
68
|
+
require 'experimental/controller_actions'
|
102
69
|
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
collection_action :inactive do
|
107
|
-
experiments_inactive
|
108
|
-
render template: 'admin/experiments/index'
|
109
|
-
end
|
70
|
+
ActiveAdmin.register Experimental::Experiment, as: "Experiment" do
|
71
|
+
actions :index, :new, :create
|
72
|
+
filter :name
|
110
73
|
|
111
|
-
|
112
|
-
|
113
|
-
|
74
|
+
controller do
|
75
|
+
class_eval do
|
76
|
+
include Experimental::ControllerActions
|
77
|
+
end
|
114
78
|
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
end
|
79
|
+
def base_resource_name
|
80
|
+
"admin_experiment"
|
81
|
+
end
|
119
82
|
|
120
|
-
|
121
|
-
|
83
|
+
def create
|
84
|
+
if experiments_create
|
85
|
+
redirect_to admin_experiments_path
|
86
|
+
else
|
87
|
+
render :new
|
122
88
|
end
|
123
|
-
|
124
|
-
form :partial => 'new'
|
125
89
|
end
|
126
90
|
|
127
|
-
|
91
|
+
def new
|
92
|
+
experiments_new
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
# collection_actions force active_admin to create a route
|
97
|
+
collection_action :set_winner, method: :post do
|
98
|
+
experiments_set_winner
|
99
|
+
end
|
100
|
+
|
101
|
+
# can do this instead of the ended_or_removed scope below
|
102
|
+
# you will need to add a link to inactive_admins_experiments_path
|
103
|
+
# in your view
|
104
|
+
collection_action :inactive do
|
105
|
+
experiments_inactive
|
106
|
+
render template: 'admin/experiments/index'
|
107
|
+
end
|
108
|
+
|
109
|
+
scope :in_progress, :default => true do |experiments|
|
110
|
+
experiments.in_progress
|
111
|
+
end
|
112
|
+
|
113
|
+
scope :ended_or_removed do |experiments|
|
114
|
+
@include_inactive = true
|
115
|
+
experiments.ended_or_removed
|
116
|
+
end
|
117
|
+
|
118
|
+
index do
|
119
|
+
render template: 'admin/experiments/index'
|
120
|
+
end
|
121
|
+
|
122
|
+
form partial: 'new'
|
123
|
+
end
|
124
|
+
```
|
128
125
|
|
129
126
|
#### Views
|
130
127
|
|
131
128
|
create an index and new view in appropriate view folder, i.e.
|
132
129
|
|
133
|
-
|
134
|
-
````
|
135
|
-
<%= render partial: 'experimental/links' %>
|
136
|
-
<%= render partial: 'experimental/index' %>
|
137
|
-
````
|
130
|
+
`app/views/admin/experiments/index.html.erb`
|
138
131
|
|
139
|
-
```
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
132
|
+
```erb
|
133
|
+
<%= render partial: 'experimental/links' %>
|
134
|
+
<%= render partial: 'experimental/index' %>
|
135
|
+
```
|
136
|
+
|
137
|
+
`app/views/admin/experiments/new.html.erb`
|
138
|
+
|
139
|
+
```erb
|
140
|
+
<%= render partial: 'experimental/links' %>
|
141
|
+
<%= render partial: 'experimental/new' %>
|
142
|
+
```
|
144
143
|
|
145
144
|
*Note: ActiveAdmin users will not need to include the links
|
146
145
|
partials*
|
@@ -149,20 +148,20 @@ create an index and new view in appropriate view folder, i.e.
|
|
149
148
|
|
150
149
|
For the class you'd like to be the subject of experiments, include the
|
151
150
|
Experimental::Subject module in a model with an id and timestamps
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
151
|
+
```ruby
|
152
|
+
class User < ActiveRecord::Base
|
153
|
+
include Experimental::Subject
|
154
|
+
# ...
|
155
|
+
end
|
156
|
+
```
|
158
157
|
|
159
158
|
## Usage
|
160
159
|
|
161
160
|
### Create an experiment
|
162
161
|
|
163
|
-
In `config/
|
162
|
+
In `config/experimental.yml`, add the name, num_buckets, and notes of the
|
164
163
|
experiment under in_code:
|
165
|
-
|
164
|
+
```yaml
|
166
165
|
in_code:
|
167
166
|
-
|
168
167
|
name: :price_experiment
|
@@ -171,22 +170,22 @@ in_code:
|
|
171
170
|
0: $22
|
172
171
|
1: $19.99
|
173
172
|
|
174
|
-
|
175
|
-
Then run `rake
|
173
|
+
```
|
174
|
+
Then run `rake experimental:sync`
|
176
175
|
|
177
176
|
### Using the experiment
|
178
177
|
|
179
178
|
To see if a user is in the experiment population AND in a bucket:
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
179
|
+
```ruby
|
180
|
+
# checks if the user is in the my_experiment population
|
181
|
+
# and if they are in bucket 0
|
182
|
+
user.in_bucket?(:my_experiment, 0)
|
183
|
+
```
|
185
184
|
|
186
185
|
To see if a user is in the experiment population **ONLY**
|
187
|
-
|
188
|
-
|
189
|
-
|
186
|
+
```ruby
|
187
|
+
user.in_experiment?(:my_experiment)
|
188
|
+
```
|
190
189
|
|
191
190
|
### Ending an experiment
|
192
191
|
|
@@ -203,128 +202,124 @@ A removed experiment is an experiment that is not referenced
|
|
203
202
|
anywhere in code. In fact, the framework will throw an exception
|
204
203
|
if you reference an experiment that is not in code.
|
205
204
|
|
206
|
-
|
207
|
-
`
|
208
|
-
remove the experiment and expire the cache.
|
205
|
+
Removing an experiment from `config/experimental.yml` and running `rake
|
206
|
+
experimental:sync` will remove the experiment and expire the cache.
|
209
207
|
|
210
|
-
|
208
|
+
```yaml
|
211
209
|
removed:
|
212
210
|
-
|
213
211
|
name: :price_experiment
|
214
|
-
|
215
|
-
Then run `rake
|
216
|
-
|
217
|
-
|
212
|
+
```
|
213
|
+
Then run `rake experimental:sync`
|
218
214
|
|
219
215
|
## Testing
|
220
216
|
|
221
217
|
### Setup
|
222
|
-
in
|
218
|
+
in `spec_helper.rb` (after inclusion of ActiveSupport)
|
223
219
|
|
224
|
-
|
225
|
-
|
226
|
-
|
220
|
+
```ruby
|
221
|
+
require 'experimental/rspec_helpers'
|
222
|
+
```
|
227
223
|
|
228
224
|
*You may want to force experiments off for all tests by default*
|
229
|
-
|
230
|
-
|
231
|
-
|
232
|
-
|
233
|
-
|
225
|
+
```ruby
|
226
|
+
config.before(:each) do
|
227
|
+
User.any_instance.stub(:in_experiment?).and_return(false)
|
228
|
+
end
|
229
|
+
```
|
234
230
|
|
235
231
|
### Testing experiments
|
236
232
|
|
237
233
|
Include the Rspec helpers in your spec class or spec_helper
|
238
|
-
|
239
|
-
|
240
|
-
|
234
|
+
```ruby
|
235
|
+
include Experimental::RspecHelpers
|
236
|
+
```
|
241
237
|
|
242
238
|
Shared contexts are available for in_experiment? and in_bucket?
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
|
247
|
-
|
248
|
-
|
239
|
+
```ruby
|
240
|
+
include_context "in experiment"
|
241
|
+
include_context "not in experiment"
|
242
|
+
|
243
|
+
include_context "in experiment bucket 0"
|
244
|
+
include_context "in experiment bucket 1"
|
249
245
|
```
|
250
246
|
|
251
247
|
Helper methods are also available:
|
252
248
|
|
253
249
|
**is_in_experiment**
|
254
|
-
|
255
|
-
|
256
|
-
experiment
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
|
261
|
-
|
262
|
-
|
263
|
-
|
264
|
-
|
265
|
-
|
266
|
-
````
|
250
|
+
```ruby
|
251
|
+
# first param is true for in experiment, false for not in experiment
|
252
|
+
# second param is the experiment name
|
253
|
+
# third param is the subject object
|
254
|
+
is_in_experiment(true, :my_experiment, my_subject)
|
255
|
+
|
256
|
+
# if user and experiment_name are defined, you can do
|
257
|
+
let(:experiment_name) { :my_experiment }
|
258
|
+
let(:user) { User.new }
|
259
|
+
is_in_experiment # true if in experiment
|
260
|
+
is_in_experiment(false) # true if NOT in experiment
|
261
|
+
```
|
267
262
|
|
268
263
|
**is_not_in_experiment**
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
|
273
|
-
|
274
|
-
|
275
|
-
|
276
|
-
|
277
|
-
|
278
|
-
|
264
|
+
```ruby
|
265
|
+
# first param is name of experiment
|
266
|
+
# second param is subject object
|
267
|
+
is_not_in_experiment(:my_experiment, my_subject)
|
268
|
+
|
269
|
+
# if user and experiment_name are defined, you can do
|
270
|
+
let(:experiment_name) { :my_experiment }
|
271
|
+
let(:user) { User.new }
|
272
|
+
is_not_in_experiment
|
273
|
+
```
|
279
274
|
|
280
275
|
**has_experiment_bucket**
|
281
|
-
|
282
|
-
|
283
|
-
|
284
|
-
# if user and experiment_name are defined, you can do
|
285
|
-
let(:experiment_name) { :my_experiment }
|
286
|
-
let(:user) { User.new }
|
287
|
-
has_experiment_bucket(1)
|
288
|
-
|
289
|
-
````
|
290
|
-
|
276
|
+
```ruby
|
277
|
+
has_experiment_bucket(1, :my_experiment, my_subject)
|
291
278
|
|
279
|
+
# if user and experiment_name are defined, you can do
|
280
|
+
let(:experiment_name) { :my_experiment }
|
281
|
+
let(:user) { User.new }
|
282
|
+
has_experiment_bucket(1)
|
283
|
+
```
|
292
284
|
|
293
285
|
## Developer Workflow
|
294
286
|
|
295
|
-
Experiments *can* be defined in config/
|
296
|
-
Running the rake task `rake
|
287
|
+
Experiments *can* be defined in `config/experimental.yml`
|
288
|
+
Running the rake task `rake experimental:sync` will load those
|
297
289
|
experiments under 'in_code' into the database and set removed_at
|
298
290
|
timestamp for those under 'removed'
|
299
291
|
|
300
292
|
You will likely want to automate the running of `rake
|
301
|
-
|
293
|
+
experimental:sync` by adding to your deploy file.
|
302
294
|
|
303
295
|
### Capistrano
|
304
296
|
In `config/deploy.rb`:
|
305
297
|
|
306
298
|
Create a namespace to run the task:
|
307
|
-
|
308
|
-
|
309
|
-
|
310
|
-
|
311
|
-
|
312
|
-
|
313
|
-
|
314
|
-
|
299
|
+
```ruby
|
300
|
+
namespace :database do
|
301
|
+
desc "Sync experiments"
|
302
|
+
task :sync_from_app, roles: :db, only: { primary: true } do
|
303
|
+
run "cd #{current_path} && RAILS_ENV=#{rails_env} bundle exec rake experimental:sync"
|
304
|
+
end
|
305
|
+
end
|
306
|
+
```
|
315
307
|
|
316
308
|
Include that in the deploy:default task:
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
|
321
|
-
|
322
|
-
|
323
|
-
|
324
|
-
|
325
|
-
|
326
|
-
|
327
|
-
|
309
|
+
```ruby
|
310
|
+
namespace :deploy do
|
311
|
+
#...
|
312
|
+
task :default do
|
313
|
+
begin
|
314
|
+
update_code
|
315
|
+
migrate
|
316
|
+
database.sync_from_app
|
317
|
+
restart
|
318
|
+
#...
|
319
|
+
end
|
320
|
+
end
|
321
|
+
end
|
322
|
+
```
|
328
323
|
|
329
324
|
### Admin created experiments
|
330
325
|
|
@@ -332,6 +327,5 @@ The purpose of Admin created experiments are for experiments
|
|
332
327
|
that will flow through to another system, such as an email provider.
|
333
328
|
They likely start with a known string and are dynamically sent in
|
334
329
|
code.
|
335
|
-
Otherwise, Admin created experiments will do nothing as
|
330
|
+
Otherwise, Admin created experiments will do nothing as there is no
|
336
331
|
code attached to them.
|
337
|
-
|
@@ -2,10 +2,7 @@ module Experimental
|
|
2
2
|
class Experiment < ActiveRecord::Base
|
3
3
|
extend Population::Filter
|
4
4
|
|
5
|
-
|
6
|
-
@@use_cache = true
|
7
|
-
|
8
|
-
attr_accessible :name, :num_buckets, :notes
|
5
|
+
attr_accessible :name, :num_buckets, :notes, :population
|
9
6
|
|
10
7
|
validates_presence_of :name, :num_buckets
|
11
8
|
validates_numericality_of :num_buckets, :greater_than_or_equal_to => 2
|
@@ -14,8 +11,6 @@ module Experimental
|
|
14
11
|
:less_than => :num_buckets,
|
15
12
|
:if => :ended?
|
16
13
|
|
17
|
-
after_create :expire_cache
|
18
|
-
|
19
14
|
def self.in_code
|
20
15
|
where(:removed_at => nil)
|
21
16
|
end
|
@@ -37,38 +32,43 @@ module Experimental
|
|
37
32
|
end
|
38
33
|
|
39
34
|
def self.[](experiment_name)
|
40
|
-
|
41
|
-
Cache.get(experiment_name)
|
42
|
-
else
|
43
|
-
find_by_name(experiment_name.to_s)
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
def self.expire_cache
|
48
|
-
Cache.expire_last_updated if use_cache
|
49
|
-
end
|
50
|
-
|
51
|
-
def expire_cache
|
52
|
-
Experiment.expire_cache
|
35
|
+
Experimental.source[experiment_name.to_s]
|
53
36
|
end
|
54
37
|
|
55
38
|
def bucket(subject)
|
56
|
-
|
39
|
+
if ended? || removed?
|
40
|
+
winning_bucket
|
41
|
+
elsif Experimental.overrides.include?(subject, name)
|
42
|
+
Experimental.overrides[subject, name]
|
43
|
+
else
|
44
|
+
bucket_number(subject)
|
45
|
+
end
|
57
46
|
end
|
58
47
|
|
59
48
|
def in?(subject)
|
60
|
-
|
61
|
-
|
49
|
+
if removed?
|
50
|
+
false
|
51
|
+
elsif Experimental.overrides.include?(subject, name)
|
52
|
+
!!Experimental.overrides[subject, name]
|
53
|
+
else
|
54
|
+
population_filter.in?(subject, self)
|
55
|
+
end
|
62
56
|
end
|
63
57
|
|
64
58
|
def end(winning_num)
|
65
59
|
self.winning_bucket = winning_num
|
66
60
|
self.end_date = Time.now
|
67
|
-
|
61
|
+
save
|
62
|
+
end
|
68
63
|
|
69
|
-
|
64
|
+
def restart
|
65
|
+
return unless ended?
|
70
66
|
|
71
|
-
|
67
|
+
self.winning_bucket = nil
|
68
|
+
self.start_date = Time.now
|
69
|
+
self.end_date = nil
|
70
|
+
|
71
|
+
save
|
72
72
|
end
|
73
73
|
|
74
74
|
def remove
|
@@ -78,7 +78,6 @@ module Experimental
|
|
78
78
|
result = update_attributes(
|
79
79
|
{ removed_at: Time.now }, without_protection: true
|
80
80
|
)
|
81
|
-
expire_cache if result
|
82
81
|
end
|
83
82
|
|
84
83
|
result
|
@@ -96,6 +95,10 @@ module Experimental
|
|
96
95
|
!removed? && !ended?
|
97
96
|
end
|
98
97
|
|
98
|
+
def self.active
|
99
|
+
where(['removed_at IS NULL AND (end_date IS NULL OR ? <= end_date)', Time.now])
|
100
|
+
end
|
101
|
+
|
99
102
|
def to_sql_formula(subject_table = "users")
|
100
103
|
"CONV(SUBSTR(SHA1(CONCAT(\"#{name}\",#{subject_table}.id)),1,8),16,10) % #{num_buckets}"
|
101
104
|
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
module Experimental
|
2
2
|
module Subject
|
3
3
|
def in_experiment?(name)
|
4
|
-
|
4
|
+
Experimental.source[name].try { |e| e.in?(self) }
|
5
5
|
end
|
6
6
|
|
7
7
|
def not_in_experiment?(name)
|
@@ -9,7 +9,7 @@ module Experimental
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def experiment_bucket(name)
|
12
|
-
|
12
|
+
Experimental.source[name].try { |e| e.in?(self) ? e.bucket(self) : nil }
|
13
13
|
end
|
14
14
|
|
15
15
|
def in_bucket?(name, bucket)
|
@@ -0,0 +1,34 @@
|
|
1
|
+
require 'logger'
|
2
|
+
|
3
|
+
module Experimental
|
4
|
+
class Loader
|
5
|
+
def initialize(options = {})
|
6
|
+
@logger = options[:logger] || Logger.new('/dev/null')
|
7
|
+
end
|
8
|
+
|
9
|
+
attr_reader :logger
|
10
|
+
|
11
|
+
def sync
|
12
|
+
logger.info "Synchronizing experiments..."
|
13
|
+
|
14
|
+
Experimental::Experiment.transaction do
|
15
|
+
active = Experimental.experiment_data.map do |name, attributes|
|
16
|
+
experiment = Experimental::Experiment.find_or_initialize_by_name(name)
|
17
|
+
logger.info " * #{experiment.id ? 'updating' : 'creating'} #{name}"
|
18
|
+
experiment.assign_attributes(attributes)
|
19
|
+
experiment.start_date ||= Time.now
|
20
|
+
experiment.tap(&:save!)
|
21
|
+
end
|
22
|
+
|
23
|
+
scope = Experimental::Experiment.in_code
|
24
|
+
scope = scope.where('id NOT IN (?)', active.map(&:id)) unless active.empty?
|
25
|
+
scope.find_each do |experiment|
|
26
|
+
logger.info " * removing #{experiment.name}"
|
27
|
+
experiment.remove
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
logger.info "Done."
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
module Experimental
|
2
|
+
class Overrides
|
3
|
+
def initialize
|
4
|
+
@overrides ||= Hash.new do |experiment_overrides, experiment_name|
|
5
|
+
experiment_overrides[experiment_name] = {}
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
def include?(subject, experiment_name)
|
10
|
+
experiment_name = experiment_name.to_s
|
11
|
+
@overrides.key?(experiment_name) && @overrides[experiment_name].key?(subject)
|
12
|
+
end
|
13
|
+
|
14
|
+
def [](subject, experiment_name)
|
15
|
+
experiment_name = experiment_name.to_s
|
16
|
+
@overrides[experiment_name][subject]
|
17
|
+
end
|
18
|
+
|
19
|
+
def []=(subject, experiment_name, bucket)
|
20
|
+
experiment_name = experiment_name.to_s
|
21
|
+
@overrides[experiment_name][subject] = bucket
|
22
|
+
end
|
23
|
+
|
24
|
+
def reset
|
25
|
+
@overrides.clear
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
module Experimental
|
2
|
+
class Railtie < Rails::Railtie
|
3
|
+
initializer "experimental.initialize" do
|
4
|
+
config_path = "#{Rails.root}/config/experimental.yml"
|
5
|
+
if File.exist?(config_path)
|
6
|
+
full = YAML.load_file(config_path)
|
7
|
+
configuration = full[Rails.env] || {}
|
8
|
+
configuration.update(full.slice('experiments'))
|
9
|
+
Experimental.configure(configuration)
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
module Experimental
|
2
|
+
module Source
|
3
|
+
class Base
|
4
|
+
# Return the experiment with the given name (Symbol or String)
|
5
|
+
def [](name)
|
6
|
+
raise NotImplementedError, 'abstract'
|
7
|
+
end
|
8
|
+
|
9
|
+
# Return all active experiments.
|
10
|
+
def active
|
11
|
+
raise NotImplementedError, 'abstract'
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
module Experimental
|
2
|
+
module Source
|
3
|
+
class Cache < Base
|
4
|
+
# A cache source provides in memory caching around another source.
|
5
|
+
#
|
6
|
+
# If a +:ttl+ option is passed, experiments will only be cached for that
|
7
|
+
# many seconds, otherwise it is cached forever.
|
8
|
+
def initialize(source, options = {})
|
9
|
+
@source = source
|
10
|
+
@ttl = options[:ttl]
|
11
|
+
@last_update = nil
|
12
|
+
@cache = {}
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :source, :ttl
|
16
|
+
|
17
|
+
def [](name)
|
18
|
+
refresh if dirty?
|
19
|
+
cache[name.to_s]
|
20
|
+
end
|
21
|
+
|
22
|
+
def active
|
23
|
+
refresh if dirty?
|
24
|
+
cache.values
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
attr_accessor :cache, :last_update
|
30
|
+
|
31
|
+
def dirty?
|
32
|
+
return true if last_update.nil?
|
33
|
+
ttl ? (Time.now.to_f - last_update > ttl) : false
|
34
|
+
end
|
35
|
+
|
36
|
+
def refresh
|
37
|
+
cache.clear
|
38
|
+
source.active.each do |experiment|
|
39
|
+
cache[experiment.name] = experiment
|
40
|
+
end
|
41
|
+
self.last_update = Time.now.to_f
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Experimental
|
2
|
+
module Source
|
3
|
+
class Configuration < Base
|
4
|
+
def initialize
|
5
|
+
@experiments = {}
|
6
|
+
Experimental.experiment_data.each do |name, attributes|
|
7
|
+
experiment = Experiment.new(attributes) { |e| e.name = name }
|
8
|
+
@experiments[experiment.name] = experiment
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
def [](name)
|
13
|
+
@experiments[name.to_s]
|
14
|
+
end
|
15
|
+
|
16
|
+
def active
|
17
|
+
@experiments.values
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Experimental
|
2
|
+
module Test
|
3
|
+
def self.included(base)
|
4
|
+
base.before do
|
5
|
+
Experimental.source = Experimental::Source::Configuration.new
|
6
|
+
Experimental.overrides.reset
|
7
|
+
end
|
8
|
+
|
9
|
+
base.after do
|
10
|
+
Experimental.overrides.reset
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
# Force the given subject into the given +bucket+ of the given +experiment+.
|
15
|
+
#
|
16
|
+
# If +bucket+ is nil, exclude the user from the experiment.
|
17
|
+
def set_experimental_bucket(subject, experiment_name, bucket)
|
18
|
+
Experimental.overrides[subject, experiment_name] = bucket
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
data/lib/experimental/version.rb
CHANGED
data/lib/experimental.rb
CHANGED
@@ -4,8 +4,51 @@ require 'experimental/engine'
|
|
4
4
|
module Experimental
|
5
5
|
autoload :VERSION, 'experimental/version'
|
6
6
|
autoload :ControllerActions, 'experimental/controller_actions'
|
7
|
+
autoload :Loader, 'experimental/loader'
|
8
|
+
autoload :Overrides, 'experimental/overrides'
|
9
|
+
autoload :Source, 'experimental/source'
|
10
|
+
autoload :Test, 'experimental/test'
|
7
11
|
|
8
|
-
|
9
|
-
|
12
|
+
class << self
|
13
|
+
def configure(configuration)
|
14
|
+
source = Source::ActiveRecord.new
|
15
|
+
if (ttl = configuration['cache_for'])
|
16
|
+
source = Source::Cache.new(source, ttl: ttl)
|
17
|
+
end
|
18
|
+
self.experiment_data = configuration['experiments']
|
19
|
+
self.source = source
|
20
|
+
end
|
21
|
+
|
22
|
+
def register_population_filter(name, filter_class)
|
23
|
+
Experiment.register_population_filter(name, filter_class)
|
24
|
+
end
|
25
|
+
|
26
|
+
def source=(source)
|
27
|
+
@experimental_source = source
|
28
|
+
end
|
29
|
+
|
30
|
+
def source
|
31
|
+
@experimental_source ||= Source::ActiveRecord.new
|
32
|
+
end
|
33
|
+
|
34
|
+
def experiment_data=(data)
|
35
|
+
@experimental_data = data
|
36
|
+
end
|
37
|
+
|
38
|
+
def experiment_data
|
39
|
+
@experimental_data ||= {}
|
40
|
+
end
|
41
|
+
|
42
|
+
def reset
|
43
|
+
self.source = nil
|
44
|
+
self.experiment_data = nil
|
45
|
+
Experiment.reset_population_filters
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def self.overrides
|
50
|
+
Thread.current[:experimental_overrides] ||= Overrides.new
|
10
51
|
end
|
11
52
|
end
|
53
|
+
|
54
|
+
require 'experimental/railtie' if defined?(Rails)
|
@@ -1,6 +1,7 @@
|
|
1
|
-
namespace :
|
1
|
+
namespace :experimental do
|
2
2
|
desc "sync experiments from config/experiment.yml into the database"
|
3
3
|
task :sync => :environment do
|
4
|
-
|
4
|
+
logger = Logger.new(STDOUT)
|
5
|
+
Experimental::Loader.new(logger: logger).sync
|
5
6
|
end
|
6
7
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: experimental
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- HowAboutWe.com
|
@@ -10,104 +10,118 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: bin
|
12
12
|
cert_chain: []
|
13
|
-
date: 2013-
|
13
|
+
date: 2013-10-22 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: rails
|
17
17
|
requirement: !ruby/object:Gem::Requirement
|
18
18
|
requirements:
|
19
|
-
- -
|
19
|
+
- - <=
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
21
|
+
version: '5.0'
|
22
22
|
type: :runtime
|
23
23
|
prerelease: false
|
24
24
|
version_requirements: !ruby/object:Gem::Requirement
|
25
25
|
requirements:
|
26
|
-
- -
|
26
|
+
- - <=
|
27
27
|
- !ruby/object:Gem::Version
|
28
|
-
version:
|
28
|
+
version: '5.0'
|
29
29
|
- !ruby/object:Gem::Dependency
|
30
30
|
name: jquery-rails
|
31
31
|
requirement: !ruby/object:Gem::Requirement
|
32
32
|
requirements:
|
33
|
-
- -
|
33
|
+
- - '>='
|
34
34
|
- !ruby/object:Gem::Version
|
35
35
|
version: '0'
|
36
36
|
type: :runtime
|
37
37
|
prerelease: false
|
38
38
|
version_requirements: !ruby/object:Gem::Requirement
|
39
39
|
requirements:
|
40
|
-
- -
|
40
|
+
- - '>='
|
41
41
|
- !ruby/object:Gem::Version
|
42
42
|
version: '0'
|
43
43
|
- !ruby/object:Gem::Dependency
|
44
44
|
name: sqlite3
|
45
45
|
requirement: !ruby/object:Gem::Requirement
|
46
46
|
requirements:
|
47
|
-
- -
|
47
|
+
- - '>='
|
48
48
|
- !ruby/object:Gem::Version
|
49
49
|
version: '0'
|
50
50
|
type: :development
|
51
51
|
prerelease: false
|
52
52
|
version_requirements: !ruby/object:Gem::Requirement
|
53
53
|
requirements:
|
54
|
-
- -
|
54
|
+
- - '>='
|
55
|
+
- !ruby/object:Gem::Version
|
56
|
+
version: '0'
|
57
|
+
- !ruby/object:Gem::Dependency
|
58
|
+
name: timecop
|
59
|
+
requirement: !ruby/object:Gem::Requirement
|
60
|
+
requirements:
|
61
|
+
- - '>='
|
62
|
+
- !ruby/object:Gem::Version
|
63
|
+
version: '0'
|
64
|
+
type: :development
|
65
|
+
prerelease: false
|
66
|
+
version_requirements: !ruby/object:Gem::Requirement
|
67
|
+
requirements:
|
68
|
+
- - '>='
|
55
69
|
- !ruby/object:Gem::Version
|
56
70
|
version: '0'
|
57
71
|
- !ruby/object:Gem::Dependency
|
58
72
|
name: rspec-rails
|
59
73
|
requirement: !ruby/object:Gem::Requirement
|
60
74
|
requirements:
|
61
|
-
- -
|
75
|
+
- - '>='
|
62
76
|
- !ruby/object:Gem::Version
|
63
77
|
version: '0'
|
64
78
|
type: :development
|
65
79
|
prerelease: false
|
66
80
|
version_requirements: !ruby/object:Gem::Requirement
|
67
81
|
requirements:
|
68
|
-
- -
|
82
|
+
- - '>='
|
69
83
|
- !ruby/object:Gem::Version
|
70
84
|
version: '0'
|
71
85
|
- !ruby/object:Gem::Dependency
|
72
86
|
name: activeadmin
|
73
87
|
requirement: !ruby/object:Gem::Requirement
|
74
88
|
requirements:
|
75
|
-
- -
|
89
|
+
- - '>='
|
76
90
|
- !ruby/object:Gem::Version
|
77
91
|
version: '0'
|
78
92
|
type: :development
|
79
93
|
prerelease: false
|
80
94
|
version_requirements: !ruby/object:Gem::Requirement
|
81
95
|
requirements:
|
82
|
-
- -
|
96
|
+
- - '>='
|
83
97
|
- !ruby/object:Gem::Version
|
84
98
|
version: '0'
|
85
99
|
- !ruby/object:Gem::Dependency
|
86
100
|
name: sass-rails
|
87
101
|
requirement: !ruby/object:Gem::Requirement
|
88
102
|
requirements:
|
89
|
-
- -
|
103
|
+
- - '>='
|
90
104
|
- !ruby/object:Gem::Version
|
91
105
|
version: '0'
|
92
106
|
type: :development
|
93
107
|
prerelease: false
|
94
108
|
version_requirements: !ruby/object:Gem::Requirement
|
95
109
|
requirements:
|
96
|
-
- -
|
110
|
+
- - '>='
|
97
111
|
- !ruby/object:Gem::Version
|
98
112
|
version: '0'
|
99
113
|
- !ruby/object:Gem::Dependency
|
100
114
|
name: coffee-rails
|
101
115
|
requirement: !ruby/object:Gem::Requirement
|
102
116
|
requirements:
|
103
|
-
- -
|
117
|
+
- - '>='
|
104
118
|
- !ruby/object:Gem::Version
|
105
119
|
version: '0'
|
106
120
|
type: :development
|
107
121
|
prerelease: false
|
108
122
|
version_requirements: !ruby/object:Gem::Requirement
|
109
123
|
requirements:
|
110
|
-
- -
|
124
|
+
- - '>='
|
111
125
|
- !ruby/object:Gem::Version
|
112
126
|
version: '0'
|
113
127
|
description: AB Test framework for Rails
|
@@ -117,9 +131,7 @@ executables: []
|
|
117
131
|
extensions: []
|
118
132
|
extra_rdoc_files: []
|
119
133
|
files:
|
120
|
-
- app/models/experimental/cache.rb
|
121
134
|
- app/models/experimental/experiment.rb
|
122
|
-
- app/models/experimental/loader.rb
|
123
135
|
- app/models/experimental/population/default.rb
|
124
136
|
- app/models/experimental/population/filter.rb
|
125
137
|
- app/models/experimental/population/new_users.rb
|
@@ -131,18 +143,27 @@ files:
|
|
131
143
|
- config/routes.rb
|
132
144
|
- lib/experimental/controller_actions.rb
|
133
145
|
- lib/experimental/engine.rb
|
146
|
+
- lib/experimental/loader.rb
|
147
|
+
- lib/experimental/overrides.rb
|
148
|
+
- lib/experimental/railtie.rb
|
134
149
|
- lib/experimental/rspec_helpers.rb
|
150
|
+
- lib/experimental/source/active_record.rb
|
151
|
+
- lib/experimental/source/base.rb
|
152
|
+
- lib/experimental/source/cache.rb
|
153
|
+
- lib/experimental/source/configuration.rb
|
154
|
+
- lib/experimental/source.rb
|
155
|
+
- lib/experimental/test.rb
|
135
156
|
- lib/experimental/version.rb
|
136
157
|
- lib/experimental.rb
|
137
158
|
- lib/generators/experimental/experimental_generator.rb
|
138
159
|
- lib/generators/experimental/templates/create_experiments_table.rb
|
139
160
|
- lib/generators/experimental/templates/experimental.rb
|
140
161
|
- lib/generators/experimental/templates/experiments.yml
|
141
|
-
- lib/tasks/
|
162
|
+
- lib/tasks/experimental.rake
|
142
163
|
- MIT-LICENSE
|
143
164
|
- Rakefile
|
144
165
|
- README.markdown
|
145
|
-
homepage:
|
166
|
+
homepage: https://github.com/howaboutwe/experimental
|
146
167
|
licenses:
|
147
168
|
- MIT
|
148
169
|
metadata: {}
|
@@ -152,17 +173,17 @@ require_paths:
|
|
152
173
|
- lib
|
153
174
|
required_ruby_version: !ruby/object:Gem::Requirement
|
154
175
|
requirements:
|
155
|
-
- -
|
176
|
+
- - '>='
|
156
177
|
- !ruby/object:Gem::Version
|
157
178
|
version: '0'
|
158
179
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
159
180
|
requirements:
|
160
|
-
- -
|
181
|
+
- - '>='
|
161
182
|
- !ruby/object:Gem::Version
|
162
183
|
version: '0'
|
163
184
|
requirements: []
|
164
185
|
rubyforge_project:
|
165
|
-
rubygems_version: 2.
|
186
|
+
rubygems_version: 2.1.9
|
166
187
|
signing_key:
|
167
188
|
specification_version: 4
|
168
189
|
summary: Adds support for database-backed AB tests in Rails apps
|
@@ -1,69 +0,0 @@
|
|
1
|
-
module Experimental
|
2
|
-
class Cache
|
3
|
-
cattr_accessor :interval
|
4
|
-
cattr_reader :last_check
|
5
|
-
cattr_reader :last_update
|
6
|
-
cattr_reader :experiments
|
7
|
-
cattr_accessor :cache_race_condition_ttl
|
8
|
-
@@cache_race_condition_ttl = 10
|
9
|
-
cattr_accessor :cache_key
|
10
|
-
|
11
|
-
class << self
|
12
|
-
def get(name)
|
13
|
-
unless within_interval?
|
14
|
-
if need_update?(last_cached_update)
|
15
|
-
@@experiments = experiments_to_hash(Experiment.in_code)
|
16
|
-
@@last_update = Time.now
|
17
|
-
end
|
18
|
-
end
|
19
|
-
experiments[name.to_sym]
|
20
|
-
end
|
21
|
-
|
22
|
-
def [](name)
|
23
|
-
get(name)
|
24
|
-
end
|
25
|
-
|
26
|
-
def interval
|
27
|
-
@@interval ||= 5.minutes
|
28
|
-
end
|
29
|
-
|
30
|
-
def last_check
|
31
|
-
@@last_check ||=
|
32
|
-
Time.now - interval - 1.minute
|
33
|
-
end
|
34
|
-
|
35
|
-
def last_update
|
36
|
-
@@last_update ||= last_cached_update
|
37
|
-
end
|
38
|
-
|
39
|
-
def cache_key
|
40
|
-
@@cache_key ||= "experiments_last_update"
|
41
|
-
end
|
42
|
-
|
43
|
-
def within_interval?
|
44
|
-
(Time.now - last_check) < interval
|
45
|
-
end
|
46
|
-
|
47
|
-
def need_update?(last_cached_update_time)
|
48
|
-
experiments.nil? || last_update < last_cached_update_time
|
49
|
-
end
|
50
|
-
|
51
|
-
def experiments_to_hash(experiments)
|
52
|
-
HashWithIndifferentAccess.new.tap do |h|
|
53
|
-
experiments.each { |e| h[e.name.to_sym] = e }
|
54
|
-
end
|
55
|
-
end
|
56
|
-
|
57
|
-
def last_cached_update
|
58
|
-
# setting a default to 1 minute ago will force all servers to update
|
59
|
-
Rails.cache.fetch(cache_key, :race_condition_ttl => cache_race_condition_ttl ) do
|
60
|
-
Experiment.last_updated_at || 1.minute.ago
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
def expire_last_updated
|
65
|
-
Rails.cache.delete(cache_key)
|
66
|
-
end
|
67
|
-
end
|
68
|
-
end
|
69
|
-
end
|
@@ -1,64 +0,0 @@
|
|
1
|
-
module Experimental
|
2
|
-
class Loader
|
3
|
-
cattr_accessor :file_path
|
4
|
-
@@file_path = 'config/experiments.yml'
|
5
|
-
|
6
|
-
@@whitelisted_attributes = [:name, :num_buckets, :notes, :population]
|
7
|
-
|
8
|
-
class << self
|
9
|
-
def sync(verbose = false)
|
10
|
-
puts "Loading experiments ..." if verbose
|
11
|
-
experiments = load_experiments
|
12
|
-
|
13
|
-
#new/active
|
14
|
-
experiments.in_code.each do |exp|
|
15
|
-
name = exp.with_indifferent_access[:name]
|
16
|
-
puts "\tUpdating #{name} ..." if verbose
|
17
|
-
|
18
|
-
exp = whitelisted_attrs(exp)
|
19
|
-
e = Experimental::Experiment.find_by_name(name) || Experimental::Experiment.new
|
20
|
-
|
21
|
-
puts "\t\tcreating ..." if verbose && e.id.nil?
|
22
|
-
puts "\t\tupdating ..." if verbose && !e.id.nil?
|
23
|
-
|
24
|
-
exp.merge!({ start_date: Time.now }) if e.start_date.nil?
|
25
|
-
e.update_attributes!(exp, without_protection: true)
|
26
|
-
end unless experiments.in_code.nil?
|
27
|
-
|
28
|
-
#removed
|
29
|
-
experiments.removed.each do |exp|
|
30
|
-
name = exp.with_indifferent_access[:name]
|
31
|
-
puts "\tRemoving #{name} ..." if verbose
|
32
|
-
|
33
|
-
exp = whitelisted_attrs(exp)
|
34
|
-
e = Experimental::Experiment.find_by_name(name)
|
35
|
-
|
36
|
-
if e && e.removed_at.nil?
|
37
|
-
puts "\t\t#{name} exists, removing ..." if verbose
|
38
|
-
result = e.remove
|
39
|
-
puts "\t\t\t#{result}" if verbose
|
40
|
-
else
|
41
|
-
puts "\t\t#{name} doesn't exist!" if verbose
|
42
|
-
end
|
43
|
-
end unless experiments.removed.nil?
|
44
|
-
|
45
|
-
puts "Expiring cache ..." if verbose
|
46
|
-
Experimental::Experiment.expire_cache
|
47
|
-
|
48
|
-
puts "Done syncing experiments!" if verbose
|
49
|
-
end
|
50
|
-
|
51
|
-
private
|
52
|
-
|
53
|
-
def load_experiments
|
54
|
-
OpenStruct.new YAML.load_file(File.join(Rails.root, file_path))
|
55
|
-
end
|
56
|
-
|
57
|
-
def whitelisted_attrs(exp)
|
58
|
-
exp.
|
59
|
-
select { |k, v| @@whitelisted_attributes.include?(k.to_sym) }.
|
60
|
-
with_indifferent_access
|
61
|
-
end
|
62
|
-
end
|
63
|
-
end
|
64
|
-
end
|