kgrift 1.3.108
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/KGrift/Gemfile +22 -0
- data/KGrift/README.md +66 -0
- data/KGrift/bin/kgrift +11 -0
- data/KGrift/grifter.yml +224 -0
- data/KGrift/internal_test_graphs/basic_test_graph_definition.yml +2915 -0
- data/KGrift/internal_test_graphs/unicode_test_graph_definition.yml +3070 -0
- data/KGrift/knewton_grifts/analytics_grifts.rb +103 -0
- data/KGrift/knewton_grifts/async_helper_grifts.rb +63 -0
- data/KGrift/knewton_grifts/authenticator_grifts.rb +46 -0
- data/KGrift/knewton_grifts/basic_grifts.rb +29 -0
- data/KGrift/knewton_grifts/batch_grifts.rb +14 -0
- data/KGrift/knewton_grifts/content_collection_grifts.rb +204 -0
- data/KGrift/knewton_grifts/content_collection_v1_grifts.rb +521 -0
- data/KGrift/knewton_grifts/content_eid_grifts.rb +41 -0
- data/KGrift/knewton_grifts/copy_grifts.rb +151 -0
- data/KGrift/knewton_grifts/deprecated_graph_and_taxonomy_grifts.rb +353 -0
- data/KGrift/knewton_grifts/goal_grifts.rb +203 -0
- data/KGrift/knewton_grifts/graph_and_taxonomy_grifts.rb +136 -0
- data/KGrift/knewton_grifts/graph_create_grifts.rb +34 -0
- data/KGrift/knewton_grifts/graph_query_grifts.rb +448 -0
- data/KGrift/knewton_grifts/graph_tools_grifts.rb +151 -0
- data/KGrift/knewton_grifts/graph_validation_grifts.rb +447 -0
- data/KGrift/knewton_grifts/helper_grifts.rb +92 -0
- data/KGrift/knewton_grifts/jmeter_data_grifts.rb +56 -0
- data/KGrift/knewton_grifts/learning_instance_grifts.rb +46 -0
- data/KGrift/knewton_grifts/looper_grifts.rb +34 -0
- data/KGrift/knewton_grifts/moxy_grifts.rb +64 -0
- data/KGrift/knewton_grifts/oauth_grifts.rb +182 -0
- data/KGrift/knewton_grifts/partner_grifts.rb +70 -0
- data/KGrift/knewton_grifts/partner_support_grifts.rb +85 -0
- data/KGrift/knewton_grifts/recommendation_setup_grifts.rb +215 -0
- data/KGrift/knewton_grifts/registration_grifts.rb +159 -0
- data/KGrift/knewton_grifts/registration_info_grifts.rb +23 -0
- data/KGrift/knewton_grifts/report_grifts.rb +122 -0
- data/KGrift/knewton_grifts/shell_command_grifts.rb +21 -0
- data/KGrift/knewton_grifts/student_flow_grifts.rb +560 -0
- data/KGrift/knewton_grifts/tag_grifts.rb +41 -0
- data/KGrift/knewton_grifts/test_data_grifts.rb +328 -0
- data/KGrift/knewton_grifts/test_user_grifts.rb +264 -0
- data/KGrift/lib/dtrace.rb +20 -0
- data/KGrift/lib/kgrift.rb +7 -0
- data/KGrift/test_data_generators/basic_book_and_taxonomies.rb +35 -0
- data/KGrift/test_data_generators/lo_test_graph.rb +34 -0
- data/KGrift/test_data_generators/partner_owned_book_and_taxonomies.rb +28 -0
- data/KGrift/test_data_generators/partner_owned_book_and_taxonomies_unicode.rb +28 -0
- data/KGrift/test_data_generators/sandcastle_book_and_taxonomies.rb +13 -0
- data/KGrift/test_data_generators/sandcastle_book_and_taxonomies.yml +3709 -0
- data/KGrift/test_data_generators/sandcastle_graph.rb +8 -0
- data/KGrift/test_data_generators/sandcastle_graph_definition.json +4483 -0
- data/KGrift/test_data_generators/sandcastle_graph_full.rb +7 -0
- data/KGrift/test_data_generators/sandcastle_taxonomies.yml +378 -0
- data/KGrift/test_data_generators/sandcastle_with_taxons.rb +56 -0
- data/KGrift/test_data_generators/sandcastle_with_taxons.yml +3994 -0
- data/KGrift/test_data_generators/test_users_and_partners.rb +76 -0
- data/kgrift.gemspec +43 -0
- metadata +144 -0
@@ -0,0 +1,23 @@
|
|
1
|
+
def print_goals_for_registration_report registration_id, goal_id=nil
|
2
|
+
reg = get_registration registration_id
|
3
|
+
|
4
|
+
goals = get_goals reg['learning_instance_id']
|
5
|
+
|
6
|
+
if goal_id
|
7
|
+
goals = goals.select{|g| g['id'] == goal_id}
|
8
|
+
end
|
9
|
+
|
10
|
+
puts " "
|
11
|
+
puts "===================="
|
12
|
+
puts "Learning instance: #{reg['learning_instance_id']}"
|
13
|
+
puts ""
|
14
|
+
goals.each do |goal|
|
15
|
+
puts "--------------------"
|
16
|
+
puts "active: #{check_if_activated(reg['learning_instance_id'], registration_id, goal['id'])}"
|
17
|
+
puts "#{goal.to_yaml}"
|
18
|
+
puts " "
|
19
|
+
|
20
|
+
end
|
21
|
+
|
22
|
+
return ''
|
23
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# Generic methods for writing some reports
|
2
|
+
# The reports that can be made are:
|
3
|
+
# - all_requests_report - a simple csv noting some details about every request
|
4
|
+
# - requests by endpoint - compiles response time stats about each endpoint that has been used
|
5
|
+
#
|
6
|
+
|
7
|
+
require 'fileutils'
|
8
|
+
|
9
|
+
# construct a path to the report directory, and ensure it exists
|
10
|
+
def report_dir
|
11
|
+
report_dir = File.join(Dir.pwd, 'reports')
|
12
|
+
FileUtils.mkdir_p report_dir
|
13
|
+
report_dir
|
14
|
+
end
|
15
|
+
|
16
|
+
# write a csv that lists all requests
|
17
|
+
def write_all_requests_report filename=nil
|
18
|
+
# default the filename to reports/all_requests.csv
|
19
|
+
filename = File.join(report_dir, 'all_requests.csv') unless filename
|
20
|
+
|
21
|
+
requests = metrics_all_requests
|
22
|
+
Log.info "Writing all #{requests.length} requests report to filename: #{filename}"
|
23
|
+
|
24
|
+
columns = [:service, :method, :path, :status, :duration_ms, :end_time]
|
25
|
+
fmt_str = '%-8.8s,%-8.8s,%-61.60s,%-8.8s,%-12.12s,%-28.28s'
|
26
|
+
File.open(filename, 'w') do |output|
|
27
|
+
output.puts(fmt_str % columns)
|
28
|
+
requests.each do |data|
|
29
|
+
output.puts(fmt_str % data.to_a)
|
30
|
+
end
|
31
|
+
end
|
32
|
+
Log.debug "Done writing all request report"
|
33
|
+
filename
|
34
|
+
end
|
35
|
+
|
36
|
+
|
37
|
+
# write a csv that lists all unique endpoints
|
38
|
+
# and for each endpoint, some stats about response time
|
39
|
+
def write_metrics_by_endpoint_report filename=nil
|
40
|
+
filename = File.join(report_dir, 'metrics_by_endpoint.csv') unless filename
|
41
|
+
|
42
|
+
requests = metrics_all_requests
|
43
|
+
|
44
|
+
# now compile the list of requests into a list of specific endoints that were hit
|
45
|
+
# this means substituting specific IDs in requests paths with the string ':id'
|
46
|
+
metrics_by_endpoint = {}
|
47
|
+
requests.each do |sample|
|
48
|
+
method = sample.method.upcase # GET, POST, PUT, etc...
|
49
|
+
path = sample.path.gsub(/\?.*$/,'') # the path, chop off query params
|
50
|
+
|
51
|
+
#replace uuids with :id
|
52
|
+
path.gsub!(/[0-9a-f\-]{36}/,':id')
|
53
|
+
#replace any url path component with a capital letter, number or % to :id
|
54
|
+
path.gsub!(/[^\/]+[A-Z0-9% ][^\/]+/,':id')
|
55
|
+
|
56
|
+
endpoint = "#{'%-7s' % method} #{path}"
|
57
|
+
|
58
|
+
metrics_by_endpoint[endpoint] ||= {
|
59
|
+
durations: [],
|
60
|
+
count: 0
|
61
|
+
}
|
62
|
+
metrics_by_endpoint[endpoint][:count] += 1
|
63
|
+
metrics_by_endpoint[endpoint][:durations] << sample.duration_ms
|
64
|
+
end
|
65
|
+
|
66
|
+
Log.info "Writing metrics by endpoint report for #{metrics_by_endpoint.length} endpoints to filename: #{filename}"
|
67
|
+
File.open(filename, 'w') do |output|
|
68
|
+
columns = [:endpoint, :count, :mean_duration, :median_duration, :standard_deviation, :min_duration, :percentile_95, :max_duration]
|
69
|
+
fmt_str = '%-80s' + columns[1..-1].inject(''){|s, c| s + ",%-#{c.length+1}s"}
|
70
|
+
output.puts fmt_str % columns
|
71
|
+
|
72
|
+
metrics_by_endpoint.each_pair do |endpoint, data|
|
73
|
+
durations = data[:durations].sort
|
74
|
+
output.puts fmt_str % [endpoint, data[:count], durations.mean.to_i, durations[durations.length/2], durations.sample_standard_deviation.to_i, durations.first, durations.percentile(0.95).to_i, durations.last]
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
Log.debug "Done writing metrics by endpoint report"
|
79
|
+
filename
|
80
|
+
end
|
81
|
+
|
82
|
+
|
83
|
+
# MONKEY PATCH ARRAY:
|
84
|
+
# methods for doing basic stats calculations on Arrays
|
85
|
+
# used in above metrics_by_endpoint_report method
|
86
|
+
class ::Array
|
87
|
+
def sum
|
88
|
+
self.inject(:+).to_f
|
89
|
+
end
|
90
|
+
|
91
|
+
def mean
|
92
|
+
return 0.0 if self.length < 1
|
93
|
+
self.sum / self.length.to_f
|
94
|
+
end
|
95
|
+
|
96
|
+
def sample_standard_deviation
|
97
|
+
#unless we have at least 2 items in list, std dev is 0 by convention
|
98
|
+
return 0.0 if self.length < 2
|
99
|
+
|
100
|
+
mean = self.mean
|
101
|
+
sum = self.inject(0.0) { |acc, i| acc + ((i - mean)**2) }
|
102
|
+
|
103
|
+
Math.sqrt( sum / (self.length - 1))
|
104
|
+
end
|
105
|
+
|
106
|
+
def standard_error_of_the_mean
|
107
|
+
return 0.0 if self.length < 2
|
108
|
+
(self.sample_standard_deviation / Math.sqrt(self.length))
|
109
|
+
end
|
110
|
+
|
111
|
+
def percentile(percentile)
|
112
|
+
return 0.0 if self.length < 1
|
113
|
+
return self[0] if self.length < 2
|
114
|
+
values_sorted = self.sort
|
115
|
+
k = (percentile*(values_sorted.length-1)+1).floor - 1
|
116
|
+
f = (percentile*(values_sorted.length-1)+1).modulo(1)
|
117
|
+
return values_sorted[k] + (f * (values_sorted[k+1] - values_sorted[k]))
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
|
122
|
+
|
@@ -0,0 +1,21 @@
|
|
1
|
+
"""
|
2
|
+
This file contains methods that are used to deal with
|
3
|
+
shell commands.
|
4
|
+
|
5
|
+
Nothing should really be using these anymore (it is a holdover
|
6
|
+
from how test data used to be managed.) Nonetheless, its possible
|
7
|
+
some client of kgrift is out there using this, so its being kept around
|
8
|
+
|
9
|
+
"""
|
10
|
+
|
11
|
+
class ::ShellCommandFailed < Exception; end
|
12
|
+
|
13
|
+
def execute_shell_command cmd_string
|
14
|
+
Log.debug "Executing shell command:\n\t#{cmd_string}"
|
15
|
+
output = %x(#{cmd_string} 2>&1).strip #catch stderr so we can log it properly, strip whitespace to make processing easier
|
16
|
+
Log.debug "Shell command returned:\n\t#{output}"
|
17
|
+
raise ShellCommandFailed.new("Shell cmd failed.\nCommand: #{cmd_string}\nOutput: #{output}") unless $?.success?
|
18
|
+
output
|
19
|
+
end
|
20
|
+
|
21
|
+
|
@@ -0,0 +1,560 @@
|
|
1
|
+
#
|
2
|
+
# Student flow grifts is all about emulating different kinds of student flows for a given goal
|
3
|
+
#
|
4
|
+
|
5
|
+
#based on a source learning instance and goal
|
6
|
+
#copy the goal to a new learning instance
|
7
|
+
#and send student events per some algorithmn
|
8
|
+
# To run multiple goals for a single student, call with the goal_id
|
9
|
+
# parameter set to nil and goal_id_list: [ <list of ids> ]
|
10
|
+
def copy_and_complete_goal learning_instance_id, goal_id, source_env=nil, options={}
|
11
|
+
options ={
|
12
|
+
:target_score_override => nil,
|
13
|
+
:days_from_now_override => nil,
|
14
|
+
:goal_id_list => nil,
|
15
|
+
:wait_after_init => false,
|
16
|
+
:force_metrics => true,
|
17
|
+
:taxon_id => nil
|
18
|
+
}.merge(options)
|
19
|
+
if goal_id
|
20
|
+
puts "Initializing goal bot. Will copy and complete learning instance '#{learning_instance_id}' and goal '#{goal_id}'"
|
21
|
+
else
|
22
|
+
puts "Initializing goal bot. Will copy and complete learning instance '#{learning_instance_id}' and goals '#{options[:goal_id_list]}'"
|
23
|
+
end
|
24
|
+
|
25
|
+
set_account :knerd
|
26
|
+
copy_data = copy_learning_instance 'source_environment' => source_env,
|
27
|
+
'learning_instance_id' => learning_instance_id,
|
28
|
+
'goal_id' => goal_id,
|
29
|
+
'goal_id_list' => options[:goal_id_list],
|
30
|
+
'goal_target_score_override' => options[:target_score_override],
|
31
|
+
'goal_days_from_now_override' => options[:days_from_now_override],
|
32
|
+
'force_metrics' => options[:force_metrics]
|
33
|
+
learning_instance = copy_data['learning_instance']
|
34
|
+
learning_instance_id = learning_instance['id']
|
35
|
+
graph_id = get_learning_instance(learning_instance_id)['graph_id']
|
36
|
+
|
37
|
+
test_student_profile = add_new_student_to_learning_instance learning_instance_id
|
38
|
+
registration_id = test_student_profile['registration_id']
|
39
|
+
|
40
|
+
if goal_id
|
41
|
+
goal = copy_data['goal']
|
42
|
+
goal_id = goal['id']
|
43
|
+
activate_registration_on_goal learning_instance_id, registration_id, goal_id
|
44
|
+
puts "Successfully copied learning instance and goal and created a fresh registration.\n - Learning instance id: #{learning_instance_id}\n - Goal id: #{goal_id}\n - registration id: #{registration_id}\n\n"
|
45
|
+
puts "Initialization complete\n"
|
46
|
+
|
47
|
+
# Wait to give user a chance to send events to canary
|
48
|
+
if options[:wait_after_init]
|
49
|
+
puts "Hit return key to start sending events"
|
50
|
+
STDIN.gets
|
51
|
+
end
|
52
|
+
|
53
|
+
to_return = complete_goal registration_id, goal_id, options
|
54
|
+
elsif options[:goal_id_list]
|
55
|
+
goal_list = copy_data['goal_list']
|
56
|
+
goal_id_list = goal_list.collect { |goal| goal['id'] }
|
57
|
+
goal_id_list.each { |goal_id| activate_registration_on_goal learning_instance_id, registration_id, goal_id }
|
58
|
+
puts "Successfully copied learning instance and goals and created a fresh registration.\n - Learning instance id: #{learning_instance_id}\n - Goal ids: #{goal_id_list}\n - registration id: #{registration_id}\n\n"
|
59
|
+
puts "Initialization complete\n"
|
60
|
+
|
61
|
+
# Wait to give user a chance to send events to canary
|
62
|
+
if options[:wait_after_init]
|
63
|
+
puts "Hit return key to start sending events"
|
64
|
+
STDIN.gets
|
65
|
+
end
|
66
|
+
|
67
|
+
to_return = goal_id_list.collect { |goal_id| complete_goal registration_id, goal_id, options }
|
68
|
+
end
|
69
|
+
to_return
|
70
|
+
end
|
71
|
+
|
72
|
+
def goal_bot_with_pause learning_instance_id, goal_id, source_env=nil, options={}
|
73
|
+
options = { :wait_after_init => true }.merge(options)
|
74
|
+
copy_and_complete_goal learning_instance_id, goal_id, source_env, options
|
75
|
+
end
|
76
|
+
|
77
|
+
def complete_goal registration_id, goal_id, options={}
|
78
|
+
default_options = {
|
79
|
+
:max_cycles => 100,
|
80
|
+
:starting_event_threshold => 0.00,
|
81
|
+
:event_threshold_multiplier => 0.96,
|
82
|
+
:event_selection_mode => :pick_top_recommended,
|
83
|
+
:initial_event_behavior => :send_focus_event,
|
84
|
+
:concept_level_thresholds => nil, #this is a file with per-concept data
|
85
|
+
:concept_thresholds => nil, #this is a data structure with per-concept data
|
86
|
+
:num_fails_before_success => 0,
|
87
|
+
:pause_after_sending_event => 6, # increased from 5 to 6 due to reduced rate limit
|
88
|
+
:target_score_override => nil,
|
89
|
+
:days_from_now_override => nil,
|
90
|
+
:goal_id_list => nil,
|
91
|
+
:wait_after_init => false,
|
92
|
+
:force_metrics => true,
|
93
|
+
:taxon_id => nil
|
94
|
+
}
|
95
|
+
|
96
|
+
options.keys.each do |k|
|
97
|
+
if !default_options.keys.include?(k)
|
98
|
+
raise "Invalid option #{k} passed to complete_goal (NOTE: options are symbols)"
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
options= default_options.merge(options)
|
103
|
+
|
104
|
+
registration_id = registration_id
|
105
|
+
goal_id = goal_id
|
106
|
+
taxon_id = options[:taxon_id]
|
107
|
+
|
108
|
+
set_account :knerd
|
109
|
+
registration = get_registration(registration_id)
|
110
|
+
account_id = registration['account_id']
|
111
|
+
learning_instance = get_learning_instance(get_registration(registration_id)['learning_instance_id'])
|
112
|
+
learning_instance_id = learning_instance['id']
|
113
|
+
goal = get_goal learning_instance['id'], goal_id
|
114
|
+
metrics_enabled_goal = goal['metrics_enabled']
|
115
|
+
graph = get_graph_and_cache_it learning_instance['graph_id']
|
116
|
+
|
117
|
+
if !metrics_enabled_goal
|
118
|
+
puts "WARNING: Goal doesn't have metrics_enabled, will NOT be getting ANY metrics."
|
119
|
+
end
|
120
|
+
|
121
|
+
cycle_num ||= 0
|
122
|
+
event_num ||= 0
|
123
|
+
mod_ids_sent ||= []
|
124
|
+
event_bodies_sent ||= []
|
125
|
+
|
126
|
+
cur_event_threshold = options[:starting_event_threshold]
|
127
|
+
num_fails_before_success = options[:num_fails_before_success]
|
128
|
+
|
129
|
+
# Get concept-level thresholds from a file
|
130
|
+
concept_thresholds = []
|
131
|
+
if options[:concept_level_thresholds] != nil
|
132
|
+
File.open(options[:concept_level_thresholds]) do |f|
|
133
|
+
f.each_line do |line|
|
134
|
+
line_array = line.split
|
135
|
+
concept_thresh = {'id'=> line_array[0],
|
136
|
+
'current_threshold' => line_array[1].to_f,
|
137
|
+
'threshold_multiplier' => line_array[2].to_f,
|
138
|
+
'num_fails_before_success' => line_array[3].to_i,
|
139
|
+
}
|
140
|
+
concept_thresholds << concept_thresh
|
141
|
+
end
|
142
|
+
end
|
143
|
+
# Or get concept-level thresholds straight from a datastructure
|
144
|
+
elsif options[:concept_thresholds] != nil
|
145
|
+
concept_thresholds = options[:concept_thresholds]
|
146
|
+
end
|
147
|
+
|
148
|
+
results = {
|
149
|
+
"readiness_forecasts" => {
|
150
|
+
"learning_instance_id" => learning_instance_id,
|
151
|
+
"registrations" => [
|
152
|
+
"registration_id" => registration_id,
|
153
|
+
"goals" => [
|
154
|
+
{
|
155
|
+
"goal_id" => goal_id,
|
156
|
+
"readiness_forecasts" => []
|
157
|
+
}
|
158
|
+
]
|
159
|
+
]
|
160
|
+
},
|
161
|
+
"expected_score" => {
|
162
|
+
"learning_instance_id" => learning_instance_id,
|
163
|
+
"registrations" => [
|
164
|
+
"registration_id" => registration_id,
|
165
|
+
"goals" => [
|
166
|
+
{
|
167
|
+
"goal_id" => goal_id,
|
168
|
+
"target_modules" => []
|
169
|
+
}
|
170
|
+
]
|
171
|
+
]
|
172
|
+
},
|
173
|
+
"proficiency" => {
|
174
|
+
"learning_instance_id" => learning_instance_id,
|
175
|
+
"accounts" => [
|
176
|
+
"account_id" => account_id,
|
177
|
+
"taxons" => [
|
178
|
+
{
|
179
|
+
"taxon_id" => taxon_id,
|
180
|
+
"proficiencies" => []
|
181
|
+
}
|
182
|
+
]
|
183
|
+
]
|
184
|
+
}
|
185
|
+
}
|
186
|
+
|
187
|
+
time_format_string = "%Y-%m-%dT%H:%M:%S"
|
188
|
+
current_recommendation = nil
|
189
|
+
current_readiness_forecast = nil
|
190
|
+
current_expected_score = nil
|
191
|
+
current_proficiency = nil
|
192
|
+
start_date = Time.now.strftime(time_format_string)
|
193
|
+
|
194
|
+
readiness_forecasts = []
|
195
|
+
expected_scores = Hash.new {|h, k| h[k] = []}
|
196
|
+
proficiencies = []
|
197
|
+
|
198
|
+
loop do
|
199
|
+
if cycle_num > options[:max_cycles]
|
200
|
+
puts "WARNING: goal did not complete after #{options[:max_cycles]} adaptive cycles"
|
201
|
+
break
|
202
|
+
end
|
203
|
+
|
204
|
+
event_bodies = send_events_for_recommendation current_recommendation,
|
205
|
+
registration_id, goal, graph,
|
206
|
+
(options.merge incorrect_threshold: cur_event_threshold,
|
207
|
+
concept_thresholds: concept_thresholds,
|
208
|
+
num_fails_before_success: num_fails_before_success)
|
209
|
+
|
210
|
+
if event_bodies.is_a? Array
|
211
|
+
event_bodies_sent += event_bodies
|
212
|
+
mod_ids = event_bodies.map{|eb| eb['module_id']}
|
213
|
+
mod_ids_sent += mod_ids
|
214
|
+
event_num += mod_ids.length
|
215
|
+
end
|
216
|
+
|
217
|
+
puts "\n=====================\nAdaptive cycle number #{cycle_num}\n"
|
218
|
+
current_recommendation = get_next_recommendation_in_flow registration_id, goal_id, current_recommendation, options
|
219
|
+
puts "Got new recommendation:\n#{current_recommendation.to_yaml}\n"
|
220
|
+
|
221
|
+
cur_event_threshold *= options[:event_threshold_multiplier]
|
222
|
+
concept_thresholds.map{|c| c['current_threshold'] *= c['threshold_multiplier']}
|
223
|
+
|
224
|
+
current_recommendation['module_ids'].each do |rec_mod_id|
|
225
|
+
if mod_ids_sent.include? rec_mod_id
|
226
|
+
puts "WARNING: recommendation includes a module we have sent an event for: #{rec_mod_id}"
|
227
|
+
#todo add this to an errors array
|
228
|
+
end
|
229
|
+
end
|
230
|
+
|
231
|
+
if metrics_enabled_goal
|
232
|
+
# use an eventually here to poll until a full payload is returned
|
233
|
+
current_readiness_forecast = nil
|
234
|
+
eventually timeout: 60, interval: 5 do
|
235
|
+
current_readiness_forecast = kapi.post "v0/registrations/#{registration_id}/metrics/readiness-forecast/rows", {"goal_ids"=>[goal_id]}
|
236
|
+
# in order to test if analytics has computed something yet, see if we can reference what we want.
|
237
|
+
# If we can't, throw an ExpectationError which causes eventually to try again (until timeout)
|
238
|
+
begin
|
239
|
+
current_readiness_forecast["rows"][0]["date"]
|
240
|
+
rescue Exception => e
|
241
|
+
raise ExpectationError.new('Analytics has not yet returned a readiness forcast')
|
242
|
+
end
|
243
|
+
end
|
244
|
+
current_readiness_forecast["rows"][0]["date"] = Time.now.strftime(time_format_string)
|
245
|
+
readiness_forecasts << current_readiness_forecast["rows"][0]["readiness_forecasts"][0]
|
246
|
+
puts "Got new readiness forecast:\n#{current_readiness_forecast.to_yaml}\n"
|
247
|
+
|
248
|
+
current_expected_score = nil
|
249
|
+
eventually timeout: 60, interval: 5 do
|
250
|
+
current_expected_score = kapi.get "v0/registrations/#{registration_id}/metrics/expected-score?goal_id=#{goal_id}"
|
251
|
+
begin
|
252
|
+
current_expected_score["goals"][0]["target_modules"]
|
253
|
+
rescue Exception => e
|
254
|
+
raise ExpectationError.new('Analytics has not yet returned an expected score')
|
255
|
+
end
|
256
|
+
end
|
257
|
+
current_expected_score["goals"][0]["target_modules"].each { |value|
|
258
|
+
value["expected_scores"][0]["estimate_date"] = Time.now.strftime(time_format_string)
|
259
|
+
expected_scores[value["module_id"]] << value["expected_scores"][0]
|
260
|
+
}
|
261
|
+
puts "Got new expected score:\n#{current_expected_score.to_yaml}\n"
|
262
|
+
|
263
|
+
if taxon_id != nil
|
264
|
+
current_proficiency = nil
|
265
|
+
eventually timeout: 60, interval: 5 do
|
266
|
+
current_proficiency = kapi.post "v0/accounts/#{account_id}/metrics/proficiency/rows", {"taxon_ids"=>[taxon_id]}
|
267
|
+
begin
|
268
|
+
current_proficiency["rows"][0]["proficiencies"][0]
|
269
|
+
rescue Exception => e
|
270
|
+
raise ExpectationError.new('Analytics has not yet returned a proficiency')
|
271
|
+
end
|
272
|
+
end
|
273
|
+
current_proficiency["rows"][0]["date"] = Time.now.strftime(time_format_string)
|
274
|
+
proficiencies << current_proficiency["rows"][0]
|
275
|
+
puts "Got new current_proficiency:\n#{current_proficiency['rows'].to_yaml}\n"
|
276
|
+
end
|
277
|
+
|
278
|
+
end
|
279
|
+
|
280
|
+
if current_recommendation['module_ids'].empty?
|
281
|
+
puts "Got empty rec set after sending #{mod_ids_sent.length} events; #{cycle_num} adaptive cycles\n"
|
282
|
+
puts "Mod IDs we sent: #{mod_ids_sent.join(', ')}\n"
|
283
|
+
break
|
284
|
+
end
|
285
|
+
|
286
|
+
cycle_num += 1
|
287
|
+
puts "\n--------------------\n"
|
288
|
+
|
289
|
+
end
|
290
|
+
|
291
|
+
if metrics_enabled_goal
|
292
|
+
results["readiness_forecasts"]["registrations"][0]["goals"][0]["readiness_forecasts"] = readiness_forecasts
|
293
|
+
expected_scores.each { |module_id, expected_score_values|
|
294
|
+
results["expected_score"]["registrations"][0]["goals"][0]["target_modules"] << {
|
295
|
+
"module_id" => module_id,
|
296
|
+
"expected_scores" => expected_score_values
|
297
|
+
}
|
298
|
+
}
|
299
|
+
results["proficiency"]["accounts"][0]["taxons"][0]["proficiencies"] = proficiencies.map{ |proficiency|
|
300
|
+
{
|
301
|
+
"relative_estimate" => proficiency["proficiencies"][0]["relative_estimate"],
|
302
|
+
"estimate_status" => proficiency["proficiencies"][0]["estimate_status"],
|
303
|
+
"confidence" => proficiency["proficiencies"][0]["conf"],
|
304
|
+
"estimate_date" => proficiency["date"]
|
305
|
+
}
|
306
|
+
}
|
307
|
+
|
308
|
+
test_data = {
|
309
|
+
"learner_registrations" => {
|
310
|
+
"student_1" => {
|
311
|
+
"id" => registration_id
|
312
|
+
}
|
313
|
+
},
|
314
|
+
"taxon_temp_id_to_id" => {
|
315
|
+
},
|
316
|
+
"goals" => {
|
317
|
+
"goal_1" => {
|
318
|
+
"id" => goal_id
|
319
|
+
}
|
320
|
+
},
|
321
|
+
"start_date" => start_date,
|
322
|
+
"end_date" => Time.now.strftime(time_format_string),
|
323
|
+
"learning_instance" => {
|
324
|
+
"id" => learning_instance_id
|
325
|
+
}
|
326
|
+
}
|
327
|
+
|
328
|
+
write_yaml test_data, results
|
329
|
+
end
|
330
|
+
|
331
|
+
response = {
|
332
|
+
'goal_id' => goal_id,
|
333
|
+
'registration_id' => registration_id,
|
334
|
+
'adaptive_cycles' => cycle_num,
|
335
|
+
'event_count' => event_num,
|
336
|
+
'module_ids_sent' => mod_ids_sent,
|
337
|
+
'event_bodies_sent' => event_bodies_sent,
|
338
|
+
'expected_scores' => expected_scores,
|
339
|
+
'readiness_forecasts' => readiness_forecasts,
|
340
|
+
'proficiencies' => results["proficiency"]["accounts"][0]["taxons"][0]["proficiencies"]
|
341
|
+
}
|
342
|
+
Log.debug "Goal did complete. Details:\n#{response.to_yaml}\n"
|
343
|
+
response
|
344
|
+
end
|
345
|
+
|
346
|
+
|
347
|
+
def send_events_for_recommendation current_recommendation, registration_id, goal, graph, options={}
|
348
|
+
|
349
|
+
#the first event we send shall be a random one from the goal
|
350
|
+
module_id = if current_recommendation
|
351
|
+
case options[:event_selection_mode]
|
352
|
+
when :pick_top_recommended
|
353
|
+
current_recommendation['module_ids'].first
|
354
|
+
when :pick_random_recommended
|
355
|
+
current_recommendation['module_ids'].sample
|
356
|
+
else
|
357
|
+
goal['recommendable_modules'].sample['module_id']
|
358
|
+
end
|
359
|
+
else
|
360
|
+
#this is the first event sent
|
361
|
+
case options[:initial_event_behavior]
|
362
|
+
when :send_focus_event
|
363
|
+
:focus
|
364
|
+
when :random_recommendable_module
|
365
|
+
goal['recommendable_modules'].sample['module_id']
|
366
|
+
else
|
367
|
+
nil
|
368
|
+
end
|
369
|
+
end
|
370
|
+
|
371
|
+
return if module_id.nil?
|
372
|
+
|
373
|
+
if module_id == :focus
|
374
|
+
puts "sending focus event for registration #{registration_id} and goal #{goal['id']}"
|
375
|
+
send_focus_event registration_id, goal['id']
|
376
|
+
# Sleep after focus event to ensure new rec
|
377
|
+
sleep options[:pause_after_sending_event]
|
378
|
+
return nil
|
379
|
+
end
|
380
|
+
|
381
|
+
module_id = internalize_module_id graph, module_id
|
382
|
+
|
383
|
+
node_type = get_node_type graph, module_id
|
384
|
+
|
385
|
+
puts "Following recommendation for module: #{module_id} #{node_type}"
|
386
|
+
atom_ids = case node_type
|
387
|
+
when 'atom'
|
388
|
+
[module_id]
|
389
|
+
when 'bndl'
|
390
|
+
[module_id] + graph['edges'].select{|e| e['type'] == 'contains' and e['start'] == module_id}.map{|e| e['end']}
|
391
|
+
end
|
392
|
+
|
393
|
+
puts "Will send events for these modules: #{atom_ids.join(', ')}"
|
394
|
+
|
395
|
+
event_bodies = []
|
396
|
+
atom_ids.each do |atom|
|
397
|
+
|
398
|
+
concepts = find_associated_concept_ids graph, atom
|
399
|
+
|
400
|
+
# Use the concept threshold for the recommended concept, if applicable
|
401
|
+
# In the unexpected event that there are multiple concepts, use the threshold for the first
|
402
|
+
current_concept_threshold = options[:concept_thresholds].find{|con| con['id'] == concepts[0]}
|
403
|
+
if current_concept_threshold != nil
|
404
|
+
options[:incorrect_threshold] = current_concept_threshold['current_threshold']
|
405
|
+
options[:num_fails_before_success] = current_concept_threshold['num_fails_before_success']
|
406
|
+
end
|
407
|
+
|
408
|
+
event_body = nil
|
409
|
+
|
410
|
+
content_types = get_content_types graph, atom
|
411
|
+
event_type = nil
|
412
|
+
if content_types.include? 'assessment'
|
413
|
+
event_type = :graded
|
414
|
+
elsif content_types.include? 'instructional'
|
415
|
+
event_type = :ungraded
|
416
|
+
end
|
417
|
+
|
418
|
+
options[:is_correct] = true
|
419
|
+
if (rand < options[:incorrect_threshold])
|
420
|
+
options[:is_correct] = false
|
421
|
+
end
|
422
|
+
|
423
|
+
if event_type != nil
|
424
|
+
|
425
|
+
event_bodies << generate_and_send_event(registration_id, atom, event_type, options)
|
426
|
+
sleep options[:pause_after_sending_event]
|
427
|
+
|
428
|
+
if not options[:is_correct] and options[:num_fails_before_success] > 0
|
429
|
+
|
430
|
+
# we already failed once, so let's fail n-1 more times, then succeed
|
431
|
+
num_more_fails_before_success = options[:num_fails_before_success] - 1
|
432
|
+
puts "-------\n"
|
433
|
+
puts "Student will fail #{num_more_fails_before_success} more times before succeeding\n"
|
434
|
+
puts "-------\n\n"
|
435
|
+
|
436
|
+
for i in 1..num_more_fails_before_success do
|
437
|
+
sleep options[:pause_after_sending_event] # wait a bit before sending off the next event
|
438
|
+
event_bodies << generate_and_send_event(registration_id, atom, event_type, options)
|
439
|
+
end
|
440
|
+
|
441
|
+
#student gets is correct!
|
442
|
+
options[:is_correct] = true
|
443
|
+
sleep options[:pause_after_sending_event] # wait a bit before sending off the next event
|
444
|
+
event_bodies << generate_and_send_event(registration_id, atom, event_type, options)
|
445
|
+
|
446
|
+
end
|
447
|
+
|
448
|
+
end
|
449
|
+
end
|
450
|
+
event_bodies
|
451
|
+
end
|
452
|
+
|
453
|
+
def generate_and_send_event registration_id=nil, atom=nil, event_type=:graded, options={}
|
454
|
+
event_body = generate_event_body(event_type, options).merge({'module_id' => atom})
|
455
|
+
puts "sending #{event_type} for registration #{registration_id}:\n#{event_body.to_yaml}\n"
|
456
|
+
if event_type == :graded
|
457
|
+
send_graded_event registration_id, nil, event_body
|
458
|
+
else
|
459
|
+
send_ungraded_event registration_id, nil, event_body
|
460
|
+
end
|
461
|
+
return event_body
|
462
|
+
end
|
463
|
+
|
464
|
+
#return a generated event body, either randomly or according to the is_correct options
|
465
|
+
def generate_event_body type=:graded, options={}
|
466
|
+
options = {
|
467
|
+
:incorrect_threshold => 0.00,
|
468
|
+
:is_correct => nil,
|
469
|
+
}.merge(options)
|
470
|
+
#puts "Current incorrect probability threshold: #{cur_event_threshold}"
|
471
|
+
|
472
|
+
# If user doesn't pass a correctness option, randomly determine correctness
|
473
|
+
if options[:is_correct] == nil
|
474
|
+
if (rand < options[:incorrect_threshold])
|
475
|
+
options[:is_correct] = false
|
476
|
+
else
|
477
|
+
options[:is_correct] = true
|
478
|
+
end
|
479
|
+
end
|
480
|
+
|
481
|
+
rand_fields = case type
|
482
|
+
when :graded
|
483
|
+
if (options[:is_correct])
|
484
|
+
#the correct case
|
485
|
+
{ 'is_correct' => true, 'score' => 1.0, 'response' => rand(1000).to_s }
|
486
|
+
else
|
487
|
+
#the incorrect case
|
488
|
+
{ 'is_correct' => false, 'score' => 0.0, 'response' => rand(1000).to_s }
|
489
|
+
end
|
490
|
+
|
491
|
+
else
|
492
|
+
{}
|
493
|
+
end
|
494
|
+
|
495
|
+
# calculate a more "realistic" duration based off the sleep between events time
|
496
|
+
duration = rand(1000)
|
497
|
+
if options[:pause_after_sending_event] != nil
|
498
|
+
total_time = options[:pause_after_sending_event] * 1000
|
499
|
+
duration = (0.6 * total_time + rand(0.3 * total_time)).to_int
|
500
|
+
end
|
501
|
+
calculated_fields = { 'duration' => duration }
|
502
|
+
|
503
|
+
const_fields = { 'is_complete' => true, 'interaction_end_time' => datetime}
|
504
|
+
return rand_fields.merge(calculated_fields).merge(const_fields)
|
505
|
+
end
|
506
|
+
|
507
|
+
def get_next_recommendation_in_flow registration_id, goal_id, current_recommendation=nil, options={}
|
508
|
+
options = {
|
509
|
+
:new_rec_timeout => 30,
|
510
|
+
}.merge(options)
|
511
|
+
|
512
|
+
eventually timeout: options[:new_rec_timeout] do
|
513
|
+
recommendation = get_recommendation registration_id, goal_id
|
514
|
+
if (!current_recommendation) or (recommendation['recommendation_id'] != current_recommendation['recommendation_id'])
|
515
|
+
current_recommendation = recommendation
|
516
|
+
else
|
517
|
+
raise ExpectationError.new("RecService did not return new recommendation within #{options[:new_rec_timeout]}")
|
518
|
+
end
|
519
|
+
end
|
520
|
+
current_recommendation
|
521
|
+
end
|
522
|
+
|
523
|
+
def write_yaml test_data, results
|
524
|
+
#Setup output directory for current run
|
525
|
+
output_dir = "reports/goalbot_data"
|
526
|
+
FileUtils.mkpath output_dir
|
527
|
+
|
528
|
+
yaml_data = {}
|
529
|
+
yaml_data['students'] = {}
|
530
|
+
yaml_data['env'] = grifter_configuration[:environment].to_s
|
531
|
+
test_data['learner_registrations'].keys.each do |student_name|
|
532
|
+
yaml_data['students'][student_name] = test_data['learner_registrations'][student_name]['id']
|
533
|
+
end
|
534
|
+
yaml_data['taxons'] = {}
|
535
|
+
test_data['taxon_temp_id_to_id'].keys.each do |key|
|
536
|
+
yaml_data['taxons'][test_data['taxon_temp_id_to_name'][key]] = test_data['taxon_temp_id_to_id'][key]
|
537
|
+
end
|
538
|
+
yaml_data['taxon_ids'] = {}
|
539
|
+
test_data['taxon_temp_id_to_id'].keys.each do |key|
|
540
|
+
yaml_data['taxon_ids'][test_data['taxon_temp_id_to_id'][key]] = test_data['taxon_temp_id_to_name'][key]
|
541
|
+
end
|
542
|
+
yaml_data['taxon_temp_ids'] = test_data['taxon_temp_id_to_id']
|
543
|
+
yaml_data['taxon_ids_to_temp'] = test_data['taxon_temp_id_to_id'].invert
|
544
|
+
yaml_data['goals'] = {}
|
545
|
+
test_data['goals'].keys.each do |key|
|
546
|
+
yaml_data['goals'][key] = test_data['goals'][key]['id']
|
547
|
+
end
|
548
|
+
yaml_data['start_date'] = test_data['start_date']
|
549
|
+
yaml_data['end_date'] = test_data['end_date']
|
550
|
+
yaml_data['learning_instance'] = test_data['learning_instance']['id']
|
551
|
+
yaml_data['results'] = results
|
552
|
+
|
553
|
+
output_file = "#{output_dir}/goalbot_run_#{yaml_data['env']}_#{DateTime.now}.yml.gz"
|
554
|
+
|
555
|
+
Log.info "Writing run data to #{output_file}"
|
556
|
+
Zlib::GzipWriter.open(output_file) {|f| f.write YAML::dump(yaml_data) }
|
557
|
+
end
|
558
|
+
|
559
|
+
alias :goal_bot :copy_and_complete_goal
|
560
|
+
alias :get_new_recommendation :get_next_recommendation_in_flow
|