kgrift 1.3.108

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. checksums.yaml +7 -0
  2. data/KGrift/Gemfile +22 -0
  3. data/KGrift/README.md +66 -0
  4. data/KGrift/bin/kgrift +11 -0
  5. data/KGrift/grifter.yml +224 -0
  6. data/KGrift/internal_test_graphs/basic_test_graph_definition.yml +2915 -0
  7. data/KGrift/internal_test_graphs/unicode_test_graph_definition.yml +3070 -0
  8. data/KGrift/knewton_grifts/analytics_grifts.rb +103 -0
  9. data/KGrift/knewton_grifts/async_helper_grifts.rb +63 -0
  10. data/KGrift/knewton_grifts/authenticator_grifts.rb +46 -0
  11. data/KGrift/knewton_grifts/basic_grifts.rb +29 -0
  12. data/KGrift/knewton_grifts/batch_grifts.rb +14 -0
  13. data/KGrift/knewton_grifts/content_collection_grifts.rb +204 -0
  14. data/KGrift/knewton_grifts/content_collection_v1_grifts.rb +521 -0
  15. data/KGrift/knewton_grifts/content_eid_grifts.rb +41 -0
  16. data/KGrift/knewton_grifts/copy_grifts.rb +151 -0
  17. data/KGrift/knewton_grifts/deprecated_graph_and_taxonomy_grifts.rb +353 -0
  18. data/KGrift/knewton_grifts/goal_grifts.rb +203 -0
  19. data/KGrift/knewton_grifts/graph_and_taxonomy_grifts.rb +136 -0
  20. data/KGrift/knewton_grifts/graph_create_grifts.rb +34 -0
  21. data/KGrift/knewton_grifts/graph_query_grifts.rb +448 -0
  22. data/KGrift/knewton_grifts/graph_tools_grifts.rb +151 -0
  23. data/KGrift/knewton_grifts/graph_validation_grifts.rb +447 -0
  24. data/KGrift/knewton_grifts/helper_grifts.rb +92 -0
  25. data/KGrift/knewton_grifts/jmeter_data_grifts.rb +56 -0
  26. data/KGrift/knewton_grifts/learning_instance_grifts.rb +46 -0
  27. data/KGrift/knewton_grifts/looper_grifts.rb +34 -0
  28. data/KGrift/knewton_grifts/moxy_grifts.rb +64 -0
  29. data/KGrift/knewton_grifts/oauth_grifts.rb +182 -0
  30. data/KGrift/knewton_grifts/partner_grifts.rb +70 -0
  31. data/KGrift/knewton_grifts/partner_support_grifts.rb +85 -0
  32. data/KGrift/knewton_grifts/recommendation_setup_grifts.rb +215 -0
  33. data/KGrift/knewton_grifts/registration_grifts.rb +159 -0
  34. data/KGrift/knewton_grifts/registration_info_grifts.rb +23 -0
  35. data/KGrift/knewton_grifts/report_grifts.rb +122 -0
  36. data/KGrift/knewton_grifts/shell_command_grifts.rb +21 -0
  37. data/KGrift/knewton_grifts/student_flow_grifts.rb +560 -0
  38. data/KGrift/knewton_grifts/tag_grifts.rb +41 -0
  39. data/KGrift/knewton_grifts/test_data_grifts.rb +328 -0
  40. data/KGrift/knewton_grifts/test_user_grifts.rb +264 -0
  41. data/KGrift/lib/dtrace.rb +20 -0
  42. data/KGrift/lib/kgrift.rb +7 -0
  43. data/KGrift/test_data_generators/basic_book_and_taxonomies.rb +35 -0
  44. data/KGrift/test_data_generators/lo_test_graph.rb +34 -0
  45. data/KGrift/test_data_generators/partner_owned_book_and_taxonomies.rb +28 -0
  46. data/KGrift/test_data_generators/partner_owned_book_and_taxonomies_unicode.rb +28 -0
  47. data/KGrift/test_data_generators/sandcastle_book_and_taxonomies.rb +13 -0
  48. data/KGrift/test_data_generators/sandcastle_book_and_taxonomies.yml +3709 -0
  49. data/KGrift/test_data_generators/sandcastle_graph.rb +8 -0
  50. data/KGrift/test_data_generators/sandcastle_graph_definition.json +4483 -0
  51. data/KGrift/test_data_generators/sandcastle_graph_full.rb +7 -0
  52. data/KGrift/test_data_generators/sandcastle_taxonomies.yml +378 -0
  53. data/KGrift/test_data_generators/sandcastle_with_taxons.rb +56 -0
  54. data/KGrift/test_data_generators/sandcastle_with_taxons.yml +3994 -0
  55. data/KGrift/test_data_generators/test_users_and_partners.rb +76 -0
  56. data/kgrift.gemspec +43 -0
  57. metadata +144 -0
@@ -0,0 +1,23 @@
1
+ def print_goals_for_registration_report registration_id, goal_id=nil
2
+ reg = get_registration registration_id
3
+
4
+ goals = get_goals reg['learning_instance_id']
5
+
6
+ if goal_id
7
+ goals = goals.select{|g| g['id'] == goal_id}
8
+ end
9
+
10
+ puts " "
11
+ puts "===================="
12
+ puts "Learning instance: #{reg['learning_instance_id']}"
13
+ puts ""
14
+ goals.each do |goal|
15
+ puts "--------------------"
16
+ puts "active: #{check_if_activated(reg['learning_instance_id'], registration_id, goal['id'])}"
17
+ puts "#{goal.to_yaml}"
18
+ puts " "
19
+
20
+ end
21
+
22
+ return ''
23
+ end
@@ -0,0 +1,122 @@
1
+ # Generic methods for writing some reports
2
+ # The reports that can be made are:
3
+ # - all_requests_report - a simple csv noting some details about every request
4
+ # - requests by endpoint - compiles response time stats about each endpoint that has been used
5
+ #
6
+
7
+ require 'fileutils'
8
+
9
+ # construct a path to the report directory, and ensure it exists
10
+ def report_dir
11
+ report_dir = File.join(Dir.pwd, 'reports')
12
+ FileUtils.mkdir_p report_dir
13
+ report_dir
14
+ end
15
+
16
+ # write a csv that lists all requests
17
+ def write_all_requests_report filename=nil
18
+ # default the filename to reports/all_requests.csv
19
+ filename = File.join(report_dir, 'all_requests.csv') unless filename
20
+
21
+ requests = metrics_all_requests
22
+ Log.info "Writing all #{requests.length} requests report to filename: #{filename}"
23
+
24
+ columns = [:service, :method, :path, :status, :duration_ms, :end_time]
25
+ fmt_str = '%-8.8s,%-8.8s,%-61.60s,%-8.8s,%-12.12s,%-28.28s'
26
+ File.open(filename, 'w') do |output|
27
+ output.puts(fmt_str % columns)
28
+ requests.each do |data|
29
+ output.puts(fmt_str % data.to_a)
30
+ end
31
+ end
32
+ Log.debug "Done writing all request report"
33
+ filename
34
+ end
35
+
36
+
37
+ # write a csv that lists all unique endpoints
38
+ # and for each endpoint, some stats about response time
39
+ def write_metrics_by_endpoint_report filename=nil
40
+ filename = File.join(report_dir, 'metrics_by_endpoint.csv') unless filename
41
+
42
+ requests = metrics_all_requests
43
+
44
+ # now compile the list of requests into a list of specific endoints that were hit
45
+ # this means substituting specific IDs in requests paths with the string ':id'
46
+ metrics_by_endpoint = {}
47
+ requests.each do |sample|
48
+ method = sample.method.upcase # GET, POST, PUT, etc...
49
+ path = sample.path.gsub(/\?.*$/,'') # the path, chop off query params
50
+
51
+ #replace uuids with :id
52
+ path.gsub!(/[0-9a-f\-]{36}/,':id')
53
+ #replace any url path component with a capital letter, number or % to :id
54
+ path.gsub!(/[^\/]+[A-Z0-9% ][^\/]+/,':id')
55
+
56
+ endpoint = "#{'%-7s' % method} #{path}"
57
+
58
+ metrics_by_endpoint[endpoint] ||= {
59
+ durations: [],
60
+ count: 0
61
+ }
62
+ metrics_by_endpoint[endpoint][:count] += 1
63
+ metrics_by_endpoint[endpoint][:durations] << sample.duration_ms
64
+ end
65
+
66
+ Log.info "Writing metrics by endpoint report for #{metrics_by_endpoint.length} endpoints to filename: #{filename}"
67
+ File.open(filename, 'w') do |output|
68
+ columns = [:endpoint, :count, :mean_duration, :median_duration, :standard_deviation, :min_duration, :percentile_95, :max_duration]
69
+ fmt_str = '%-80s' + columns[1..-1].inject(''){|s, c| s + ",%-#{c.length+1}s"}
70
+ output.puts fmt_str % columns
71
+
72
+ metrics_by_endpoint.each_pair do |endpoint, data|
73
+ durations = data[:durations].sort
74
+ output.puts fmt_str % [endpoint, data[:count], durations.mean.to_i, durations[durations.length/2], durations.sample_standard_deviation.to_i, durations.first, durations.percentile(0.95).to_i, durations.last]
75
+ end
76
+ end
77
+
78
+ Log.debug "Done writing metrics by endpoint report"
79
+ filename
80
+ end
81
+
82
+
83
+ # MONKEY PATCH ARRAY:
84
+ # methods for doing basic stats calculations on Arrays
85
+ # used in above metrics_by_endpoint_report method
86
+ class ::Array
87
+ def sum
88
+ self.inject(:+).to_f
89
+ end
90
+
91
+ def mean
92
+ return 0.0 if self.length < 1
93
+ self.sum / self.length.to_f
94
+ end
95
+
96
+ def sample_standard_deviation
97
+ #unless we have at least 2 items in list, std dev is 0 by convention
98
+ return 0.0 if self.length < 2
99
+
100
+ mean = self.mean
101
+ sum = self.inject(0.0) { |acc, i| acc + ((i - mean)**2) }
102
+
103
+ Math.sqrt( sum / (self.length - 1))
104
+ end
105
+
106
+ def standard_error_of_the_mean
107
+ return 0.0 if self.length < 2
108
+ (self.sample_standard_deviation / Math.sqrt(self.length))
109
+ end
110
+
111
+ def percentile(percentile)
112
+ return 0.0 if self.length < 1
113
+ return self[0] if self.length < 2
114
+ values_sorted = self.sort
115
+ k = (percentile*(values_sorted.length-1)+1).floor - 1
116
+ f = (percentile*(values_sorted.length-1)+1).modulo(1)
117
+ return values_sorted[k] + (f * (values_sorted[k+1] - values_sorted[k]))
118
+ end
119
+ end
120
+
121
+
122
+
@@ -0,0 +1,21 @@
1
+ """
2
+ This file contains methods that are used to deal with
3
+ shell commands.
4
+
5
+ Nothing should really be using these anymore (it is a holdover
6
+ from how test data used to be managed.) Nonetheless, its possible
7
+ some client of kgrift is out there using this, so its being kept around
8
+
9
+ """
10
+
11
+ class ::ShellCommandFailed < Exception; end
12
+
13
+ def execute_shell_command cmd_string
14
+ Log.debug "Executing shell command:\n\t#{cmd_string}"
15
+ output = %x(#{cmd_string} 2>&1).strip #catch stderr so we can log it properly, strip whitespace to make processing easier
16
+ Log.debug "Shell command returned:\n\t#{output}"
17
+ raise ShellCommandFailed.new("Shell cmd failed.\nCommand: #{cmd_string}\nOutput: #{output}") unless $?.success?
18
+ output
19
+ end
20
+
21
+
@@ -0,0 +1,560 @@
1
+ #
2
+ # Student flow grifts is all about emulating different kinds of student flows for a given goal
3
+ #
4
+
5
+ #based on a source learning instance and goal
6
+ #copy the goal to a new learning instance
7
+ #and send student events per some algorithmn
8
+ # To run multiple goals for a single student, call with the goal_id
9
+ # parameter set to nil and goal_id_list: [ <list of ids> ]
10
+ def copy_and_complete_goal learning_instance_id, goal_id, source_env=nil, options={}
11
+ options ={
12
+ :target_score_override => nil,
13
+ :days_from_now_override => nil,
14
+ :goal_id_list => nil,
15
+ :wait_after_init => false,
16
+ :force_metrics => true,
17
+ :taxon_id => nil
18
+ }.merge(options)
19
+ if goal_id
20
+ puts "Initializing goal bot. Will copy and complete learning instance '#{learning_instance_id}' and goal '#{goal_id}'"
21
+ else
22
+ puts "Initializing goal bot. Will copy and complete learning instance '#{learning_instance_id}' and goals '#{options[:goal_id_list]}'"
23
+ end
24
+
25
+ set_account :knerd
26
+ copy_data = copy_learning_instance 'source_environment' => source_env,
27
+ 'learning_instance_id' => learning_instance_id,
28
+ 'goal_id' => goal_id,
29
+ 'goal_id_list' => options[:goal_id_list],
30
+ 'goal_target_score_override' => options[:target_score_override],
31
+ 'goal_days_from_now_override' => options[:days_from_now_override],
32
+ 'force_metrics' => options[:force_metrics]
33
+ learning_instance = copy_data['learning_instance']
34
+ learning_instance_id = learning_instance['id']
35
+ graph_id = get_learning_instance(learning_instance_id)['graph_id']
36
+
37
+ test_student_profile = add_new_student_to_learning_instance learning_instance_id
38
+ registration_id = test_student_profile['registration_id']
39
+
40
+ if goal_id
41
+ goal = copy_data['goal']
42
+ goal_id = goal['id']
43
+ activate_registration_on_goal learning_instance_id, registration_id, goal_id
44
+ puts "Successfully copied learning instance and goal and created a fresh registration.\n - Learning instance id: #{learning_instance_id}\n - Goal id: #{goal_id}\n - registration id: #{registration_id}\n\n"
45
+ puts "Initialization complete\n"
46
+
47
+ # Wait to give user a chance to send events to canary
48
+ if options[:wait_after_init]
49
+ puts "Hit return key to start sending events"
50
+ STDIN.gets
51
+ end
52
+
53
+ to_return = complete_goal registration_id, goal_id, options
54
+ elsif options[:goal_id_list]
55
+ goal_list = copy_data['goal_list']
56
+ goal_id_list = goal_list.collect { |goal| goal['id'] }
57
+ goal_id_list.each { |goal_id| activate_registration_on_goal learning_instance_id, registration_id, goal_id }
58
+ puts "Successfully copied learning instance and goals and created a fresh registration.\n - Learning instance id: #{learning_instance_id}\n - Goal ids: #{goal_id_list}\n - registration id: #{registration_id}\n\n"
59
+ puts "Initialization complete\n"
60
+
61
+ # Wait to give user a chance to send events to canary
62
+ if options[:wait_after_init]
63
+ puts "Hit return key to start sending events"
64
+ STDIN.gets
65
+ end
66
+
67
+ to_return = goal_id_list.collect { |goal_id| complete_goal registration_id, goal_id, options }
68
+ end
69
+ to_return
70
+ end
71
+
72
+ def goal_bot_with_pause learning_instance_id, goal_id, source_env=nil, options={}
73
+ options = { :wait_after_init => true }.merge(options)
74
+ copy_and_complete_goal learning_instance_id, goal_id, source_env, options
75
+ end
76
+
77
+ def complete_goal registration_id, goal_id, options={}
78
+ default_options = {
79
+ :max_cycles => 100,
80
+ :starting_event_threshold => 0.00,
81
+ :event_threshold_multiplier => 0.96,
82
+ :event_selection_mode => :pick_top_recommended,
83
+ :initial_event_behavior => :send_focus_event,
84
+ :concept_level_thresholds => nil, #this is a file with per-concept data
85
+ :concept_thresholds => nil, #this is a data structure with per-concept data
86
+ :num_fails_before_success => 0,
87
+ :pause_after_sending_event => 6, # increased from 5 to 6 due to reduced rate limit
88
+ :target_score_override => nil,
89
+ :days_from_now_override => nil,
90
+ :goal_id_list => nil,
91
+ :wait_after_init => false,
92
+ :force_metrics => true,
93
+ :taxon_id => nil
94
+ }
95
+
96
+ options.keys.each do |k|
97
+ if !default_options.keys.include?(k)
98
+ raise "Invalid option #{k} passed to complete_goal (NOTE: options are symbols)"
99
+ end
100
+ end
101
+
102
+ options= default_options.merge(options)
103
+
104
+ registration_id = registration_id
105
+ goal_id = goal_id
106
+ taxon_id = options[:taxon_id]
107
+
108
+ set_account :knerd
109
+ registration = get_registration(registration_id)
110
+ account_id = registration['account_id']
111
+ learning_instance = get_learning_instance(get_registration(registration_id)['learning_instance_id'])
112
+ learning_instance_id = learning_instance['id']
113
+ goal = get_goal learning_instance['id'], goal_id
114
+ metrics_enabled_goal = goal['metrics_enabled']
115
+ graph = get_graph_and_cache_it learning_instance['graph_id']
116
+
117
+ if !metrics_enabled_goal
118
+ puts "WARNING: Goal doesn't have metrics_enabled, will NOT be getting ANY metrics."
119
+ end
120
+
121
+ cycle_num ||= 0
122
+ event_num ||= 0
123
+ mod_ids_sent ||= []
124
+ event_bodies_sent ||= []
125
+
126
+ cur_event_threshold = options[:starting_event_threshold]
127
+ num_fails_before_success = options[:num_fails_before_success]
128
+
129
+ # Get concept-level thresholds from a file
130
+ concept_thresholds = []
131
+ if options[:concept_level_thresholds] != nil
132
+ File.open(options[:concept_level_thresholds]) do |f|
133
+ f.each_line do |line|
134
+ line_array = line.split
135
+ concept_thresh = {'id'=> line_array[0],
136
+ 'current_threshold' => line_array[1].to_f,
137
+ 'threshold_multiplier' => line_array[2].to_f,
138
+ 'num_fails_before_success' => line_array[3].to_i,
139
+ }
140
+ concept_thresholds << concept_thresh
141
+ end
142
+ end
143
+ # Or get concept-level thresholds straight from a datastructure
144
+ elsif options[:concept_thresholds] != nil
145
+ concept_thresholds = options[:concept_thresholds]
146
+ end
147
+
148
+ results = {
149
+ "readiness_forecasts" => {
150
+ "learning_instance_id" => learning_instance_id,
151
+ "registrations" => [
152
+ "registration_id" => registration_id,
153
+ "goals" => [
154
+ {
155
+ "goal_id" => goal_id,
156
+ "readiness_forecasts" => []
157
+ }
158
+ ]
159
+ ]
160
+ },
161
+ "expected_score" => {
162
+ "learning_instance_id" => learning_instance_id,
163
+ "registrations" => [
164
+ "registration_id" => registration_id,
165
+ "goals" => [
166
+ {
167
+ "goal_id" => goal_id,
168
+ "target_modules" => []
169
+ }
170
+ ]
171
+ ]
172
+ },
173
+ "proficiency" => {
174
+ "learning_instance_id" => learning_instance_id,
175
+ "accounts" => [
176
+ "account_id" => account_id,
177
+ "taxons" => [
178
+ {
179
+ "taxon_id" => taxon_id,
180
+ "proficiencies" => []
181
+ }
182
+ ]
183
+ ]
184
+ }
185
+ }
186
+
187
+ time_format_string = "%Y-%m-%dT%H:%M:%S"
188
+ current_recommendation = nil
189
+ current_readiness_forecast = nil
190
+ current_expected_score = nil
191
+ current_proficiency = nil
192
+ start_date = Time.now.strftime(time_format_string)
193
+
194
+ readiness_forecasts = []
195
+ expected_scores = Hash.new {|h, k| h[k] = []}
196
+ proficiencies = []
197
+
198
+ loop do
199
+ if cycle_num > options[:max_cycles]
200
+ puts "WARNING: goal did not complete after #{options[:max_cycles]} adaptive cycles"
201
+ break
202
+ end
203
+
204
+ event_bodies = send_events_for_recommendation current_recommendation,
205
+ registration_id, goal, graph,
206
+ (options.merge incorrect_threshold: cur_event_threshold,
207
+ concept_thresholds: concept_thresholds,
208
+ num_fails_before_success: num_fails_before_success)
209
+
210
+ if event_bodies.is_a? Array
211
+ event_bodies_sent += event_bodies
212
+ mod_ids = event_bodies.map{|eb| eb['module_id']}
213
+ mod_ids_sent += mod_ids
214
+ event_num += mod_ids.length
215
+ end
216
+
217
+ puts "\n=====================\nAdaptive cycle number #{cycle_num}\n"
218
+ current_recommendation = get_next_recommendation_in_flow registration_id, goal_id, current_recommendation, options
219
+ puts "Got new recommendation:\n#{current_recommendation.to_yaml}\n"
220
+
221
+ cur_event_threshold *= options[:event_threshold_multiplier]
222
+ concept_thresholds.map{|c| c['current_threshold'] *= c['threshold_multiplier']}
223
+
224
+ current_recommendation['module_ids'].each do |rec_mod_id|
225
+ if mod_ids_sent.include? rec_mod_id
226
+ puts "WARNING: recommendation includes a module we have sent an event for: #{rec_mod_id}"
227
+ #todo add this to an errors array
228
+ end
229
+ end
230
+
231
+ if metrics_enabled_goal
232
+ # use an eventually here to poll until a full payload is returned
233
+ current_readiness_forecast = nil
234
+ eventually timeout: 60, interval: 5 do
235
+ current_readiness_forecast = kapi.post "v0/registrations/#{registration_id}/metrics/readiness-forecast/rows", {"goal_ids"=>[goal_id]}
236
+ # in order to test if analytics has computed something yet, see if we can reference what we want.
237
+ # If we can't, throw an ExpectationError which causes eventually to try again (until timeout)
238
+ begin
239
+ current_readiness_forecast["rows"][0]["date"]
240
+ rescue Exception => e
241
+ raise ExpectationError.new('Analytics has not yet returned a readiness forcast')
242
+ end
243
+ end
244
+ current_readiness_forecast["rows"][0]["date"] = Time.now.strftime(time_format_string)
245
+ readiness_forecasts << current_readiness_forecast["rows"][0]["readiness_forecasts"][0]
246
+ puts "Got new readiness forecast:\n#{current_readiness_forecast.to_yaml}\n"
247
+
248
+ current_expected_score = nil
249
+ eventually timeout: 60, interval: 5 do
250
+ current_expected_score = kapi.get "v0/registrations/#{registration_id}/metrics/expected-score?goal_id=#{goal_id}"
251
+ begin
252
+ current_expected_score["goals"][0]["target_modules"]
253
+ rescue Exception => e
254
+ raise ExpectationError.new('Analytics has not yet returned an expected score')
255
+ end
256
+ end
257
+ current_expected_score["goals"][0]["target_modules"].each { |value|
258
+ value["expected_scores"][0]["estimate_date"] = Time.now.strftime(time_format_string)
259
+ expected_scores[value["module_id"]] << value["expected_scores"][0]
260
+ }
261
+ puts "Got new expected score:\n#{current_expected_score.to_yaml}\n"
262
+
263
+ if taxon_id != nil
264
+ current_proficiency = nil
265
+ eventually timeout: 60, interval: 5 do
266
+ current_proficiency = kapi.post "v0/accounts/#{account_id}/metrics/proficiency/rows", {"taxon_ids"=>[taxon_id]}
267
+ begin
268
+ current_proficiency["rows"][0]["proficiencies"][0]
269
+ rescue Exception => e
270
+ raise ExpectationError.new('Analytics has not yet returned a proficiency')
271
+ end
272
+ end
273
+ current_proficiency["rows"][0]["date"] = Time.now.strftime(time_format_string)
274
+ proficiencies << current_proficiency["rows"][0]
275
+ puts "Got new current_proficiency:\n#{current_proficiency['rows'].to_yaml}\n"
276
+ end
277
+
278
+ end
279
+
280
+ if current_recommendation['module_ids'].empty?
281
+ puts "Got empty rec set after sending #{mod_ids_sent.length} events; #{cycle_num} adaptive cycles\n"
282
+ puts "Mod IDs we sent: #{mod_ids_sent.join(', ')}\n"
283
+ break
284
+ end
285
+
286
+ cycle_num += 1
287
+ puts "\n--------------------\n"
288
+
289
+ end
290
+
291
+ if metrics_enabled_goal
292
+ results["readiness_forecasts"]["registrations"][0]["goals"][0]["readiness_forecasts"] = readiness_forecasts
293
+ expected_scores.each { |module_id, expected_score_values|
294
+ results["expected_score"]["registrations"][0]["goals"][0]["target_modules"] << {
295
+ "module_id" => module_id,
296
+ "expected_scores" => expected_score_values
297
+ }
298
+ }
299
+ results["proficiency"]["accounts"][0]["taxons"][0]["proficiencies"] = proficiencies.map{ |proficiency|
300
+ {
301
+ "relative_estimate" => proficiency["proficiencies"][0]["relative_estimate"],
302
+ "estimate_status" => proficiency["proficiencies"][0]["estimate_status"],
303
+ "confidence" => proficiency["proficiencies"][0]["conf"],
304
+ "estimate_date" => proficiency["date"]
305
+ }
306
+ }
307
+
308
+ test_data = {
309
+ "learner_registrations" => {
310
+ "student_1" => {
311
+ "id" => registration_id
312
+ }
313
+ },
314
+ "taxon_temp_id_to_id" => {
315
+ },
316
+ "goals" => {
317
+ "goal_1" => {
318
+ "id" => goal_id
319
+ }
320
+ },
321
+ "start_date" => start_date,
322
+ "end_date" => Time.now.strftime(time_format_string),
323
+ "learning_instance" => {
324
+ "id" => learning_instance_id
325
+ }
326
+ }
327
+
328
+ write_yaml test_data, results
329
+ end
330
+
331
+ response = {
332
+ 'goal_id' => goal_id,
333
+ 'registration_id' => registration_id,
334
+ 'adaptive_cycles' => cycle_num,
335
+ 'event_count' => event_num,
336
+ 'module_ids_sent' => mod_ids_sent,
337
+ 'event_bodies_sent' => event_bodies_sent,
338
+ 'expected_scores' => expected_scores,
339
+ 'readiness_forecasts' => readiness_forecasts,
340
+ 'proficiencies' => results["proficiency"]["accounts"][0]["taxons"][0]["proficiencies"]
341
+ }
342
+ Log.debug "Goal did complete. Details:\n#{response.to_yaml}\n"
343
+ response
344
+ end
345
+
346
+
347
+ def send_events_for_recommendation current_recommendation, registration_id, goal, graph, options={}
348
+
349
+ #the first event we send shall be a random one from the goal
350
+ module_id = if current_recommendation
351
+ case options[:event_selection_mode]
352
+ when :pick_top_recommended
353
+ current_recommendation['module_ids'].first
354
+ when :pick_random_recommended
355
+ current_recommendation['module_ids'].sample
356
+ else
357
+ goal['recommendable_modules'].sample['module_id']
358
+ end
359
+ else
360
+ #this is the first event sent
361
+ case options[:initial_event_behavior]
362
+ when :send_focus_event
363
+ :focus
364
+ when :random_recommendable_module
365
+ goal['recommendable_modules'].sample['module_id']
366
+ else
367
+ nil
368
+ end
369
+ end
370
+
371
+ return if module_id.nil?
372
+
373
+ if module_id == :focus
374
+ puts "sending focus event for registration #{registration_id} and goal #{goal['id']}"
375
+ send_focus_event registration_id, goal['id']
376
+ # Sleep after focus event to ensure new rec
377
+ sleep options[:pause_after_sending_event]
378
+ return nil
379
+ end
380
+
381
+ module_id = internalize_module_id graph, module_id
382
+
383
+ node_type = get_node_type graph, module_id
384
+
385
+ puts "Following recommendation for module: #{module_id} #{node_type}"
386
+ atom_ids = case node_type
387
+ when 'atom'
388
+ [module_id]
389
+ when 'bndl'
390
+ [module_id] + graph['edges'].select{|e| e['type'] == 'contains' and e['start'] == module_id}.map{|e| e['end']}
391
+ end
392
+
393
+ puts "Will send events for these modules: #{atom_ids.join(', ')}"
394
+
395
+ event_bodies = []
396
+ atom_ids.each do |atom|
397
+
398
+ concepts = find_associated_concept_ids graph, atom
399
+
400
+ # Use the concept threshold for the recommended concept, if applicable
401
+ # In the unexpected event that there are multiple concepts, use the threshold for the first
402
+ current_concept_threshold = options[:concept_thresholds].find{|con| con['id'] == concepts[0]}
403
+ if current_concept_threshold != nil
404
+ options[:incorrect_threshold] = current_concept_threshold['current_threshold']
405
+ options[:num_fails_before_success] = current_concept_threshold['num_fails_before_success']
406
+ end
407
+
408
+ event_body = nil
409
+
410
+ content_types = get_content_types graph, atom
411
+ event_type = nil
412
+ if content_types.include? 'assessment'
413
+ event_type = :graded
414
+ elsif content_types.include? 'instructional'
415
+ event_type = :ungraded
416
+ end
417
+
418
+ options[:is_correct] = true
419
+ if (rand < options[:incorrect_threshold])
420
+ options[:is_correct] = false
421
+ end
422
+
423
+ if event_type != nil
424
+
425
+ event_bodies << generate_and_send_event(registration_id, atom, event_type, options)
426
+ sleep options[:pause_after_sending_event]
427
+
428
+ if not options[:is_correct] and options[:num_fails_before_success] > 0
429
+
430
+ # we already failed once, so let's fail n-1 more times, then succeed
431
+ num_more_fails_before_success = options[:num_fails_before_success] - 1
432
+ puts "-------\n"
433
+ puts "Student will fail #{num_more_fails_before_success} more times before succeeding\n"
434
+ puts "-------\n\n"
435
+
436
+ for i in 1..num_more_fails_before_success do
437
+ sleep options[:pause_after_sending_event] # wait a bit before sending off the next event
438
+ event_bodies << generate_and_send_event(registration_id, atom, event_type, options)
439
+ end
440
+
441
+ #student gets is correct!
442
+ options[:is_correct] = true
443
+ sleep options[:pause_after_sending_event] # wait a bit before sending off the next event
444
+ event_bodies << generate_and_send_event(registration_id, atom, event_type, options)
445
+
446
+ end
447
+
448
+ end
449
+ end
450
+ event_bodies
451
+ end
452
+
453
+ def generate_and_send_event registration_id=nil, atom=nil, event_type=:graded, options={}
454
+ event_body = generate_event_body(event_type, options).merge({'module_id' => atom})
455
+ puts "sending #{event_type} for registration #{registration_id}:\n#{event_body.to_yaml}\n"
456
+ if event_type == :graded
457
+ send_graded_event registration_id, nil, event_body
458
+ else
459
+ send_ungraded_event registration_id, nil, event_body
460
+ end
461
+ return event_body
462
+ end
463
+
464
+ #return a generated event body, either randomly or according to the is_correct options
465
+ def generate_event_body type=:graded, options={}
466
+ options = {
467
+ :incorrect_threshold => 0.00,
468
+ :is_correct => nil,
469
+ }.merge(options)
470
+ #puts "Current incorrect probability threshold: #{cur_event_threshold}"
471
+
472
+ # If user doesn't pass a correctness option, randomly determine correctness
473
+ if options[:is_correct] == nil
474
+ if (rand < options[:incorrect_threshold])
475
+ options[:is_correct] = false
476
+ else
477
+ options[:is_correct] = true
478
+ end
479
+ end
480
+
481
+ rand_fields = case type
482
+ when :graded
483
+ if (options[:is_correct])
484
+ #the correct case
485
+ { 'is_correct' => true, 'score' => 1.0, 'response' => rand(1000).to_s }
486
+ else
487
+ #the incorrect case
488
+ { 'is_correct' => false, 'score' => 0.0, 'response' => rand(1000).to_s }
489
+ end
490
+
491
+ else
492
+ {}
493
+ end
494
+
495
+ # calculate a more "realistic" duration based off the sleep between events time
496
+ duration = rand(1000)
497
+ if options[:pause_after_sending_event] != nil
498
+ total_time = options[:pause_after_sending_event] * 1000
499
+ duration = (0.6 * total_time + rand(0.3 * total_time)).to_int
500
+ end
501
+ calculated_fields = { 'duration' => duration }
502
+
503
+ const_fields = { 'is_complete' => true, 'interaction_end_time' => datetime}
504
+ return rand_fields.merge(calculated_fields).merge(const_fields)
505
+ end
506
+
507
+ def get_next_recommendation_in_flow registration_id, goal_id, current_recommendation=nil, options={}
508
+ options = {
509
+ :new_rec_timeout => 30,
510
+ }.merge(options)
511
+
512
+ eventually timeout: options[:new_rec_timeout] do
513
+ recommendation = get_recommendation registration_id, goal_id
514
+ if (!current_recommendation) or (recommendation['recommendation_id'] != current_recommendation['recommendation_id'])
515
+ current_recommendation = recommendation
516
+ else
517
+ raise ExpectationError.new("RecService did not return new recommendation within #{options[:new_rec_timeout]}")
518
+ end
519
+ end
520
+ current_recommendation
521
+ end
522
+
523
+ def write_yaml test_data, results
524
+ #Setup output directory for current run
525
+ output_dir = "reports/goalbot_data"
526
+ FileUtils.mkpath output_dir
527
+
528
+ yaml_data = {}
529
+ yaml_data['students'] = {}
530
+ yaml_data['env'] = grifter_configuration[:environment].to_s
531
+ test_data['learner_registrations'].keys.each do |student_name|
532
+ yaml_data['students'][student_name] = test_data['learner_registrations'][student_name]['id']
533
+ end
534
+ yaml_data['taxons'] = {}
535
+ test_data['taxon_temp_id_to_id'].keys.each do |key|
536
+ yaml_data['taxons'][test_data['taxon_temp_id_to_name'][key]] = test_data['taxon_temp_id_to_id'][key]
537
+ end
538
+ yaml_data['taxon_ids'] = {}
539
+ test_data['taxon_temp_id_to_id'].keys.each do |key|
540
+ yaml_data['taxon_ids'][test_data['taxon_temp_id_to_id'][key]] = test_data['taxon_temp_id_to_name'][key]
541
+ end
542
+ yaml_data['taxon_temp_ids'] = test_data['taxon_temp_id_to_id']
543
+ yaml_data['taxon_ids_to_temp'] = test_data['taxon_temp_id_to_id'].invert
544
+ yaml_data['goals'] = {}
545
+ test_data['goals'].keys.each do |key|
546
+ yaml_data['goals'][key] = test_data['goals'][key]['id']
547
+ end
548
+ yaml_data['start_date'] = test_data['start_date']
549
+ yaml_data['end_date'] = test_data['end_date']
550
+ yaml_data['learning_instance'] = test_data['learning_instance']['id']
551
+ yaml_data['results'] = results
552
+
553
+ output_file = "#{output_dir}/goalbot_run_#{yaml_data['env']}_#{DateTime.now}.yml.gz"
554
+
555
+ Log.info "Writing run data to #{output_file}"
556
+ Zlib::GzipWriter.open(output_file) {|f| f.write YAML::dump(yaml_data) }
557
+ end
558
+
559
+ alias :goal_bot :copy_and_complete_goal
560
+ alias :get_new_recommendation :get_next_recommendation_in_flow