mooc-data-parser 0.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: ef30888b18e521abbe650bbb3f66655a28ac5058
4
+ data.tar.gz: 237e3ebef04827ac78c9e6566b0cdedaaf335541
5
+ SHA512:
6
+ metadata.gz: be54dfa5face8632270d842f5bd6e8510be40fa3209b53816347d25881a7f9e230639ce2a7cf660c80be2b03dd380fb79f719eb42cbd2cec258d3722700f56e8
7
+ data.tar.gz: 355deeba3ab80be18c49895ddd9971aabf677ddbbb2f73bccfc93cadf8c55173eb50a98f7cb3646bd5097944acb5d412c4a86c75ac5fdb7204a02b6462425e78
data/.gitignore ADDED
@@ -0,0 +1,22 @@
1
+ *.gem
2
+ *.rbc
3
+ .bundle
4
+ .config
5
+ .yardoc
6
+ Gemfile.lock
7
+ InstalledFiles
8
+ _yardoc
9
+ coverage
10
+ doc/
11
+ lib/bundler/man
12
+ pkg
13
+ rdoc
14
+ spec/reports
15
+ test/tmp
16
+ test/version_tmp
17
+ tmp
18
+ *.bundle
19
+ *.so
20
+ *.o
21
+ *.a
22
+ mkmf.log
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in mooc-data-parser.gemspec
4
+ gemspec
data/LICENSE.txt ADDED
@@ -0,0 +1,22 @@
1
+ Copyright (c) 2014 Jarmo Isotalo
2
+
3
+ MIT License
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining
6
+ a copy of this software and associated documentation files (the
7
+ "Software"), to deal in the Software without restriction, including
8
+ without limitation the rights to use, copy, modify, merge, publish,
9
+ distribute, sublicense, and/or sell copies of the Software, and to
10
+ permit persons to whom the Software is furnished to do so, subject to
11
+ the following conditions:
12
+
13
+ The above copyright notice and this permission notice shall be
14
+ included in all copies or substantial portions of the Software.
15
+
16
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+ EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+ MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
19
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
20
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
21
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
22
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
data/Rakefile ADDED
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+
data/Readme.md ADDED
@@ -0,0 +1,64 @@
1
+ # Mooc Data Parser
2
+
3
+ ## Installation
4
+
5
+ Or install it yourself as:
6
+
7
+ $ gem install mooc-data-parser
8
+ Scripts made to process some data for our Massive Open Online Courses.
9
+
10
+ ## Usage
11
+
12
+ ```bash
13
+ Usage: show-mooc-details.rb [options]
14
+ -f, --force Reload data from server
15
+ -u, --user username Show details for user
16
+ -m, --missing-points Show missing compulsary points
17
+ -c, --completion-precentige Show completition percentige
18
+ -e, --email emailaddress Show details for user
19
+ -t, --tmc-account tmc-account Show details for user
20
+ -l, --list Show the basic list
21
+ -h, --help Show this message
22
+ ```
23
+
24
+
25
+ Since refreshing data for each search is unnecessary this will cache everything that requires a http request,
26
+ thus making the functionality much faster.
27
+
28
+ To get fresh data, use `-f` command line parameter.
29
+
30
+
31
+ Show basic info for applicants
32
+ ```bash
33
+ ./show-mooc-data.rb -l
34
+ ```
35
+
36
+ Show basic info for applicants and percents for each week
37
+ ```bash
38
+ ./show-mooc-data.rb -l -c
39
+ ```
40
+
41
+ Show basic info for applicants and missing points
42
+ ```bash
43
+ ./show-mooc-data.rb -l -m
44
+ ```
45
+
46
+ Show basic info for applicants and percents for each week and missing points
47
+ ```bash
48
+ ./show-mooc-data.rb -l -c -m
49
+ ```
50
+
51
+ Find info about a user
52
+ * by email:
53
+ ```bash
54
+ ./show-mooc-data.rb -e <email>
55
+ ```
56
+
57
+ * by username:
58
+ ```bash
59
+ ./show-mooc-data.rb -u <username>
60
+ ```
61
+
62
+ Flags `-c` and `-l` can be used with these too.
63
+
64
+
@@ -0,0 +1,11 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ begin
4
+ require 'mooc-data-parser'
5
+ rescue LoadError
6
+ require 'rubygems'
7
+ require 'mooc-data-parser'
8
+ end
9
+
10
+ Mooc::Data::Parser::App.new.run(ARGV)
11
+
@@ -0,0 +1,15 @@
1
+ require "mooc/data/parser/app"
2
+ require "mooc/data/parser/dummy_cacher"
3
+ require "mooc/data/parser/version"
4
+ module Mooc
5
+ module Data
6
+ require 'optparse'
7
+ require 'ostruct'
8
+ require 'httparty'
9
+ require 'json'
10
+ require 'io/console'
11
+ module Parser
12
+ end
13
+ end
14
+ end
15
+
@@ -0,0 +1,262 @@
1
+ #!/usr/bin/env ruby -w
2
+ module Mooc
3
+ module Data
4
+ require 'optparse'
5
+ require 'ostruct'
6
+ require 'httparty'
7
+ require 'json'
8
+ require 'io/console'
9
+ module Parser
10
+ class App
11
+ def run(args)
12
+ $cache ||= Mooc::Data::Parser::DummyCacher.new
13
+ @notes = begin JSON.parse($cache.read_file_from_cache('data.json')) rescue {} end
14
+
15
+ @options = OpenStruct.new
16
+ opt = OptionParser.new do |opts|
17
+ opts.banner = "Usage: show-mooc-details.rb [options]"
18
+
19
+ opts.on("-f", "--force", "Reload data from server") do |v|
20
+ @options.reload = true
21
+ end
22
+ opts.on("-u", "--user username", "Show details for user") do |v|
23
+ @options.user = v
24
+ end
25
+ opts.on("-m", "--missing-points", "Show missing compulsary points") do |v|
26
+ @options.show_missing_compulsory_points = true
27
+ end
28
+ opts.on("-c", "--completion-precentige", "Show completition percentige") do |v|
29
+ @options.show_completion_percentige = true
30
+ end
31
+ opts.on("-e", "--email emailaddress", "Show details for user") do |v|
32
+ @options.user_email = v
33
+ end
34
+ opts.on("-t", "--tmc-account tmc-account", "Show details for user") do |v|
35
+ @options.user_tmc_username = v
36
+ end
37
+ opts.on("-l", "--list", "Show the basic list") do |v|
38
+ @options.list = true
39
+ end
40
+ opts.on_tail("-h", "--help", "Show this message") do
41
+ puts opts
42
+ exit
43
+ end
44
+ end
45
+ opt.parse!(args)
46
+
47
+ json = maybe_fetch_json()
48
+ if @options.user
49
+ show_info_about(@options.user, 'username', json)
50
+ elsif @options.user_email
51
+ show_info_about(@options.user_email, 'email', json)
52
+ elsif @options.user_tmc_username
53
+ show_info_about(@options.user_tmc_username, 'username', json)
54
+ elsif @options.list
55
+ list_and_filter_participants(json)
56
+ else
57
+ $cache.write_file_to_cache('data.json', @notes.to_json)
58
+ puts opt
59
+ abort
60
+ end
61
+ $cache.write_file_to_cache('data.json', @notes.to_json)
62
+ end
63
+
64
+ def get_auth
65
+ print 'username: '
66
+ username = $stdin.gets.strip
67
+ print 'password: '
68
+ password = $stdin.noecho(&:gets).strip
69
+ puts
70
+ {username: username, password: password}
71
+ end
72
+
73
+ def maybe_fetch_json()
74
+ if @options.reload or @notes['user_info'].nil? or @notes['week_data'].nil?
75
+
76
+ t = -> do
77
+ loop do
78
+ print '.'
79
+ sleep 0.5
80
+ end
81
+ puts
82
+ end
83
+
84
+ auth = get_auth()
85
+ th = Thread.new(&t)
86
+
87
+ url = "http://tmc.mooc.fi/mooc/participants.json?api_version=7&utf8=%E2%9C%93&filter_koko_nimi=&column_username=1&column_email=1&column_koko_nimi=1&column_hakee_yliopistoon_2014=1&group_completion_course_id=18"
88
+ user_info = JSON.parse(HTTParty.get(url, basic_auth: auth).body)['participants']
89
+ week_data = fetch_week_datas(auth)
90
+ @notes['user_info'] = user_info.clone
91
+ @notes['week_data'] = week_data.clone
92
+ th.kill
93
+ puts
94
+ {participants: user_info, week_data: week_data}
95
+ else
96
+ {participants: @notes['user_info'].clone, week_data: @notes['week_data'].clone}
97
+ end
98
+ end
99
+
100
+ def show_info_about(user, user_field = 'username', json)
101
+ participants = json[:participants]
102
+ week_data = json[:week_data]
103
+ my_user = participants.find{|a| a[user_field] == user }
104
+ if my_user.nil?
105
+ abort "User not found"
106
+ end
107
+ formatted_print_user_details ["Username", my_user['username']]
108
+ formatted_print_user_details ["Email", my_user['email']]
109
+ formatted_print_user_details ["Hakee yliopistoon", my_user['hakee_yliopistoon_2014']]
110
+ formatted_print_user_details ["Koko nimi", my_user['koko_nimi']]
111
+
112
+ if @options.show_completion_percentige
113
+ formatted_print_user_details ["Points per week"]
114
+ done_exercise_percents(my_user, participants).each do |k|
115
+ begin
116
+ k = k.first
117
+ formatted_print_user_details [k[0], k[1]]
118
+ rescue
119
+ nil
120
+ end
121
+ end
122
+ end
123
+
124
+ if @options.show_missing_compulsory_points
125
+ formatted_print_user_details ["Compulsory points"]
126
+ get_points_info_for_user(my_user, week_data).each do |k,v|
127
+ formatted_print_user_details [k, v.join(", ")]
128
+ end
129
+ end
130
+
131
+ end
132
+
133
+ def formatted_print_user_details(details)
134
+ case details.size
135
+ when 1
136
+ puts "%18s" % details
137
+ when 2
138
+ puts "%18s: %-20s" % details
139
+ end
140
+ end
141
+
142
+ def fetch_week_datas(auth)
143
+ base_url = "http://tmc.mooc.fi/mooc/courses/18/points/"
144
+ weeks = %w(1 2 3 4 5 6 7 8 9 10 11 12)
145
+ rest = ".json?api_version=7"
146
+ week_data = {}
147
+ weeks.each do |week|
148
+ week_data[week] = JSON.parse(HTTParty.get(base_url + week + rest, basic_auth: auth).body)['users_to_points']
149
+ end
150
+ week_data
151
+ end
152
+
153
+ def list_and_filter_participants(json)
154
+ wanted_fields = %W(username email koko_nimi)
155
+
156
+ participants = json[:participants]
157
+ week_data = json[:week_data]
158
+ everyone_in_course = participants.size
159
+ only_applying!(participants)
160
+ hakee_yliopistoon = participants.size
161
+
162
+ puts "%-20s %-35s %-25s %-120s" % ["Username", "Email", "Real name", "Missing points"]
163
+ puts '-'*200
164
+ participants.each do |participant|
165
+ nice_string_in_array = wanted_fields.map do |key|
166
+ participant[key]
167
+ end
168
+ if @options.show_completion_percentige
169
+ nice_string_in_array << format_done_exercises_percents(done_exercise_percents(participant, participants))
170
+ end
171
+ if @options.show_missing_compulsory_points
172
+ nice_string_in_array << missing_points_to_list_string(get_points_info_for_user(participant, week_data))
173
+ end
174
+
175
+
176
+ to_be_printed = "%-20s %-35s %-25s "
177
+ to_be_printed << "%-180s " if @options.show_completion_percentige
178
+ to_be_printed << "%-120s" if @options.show_missing_compulsory_points
179
+
180
+ puts to_be_printed % nice_string_in_array
181
+ end
182
+
183
+ puts
184
+ puts
185
+ puts "Stats: "
186
+ puts "%25s: %4d" % ["Kaikenkaikkiaan kurssilla", everyone_in_course]
187
+ puts "%25s: %4d" % ["Hakee yliopistoon", hakee_yliopistoon]
188
+
189
+ end
190
+
191
+
192
+ def format_done_exercises_percents(hash)
193
+ hash.map do |k|
194
+ begin
195
+ k = k.first
196
+ "#{k[0].scan(/\d+/).first}: #{k[1]}"
197
+ rescue
198
+ nil
199
+ end
200
+ end.compact.join(", ")
201
+ end
202
+
203
+
204
+ def done_exercise_percents(participant, participants_data)
205
+ user_info = participants_data.find{ |p| p['username'] == participant['username'] }
206
+ exercise_weeks = user_info['groups']
207
+ week_keys = (1..12).map{|i| "viikko#{i}"}
208
+
209
+ week_keys.map do |week|
210
+ details = exercise_weeks[week]
211
+ unless details.nil?
212
+ {week => ("%3.1f%" % [(details['points'].to_f / details['total'].to_f) * 100])}
213
+ end
214
+ end
215
+ end
216
+
217
+ def missing_points_to_list_string(missing_by_week)
218
+ str = ""
219
+ missing_by_week.keys.each do |week|
220
+ missing = missing_by_week[week]
221
+ unless missing.nil? or missing.length == 0
222
+ str << week
223
+ str << ": "
224
+ str << missing.join(",")
225
+ str << " "
226
+ end
227
+ end
228
+
229
+ str
230
+
231
+ end
232
+
233
+ def get_points_info_for_user(participant, week_data)
234
+ # TODO: täydennä data viikolle 12
235
+ compulsory_exercises = {'6' => %w(102.1 102.2 102.3 103.1 103.2 103.3), '7' => %w(116.1 116.2 116.3), '8' => %w(124.1 124.2 124.3 124.4),
236
+ '9' => %w(134.1 134.2 134.3 134.4 134.5), '10' => %w(141.1 141.2 141.3 141.4), '11' => %w(151.1 151.2 151.3 151.4), '12' => %w()}
237
+ points_by_week = {}
238
+ week_data.keys.each do |week|
239
+ points_by_week[week] = week_data[week][participant['username']]
240
+ end
241
+
242
+
243
+ missing_by_week = {}
244
+ points_by_week.keys.each do |week|
245
+ weeks_points = points_by_week[week] || [] #palauttaa arrayn
246
+ weeks_compulsory_points = compulsory_exercises[week] || []
247
+ missing_by_week[week] = weeks_compulsory_points - weeks_points
248
+ end
249
+
250
+ missing_by_week
251
+ end
252
+
253
+ def only_applying!(participants)
254
+ participants.select! do |participant|
255
+ participant['hakee_yliopistoon_2014']
256
+ end
257
+ end
258
+ end
259
+ end
260
+ end
261
+ end
262
+ __END__
@@ -0,0 +1,65 @@
1
+
2
+ module Mooc
3
+ module Data
4
+ module Parser
5
+ require 'fileutils'
6
+ class DummyCacher
7
+ def initialize
8
+ FileUtils.mkdir_p(path)
9
+ end
10
+ # Yep, well just overwrite it if it exists
11
+ def cache_file(file)
12
+ FileUtils.cp(file, path)
13
+ end
14
+
15
+ def write_file_to_cache(filename, contents)
16
+ Dir.chdir(path) do
17
+ File.open(filename, "wb") { |file| file.write(contents) }
18
+ end
19
+ end
20
+
21
+ def read_file_from_cache(filename)
22
+ Dir.chdir(path) do
23
+ File.read(filename)
24
+ end
25
+ end
26
+
27
+ def get_from_cache(file_name)
28
+ File.join(path, file_name)
29
+ end
30
+
31
+ def file_exists?(file_name)
32
+ File.exists? get_from_cache(file_name)
33
+ end
34
+
35
+ def tmpdir_path
36
+ Dir.tmpdir
37
+ end
38
+
39
+ def unzip_file(file_name)
40
+ Dir.chdir(path) do
41
+ # Because I have found rubyzip to be buggy, we rely that your system has zip-command available
42
+ `unzip -o #{file_name}`
43
+ end
44
+ end
45
+
46
+ def find_files_matching(matcher)
47
+ files = []
48
+ Dir.chdir(path) do
49
+ files = Dir.glob(matcher)
50
+ end
51
+ files
52
+ end
53
+
54
+ def path
55
+ File.join(Dir.tmpdir, "mooc-data-analyser")
56
+ end
57
+
58
+ def clean!
59
+ FileUtils.rm_rf path
60
+ end
61
+
62
+ end
63
+ end
64
+ end
65
+ end
@@ -0,0 +1,7 @@
1
+ module Mooc
2
+ module Data
3
+ module Parser
4
+ VERSION = "0.0.1"
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,23 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+ require 'mooc/data/parser/version'
5
+
6
+ Gem::Specification.new do |spec|
7
+ spec.name = "mooc-data-parser"
8
+ spec.version = Mooc::Data::Parser::VERSION
9
+ spec.authors = ["Jarmo Isotalo"]
10
+ spec.email = ["jamo@isotalo.fi"]
11
+ spec.summary = %q{A small command line utility to show data from our tmc-server.}
12
+ spec.homepage = ""
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files -z`.split("\x0")
16
+ spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
17
+ spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
18
+ spec.require_paths = ["lib"]
19
+
20
+ spec.add_dependency 'httparty'
21
+ spec.add_development_dependency "bundler", "~> 1.6"
22
+ spec.add_development_dependency "rake"
23
+ end
metadata ADDED
@@ -0,0 +1,98 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: mooc-data-parser
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.1
5
+ platform: ruby
6
+ authors:
7
+ - Jarmo Isotalo
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-04-15 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: httparty
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0'
20
+ type: :runtime
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - ">="
25
+ - !ruby/object:Gem::Version
26
+ version: '0'
27
+ - !ruby/object:Gem::Dependency
28
+ name: bundler
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '1.6'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '1.6'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - ">="
53
+ - !ruby/object:Gem::Version
54
+ version: '0'
55
+ description:
56
+ email:
57
+ - jamo@isotalo.fi
58
+ executables:
59
+ - mooc-data-parser
60
+ extensions: []
61
+ extra_rdoc_files: []
62
+ files:
63
+ - ".gitignore"
64
+ - Gemfile
65
+ - LICENSE.txt
66
+ - Rakefile
67
+ - Readme.md
68
+ - bin/mooc-data-parser
69
+ - lib/mooc/data/parser.rb
70
+ - lib/mooc/data/parser/App.rb
71
+ - lib/mooc/data/parser/dummy_cacher.rb
72
+ - lib/mooc/data/parser/version.rb
73
+ - mooc-data-parser.gemspec
74
+ homepage: ''
75
+ licenses:
76
+ - MIT
77
+ metadata: {}
78
+ post_install_message:
79
+ rdoc_options: []
80
+ require_paths:
81
+ - lib
82
+ required_ruby_version: !ruby/object:Gem::Requirement
83
+ requirements:
84
+ - - ">="
85
+ - !ruby/object:Gem::Version
86
+ version: '0'
87
+ required_rubygems_version: !ruby/object:Gem::Requirement
88
+ requirements:
89
+ - - ">="
90
+ - !ruby/object:Gem::Version
91
+ version: '0'
92
+ requirements: []
93
+ rubyforge_project:
94
+ rubygems_version: 2.2.2
95
+ signing_key:
96
+ specification_version: 4
97
+ summary: A small command line utility to show data from our tmc-server.
98
+ test_files: []