mooc-data-parser 0.0.1 → 0.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/mooc-data-parser +3 -3
- data/lib/mooc_data_parser.rb +11 -0
- data/lib/mooc_data_parser/App.rb +257 -0
- data/lib/mooc_data_parser/dummy_cacher.rb +62 -0
- data/lib/mooc_data_parser/version.rb +3 -0
- data/mooc-data-parser.gemspec +3 -3
- metadata +6 -6
- data/lib/mooc/data/parser.rb +0 -15
- data/lib/mooc/data/parser/App.rb +0 -262
- data/lib/mooc/data/parser/dummy_cacher.rb +0 -65
- data/lib/mooc/data/parser/version.rb +0 -7
    
        checksums.yaml
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            ---
         | 
| 2 2 | 
             
            SHA1:
         | 
| 3 | 
            -
              metadata.gz:  | 
| 4 | 
            -
              data.tar.gz:  | 
| 3 | 
            +
              metadata.gz: 23122ed3f4e7051ac6a952ff2adf73eb2baf8272
         | 
| 4 | 
            +
              data.tar.gz: 770b789ec3c408141a70f28f7aaaafcb496f0997
         | 
| 5 5 | 
             
            SHA512:
         | 
| 6 | 
            -
              metadata.gz:  | 
| 7 | 
            -
              data.tar.gz:  | 
| 6 | 
            +
              metadata.gz: 92d0ed65d8546d19fe4158a75103675700e99b7e7655fb4b0183b275a11e06fc198f1b40efddf3d0982e60593a677b196d9e1e600202f1069c14a3249cf98665
         | 
| 7 | 
            +
              data.tar.gz: 82c2cf673b4aa5ec4462e1d86eb4b918372ad3dbe8a9bae078b8a025b16f81ac09ceb2e7415829eff493a1367120e98c5b33b248cb9d78e64a44f84dcdc990f5
         | 
    
        data/bin/mooc-data-parser
    CHANGED
    
    | @@ -1,11 +1,11 @@ | |
| 1 1 | 
             
            #!/usr/bin/env ruby
         | 
| 2 2 |  | 
| 3 3 | 
             
            begin
         | 
| 4 | 
            -
              require ' | 
| 4 | 
            +
              require 'mooc_data_parser'
         | 
| 5 5 | 
             
             rescue LoadError
         | 
| 6 6 | 
             
              require 'rubygems'
         | 
| 7 | 
            -
              require ' | 
| 7 | 
            +
              require 'mooc_data_parser'
         | 
| 8 8 | 
             
            end
         | 
| 9 9 |  | 
| 10 | 
            -
             | 
| 10 | 
            +
            MoocDataParser::App.new.run(ARGV)
         | 
| 11 11 |  | 
| @@ -0,0 +1,257 @@ | |
| 1 | 
            +
            #!/usr/bin/env ruby -w
         | 
| 2 | 
            +
            module MoocDataParser
         | 
| 3 | 
            +
              require 'optparse'
         | 
| 4 | 
            +
              require 'ostruct'
         | 
| 5 | 
            +
              require 'httparty'
         | 
| 6 | 
            +
              require 'json'
         | 
| 7 | 
            +
              require 'io/console'
         | 
| 8 | 
            +
              class App
         | 
| 9 | 
            +
                def run(args)
         | 
| 10 | 
            +
                  $cache ||= MoocDataParser::DummyCacher.new
         | 
| 11 | 
            +
                  @notes = begin JSON.parse($cache.read_file_from_cache('data.json')) rescue  {} end
         | 
| 12 | 
            +
             | 
| 13 | 
            +
                  @options = OpenStruct.new
         | 
| 14 | 
            +
                  opt = OptionParser.new do |opts|
         | 
| 15 | 
            +
                    opts.banner = "Usage: show-mooc-details.rb [options]"
         | 
| 16 | 
            +
             | 
| 17 | 
            +
                    opts.on("-f", "--force", "Reload data from server") do |v|
         | 
| 18 | 
            +
                      @options.reload = true
         | 
| 19 | 
            +
                    end
         | 
| 20 | 
            +
                    opts.on("-u", "--user username", "Show details for user") do |v|
         | 
| 21 | 
            +
                      @options.user = v
         | 
| 22 | 
            +
                    end
         | 
| 23 | 
            +
                    opts.on("-m", "--missing-points", "Show missing compulsary points") do |v|
         | 
| 24 | 
            +
                      @options.show_missing_compulsory_points = true
         | 
| 25 | 
            +
                    end
         | 
| 26 | 
            +
                    opts.on("-c", "--completion-precentige", "Show completition percentige") do |v|
         | 
| 27 | 
            +
                      @options.show_completion_percentige = true
         | 
| 28 | 
            +
                    end
         | 
| 29 | 
            +
                    opts.on("-e", "--email emailaddress", "Show details for user") do |v|
         | 
| 30 | 
            +
                      @options.user_email = v
         | 
| 31 | 
            +
                    end
         | 
| 32 | 
            +
                    opts.on("-t", "--tmc-account tmc-account", "Show details for user") do |v|
         | 
| 33 | 
            +
                      @options.user_tmc_username = v
         | 
| 34 | 
            +
                    end
         | 
| 35 | 
            +
                    opts.on("-l", "--list", "Show the basic list") do |v|
         | 
| 36 | 
            +
                      @options.list = true
         | 
| 37 | 
            +
                    end
         | 
| 38 | 
            +
                    opts.on_tail("-h", "--help", "Show this message") do
         | 
| 39 | 
            +
                      puts opts
         | 
| 40 | 
            +
                      exit
         | 
| 41 | 
            +
                    end
         | 
| 42 | 
            +
                  end
         | 
| 43 | 
            +
                  opt.parse!(args)
         | 
| 44 | 
            +
             | 
| 45 | 
            +
                  json = maybe_fetch_json()
         | 
| 46 | 
            +
                  if @options.user
         | 
| 47 | 
            +
                    show_info_about(@options.user, 'username', json)
         | 
| 48 | 
            +
                  elsif @options.user_email
         | 
| 49 | 
            +
                    show_info_about(@options.user_email, 'email', json)
         | 
| 50 | 
            +
                  elsif @options.user_tmc_username
         | 
| 51 | 
            +
                    show_info_about(@options.user_tmc_username, 'username', json)
         | 
| 52 | 
            +
                  elsif @options.list
         | 
| 53 | 
            +
                    list_and_filter_participants(json)
         | 
| 54 | 
            +
                  else
         | 
| 55 | 
            +
                    $cache.write_file_to_cache('data.json', @notes.to_json)
         | 
| 56 | 
            +
                    puts opt
         | 
| 57 | 
            +
                    abort
         | 
| 58 | 
            +
                  end
         | 
| 59 | 
            +
                  $cache.write_file_to_cache('data.json', @notes.to_json)
         | 
| 60 | 
            +
                end
         | 
| 61 | 
            +
             | 
| 62 | 
            +
                def get_auth
         | 
| 63 | 
            +
                  print 'username: '
         | 
| 64 | 
            +
                  username = $stdin.gets.strip
         | 
| 65 | 
            +
                  print 'password: '
         | 
| 66 | 
            +
                  password = $stdin.noecho(&:gets).strip
         | 
| 67 | 
            +
                  puts
         | 
| 68 | 
            +
                  {username: username, password: password}
         | 
| 69 | 
            +
                end
         | 
| 70 | 
            +
             | 
| 71 | 
            +
                def maybe_fetch_json()
         | 
| 72 | 
            +
                  if @options.reload or @notes['user_info'].nil? or @notes['week_data'].nil?
         | 
| 73 | 
            +
             | 
| 74 | 
            +
                    t = -> do
         | 
| 75 | 
            +
                      loop do
         | 
| 76 | 
            +
                        print '.'
         | 
| 77 | 
            +
                        sleep 0.5
         | 
| 78 | 
            +
                      end
         | 
| 79 | 
            +
                      puts
         | 
| 80 | 
            +
                    end
         | 
| 81 | 
            +
             | 
| 82 | 
            +
                    auth = get_auth()
         | 
| 83 | 
            +
                    th = Thread.new(&t)
         | 
| 84 | 
            +
             | 
| 85 | 
            +
                    url = "http://tmc.mooc.fi/mooc/participants.json?api_version=7&utf8=%E2%9C%93&filter_koko_nimi=&column_username=1&column_email=1&column_koko_nimi=1&column_hakee_yliopistoon_2014=1&group_completion_course_id=18"
         | 
| 86 | 
            +
                    user_info = JSON.parse(HTTParty.get(url, basic_auth: auth).body)['participants']
         | 
| 87 | 
            +
                    week_data = fetch_week_datas(auth)
         | 
| 88 | 
            +
                    @notes['user_info'] = user_info.clone
         | 
| 89 | 
            +
                    @notes['week_data'] = week_data.clone
         | 
| 90 | 
            +
                    th.kill
         | 
| 91 | 
            +
                    puts
         | 
| 92 | 
            +
                    {participants: user_info, week_data: week_data}
         | 
| 93 | 
            +
                  else
         | 
| 94 | 
            +
                    {participants: @notes['user_info'].clone, week_data: @notes['week_data'].clone}
         | 
| 95 | 
            +
                  end
         | 
| 96 | 
            +
                end
         | 
| 97 | 
            +
             | 
| 98 | 
            +
                def show_info_about(user, user_field = 'username', json)
         | 
| 99 | 
            +
                  participants = json[:participants]
         | 
| 100 | 
            +
                  week_data = json[:week_data]
         | 
| 101 | 
            +
                  my_user = participants.find{|a| a[user_field] == user }
         | 
| 102 | 
            +
                  if my_user.nil?
         | 
| 103 | 
            +
                    abort "User not found"
         | 
| 104 | 
            +
                  end
         | 
| 105 | 
            +
                  formatted_print_user_details ["Username", my_user['username']]
         | 
| 106 | 
            +
                  formatted_print_user_details ["Email", my_user['email']]
         | 
| 107 | 
            +
                  formatted_print_user_details ["Hakee yliopistoon", my_user['hakee_yliopistoon_2014']]
         | 
| 108 | 
            +
                  formatted_print_user_details ["Koko nimi", my_user['koko_nimi']]
         | 
| 109 | 
            +
             | 
| 110 | 
            +
                  if @options.show_completion_percentige
         | 
| 111 | 
            +
                    formatted_print_user_details ["Points per week"]
         | 
| 112 | 
            +
                    done_exercise_percents(my_user, participants).each do |k|
         | 
| 113 | 
            +
                      begin
         | 
| 114 | 
            +
                        k = k.first
         | 
| 115 | 
            +
                        formatted_print_user_details [k[0], k[1]]
         | 
| 116 | 
            +
                      rescue
         | 
| 117 | 
            +
                        nil
         | 
| 118 | 
            +
                      end
         | 
| 119 | 
            +
                    end
         | 
| 120 | 
            +
                  end
         | 
| 121 | 
            +
             | 
| 122 | 
            +
                  if @options.show_missing_compulsory_points
         | 
| 123 | 
            +
                    formatted_print_user_details ["Compulsory points"]
         | 
| 124 | 
            +
                    get_points_info_for_user(my_user, week_data).each do |k,v|
         | 
| 125 | 
            +
                      formatted_print_user_details [k, v.join(", ")]
         | 
| 126 | 
            +
                    end
         | 
| 127 | 
            +
                  end
         | 
| 128 | 
            +
             | 
| 129 | 
            +
                end
         | 
| 130 | 
            +
             | 
| 131 | 
            +
                def formatted_print_user_details(details)
         | 
| 132 | 
            +
                  case details.size
         | 
| 133 | 
            +
                  when 1
         | 
| 134 | 
            +
                    puts "%18s" % details
         | 
| 135 | 
            +
                  when 2
         | 
| 136 | 
            +
                    puts "%18s: %-20s" % details
         | 
| 137 | 
            +
                  end
         | 
| 138 | 
            +
                end
         | 
| 139 | 
            +
             | 
| 140 | 
            +
                def fetch_week_datas(auth)
         | 
| 141 | 
            +
                  base_url = "http://tmc.mooc.fi/mooc/courses/18/points/"
         | 
| 142 | 
            +
                  weeks = %w(1 2 3 4 5 6 7 8 9 10 11 12)
         | 
| 143 | 
            +
                  rest = ".json?api_version=7"
         | 
| 144 | 
            +
                  week_data = {}
         | 
| 145 | 
            +
                  weeks.each do |week|
         | 
| 146 | 
            +
                    week_data[week] = JSON.parse(HTTParty.get(base_url + week + rest, basic_auth: auth).body)['users_to_points']
         | 
| 147 | 
            +
                  end
         | 
| 148 | 
            +
                  week_data
         | 
| 149 | 
            +
                end
         | 
| 150 | 
            +
             | 
| 151 | 
            +
                def list_and_filter_participants(json)
         | 
| 152 | 
            +
                  wanted_fields = %W(username email koko_nimi)
         | 
| 153 | 
            +
             | 
| 154 | 
            +
                  participants = json[:participants]
         | 
| 155 | 
            +
                  week_data = json[:week_data]
         | 
| 156 | 
            +
                  everyone_in_course = participants.size
         | 
| 157 | 
            +
                  only_applying!(participants)
         | 
| 158 | 
            +
                  hakee_yliopistoon = participants.size
         | 
| 159 | 
            +
             | 
| 160 | 
            +
                  puts "%-20s %-35s %-25s %-120s" % ["Username", "Email", "Real name", "Missing points"]
         | 
| 161 | 
            +
                  puts '-'*200
         | 
| 162 | 
            +
                  participants.each do |participant|
         | 
| 163 | 
            +
                    nice_string_in_array = wanted_fields.map do |key|
         | 
| 164 | 
            +
                      participant[key]
         | 
| 165 | 
            +
                    end
         | 
| 166 | 
            +
                    if @options.show_completion_percentige
         | 
| 167 | 
            +
                      nice_string_in_array << format_done_exercises_percents(done_exercise_percents(participant, participants))
         | 
| 168 | 
            +
                    end
         | 
| 169 | 
            +
                    if @options.show_missing_compulsory_points
         | 
| 170 | 
            +
                      nice_string_in_array << missing_points_to_list_string(get_points_info_for_user(participant, week_data))
         | 
| 171 | 
            +
                    end
         | 
| 172 | 
            +
             | 
| 173 | 
            +
             | 
| 174 | 
            +
                    to_be_printed = "%-20s %-35s %-25s "
         | 
| 175 | 
            +
                    to_be_printed << "%-180s " if @options.show_completion_percentige
         | 
| 176 | 
            +
                    to_be_printed << "%-120s" if @options.show_missing_compulsory_points
         | 
| 177 | 
            +
             | 
| 178 | 
            +
                    puts to_be_printed % nice_string_in_array
         | 
| 179 | 
            +
                  end
         | 
| 180 | 
            +
             | 
| 181 | 
            +
                  puts
         | 
| 182 | 
            +
                  puts
         | 
| 183 | 
            +
                  puts "Stats: "
         | 
| 184 | 
            +
                  puts "%25s: %4d" % ["Kaikenkaikkiaan kurssilla", everyone_in_course]
         | 
| 185 | 
            +
                  puts "%25s: %4d" % ["Hakee yliopistoon", hakee_yliopistoon]
         | 
| 186 | 
            +
             | 
| 187 | 
            +
                end
         | 
| 188 | 
            +
             | 
| 189 | 
            +
             | 
| 190 | 
            +
                def format_done_exercises_percents(hash)
         | 
| 191 | 
            +
                  hash.map do |k|
         | 
| 192 | 
            +
                    begin
         | 
| 193 | 
            +
                      k = k.first
         | 
| 194 | 
            +
                      "#{k[0].scan(/\d+/).first}: #{k[1]}"
         | 
| 195 | 
            +
                    rescue
         | 
| 196 | 
            +
                      nil
         | 
| 197 | 
            +
                    end
         | 
| 198 | 
            +
                  end.compact.join(", ")
         | 
| 199 | 
            +
                end
         | 
| 200 | 
            +
             | 
| 201 | 
            +
             | 
| 202 | 
            +
                def done_exercise_percents(participant, participants_data)
         | 
| 203 | 
            +
                  user_info = participants_data.find{ |p| p['username'] == participant['username'] }
         | 
| 204 | 
            +
                  exercise_weeks = user_info['groups']
         | 
| 205 | 
            +
                  week_keys = (1..12).map{|i| "viikko#{i}"}
         | 
| 206 | 
            +
             | 
| 207 | 
            +
                  week_keys.map do |week|
         | 
| 208 | 
            +
                    details = exercise_weeks[week]
         | 
| 209 | 
            +
                    unless details.nil?
         | 
| 210 | 
            +
                      {week => ("%3.1f%" % [(details['points'].to_f / details['total'].to_f) * 100])}
         | 
| 211 | 
            +
                    end
         | 
| 212 | 
            +
                  end
         | 
| 213 | 
            +
                end
         | 
| 214 | 
            +
             | 
| 215 | 
            +
                def missing_points_to_list_string(missing_by_week)
         | 
| 216 | 
            +
                  str = ""
         | 
| 217 | 
            +
                  missing_by_week.keys.each do |week|
         | 
| 218 | 
            +
                    missing = missing_by_week[week]
         | 
| 219 | 
            +
                    unless missing.nil? or missing.length == 0
         | 
| 220 | 
            +
                      str << week
         | 
| 221 | 
            +
                      str << ": "
         | 
| 222 | 
            +
                      str << missing.join(",")
         | 
| 223 | 
            +
                      str << "  "
         | 
| 224 | 
            +
                    end
         | 
| 225 | 
            +
                  end
         | 
| 226 | 
            +
             | 
| 227 | 
            +
                  str
         | 
| 228 | 
            +
             | 
| 229 | 
            +
                end
         | 
| 230 | 
            +
             | 
| 231 | 
            +
                def get_points_info_for_user(participant, week_data)
         | 
| 232 | 
            +
                  # TODO: täydennä data viikolle 12
         | 
| 233 | 
            +
                  compulsory_exercises = {'6' => %w(102.1 102.2 102.3 103.1 103.2 103.3), '7' => %w(116.1 116.2 116.3), '8' => %w(124.1 124.2 124.3 124.4),
         | 
| 234 | 
            +
                                          '9' => %w(134.1 134.2 134.3 134.4 134.5), '10' => %w(141.1 141.2 141.3 141.4), '11' => %w(151.1 151.2 151.3 151.4), '12' => %w()}
         | 
| 235 | 
            +
                  points_by_week = {}
         | 
| 236 | 
            +
                  week_data.keys.each do |week|
         | 
| 237 | 
            +
                    points_by_week[week] = week_data[week][participant['username']]
         | 
| 238 | 
            +
                  end
         | 
| 239 | 
            +
             | 
| 240 | 
            +
             | 
| 241 | 
            +
                  missing_by_week = {}
         | 
| 242 | 
            +
                  points_by_week.keys.each  do |week|
         | 
| 243 | 
            +
                    weeks_points = points_by_week[week] || [] #palauttaa arrayn
         | 
| 244 | 
            +
                    weeks_compulsory_points = compulsory_exercises[week] || []
         | 
| 245 | 
            +
                    missing_by_week[week] = weeks_compulsory_points - weeks_points
         | 
| 246 | 
            +
                  end
         | 
| 247 | 
            +
             | 
| 248 | 
            +
                  missing_by_week
         | 
| 249 | 
            +
                end
         | 
| 250 | 
            +
             | 
| 251 | 
            +
                def only_applying!(participants)
         | 
| 252 | 
            +
                  participants.select! do |participant|
         | 
| 253 | 
            +
                    participant['hakee_yliopistoon_2014']
         | 
| 254 | 
            +
                  end
         | 
| 255 | 
            +
                end
         | 
| 256 | 
            +
              end
         | 
| 257 | 
            +
            end
         | 
| @@ -0,0 +1,62 @@ | |
| 1 | 
            +
             | 
| 2 | 
            +
            module MoocDataParser
         | 
| 3 | 
            +
              require 'fileutils'
         | 
| 4 | 
            +
              require 'tmpdir'
         | 
| 5 | 
            +
              class DummyCacher
         | 
| 6 | 
            +
                def initialize
         | 
| 7 | 
            +
                  FileUtils.mkdir_p(path)
         | 
| 8 | 
            +
                end
         | 
| 9 | 
            +
                # Yep, well just overwrite it if it exists
         | 
| 10 | 
            +
                def cache_file(file)
         | 
| 11 | 
            +
                  FileUtils.cp(file, path)
         | 
| 12 | 
            +
                end
         | 
| 13 | 
            +
             | 
| 14 | 
            +
                def write_file_to_cache(filename, contents)
         | 
| 15 | 
            +
                  Dir.chdir(path) do
         | 
| 16 | 
            +
                    File.open(filename, "wb") { |file| file.write(contents) }
         | 
| 17 | 
            +
                  end
         | 
| 18 | 
            +
                end
         | 
| 19 | 
            +
             | 
| 20 | 
            +
                def read_file_from_cache(filename)
         | 
| 21 | 
            +
                  Dir.chdir(path) do
         | 
| 22 | 
            +
                    File.read(filename)
         | 
| 23 | 
            +
                  end
         | 
| 24 | 
            +
                end
         | 
| 25 | 
            +
             | 
| 26 | 
            +
                def get_from_cache(file_name)
         | 
| 27 | 
            +
                  File.join(path,  file_name)
         | 
| 28 | 
            +
                end
         | 
| 29 | 
            +
             | 
| 30 | 
            +
                def file_exists?(file_name)
         | 
| 31 | 
            +
                  File.exists? get_from_cache(file_name)
         | 
| 32 | 
            +
                end
         | 
| 33 | 
            +
             | 
| 34 | 
            +
                def tmpdir_path
         | 
| 35 | 
            +
                  Dir.tmpdir
         | 
| 36 | 
            +
                end
         | 
| 37 | 
            +
             | 
| 38 | 
            +
                def unzip_file(file_name)
         | 
| 39 | 
            +
                  Dir.chdir(path) do
         | 
| 40 | 
            +
                    # Because I have found rubyzip to be buggy, we rely that your system has zip-command available
         | 
| 41 | 
            +
                    `unzip -o  #{file_name}`
         | 
| 42 | 
            +
                  end
         | 
| 43 | 
            +
                end
         | 
| 44 | 
            +
             | 
| 45 | 
            +
                def find_files_matching(matcher)
         | 
| 46 | 
            +
                  files = []
         | 
| 47 | 
            +
                  Dir.chdir(path) do
         | 
| 48 | 
            +
                    files = Dir.glob(matcher)
         | 
| 49 | 
            +
                  end
         | 
| 50 | 
            +
                  files
         | 
| 51 | 
            +
                end
         | 
| 52 | 
            +
             | 
| 53 | 
            +
                def path
         | 
| 54 | 
            +
                  File.join(tmpdir_path, "mooc-data-analyser")
         | 
| 55 | 
            +
                end
         | 
| 56 | 
            +
             | 
| 57 | 
            +
                def clean!
         | 
| 58 | 
            +
                  FileUtils.rm_rf path
         | 
| 59 | 
            +
                end
         | 
| 60 | 
            +
             | 
| 61 | 
            +
              end
         | 
| 62 | 
            +
            end
         | 
    
        data/mooc-data-parser.gemspec
    CHANGED
    
    | @@ -1,15 +1,15 @@ | |
| 1 1 | 
             
            # coding: utf-8
         | 
| 2 2 | 
             
            lib = File.expand_path('../lib', __FILE__)
         | 
| 3 3 | 
             
            $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
         | 
| 4 | 
            -
            require ' | 
| 4 | 
            +
            require 'mooc_data_parser/version'
         | 
| 5 5 |  | 
| 6 6 | 
             
            Gem::Specification.new do |spec|
         | 
| 7 7 | 
             
              spec.name          = "mooc-data-parser"
         | 
| 8 | 
            -
              spec.version       =  | 
| 8 | 
            +
              spec.version       = MoocDataParser::VERSION
         | 
| 9 9 | 
             
              spec.authors       = ["Jarmo Isotalo"]
         | 
| 10 10 | 
             
              spec.email         = ["jamo@isotalo.fi"]
         | 
| 11 11 | 
             
              spec.summary       = %q{A small command line utility to show data from our tmc-server.}
         | 
| 12 | 
            -
              spec.homepage      = ""
         | 
| 12 | 
            +
              spec.homepage      = "https://github.com/jamox/mooc-data-parser"
         | 
| 13 13 | 
             
              spec.license       = "MIT"
         | 
| 14 14 |  | 
| 15 15 | 
             
              spec.files         = `git ls-files -z`.split("\x0")
         | 
    
        metadata
    CHANGED
    
    | @@ -1,7 +1,7 @@ | |
| 1 1 | 
             
            --- !ruby/object:Gem::Specification
         | 
| 2 2 | 
             
            name: mooc-data-parser
         | 
| 3 3 | 
             
            version: !ruby/object:Gem::Version
         | 
| 4 | 
            -
              version: 0.0. | 
| 4 | 
            +
              version: 0.0.3
         | 
| 5 5 | 
             
            platform: ruby
         | 
| 6 6 | 
             
            authors:
         | 
| 7 7 | 
             
            - Jarmo Isotalo
         | 
| @@ -66,12 +66,12 @@ files: | |
| 66 66 | 
             
            - Rakefile
         | 
| 67 67 | 
             
            - Readme.md
         | 
| 68 68 | 
             
            - bin/mooc-data-parser
         | 
| 69 | 
            -
            - lib/ | 
| 70 | 
            -
            - lib/ | 
| 71 | 
            -
            - lib/ | 
| 72 | 
            -
            - lib/ | 
| 69 | 
            +
            - lib/mooc_data_parser.rb
         | 
| 70 | 
            +
            - lib/mooc_data_parser/App.rb
         | 
| 71 | 
            +
            - lib/mooc_data_parser/dummy_cacher.rb
         | 
| 72 | 
            +
            - lib/mooc_data_parser/version.rb
         | 
| 73 73 | 
             
            - mooc-data-parser.gemspec
         | 
| 74 | 
            -
            homepage:  | 
| 74 | 
            +
            homepage: https://github.com/jamox/mooc-data-parser
         | 
| 75 75 | 
             
            licenses:
         | 
| 76 76 | 
             
            - MIT
         | 
| 77 77 | 
             
            metadata: {}
         | 
    
        data/lib/mooc/data/parser.rb
    DELETED
    
    | @@ -1,15 +0,0 @@ | |
| 1 | 
            -
            require "mooc/data/parser/app"
         | 
| 2 | 
            -
            require "mooc/data/parser/dummy_cacher"
         | 
| 3 | 
            -
            require "mooc/data/parser/version"
         | 
| 4 | 
            -
            module Mooc
         | 
| 5 | 
            -
              module Data
         | 
| 6 | 
            -
                require 'optparse'
         | 
| 7 | 
            -
                require 'ostruct'
         | 
| 8 | 
            -
                require 'httparty'
         | 
| 9 | 
            -
                require 'json'
         | 
| 10 | 
            -
                require 'io/console'
         | 
| 11 | 
            -
                module Parser
         | 
| 12 | 
            -
                end
         | 
| 13 | 
            -
              end
         | 
| 14 | 
            -
            end
         | 
| 15 | 
            -
             | 
    
        data/lib/mooc/data/parser/App.rb
    DELETED
    
    | @@ -1,262 +0,0 @@ | |
| 1 | 
            -
            #!/usr/bin/env ruby -w
         | 
| 2 | 
            -
            module Mooc
         | 
| 3 | 
            -
              module Data
         | 
| 4 | 
            -
                require 'optparse'
         | 
| 5 | 
            -
                require 'ostruct'
         | 
| 6 | 
            -
                require 'httparty'
         | 
| 7 | 
            -
                require 'json'
         | 
| 8 | 
            -
                require 'io/console'
         | 
| 9 | 
            -
                module Parser
         | 
| 10 | 
            -
                  class App
         | 
| 11 | 
            -
                    def run(args)
         | 
| 12 | 
            -
                      $cache ||= Mooc::Data::Parser::DummyCacher.new
         | 
| 13 | 
            -
                      @notes = begin JSON.parse($cache.read_file_from_cache('data.json')) rescue  {} end
         | 
| 14 | 
            -
             | 
| 15 | 
            -
                      @options = OpenStruct.new
         | 
| 16 | 
            -
                      opt = OptionParser.new do |opts|
         | 
| 17 | 
            -
                        opts.banner = "Usage: show-mooc-details.rb [options]"
         | 
| 18 | 
            -
             | 
| 19 | 
            -
                        opts.on("-f", "--force", "Reload data from server") do |v|
         | 
| 20 | 
            -
                          @options.reload = true
         | 
| 21 | 
            -
                        end
         | 
| 22 | 
            -
                        opts.on("-u", "--user username", "Show details for user") do |v|
         | 
| 23 | 
            -
                          @options.user = v
         | 
| 24 | 
            -
                        end
         | 
| 25 | 
            -
                        opts.on("-m", "--missing-points", "Show missing compulsary points") do |v|
         | 
| 26 | 
            -
                          @options.show_missing_compulsory_points = true
         | 
| 27 | 
            -
                        end
         | 
| 28 | 
            -
                        opts.on("-c", "--completion-precentige", "Show completition percentige") do |v|
         | 
| 29 | 
            -
                          @options.show_completion_percentige = true
         | 
| 30 | 
            -
                        end
         | 
| 31 | 
            -
                        opts.on("-e", "--email emailaddress", "Show details for user") do |v|
         | 
| 32 | 
            -
                          @options.user_email = v
         | 
| 33 | 
            -
                        end
         | 
| 34 | 
            -
                        opts.on("-t", "--tmc-account tmc-account", "Show details for user") do |v|
         | 
| 35 | 
            -
                          @options.user_tmc_username = v
         | 
| 36 | 
            -
                        end
         | 
| 37 | 
            -
                        opts.on("-l", "--list", "Show the basic list") do |v|
         | 
| 38 | 
            -
                          @options.list = true
         | 
| 39 | 
            -
                        end
         | 
| 40 | 
            -
                        opts.on_tail("-h", "--help", "Show this message") do
         | 
| 41 | 
            -
                          puts opts
         | 
| 42 | 
            -
                          exit
         | 
| 43 | 
            -
                        end
         | 
| 44 | 
            -
                      end
         | 
| 45 | 
            -
                      opt.parse!(args)
         | 
| 46 | 
            -
             | 
| 47 | 
            -
                      json = maybe_fetch_json()
         | 
| 48 | 
            -
                      if @options.user
         | 
| 49 | 
            -
                        show_info_about(@options.user, 'username', json)
         | 
| 50 | 
            -
                      elsif @options.user_email
         | 
| 51 | 
            -
                        show_info_about(@options.user_email, 'email', json)
         | 
| 52 | 
            -
                      elsif @options.user_tmc_username
         | 
| 53 | 
            -
                        show_info_about(@options.user_tmc_username, 'username', json)
         | 
| 54 | 
            -
                      elsif @options.list
         | 
| 55 | 
            -
                        list_and_filter_participants(json)
         | 
| 56 | 
            -
                      else
         | 
| 57 | 
            -
                        $cache.write_file_to_cache('data.json', @notes.to_json)
         | 
| 58 | 
            -
                        puts opt
         | 
| 59 | 
            -
                        abort
         | 
| 60 | 
            -
                      end
         | 
| 61 | 
            -
                      $cache.write_file_to_cache('data.json', @notes.to_json)
         | 
| 62 | 
            -
                    end
         | 
| 63 | 
            -
             | 
| 64 | 
            -
                    def get_auth
         | 
| 65 | 
            -
                      print 'username: '
         | 
| 66 | 
            -
                      username = $stdin.gets.strip
         | 
| 67 | 
            -
                      print 'password: '
         | 
| 68 | 
            -
                      password = $stdin.noecho(&:gets).strip
         | 
| 69 | 
            -
                      puts
         | 
| 70 | 
            -
                      {username: username, password: password}
         | 
| 71 | 
            -
                    end
         | 
| 72 | 
            -
             | 
| 73 | 
            -
                    def maybe_fetch_json()
         | 
| 74 | 
            -
                      if @options.reload or @notes['user_info'].nil? or @notes['week_data'].nil?
         | 
| 75 | 
            -
             | 
| 76 | 
            -
                        t = -> do
         | 
| 77 | 
            -
                          loop do
         | 
| 78 | 
            -
                            print '.'
         | 
| 79 | 
            -
                            sleep 0.5
         | 
| 80 | 
            -
                          end
         | 
| 81 | 
            -
                          puts
         | 
| 82 | 
            -
                        end
         | 
| 83 | 
            -
             | 
| 84 | 
            -
                        auth = get_auth()
         | 
| 85 | 
            -
                        th = Thread.new(&t)
         | 
| 86 | 
            -
             | 
| 87 | 
            -
                        url = "http://tmc.mooc.fi/mooc/participants.json?api_version=7&utf8=%E2%9C%93&filter_koko_nimi=&column_username=1&column_email=1&column_koko_nimi=1&column_hakee_yliopistoon_2014=1&group_completion_course_id=18"
         | 
| 88 | 
            -
                        user_info = JSON.parse(HTTParty.get(url, basic_auth: auth).body)['participants']
         | 
| 89 | 
            -
                        week_data = fetch_week_datas(auth)
         | 
| 90 | 
            -
                        @notes['user_info'] = user_info.clone
         | 
| 91 | 
            -
                        @notes['week_data'] = week_data.clone
         | 
| 92 | 
            -
                        th.kill
         | 
| 93 | 
            -
                        puts
         | 
| 94 | 
            -
                        {participants: user_info, week_data: week_data}
         | 
| 95 | 
            -
                      else
         | 
| 96 | 
            -
                        {participants: @notes['user_info'].clone, week_data: @notes['week_data'].clone}
         | 
| 97 | 
            -
                      end
         | 
| 98 | 
            -
                    end
         | 
| 99 | 
            -
             | 
| 100 | 
            -
                    def show_info_about(user, user_field = 'username', json)
         | 
| 101 | 
            -
                      participants = json[:participants]
         | 
| 102 | 
            -
                      week_data = json[:week_data]
         | 
| 103 | 
            -
                      my_user = participants.find{|a| a[user_field] == user }
         | 
| 104 | 
            -
                      if my_user.nil?
         | 
| 105 | 
            -
                        abort "User not found"
         | 
| 106 | 
            -
                      end
         | 
| 107 | 
            -
                      formatted_print_user_details ["Username", my_user['username']]
         | 
| 108 | 
            -
                      formatted_print_user_details ["Email", my_user['email']]
         | 
| 109 | 
            -
                      formatted_print_user_details ["Hakee yliopistoon", my_user['hakee_yliopistoon_2014']]
         | 
| 110 | 
            -
                      formatted_print_user_details ["Koko nimi", my_user['koko_nimi']]
         | 
| 111 | 
            -
             | 
| 112 | 
            -
                      if @options.show_completion_percentige
         | 
| 113 | 
            -
                        formatted_print_user_details ["Points per week"]
         | 
| 114 | 
            -
                        done_exercise_percents(my_user, participants).each do |k|
         | 
| 115 | 
            -
                          begin
         | 
| 116 | 
            -
                            k = k.first
         | 
| 117 | 
            -
                            formatted_print_user_details [k[0], k[1]]
         | 
| 118 | 
            -
                          rescue
         | 
| 119 | 
            -
                            nil
         | 
| 120 | 
            -
                          end
         | 
| 121 | 
            -
                        end
         | 
| 122 | 
            -
                      end
         | 
| 123 | 
            -
             | 
| 124 | 
            -
                      if @options.show_missing_compulsory_points
         | 
| 125 | 
            -
                        formatted_print_user_details ["Compulsory points"]
         | 
| 126 | 
            -
                        get_points_info_for_user(my_user, week_data).each do |k,v|
         | 
| 127 | 
            -
                          formatted_print_user_details [k, v.join(", ")]
         | 
| 128 | 
            -
                        end
         | 
| 129 | 
            -
                      end
         | 
| 130 | 
            -
             | 
| 131 | 
            -
                    end
         | 
| 132 | 
            -
             | 
| 133 | 
            -
                    def formatted_print_user_details(details)
         | 
| 134 | 
            -
                      case details.size
         | 
| 135 | 
            -
                      when 1
         | 
| 136 | 
            -
                        puts "%18s" % details
         | 
| 137 | 
            -
                      when 2
         | 
| 138 | 
            -
                        puts "%18s: %-20s" % details
         | 
| 139 | 
            -
                      end
         | 
| 140 | 
            -
                    end
         | 
| 141 | 
            -
             | 
| 142 | 
            -
                    def fetch_week_datas(auth)
         | 
| 143 | 
            -
                      base_url = "http://tmc.mooc.fi/mooc/courses/18/points/"
         | 
| 144 | 
            -
                      weeks = %w(1 2 3 4 5 6 7 8 9 10 11 12)
         | 
| 145 | 
            -
                      rest = ".json?api_version=7"
         | 
| 146 | 
            -
                      week_data = {}
         | 
| 147 | 
            -
                      weeks.each do |week|
         | 
| 148 | 
            -
                        week_data[week] = JSON.parse(HTTParty.get(base_url + week + rest, basic_auth: auth).body)['users_to_points']
         | 
| 149 | 
            -
                      end
         | 
| 150 | 
            -
                      week_data
         | 
| 151 | 
            -
                    end
         | 
| 152 | 
            -
             | 
| 153 | 
            -
                    def list_and_filter_participants(json)
         | 
| 154 | 
            -
                      wanted_fields = %W(username email koko_nimi)
         | 
| 155 | 
            -
             | 
| 156 | 
            -
                      participants = json[:participants]
         | 
| 157 | 
            -
                      week_data = json[:week_data]
         | 
| 158 | 
            -
                      everyone_in_course = participants.size
         | 
| 159 | 
            -
                      only_applying!(participants)
         | 
| 160 | 
            -
                      hakee_yliopistoon = participants.size
         | 
| 161 | 
            -
             | 
| 162 | 
            -
                      puts "%-20s %-35s %-25s %-120s" % ["Username", "Email", "Real name", "Missing points"]
         | 
| 163 | 
            -
                      puts '-'*200
         | 
| 164 | 
            -
                      participants.each do |participant|
         | 
| 165 | 
            -
                        nice_string_in_array = wanted_fields.map do |key|
         | 
| 166 | 
            -
                          participant[key]
         | 
| 167 | 
            -
                        end
         | 
| 168 | 
            -
                        if @options.show_completion_percentige
         | 
| 169 | 
            -
                          nice_string_in_array << format_done_exercises_percents(done_exercise_percents(participant, participants))
         | 
| 170 | 
            -
                        end
         | 
| 171 | 
            -
                        if @options.show_missing_compulsory_points
         | 
| 172 | 
            -
                          nice_string_in_array << missing_points_to_list_string(get_points_info_for_user(participant, week_data))
         | 
| 173 | 
            -
                        end
         | 
| 174 | 
            -
             | 
| 175 | 
            -
             | 
| 176 | 
            -
                        to_be_printed = "%-20s %-35s %-25s "
         | 
| 177 | 
            -
                        to_be_printed << "%-180s " if @options.show_completion_percentige
         | 
| 178 | 
            -
                        to_be_printed << "%-120s" if @options.show_missing_compulsory_points
         | 
| 179 | 
            -
             | 
| 180 | 
            -
                        puts to_be_printed % nice_string_in_array
         | 
| 181 | 
            -
                      end
         | 
| 182 | 
            -
             | 
| 183 | 
            -
                      puts
         | 
| 184 | 
            -
                      puts
         | 
| 185 | 
            -
                      puts "Stats: "
         | 
| 186 | 
            -
                      puts "%25s: %4d" % ["Kaikenkaikkiaan kurssilla", everyone_in_course]
         | 
| 187 | 
            -
                      puts "%25s: %4d" % ["Hakee yliopistoon", hakee_yliopistoon]
         | 
| 188 | 
            -
             | 
| 189 | 
            -
                    end
         | 
| 190 | 
            -
             | 
| 191 | 
            -
             | 
| 192 | 
            -
                    def format_done_exercises_percents(hash)
         | 
| 193 | 
            -
                      hash.map do |k|
         | 
| 194 | 
            -
                        begin
         | 
| 195 | 
            -
                          k = k.first
         | 
| 196 | 
            -
                          "#{k[0].scan(/\d+/).first}: #{k[1]}"
         | 
| 197 | 
            -
                        rescue
         | 
| 198 | 
            -
                          nil
         | 
| 199 | 
            -
                        end
         | 
| 200 | 
            -
                      end.compact.join(", ")
         | 
| 201 | 
            -
                    end
         | 
| 202 | 
            -
             | 
| 203 | 
            -
             | 
| 204 | 
            -
                    def done_exercise_percents(participant, participants_data)
         | 
| 205 | 
            -
                      user_info = participants_data.find{ |p| p['username'] == participant['username'] }
         | 
| 206 | 
            -
                      exercise_weeks = user_info['groups']
         | 
| 207 | 
            -
                      week_keys = (1..12).map{|i| "viikko#{i}"}
         | 
| 208 | 
            -
             | 
| 209 | 
            -
                      week_keys.map do |week|
         | 
| 210 | 
            -
                        details = exercise_weeks[week]
         | 
| 211 | 
            -
                        unless details.nil?
         | 
| 212 | 
            -
                          {week => ("%3.1f%" % [(details['points'].to_f / details['total'].to_f) * 100])}
         | 
| 213 | 
            -
                        end
         | 
| 214 | 
            -
                      end
         | 
| 215 | 
            -
                    end
         | 
| 216 | 
            -
             | 
| 217 | 
            -
                    def missing_points_to_list_string(missing_by_week)
         | 
| 218 | 
            -
                      str = ""
         | 
| 219 | 
            -
                      missing_by_week.keys.each do |week|
         | 
| 220 | 
            -
                        missing = missing_by_week[week]
         | 
| 221 | 
            -
                        unless missing.nil? or missing.length == 0
         | 
| 222 | 
            -
                          str << week
         | 
| 223 | 
            -
                          str << ": "
         | 
| 224 | 
            -
                          str << missing.join(",")
         | 
| 225 | 
            -
                          str << "  "
         | 
| 226 | 
            -
                        end
         | 
| 227 | 
            -
                      end
         | 
| 228 | 
            -
             | 
| 229 | 
            -
                      str
         | 
| 230 | 
            -
             | 
| 231 | 
            -
                    end
         | 
| 232 | 
            -
             | 
| 233 | 
            -
                    def get_points_info_for_user(participant, week_data)
         | 
| 234 | 
            -
                      # TODO: täydennä data viikolle 12
         | 
| 235 | 
            -
                      compulsory_exercises = {'6' => %w(102.1 102.2 102.3 103.1 103.2 103.3), '7' => %w(116.1 116.2 116.3), '8' => %w(124.1 124.2 124.3 124.4),
         | 
| 236 | 
            -
                                              '9' => %w(134.1 134.2 134.3 134.4 134.5), '10' => %w(141.1 141.2 141.3 141.4), '11' => %w(151.1 151.2 151.3 151.4), '12' => %w()}
         | 
| 237 | 
            -
                      points_by_week = {}
         | 
| 238 | 
            -
                      week_data.keys.each do |week|
         | 
| 239 | 
            -
                        points_by_week[week] = week_data[week][participant['username']]
         | 
| 240 | 
            -
                      end
         | 
| 241 | 
            -
             | 
| 242 | 
            -
             | 
| 243 | 
            -
                      missing_by_week = {}
         | 
| 244 | 
            -
                      points_by_week.keys.each  do |week|
         | 
| 245 | 
            -
                        weeks_points = points_by_week[week] || [] #palauttaa arrayn
         | 
| 246 | 
            -
                        weeks_compulsory_points = compulsory_exercises[week] || []
         | 
| 247 | 
            -
                        missing_by_week[week] = weeks_compulsory_points - weeks_points
         | 
| 248 | 
            -
                      end
         | 
| 249 | 
            -
             | 
| 250 | 
            -
                      missing_by_week
         | 
| 251 | 
            -
                    end
         | 
| 252 | 
            -
             | 
| 253 | 
            -
                    def only_applying!(participants)
         | 
| 254 | 
            -
                      participants.select! do |participant|
         | 
| 255 | 
            -
                        participant['hakee_yliopistoon_2014']
         | 
| 256 | 
            -
                      end
         | 
| 257 | 
            -
                    end
         | 
| 258 | 
            -
                  end
         | 
| 259 | 
            -
                end
         | 
| 260 | 
            -
              end
         | 
| 261 | 
            -
            end
         | 
| 262 | 
            -
            __END__
         | 
| @@ -1,65 +0,0 @@ | |
| 1 | 
            -
             | 
| 2 | 
            -
            module Mooc
         | 
| 3 | 
            -
              module Data
         | 
| 4 | 
            -
                module Parser
         | 
| 5 | 
            -
                  require 'fileutils'
         | 
| 6 | 
            -
                  class DummyCacher
         | 
| 7 | 
            -
                    def initialize
         | 
| 8 | 
            -
                      FileUtils.mkdir_p(path)
         | 
| 9 | 
            -
                    end
         | 
| 10 | 
            -
                    # Yep, well just overwrite it if it exists
         | 
| 11 | 
            -
                    def cache_file(file)
         | 
| 12 | 
            -
                      FileUtils.cp(file, path)
         | 
| 13 | 
            -
                    end
         | 
| 14 | 
            -
             | 
| 15 | 
            -
                    def write_file_to_cache(filename, contents)
         | 
| 16 | 
            -
                      Dir.chdir(path) do
         | 
| 17 | 
            -
                        File.open(filename, "wb") { |file| file.write(contents) }
         | 
| 18 | 
            -
                      end
         | 
| 19 | 
            -
                    end
         | 
| 20 | 
            -
             | 
| 21 | 
            -
                    def read_file_from_cache(filename)
         | 
| 22 | 
            -
                      Dir.chdir(path) do
         | 
| 23 | 
            -
                        File.read(filename)
         | 
| 24 | 
            -
                      end
         | 
| 25 | 
            -
                    end
         | 
| 26 | 
            -
             | 
| 27 | 
            -
                    def get_from_cache(file_name)
         | 
| 28 | 
            -
                      File.join(path,  file_name)
         | 
| 29 | 
            -
                    end
         | 
| 30 | 
            -
             | 
| 31 | 
            -
                    def file_exists?(file_name)
         | 
| 32 | 
            -
                      File.exists? get_from_cache(file_name)
         | 
| 33 | 
            -
                    end
         | 
| 34 | 
            -
             | 
| 35 | 
            -
                    def tmpdir_path
         | 
| 36 | 
            -
                      Dir.tmpdir
         | 
| 37 | 
            -
                    end
         | 
| 38 | 
            -
             | 
| 39 | 
            -
                    def unzip_file(file_name)
         | 
| 40 | 
            -
                      Dir.chdir(path) do
         | 
| 41 | 
            -
                        # Because I have found rubyzip to be buggy, we rely that your system has zip-command available
         | 
| 42 | 
            -
                        `unzip -o  #{file_name}`
         | 
| 43 | 
            -
                      end
         | 
| 44 | 
            -
                    end
         | 
| 45 | 
            -
             | 
| 46 | 
            -
                    def find_files_matching(matcher)
         | 
| 47 | 
            -
                      files = []
         | 
| 48 | 
            -
                      Dir.chdir(path) do
         | 
| 49 | 
            -
                        files = Dir.glob(matcher)
         | 
| 50 | 
            -
                      end
         | 
| 51 | 
            -
                      files
         | 
| 52 | 
            -
                    end
         | 
| 53 | 
            -
             | 
| 54 | 
            -
                    def path
         | 
| 55 | 
            -
                      File.join(Dir.tmpdir, "mooc-data-analyser")
         | 
| 56 | 
            -
                    end
         | 
| 57 | 
            -
             | 
| 58 | 
            -
                    def clean!
         | 
| 59 | 
            -
                      FileUtils.rm_rf path
         | 
| 60 | 
            -
                    end
         | 
| 61 | 
            -
             | 
| 62 | 
            -
                  end
         | 
| 63 | 
            -
                end
         | 
| 64 | 
            -
              end
         | 
| 65 | 
            -
            end
         |