mooc-data-parser 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ef30888b18e521abbe650bbb3f66655a28ac5058
+  data.tar.gz: 237e3ebef04827ac78c9e6566b0cdedaaf335541
+SHA512:
+  metadata.gz: be54dfa5face8632270d842f5bd6e8510be40fa3209b53816347d25881a7f9e230639ce2a7cf660c80be2b03dd380fb79f719eb42cbd2cec258d3722700f56e8
+  data.tar.gz: 355deeba3ab80be18c49895ddd9971aabf677ddbbb2f73bccfc93cadf8c55173eb50a98f7cb3646bd5097944acb5d412c4a86c75ac5fdb7204a02b6462425e78

data/.gitignore

@@ -0,0 +1,22 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in mooc-data-parser.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Jarmo Isotalo
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/Readme.md

@@ -0,0 +1,64 @@
+# Mooc Data Parser
+
+## Installation
+
+Or install it yourself as:
+
+    $ gem install mooc-data-parser
+Scripts made to process some data for our Massive Open Online Courses.
+
+## Usage
+
+```bash
+Usage: show-mooc-details.rb [options]
+    -f, --force                      Reload data from server
+    -u, --user username              Show details for user
+    -m, --missing-points             Show missing compulsary points
+    -c, --completion-precentige      Show completition percentige
+    -e, --email emailaddress         Show details for user
+    -t, --tmc-account tmc-account    Show details for user
+    -l, --list                       Show the basic list
+    -h, --help                       Show this message
+```
+
+
+Since refreshing data for each search is unnecessary this will cache everything that requires a http request,
+thus making the functionality much faster.
+
+To get fresh data, use `-f` command line parameter.
+
+
+Show basic info for applicants
+```bash
+./show-mooc-data.rb -l
+```
+
+Show basic info for applicants and percents for each week
+```bash
+./show-mooc-data.rb -l -c
+```
+
+Show basic info for applicants and missing points
+```bash
+./show-mooc-data.rb -l -m
+```
+
+Show basic info for applicants and percents for each week and missing points
+```bash
+./show-mooc-data.rb -l -c -m
+```
+
+Find info about a user
+* by email:
+```bash
+./show-mooc-data.rb -e <email>
+```
+
+* by username:
+```bash
+./show-mooc-data.rb -u <username>
+```
+
+Flags `-c` and `-l` can be used with these too.
+
+

data/bin/mooc-data-parser

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+begin
+  require 'mooc-data-parser'
+ rescue LoadError
+  require 'rubygems'
+  require 'mooc-data-parser'
+end
+
+Mooc::Data::Parser::App.new.run(ARGV)
+

data/lib/mooc/data/parser.rb

@@ -0,0 +1,15 @@
+require "mooc/data/parser/app"
+require "mooc/data/parser/dummy_cacher"
+require "mooc/data/parser/version"
+module Mooc
+  module Data
+    require 'optparse'
+    require 'ostruct'
+    require 'httparty'
+    require 'json'
+    require 'io/console'
+    module Parser
+    end
+  end
+end
+

data/lib/mooc/data/parser/App.rb

@@ -0,0 +1,262 @@
+#!/usr/bin/env ruby -w
+module Mooc
+  module Data
+    require 'optparse'
+    require 'ostruct'
+    require 'httparty'
+    require 'json'
+    require 'io/console'
+    module Parser
+      class App
+        def run(args)
+          $cache ||= Mooc::Data::Parser::DummyCacher.new
+          @notes = begin JSON.parse($cache.read_file_from_cache('data.json')) rescue  {} end
+
+          @options = OpenStruct.new
+          opt = OptionParser.new do |opts|
+            opts.banner = "Usage: show-mooc-details.rb [options]"
+
+            opts.on("-f", "--force", "Reload data from server") do |v|
+              @options.reload = true
+            end
+            opts.on("-u", "--user username", "Show details for user") do |v|
+              @options.user = v
+            end
+            opts.on("-m", "--missing-points", "Show missing compulsary points") do |v|
+              @options.show_missing_compulsory_points = true
+            end
+            opts.on("-c", "--completion-precentige", "Show completition percentige") do |v|
+              @options.show_completion_percentige = true
+            end
+            opts.on("-e", "--email emailaddress", "Show details for user") do |v|
+              @options.user_email = v
+            end
+            opts.on("-t", "--tmc-account tmc-account", "Show details for user") do |v|
+              @options.user_tmc_username = v
+            end
+            opts.on("-l", "--list", "Show the basic list") do |v|
+              @options.list = true
+            end
+            opts.on_tail("-h", "--help", "Show this message") do
+              puts opts
+              exit
+            end
+          end
+          opt.parse!(args)
+
+          json = maybe_fetch_json()
+          if @options.user
+            show_info_about(@options.user, 'username', json)
+          elsif @options.user_email
+            show_info_about(@options.user_email, 'email', json)
+          elsif @options.user_tmc_username
+            show_info_about(@options.user_tmc_username, 'username', json)
+          elsif @options.list
+            list_and_filter_participants(json)
+          else
+            $cache.write_file_to_cache('data.json', @notes.to_json)
+            puts opt
+            abort
+          end
+          $cache.write_file_to_cache('data.json', @notes.to_json)
+        end
+
+        def get_auth
+          print 'username: '
+          username = $stdin.gets.strip
+          print 'password: '
+          password = $stdin.noecho(&:gets).strip
+          puts
+          {username: username, password: password}
+        end
+
+        def maybe_fetch_json()
+          if @options.reload or @notes['user_info'].nil? or @notes['week_data'].nil?
+
+            t = -> do
+              loop do
+                print '.'
+                sleep 0.5
+              end
+              puts
+            end
+
+            auth = get_auth()
+            th = Thread.new(&t)
+
+            url = "http://tmc.mooc.fi/mooc/participants.json?api_version=7&utf8=%E2%9C%93&filter_koko_nimi=&column_username=1&column_email=1&column_koko_nimi=1&column_hakee_yliopistoon_2014=1&group_completion_course_id=18"
+            user_info = JSON.parse(HTTParty.get(url, basic_auth: auth).body)['participants']
+            week_data = fetch_week_datas(auth)
+            @notes['user_info'] = user_info.clone
+            @notes['week_data'] = week_data.clone
+            th.kill
+            puts
+            {participants: user_info, week_data: week_data}
+          else
+            {participants: @notes['user_info'].clone, week_data: @notes['week_data'].clone}
+          end
+        end
+
+        def show_info_about(user, user_field = 'username', json)
+          participants = json[:participants]
+          week_data = json[:week_data]
+          my_user = participants.find{|a| a[user_field] == user }
+          if my_user.nil?
+            abort "User not found"
+          end
+          formatted_print_user_details ["Username", my_user['username']]
+          formatted_print_user_details ["Email", my_user['email']]
+          formatted_print_user_details ["Hakee yliopistoon", my_user['hakee_yliopistoon_2014']]
+          formatted_print_user_details ["Koko nimi", my_user['koko_nimi']]
+
+          if @options.show_completion_percentige
+            formatted_print_user_details ["Points per week"]
+            done_exercise_percents(my_user, participants).each do |k|
+              begin
+                k = k.first
+                formatted_print_user_details [k[0], k[1]]
+              rescue
+                nil
+              end
+            end
+          end
+
+          if @options.show_missing_compulsory_points
+            formatted_print_user_details ["Compulsory points"]
+            get_points_info_for_user(my_user, week_data).each do |k,v|
+              formatted_print_user_details [k, v.join(", ")]
+            end
+          end
+
+        end
+
+        def formatted_print_user_details(details)
+          case details.size
+          when 1
+            puts "%18s" % details
+          when 2
+            puts "%18s: %-20s" % details
+          end
+        end
+
+        def fetch_week_datas(auth)
+          base_url = "http://tmc.mooc.fi/mooc/courses/18/points/"
+          weeks = %w(1 2 3 4 5 6 7 8 9 10 11 12)
+          rest = ".json?api_version=7"
+          week_data = {}
+          weeks.each do |week|
+            week_data[week] = JSON.parse(HTTParty.get(base_url + week + rest, basic_auth: auth).body)['users_to_points']
+          end
+          week_data
+        end
+
+        def list_and_filter_participants(json)
+          wanted_fields = %W(username email koko_nimi)
+
+          participants = json[:participants]
+          week_data = json[:week_data]
+          everyone_in_course = participants.size
+          only_applying!(participants)
+          hakee_yliopistoon = participants.size
+
+          puts "%-20s %-35s %-25s %-120s" % ["Username", "Email", "Real name", "Missing points"]
+          puts '-'*200
+          participants.each do |participant|
+            nice_string_in_array = wanted_fields.map do |key|
+              participant[key]
+            end
+            if @options.show_completion_percentige
+              nice_string_in_array << format_done_exercises_percents(done_exercise_percents(participant, participants))
+            end
+            if @options.show_missing_compulsory_points
+              nice_string_in_array << missing_points_to_list_string(get_points_info_for_user(participant, week_data))
+            end
+
+
+            to_be_printed = "%-20s %-35s %-25s "
+            to_be_printed << "%-180s " if @options.show_completion_percentige
+            to_be_printed << "%-120s" if @options.show_missing_compulsory_points
+
+            puts to_be_printed % nice_string_in_array
+          end
+
+          puts
+          puts
+          puts "Stats: "
+          puts "%25s: %4d" % ["Kaikenkaikkiaan kurssilla", everyone_in_course]
+          puts "%25s: %4d" % ["Hakee yliopistoon", hakee_yliopistoon]
+
+        end
+
+
+        def format_done_exercises_percents(hash)
+          hash.map do |k|
+            begin
+              k = k.first
+              "#{k[0].scan(/\d+/).first}: #{k[1]}"
+            rescue
+              nil
+            end
+          end.compact.join(", ")
+        end
+
+
+        def done_exercise_percents(participant, participants_data)
+          user_info = participants_data.find{ |p| p['username'] == participant['username'] }
+          exercise_weeks = user_info['groups']
+          week_keys = (1..12).map{|i| "viikko#{i}"}
+
+          week_keys.map do |week|
+            details = exercise_weeks[week]
+            unless details.nil?
+              {week => ("%3.1f%" % [(details['points'].to_f / details['total'].to_f) * 100])}
+            end
+          end
+        end
+
+        def missing_points_to_list_string(missing_by_week)
+          str = ""
+          missing_by_week.keys.each do |week|
+            missing = missing_by_week[week]
+            unless missing.nil? or missing.length == 0
+              str << week
+              str << ": "
+              str << missing.join(",")
+              str << "  "
+            end
+          end
+
+          str
+
+        end
+
+        def get_points_info_for_user(participant, week_data)
+          # TODO: täydennä data viikolle 12
+          compulsory_exercises = {'6' => %w(102.1 102.2 102.3 103.1 103.2 103.3), '7' => %w(116.1 116.2 116.3), '8' => %w(124.1 124.2 124.3 124.4),
+                                  '9' => %w(134.1 134.2 134.3 134.4 134.5), '10' => %w(141.1 141.2 141.3 141.4), '11' => %w(151.1 151.2 151.3 151.4), '12' => %w()}
+          points_by_week = {}
+          week_data.keys.each do |week|
+            points_by_week[week] = week_data[week][participant['username']]
+          end
+
+
+          missing_by_week = {}
+          points_by_week.keys.each  do |week|
+            weeks_points = points_by_week[week] || [] #palauttaa arrayn
+            weeks_compulsory_points = compulsory_exercises[week] || []
+            missing_by_week[week] = weeks_compulsory_points - weeks_points
+          end
+
+          missing_by_week
+        end
+
+        def only_applying!(participants)
+          participants.select! do |participant|
+            participant['hakee_yliopistoon_2014']
+          end
+        end
+      end
+    end
+  end
+end
+__END__

data/lib/mooc/data/parser/dummy_cacher.rb

@@ -0,0 +1,65 @@
+
+module Mooc
+  module Data
+    module Parser
+      require 'fileutils'
+      class DummyCacher
+        def initialize
+          FileUtils.mkdir_p(path)
+        end
+        # Yep, well just overwrite it if it exists
+        def cache_file(file)
+          FileUtils.cp(file, path)
+        end
+
+        def write_file_to_cache(filename, contents)
+          Dir.chdir(path) do
+            File.open(filename, "wb") { |file| file.write(contents) }
+          end
+        end
+
+        def read_file_from_cache(filename)
+          Dir.chdir(path) do
+            File.read(filename)
+          end
+        end
+
+        def get_from_cache(file_name)
+          File.join(path,  file_name)
+        end
+
+        def file_exists?(file_name)
+          File.exists? get_from_cache(file_name)
+        end
+
+        def tmpdir_path
+          Dir.tmpdir
+        end
+
+        def unzip_file(file_name)
+          Dir.chdir(path) do
+            # Because I have found rubyzip to be buggy, we rely that your system has zip-command available
+            `unzip -o  #{file_name}`
+          end
+        end
+
+        def find_files_matching(matcher)
+          files = []
+          Dir.chdir(path) do
+            files = Dir.glob(matcher)
+          end
+          files
+        end
+
+        def path
+          File.join(Dir.tmpdir, "mooc-data-analyser")
+        end
+
+        def clean!
+          FileUtils.rm_rf path
+        end
+
+      end
+    end
+  end
+end

data/lib/mooc/data/parser/version.rb

@@ -0,0 +1,7 @@
+module Mooc
+  module Data
+    module Parser
+      VERSION = "0.0.1"
+    end
+  end
+end

data/mooc-data-parser.gemspec

@@ -0,0 +1,23 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mooc/data/parser/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "mooc-data-parser"
+  spec.version       = Mooc::Data::Parser::VERSION
+  spec.authors       = ["Jarmo Isotalo"]
+  spec.email         = ["jamo@isotalo.fi"]
+  spec.summary       = %q{A small command line utility to show data from our tmc-server.}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency 'httparty'
+  spec.add_development_dependency "bundler", "~> 1.6"
+  spec.add_development_dependency "rake"
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: mooc-data-parser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Jarmo Isotalo
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: httparty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- jamo@isotalo.fi
+executables:
+- mooc-data-parser
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- Rakefile
+- Readme.md
+- bin/mooc-data-parser
+- lib/mooc/data/parser.rb
+- lib/mooc/data/parser/App.rb
+- lib/mooc/data/parser/dummy_cacher.rb
+- lib/mooc/data/parser/version.rb
+- mooc-data-parser.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A small command line utility to show data from our tmc-server.
+test_files: []