coursera_downloader 1.0.0

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://www.rubygems.org"
+ 
+gemspec

data/Gemfile.lock

@@ -0,0 +1,18 @@
+PATH
+  remote: .
+  specs:
+    coursera_downloader (0.1.0)
+      curb (~> 0.8)
+      nokogiri (~> 1.5)
+
+GEM
+  remote: http://www.rubygems.org/
+  specs:
+    curb (0.8.3)
+    nokogiri (1.5.5)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  coursera_downloader!

data/LICENSE

@@ -0,0 +1,19 @@
+Copyright (C) 2012 by Nick Ewing
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.markdown

@@ -0,0 +1,63 @@
+Coursera Downloader
+===================
+
+Download static versions of Coursera course websites for offline reference.
+
+Install
+-------
+Download and install with the following command:
+
+    gem install coursera_downloader
+
+Use
+---
+After installing the Ruby gem, use the `coursera_downloader` command to download
+a course like so:
+
+    coursera_downloader course-identifier email password destination-directory [policy-file]
+
+For example, to download the first offering of the Neural Networks class, run
+the following command:
+    
+    coursera_downloader neuralnets-2012-001 foo.bar@example.com password123 neuralnets
+
+Default Policy File
+-------------------
+The policy file defines Ruby regular expressions used to determine how to handle
+new URLs found on the course page.  It is in the YAML format.
+
+The default policy file is shown below.
+
+    ---
+    whitelist:
+    - ^https?://class\.coursera\.org/[^/]+/
+    - ^https?://[^\.]+\.s3\.amazonaws\.com
+    - ^https?://s3\.amazonaws\.com
+    - ^https?://[^\.]+\.cloudfront\.net
+    blacklist:
+    - \.(exe|dmg)(\?.*)?$
+    disable:
+    # - ^https?://class\.coursera\.org/[^/]+/quiz
+    # - ^https?://class\.coursera\.org/[^/]+/forum
+    # - ^https?://class\.coursera\.org/[^/]+/lecture
+    # - ^https?://class\.coursera\.org/[^/]+/wiki
+    - ^https?://class\.coursera\.org/[^/]+/forum/thread?.*view=.*
+    - ^https?://class\.coursera\.org/[^/]+/forum/tag?.*view=.*
+    - ^https?://class\.coursera\.org/[^/]+/forum/list?.*view=.*
+    - ^https?://class\.coursera\.org/[^/]+/forum/toggle
+    - ^https?://class\.coursera\.org/[^/]+/forum/tag_modify
+    - ^https?://class\.coursera\.org/[^/]+/quiz/start
+    - ^https?://class\.coursera\.org/[^/]+/generic/apply_late_days
+    - ^https?://class\.coursera\.org/[^/]+/auth/logout
+    - ^https?://class\.coursera\.org/[^/]+/class/preferences
+
+License
+-------
+Coursera Downloader is available under an MIT-style license.
+
+See LICENSE.
+
+This software should only be used in accordance to Coursera's Terms of Service.
+Please see: https://www.coursera.org/about/terms
+
+Copyright (C) 2012 by Nick Ewing

data/bin/coursera_downloader

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+$: << File.expand_path("..", File.dirname(__FILE__))
+require "lib/coursera_downloader"
+
+CourseraDownloader::Runner.run

data/coursera_downloader.gemspec

@@ -0,0 +1,19 @@
+Gem::Specification.new do |s|
+  s.name         = "coursera_downloader"
+  s.version      = "1.0.0"
+  s.authors      = ["Nick Ewing"]
+  s.email        = ""
+  s.homepage     = "https://github.com/nickewing/coursera_downloader"
+  s.summary      = "Download static versions of Coursera course websites."
+  s.description  = "#{s.summary}."
+
+  s.files        = `git ls-files`.split("\n").reject {|path| path =~ /\.gitignore$/ }
+  s.executables  = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.platform     = Gem::Platform::RUBY
+  s.require_path = "lib"
+  s.rubyforge_project = "[none]"
+
+  s.add_dependency "curb",  "~> 0.8"
+  s.add_dependency "nokogiri",  "~> 1.5"
+  s.add_dependency "colored",  "~> 1.2"
+end

data/download_policy.yml

@@ -0,0 +1,22 @@
+---
+whitelist:
+- ^https?://class\.coursera\.org/[^/]+/
+- ^https?://[^\.]+\.s3\.amazonaws\.com
+- ^https?://s3\.amazonaws\.com
+- ^https?://[^\.]+\.cloudfront\.net
+blacklist:
+- \.(exe|dmg)(\?.*)?$
+disable:
+# - ^https?://class\.coursera\.org/[^/]+/quiz
+# - ^https?://class\.coursera\.org/[^/]+/forum
+# - ^https?://class\.coursera\.org/[^/]+/lecture
+# - ^https?://class\.coursera\.org/[^/]+/wiki
+- ^https?://class\.coursera\.org/[^/]+/forum/thread?.*view=.*
+- ^https?://class\.coursera\.org/[^/]+/forum/tag?.*view=.*
+- ^https?://class\.coursera\.org/[^/]+/forum/list?.*view=.*
+- ^https?://class\.coursera\.org/[^/]+/forum/toggle
+- ^https?://class\.coursera\.org/[^/]+/forum/tag_modify
+- ^https?://class\.coursera\.org/[^/]+/quiz/start
+- ^https?://class\.coursera\.org/[^/]+/generic/apply_late_days
+- ^https?://class\.coursera\.org/[^/]+/auth/logout
+- ^https?://class\.coursera\.org/[^/]+/class/preferences

data/lib/coursera_downloader.rb

@@ -0,0 +1,4 @@
+all_files = File.expand_path("coursera_downloader/*", File.dirname(__FILE__))
+Dir[all_files].each do |file|
+  require file
+end

data/lib/coursera_downloader/course.rb

@@ -0,0 +1,58 @@
+require "curb"
+require "uri"
+require "tempfile"
+
+module CourseraDownloader
+  class Course
+    attr_reader :cookie_file
+
+    def initialize(name)
+      @name = name
+      @cookie_file = Tempfile.new('coursera_cookies')
+    end
+
+    def login(email, password)
+      curl = Curl::Easy.new
+
+      curl.verbose = false
+      curl.enable_cookies = true
+      curl.cookiefile = @cookie_file.path
+      curl.cookiejar = @cookie_file.path
+      curl.follow_location = true
+
+      curl.url = login_redirect_url
+      curl.http_get
+
+      curl.url = curl.last_effective_url
+      curl.http_post([
+        Curl::PostField.content('email', email),
+        Curl::PostField.content('password', password),
+        Curl::PostField.content('login', "Login")
+      ])
+
+      curl.follow_location = false
+      curl.url = index_url.to_s
+      curl.http_get
+
+      response_code = curl.response_code
+      curl.close
+
+      response_code == 200
+    end
+
+    def index_url
+      URI.parse("#{host_url}/#{@name}/class/index")
+    end
+
+    private
+
+    def host_url
+      "https://class.coursera.org"
+    end
+
+    def login_redirect_url
+      "#{host_url}/#{@name}/auth/auth_redirector?type=login&subtype=normal&email=&visiting=&minimal=true"
+    end
+
+  end
+end

data/lib/coursera_downloader/document.rb

@@ -0,0 +1,26 @@
+require "uri"
+
+module CourseraDownloader
+  class Document
+    attr_reader :url, :content_type
+    attr_accessor :body
+
+    def initialize(url, body, content_type)
+      @url = url
+      @body = body
+      @content_type = content_type
+    end
+
+    def is_html?
+      content_type =~ /text\/html/
+    end
+
+    def is_css?
+      content_type =~ /text\/css/
+    end
+
+    def is_javascript?
+      content_type =~ /javascript/
+    end
+  end
+end

data/lib/coursera_downloader/document_processor.rb

@@ -0,0 +1,116 @@
+require "nokogiri"
+
+module CourseraDownloader
+  class DocumentProcessor
+    DISABLED_HREF = "javascript:alert('Link disabled during download.');"
+
+    attr_reader :resource_urls, :document
+
+    def initialize(document, store, policy)
+      @document = document
+      @store = store
+      @policy = policy
+      @resource_urls = Set.new
+    end
+
+    def process
+      if @document.is_html?
+        @document.body = process_html(@document.body)
+      elsif @document.is_css?
+        @document.body = process_css(@document.body)
+      end
+    end
+
+    private
+
+    def process_html(html)
+      doc = Nokogiri::HTML(html)
+
+      urls = Set.new
+      doc.css("a, img, script, link").each do |element|
+        url = element.attr("href") || element.attr("src")
+
+        url = normalize_url(url)
+
+        case @policy.url_action(url)
+        when :disable
+          disable_html_element(element)
+        when :download
+          urls << url unless urls.include?(url)
+          localize_element(element, url)
+        end
+      end
+
+      doc.xpath("//*[@style]").each do |element|
+        element["style"] = process_css(element.attr("style"))
+      end
+
+      @resource_urls += urls
+
+      doc.to_s
+    end
+
+    def localize_element(element, url)
+      path = @store.relative_resource_path(@document.url, url, true)
+      if element.attr("href")
+        element["href"] = path
+      elsif element.attr("src")
+        element["src"] = path
+      end
+    end
+
+    def disable_html_element(element)
+      if element.attr("href")
+        element["href"] = DISABLED_HREF
+      elsif element.attr("src")
+        element.remove
+      end
+    end
+
+    def process_css(css)
+      matches = css.scan(/url\(((")([^"]*)"|(')([^']*)'|[^\)]*)\)/)
+
+      urls = Set.new
+      matches.each do |match|
+        quote = match[1] || match[3]
+        raw_url = match[2] || match[4] || match[0]
+
+        # puts ">>>> #{raw_url}"
+        url = normalize_url(raw_url)
+
+        # p @policy.url_action(url)
+
+        case @policy.url_action(url)
+        when :disable
+          css.gsub!("url(#{quote}#{raw_url}#{quote})", "url()")
+        when :download
+          urls << url unless urls.include?(url)
+          path = @store.relative_resource_path(@document.url, url, true)
+          css.gsub!("url(#{quote}#{raw_url}#{quote})", "url('#{path}')")
+        end
+      end
+
+      @resource_urls += urls
+
+      css
+    end
+
+    def normalize_url(url)
+      return nil if !url || url.length == 0
+
+      begin
+        url = URI.parse(url)
+      rescue URI::InvalidURIError => e
+        return nil
+      end
+
+      if url.host
+        url = url
+      else
+        url = URI.join(@document.url, url)
+      end
+
+      url
+    end
+  end
+end

data/lib/coursera_downloader/downloader.rb

@@ -0,0 +1,142 @@
+require "set"
+require "yaml"
+require "uri"
+
+module CourseraDownloader
+  class Downloader
+    MAX_BATCH_SIZE = 10
+
+    def initialize(cookie_file, policy, store, logger)
+      @cookie_file = cookie_file
+      @store = store
+      @policy = policy
+      @logger = logger
+
+      @queue = []
+      @enqueued = Set.new # all URLs that have been ever enqueued during this run
+
+      read_state
+    end
+
+    def get(url)
+      url = URI.parse(url) unless url.is_a?(URI)
+
+      enqueue_new_url(url)
+      fetch_all
+      @logger.info("Downloaded #{@enqueued.length} total files")
+    end
+
+    private
+
+    def state_save_path
+      File.join(@store.containing_dir, "manifest.yml")
+    end
+
+    def write_state
+      if File.exists?(@store.containing_dir)
+        File.open(state_save_path, "wb") do |file|
+          state = {
+            :queue => @queue,
+            :enqueued => @enqueued.to_a
+          }
+          file.write(YAML::dump(state))
+        end
+      end
+    end
+
+    def read_state
+      if File.exists?(state_save_path)
+        saved_state = YAML::load(File.read(state_save_path))
+
+        @queue = saved_state[:queue]
+        @enqueued = Set.new(saved_state[:enqueued])
+      end
+    end
+
+    def handle_document(document)
+      if document.is_html? || document.is_css?
+        processor = DocumentProcessor.new(document, @store, @policy)
+        processor.process
+
+        processor.resource_urls.map do |resource_url|
+          enqueue_new_url(resource_url)
+        end
+
+        body = processor.document.body
+      end
+
+      @store.write(document)
+    end
+
+    def fetch_all
+      begin
+        interrupted = false
+        trap("INT") do
+          interrupted = true
+          show_interrupt_message
+        end
+
+        while @queue.length > 0 && !interrupted
+          batch = @queue.shift(MAX_BATCH_SIZE)
+          fetch_batch(batch)
+        end
+
+        trap("INT", "DEFAULT")
+      ensure
+        write_state
+      end
+    end
+
+    def fetch_batch(batch)
+      m = Curl::Multi.new
+
+      batch.each do |url|
+        curl = Curl::Easy.new(url.to_s) do |curl|
+          curl.enable_cookies = true
+          curl.cookiefile = @cookie_file.path
+          curl.cookiejar = @cookie_file.path
+          curl.follow_location = true
+
+          curl.on_complete do |result|
+            begin
+              if result.response_code == 200
+                @logger.info("Downloaded: #{url}")
+
+                content_type = result.content_type
+                content_type.force_encoding("ASCII") if content_type.respond_to?(:force_encoding)
+
+                document = Document.new(url, result.body_str, content_type)
+                handle_document(document)
+              else
+                @logger.warn("Failed to get URL '#{url}'.  Response code #{result.response_code}")
+              end
+            rescue => e
+              @logger.error(e.message + "\n  " + e.backtrace.join("\n  "))
+            end
+          end
+        end
+
+        m.add(curl)
+      end
+
+      m.perform
+    end
+
+    def enqueue_new_url(url)
+      if !@enqueued.include?(url)
+        @enqueued << url
+        @queue.push(url)
+      end
+    end
+
+    def show_interrupt_message
+      return if @interrupt_message_shown
+
+      @logger.warn("Finishing current download batch.")
+      @logger.warn("This download can be resumed by rerunning the same command.")
+
+      @interrupt_message_shown = true
+    end
+
+  end
+end

data/lib/coursera_downloader/file_store.rb

@@ -0,0 +1,56 @@
+require "fileutils"
+require "cgi"
+
+module CourseraDownloader
+  class FileStore
+    attr_reader :containing_dir
+
+    def initialize(containing_dir)
+      @containing_dir = containing_dir
+    end
+
+    def path(url, path_in_source = false)
+      path = url.path
+      dir_name = File.dirname(path)
+      extension = File.extname(path)
+      base_name = File.basename(path, extension)
+
+      extension = ".html" unless extension.length > 0
+
+      if url.query
+        query = "?#{url.query}"
+        query = CGI.escape(query)
+      else
+        query = ""
+      end
+
+      store_dir = File.join(@containing_dir, url.host, dir_name)
+      store_dir = Util.escape_path(store_dir) if path_in_source
+
+      file_name = "#{base_name}#{query}#{extension}"
+      file_name = Util.escape_path(file_name) if path_in_source
+
+      file_path = File.join(store_dir, file_name)
+
+      [store_dir, file_path]
+    end
+
+    def relative_resource_path(containing_url, resource_url, path_in_source = true)
+      _, containing_path = path(containing_url, path_in_source)
+      _, resource_path = path(resource_url, path_in_source)
+
+      Util.path_relative_to_path(containing_path, resource_path)
+    end
+
+    def write(document)
+      store_dir, file_path = path(document.url)
+
+      FileUtils.mkdir_p(store_dir)
+
+      File.open(file_path, "wb") do |file|
+        file.write(document.body)
+      end
+    end
+
+  end
+end

data/lib/coursera_downloader/log_formatter.rb

@@ -0,0 +1,19 @@
+require "logger"
+require "colored"
+
+module CourseraDownloader
+  class LogFormatter < ::Logger::Formatter
+    def call(severity, time, progname, msg)
+      case severity
+      when "INFO"
+        severity = severity.green
+      when "WARN"
+        severity = severity.yellow
+      when "ERROR"
+        severity = severity.red
+      end
+
+      "#{severity}: #{msg}\n"
+    end
+  end
+end

data/lib/coursera_downloader/policy.rb

@@ -0,0 +1,46 @@
+require "yaml"
+
+module CourseraDownloader
+  class Policy
+    def initialize(file)
+      @patterns = YAML::load(File.read(file))
+
+      @patterns.each_pair do |group, patterns|
+        @patterns[group] = patterns.map do |pattern|
+          Regexp.new(pattern)
+        end
+      end
+    end
+
+    def url_action(url)
+      return :none if !url || (url.scheme && !(url.scheme != "http" || url.scheme != "https"))
+
+      url = url.to_s
+      return :disable if disable_url?(url)
+      return :download if download_url?(url)
+    end
+
+    private
+
+    def download_url?(url)
+      @patterns["blacklist"].each do |pattern|
+        return false if url.match(pattern)
+      end
+
+      match = false
+      @patterns["whitelist"].each do |pattern|
+        match = true if url.match(pattern)
+      end
+
+      match
+    end
+
+    def disable_url?(url)
+      @patterns["disable"].each do |pattern|
+        return true if url.match(pattern)
+      end
+
+      false
+    end
+  end
+end

data/lib/coursera_downloader/runner.rb

@@ -0,0 +1,34 @@
+require "logger"
+
+module CourseraDownloader
+  class Runner
+    def self.run
+      if ARGV.length < 4
+        $stderr.puts "Usage:\n  coursera_downloader course-identifier email password destination-directory [policy-file]"
+        Process.exit(1)
+      end
+
+      course_name = ARGV[0]
+      email = ARGV[1]
+      password = ARGV[2]
+      file_store_dir = ARGV[3]
+
+      logger = Logger.new(STDOUT)
+      logger.formatter = LogFormatter.new
+
+      course = Course.new(course_name)
+      if course.login(email, password)
+        policy_file = ARGV[4] || File.expand_path("../download_policy.yml", File.dirname(__FILE__))
+        policy = Policy.new(policy_file)
+        store = FileStore.new(file_store_dir)
+        downloader = Downloader.new(course.cookie_file, policy, store, logger)
+        downloader.get(course.index_url)
+
+        _, index_store_path = store.path(course.index_url)
+        logger.info("The course index can be found at #{index_store_path}")
+      else
+        logger.error("Failed to login.")
+      end
+    end
+  end
+end

data/lib/coursera_downloader/util.rb

@@ -0,0 +1,21 @@
+require "cgi"
+
+module CourseraDownloader
+  module Util
+    def self.path_relative_to_path(target_path, path)
+      target_dirs = target_path.split("/")
+      dirs = path.split("/")
+
+      while target_dirs.length > 1 && target_dirs[0] == dirs[0]
+        target_dirs.shift
+        dirs.shift
+      end
+
+      File.join(Array.new(target_dirs.length - 1, "..") + dirs)
+    end
+
+    def self.escape_path(path)
+      path.split("/").map{|e| CGI.escape(e)}.join("/")
+    end
+  end
+end

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: coursera_downloader
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Nick Ewing
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-12-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: colored
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: Download static versions of Coursera course websites..
+email: ''
+executables:
+- coursera_downloader
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- LICENSE
+- README.markdown
+- bin/coursera_downloader
+- coursera_downloader.gemspec
+- download_policy.yml
+- lib/coursera_downloader.rb
+- lib/coursera_downloader/course.rb
+- lib/coursera_downloader/document.rb
+- lib/coursera_downloader/document_processor.rb
+- lib/coursera_downloader/downloader.rb
+- lib/coursera_downloader/file_store.rb
+- lib/coursera_downloader/log_formatter.rb
+- lib/coursera_downloader/policy.rb
+- lib/coursera_downloader/runner.rb
+- lib/coursera_downloader/util.rb
+homepage: https://github.com/nickewing/coursera_downloader
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: ! '[none]'
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: Download static versions of Coursera course websites.
+test_files: []