kitcrawler 0.1.0

data/README.md

@@ -0,0 +1,48 @@
+KITCrawler
+===================
+Fetch lecture PDFs with ease.
+
+
+It currently supports crawling PDFs for lectures from the studium.kit.edu page,
+but can be easily extended to fetch PDFs from other services.
+
+Requirements
+-------------------
+- ruby (>= 1.9, but 1.8 might also be okay)
+- bundler (or install the required gems (see `Gemfile`) manually)
+- linux (with curl, might also work on other Unixes)
+
+Install
+-------------------
+Simply run
+```
+	gem install kitcrawler
+```
+to install the gem (it's often a bit behind the repo).
+
+Or run it from source.
+```
+	git clone https://github.com/parttimenerd/KITCrawler
+	cd KITCrawler
+	bundle install
+``
+
+
+Usage
+-------------------
+Run
+```
+	kitcrawler add NAME
+```
+to add a new fetch job named `NAME`. This will prompt you to pass an entry URL to the site, etc.
+
+To finally run your jobs use
+```
+	kitcrawler fetch
+```
+
+It also supports some command line parameters, run `kitcrawler` to see an explanation.
+
+License
+-------------------
+The code is GNU GPL v3 licensed.

data/bin/kitcrawler

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+#require 'kitcrawler'
+require_relative '../lib/kitcrawler.rb'
+KITCrawler::run_cli

data/lib/cli.rb

@@ -0,0 +1,162 @@
+require 'pp'
+require 'json'
+require 'thor'
+require_relative 'services.rb'
+require_relative 'cli_add.rb'
+
+def kitgrawler_dir
+	`echo ~/.kitcrawler`.strip
+end
+
+class CLI < Thor
+	class_option :config_file, :type => :string, :aliases => "-c",
+			:default => "#{kitgrawler_dir}/config.json",
+			:desc => "Use CONFIG_FILE as config file location"
+	class_option :auth_file, :type => :string, :aliases => "-a",
+			:default => "#{kitgrawler_dir}/auth.json",
+			:desc => "Use AUTH_FILE as authentication location"
+	class_option :url_cache_file, :type => :string, :aliases => "-u",
+			:default => "#{kitgrawler_dir}/url_type_cache.json",
+			:desc => "Use URL_CACHE_FILE as url cache file location"
+	class_option :debug, :type => :boolean, :default => false, :aliases => "-d",
+			:desc => "Print everything to standard out"
+	class_option :verbose, :type => :boolean, :default => false, :aliases => "-v",
+			:desc => "Print a lot to standard out"
+	class_option :warn, :type => :boolean, :default => false, :aliases => "-w",
+			:desc => "Print only warnings and errors to standard out"
+	class_option :quiet, :type => :boolean, :default => false, :aliases => "-q",
+			:desc => "Print nothing to standard out"
+	
+	desc "fetch [NAME]", "Run fetch job NAME or all jobs"
+	def fetch name
+		if name == nil
+			CLIHelper.new(options).fetch_all
+		else
+			CLIHelper.new(options).fetch name
+		end
+	end
+	
+	desc "add NAME", "Add new job NAME"
+	def add name
+		CLI_ADD.add_config_ui name, options
+	end
+	
+end
+
+class CLIHelper
+	
+	def initialize options
+		@options = options
+		`mkdir -p #{kitgrawler_dir}`
+		init_logger
+		load_conf_file
+		load_auth_file
+		load_url_type_cache_file
+	end
+
+	def init_logger
+		@log = Logger.new STDOUT
+		@log_level = Logger::WARN
+		@log_level = Logger::DEBUG  if @options[:debug]
+		@log_level = Logger::INFO   if @options[:verbose]
+		@log_level = Logger::WARN   if @options[:warn]
+		@log_level = Logger::UNKOWN if @options[:quiet]
+		@log.level = @log_level
+		@log.progname = "cli"
+	end
+
+	def load_conf_file
+		@conf = {}
+		return unless File.exists?(@options[:config_file])
+		begin
+			@conf = JSON.load File.read(@options[:config_file])
+		rescue => ex
+			@log.fatal "Cannot load config file #{@options[:config_file]}"
+			@log.fatal ex
+			exit 1
+		end
+	end
+		
+	def load_auth_file
+		@auth_conf = {}
+		return unless File.exists?(@options[:auth_file])
+		begin
+			@auth_conf = JSON.load File.read(@options[:auth_file])
+		rescue => ex
+			@log.fatal "Cannot load authentication config file #{@options[:auth_file]}"
+			@log.fatal ex
+			exit 1
+		end
+	end
+		
+	def load_url_type_cache_file
+		@url_type_cache = {}
+		return unless File.exists?(@options[:url_cache_file])
+		begin
+			@url_type_cache = JSON.load File.read(@options[:url_cache_file])
+		rescue => ex
+			@log.fatal "Cannot load url type cache file #{@options[:url_cache_file]}"
+			@log.fatal ex
+			exit 1
+		end
+	end
+	
+	def fetch_all
+		begin
+			@conf.each_key do |grawl_job|
+				fetch grawl_job
+			end
+		rescue => ex
+			@log.fatal "Error grawling configured locations"
+			@log.fatal ex
+			exit 1
+		ensure
+			File.open(@options[:url_cache_file], "w") do |f|
+				f.puts JSON::pretty_generate @url_type_cache
+			end
+		end
+	end
+	
+	def fetch job_name
+		grawl_location = job_name
+		conf = @conf[job_name]
+		if conf == nil
+			print_job_name_guess job_name
+			return
+		end
+		begin
+			service = BaseService::get_service grawl_location, conf, @auth_conf, @log_level, @url_type_cache
+			begin
+				service.execute
+			rescue => ex
+				@log.error "Failed executing #{grawl_location}"
+				@log.error ex
+			end
+		rescue => ex
+			@log.error "Failed to instantiate #{grawl_location}"
+			@log.error ex
+		end
+	end
+	
+	def print_job_name_guess job_name
+		unless @options[:quiet]
+			puts "There is no job '#{job_name}'."
+			puts "Maybe you meant one of the following"
+			best_n_matches(@conf.keys, job_name, 3).each do |name|
+				puts "  #{name}"
+			end
+		end
+	end
+	
+	def best_n_matches arr, comp, n
+		require 'damerau-levenshtein'
+		map = {}
+		dl = DamerauLevenshtein
+		arr.each do |str|
+			map[str] = dl.distance(str, comp, 2)
+		end
+		return arr.sort {|a, b| map[a] <=> map[b] }
+	end
+end
+
+

data/lib/cli_add.rb

@@ -0,0 +1,151 @@
+require 'pp'
+require 'json'
+require_relative 'services.rb'
+
+module CLI_ADD
+	
+	require 'highline/import'
+	
+	def self.add_config_ui name, options
+		CLIHelper.new options
+		config_json = JSON.load File.read(options[:config_file])
+		auth_json = JSON.load File.read(options[:auth_file])
+		say("Configure #{name}")
+		name = check_name name, config_json
+		conf = {}
+		conf["entry_url"] = ask_entry_url
+		conf["type"] = ask_type conf
+		conf["pdfs"] = ask_pdfs conf
+		if BaseService.get_services["needs_auth"]
+			conf["auth"] = ask_auth name, conf, auth_json
+		end
+		config_json[name] = conf
+		say "This configuration is placed into your config files."
+		say "Your config file is #{options[:config_file]}."
+		say "Your authentication config file is #{options[:auth_file]}."
+		File.open(options[:config_file], "w") do |f|
+			f.puts JSON.pretty_generate config_json
+		end
+		File.open(options[:auth_file], "w") do |f|
+			f.puts JSON.pretty_generate auth_json
+		end
+	end
+
+	def self.check_name name, config_json
+		names = config_json.keys
+		while names.include? name
+			name = ask "Fetch job name (#{name} is already in use)? "
+		end
+		return name
+	end
+
+	def self.ask_entry_url
+		return ask_url "Entry point url? "
+	end
+
+	def self.ask_type conf
+		default = BaseService::get_service_for_url conf["entry_url"]
+ 		choose do |menu|
+			menu.prompt = "Service type [#{default}]? "
+			menu.default = default
+			BaseService::get_services.each do |name, service|
+				menu.choices("#{name} (#{service["description"]})") do |q|
+					say "You've choosen '#{name}'."
+					return name
+				end
+			end
+		end
+	end
+
+	def self.ask_pdfs conf
+		return {
+			"src_folder" => ask_non_empty("Source folder url (relative to entry url directory if starts with dot)? "),
+			"dest_folder" => ask_non_empty("Destination folder (relative to $HOME)? "),
+			"download_once" => ask_yes_no(
+				"Dowload a PDF only once (ignore changes, boost performance)? ", "yes"
+			) == "yes"
+		}
+	end
+
+	def self.ask_auth name, conf, auth_json
+		is_studium_kit = conf["type"] == "studium_kit"
+		has_s_kit_auth = auth_json["studium_kit"] != nil
+		has_name_auth = auth_json[name] != nil
+		default = ""
+		if ask_yes_no("Auth: Use existing user/password configuration? ", is_studium_kit && has_s_kit_auth ? "yes" : "no") == "yes"
+			if is_studium_kit && has_s_kit_auth
+				default = "studium_kit"
+			end
+			choose do |menu|
+				menu.prompt = "Auth: Which configuration? "
+				menu.default = default unless default.empty?
+				auth_json.each do |name, config|
+					menu.choices("#{name} (user: #{config["user"]})") do |q|
+						say "Auth: You've chosen '#{name}'."
+						return name
+					end
+				end
+			end
+		else
+			auths = auth_json.keys
+			auth_name = ""
+			if is_studium_kit && !has_s_kit_auth
+				default = "studium_kit"
+				auth_name = ask "Auth: Configuration name [studium_kit]? " do |q|
+					q.default = studium_kit
+				end
+			elsif not has_name_auth
+				auth_name = ask "Auth: Configuration name [#{name}]? " do |q|
+					q.default = name
+				end
+			else
+				auth_name = ask_non_empty "Auth: Configuration name? "
+			end
+			begin
+				auth_json[auth_name] = ask_user_pass
+			end while ask_yes_no("Auth: Confirm that you're credentials are right. Are they? ", "yes") == "no"
+			return auth_name
+		end
+	end
+
+	def self.ask_user_pass
+		user = ask "Auth: User name? "
+		pass = ""
+		pass2 = ""
+		begin
+			pass = ask("Auth: Password? ") { |q| q.echo = "x" }
+			pass2 = ask("Auth: Retype it ") { |q| q.echo = "x" }
+		end while pass != pass2
+		return {
+			"user" => user,
+			"pass" => pass
+		}
+	end
+
+	def self.ask_url question
+		str = ""
+		while str.strip.length < 4
+			str = ask(question) || ""
+		end
+		return str.strip
+	end
+
+	def self.ask_non_empty question
+		str = ""
+		while str.strip.empty?
+			str = ask(question) || ""
+		end
+		return str.strip
+	end
+
+	def self.ask_yes_no question, default = "yes"
+		choose do |menu|
+			menu.layout = :one_line
+			menu.prompt = "#{question} [#{default}] "
+			menu.default = default
+			menu.choices(:yes, :no) do |q|
+				return q.to_s
+			end
+		end
+	end
+end

data/lib/kitcrawler.rb

@@ -0,0 +1,16 @@
+#!/bin/ruby
+
+require 'logger'
+require 'json'
+require 'optparse'
+
+module KITCrawler
+
+require_relative 'services.rb'
+require_relative 'cli_add.rb'
+require_relative 'cli.rb'
+
+	def self.run_cli
+		CLI.start ARGV
+	end
+end

data/lib/services.rb

@@ -0,0 +1,381 @@
+require 'nokogiri'
+require 'cgi'
+require 'pp'
+require 'uri'
+require 'logger'
+require 'set'
+require 'time'
+require 'uri'
+require 'json'
+
+class BaseService
+
+	@conf = {}
+	@auth_app = ""
+	@file_header_cache = {} #url => splitted HTTP header lines
+	@processed_pdfs = {} #url => dest file
+	@name = ""
+	@@log
+	@@service_classes = {}
+	@type_cache = {}
+	@uri_cache = {} #url => URI
+	
+	def initialize name, conf, auth_conf = {}, log_level = Logger::WARN, url_type_cache = {}
+		@base_dir = `echo ~`.strip
+		@uri_cache = {}
+		@file_header_cache = {}
+		@type_cache = url_type_cache
+		@processed_pdfs = {}
+		@name = name
+		@log = Logger.new(STDOUT)
+		@log.progname = name
+		@log.level = log_level
+		
+		@conf = {
+			"type" => "base",
+			"exclude_file_endings" => [".css", ".js", ".txt", ".rss", ".atom"],
+			"access_pause" => { #in seconds
+				"min" => 0.1,
+				"max" => 0.3
+			},
+			"pdfs" => {
+				"src_folder" => "abc.de/a", #is relative to entry_url base dir if starts with dot
+				"dest_folder" => "abcd", 
+		        "download_once" => true
+			},
+			"cookie_jar" => "cookies.txt",
+			"user_agent" => "Mozilla/5.0 (X11; Ubuntu; Linux i686; rv:30.0) Gecko/20100101 Firefox/30.0",
+			"entry_url" => "",
+			"auth" => "base" #references auth conf or {"user" => "", "pass" => ""}
+		}
+		temp_conf = @conf.merge conf
+		unless conf["pdfs"] == nil
+			temp_conf["pdfs"] = @conf["pdfs"].merge conf["pdfs"]
+		end
+		@conf = temp_conf
+		if @conf["auth"].is_a? String
+			@conf["auth"] = auth_conf[@conf["auth"]]
+			@log.debug "Load auth from auth config #{auth_conf}"
+		end
+		
+		if @conf["pdfs"]["src_folder"].start_with? "."
+			entry_uri = get_uri @conf["entry_url"]
+			entry_path_url = entry_uri.scheme + "://" + entry_uri.host + File.dirname(entry_uri.path)
+			@conf["pdfs"]["src_folder"] = "#{entry_path_url}/#{@conf["pdfs"]["src_folder"]}"
+			@log.info "Source folder is #{@conf["pdfs"]["src_folder"]}"
+		end
+		
+		src_url_parsed = URI.parse(@conf["pdfs"]["src_folder"])
+		@conf["pdfs"]["src_path"] = src_url_parsed.path
+		@conf["pdfs"]["src_host"] = src_url_parsed.host
+		@log.info "Start authentication"
+		authenticate
+		@log.info "Authentication completed"
+	end
+	
+	def self.get_service name, conf, auth_conf={}, log_level = Logger::WARN, url_type_cache = {}
+		service = @@service_classes[conf["type"]]
+		if service == nil
+			raise "Unknown service #{conf["type"]}"
+		else
+			service["class"].new name, conf, auth_conf, log_level, url_type_cache
+		end
+	end
+	
+	def authenticate
+		""
+	end
+
+	def execute
+		@log.info "Start grawling #{@conf["entry_url"]}"
+		parse_html_page @conf["entry_url"]
+		@log.info "Completed grawling #{@conf["entry_url"]}"
+	end
+
+	def parse_html_page url, url_cache = Set.new
+		url = url_chomp url
+		return if url_cache.member?(url)
+		url_cache.add url
+		@log.info "Fetch and parse #{url}"
+		html = ""
+		begin
+			html = fetch_url url
+			access_pause_sleep
+		rescue => ex
+			@log.error "Cannot fetch #{url}"
+			@log.error ex
+			return
+		end
+		links = parse_html url, html
+		links["html"].each do |html_link|
+			parse_html_page html_link, url_cache
+		end
+		links["pdf"].each do |pdf_link|
+			process_pdf pdf_link
+		end
+	end
+
+	##
+	#Executes curl to fetch the requested url
+	#@param url requested url
+	#@param output_file output destination, if nil the output gets returned by
+	#this method
+	def fetch_url url, output_file=nil, curl_params=""
+		curl_params = "#{@auth_app} #{curl_params} --silent --user-agent \"#{@conf["user_agent"]}\""
+		curl_params += " -b #{@conf["cookie_jar"]} -c #{@conf["cookie_jar"]} -L -o \"#{output_file || "-"}\" #{url}"
+		@log.debug "Call curl on #{url}"
+		@log.debug "Curl parameters '#{curl_params}'"
+		`cd #{@base_dir}; curl #{curl_params}`
+	end
+
+	def post url, params, output_file=nil, curl_params=""
+		param_arr = []
+		params.each do |key, value|
+			param_arr << "#{CGI::escape(key)}=#{CGI::escape(value)}"
+		end
+		param = param_arr.join "&"
+		begin
+			fetch_url url, output_file, "#{curl_params} --data \"#{param}\""
+		rescue => ex
+			@log.error "Failed to POST #{url} with data #{params}"
+			@log.error ex
+			""
+		end
+	end
+
+	def parse_html url, html
+		doc = nil
+		links = {'pdf' => [], 'html' => []}
+		begin
+			doc = Nokogiri::HTML html
+		rescue => ex
+			@log.error "Parsing html from url #{url} failed"
+			return links
+		end
+		doc.css('a[href]').each do |link|
+			begin
+				link_url = url_chomp(URI.join(url, link.attributes["href"]).to_s).to_s
+				@log.debug "Process link #{link_url}"
+				if is_pdf_url link_url
+					links['pdf'] << link_url 
+					@log.debug "#{link_url} is pdf"
+				elsif is_html_url link_url
+					links['html'] << link_url
+					@log.debug "#{link_url} is html"
+				end
+			rescue => ex
+				@log.debug "Omit #{link}"
+			end
+		end
+		return links
+	end
+	
+	def get_field_value html, field
+		doc = nil
+		begin
+			doc = Nokogiri::HTML html
+		rescue => ex
+			@log.error "Parsing html failed"
+			@log.error ex
+			return ""
+		end
+		value = ""
+		doc.css("##{field}").each do |link|
+			value = link.attributes["value"].to_s
+		end
+		return value
+	end
+	
+	def get_type url
+		if is_excluded url
+			return ""
+		end
+		if @type_cache[url] == nil
+			if url.upcase.end_with?(".PDF") || 
+				get_file_header(url)["Content-Type"].start_with?("application/pdf", "application/x-pdf")
+				@type_cache[url] = "pdf"
+			elsif get_file_header(url)["Content-Type"].start_with?("text/html")
+				@type_cache[url] = "html"
+			else
+				@type_cache[url] = ""
+			end
+		end
+		return @type_cache[url]
+	end
+
+	def is_pdf_url url
+		get_type(url) == "pdf"
+	end
+
+	def is_html_url url
+		get_type(url) == "html" 
+	end
+	
+	def is_excluded url
+		parsed_url = get_uri url
+		parsed_url.path.send(:start_with?, @conf["exclude_file_endings"]) ||
+				parsed_url.host != @conf["pdfs"]["src_host"] ||
+				!parsed_url.path.start_with?(@conf["pdfs"]["src_path"])
+	end
+	
+	def access_pause_sleep
+		min = @conf["access_pause"]["min"]
+		max = @conf["access_pause"]["max"]
+		duration = Random.rand() * (max - min) + min
+		@log.debug "Sleep #{duration} seconds to behave a bit more human"
+		sleep duration
+	end
+
+	def get_file_header url
+		url = url_chomp url
+		if @file_header_cache[url] == nil
+			header = fetch_url url, "-", "-I"
+			lines = header.split("\r\n").map {|val| val.split(": ") }
+			response = {}
+			lines.each {|arr| response[arr[0]] = arr[1] }
+			@file_header_cache[url] = response
+			@log.info "Fetch header of #{url}"
+			access_pause_sleep
+		end
+		return @file_header_cache[url]
+	end
+
+	def get_path_url url
+		parsed = get_uri url
+		parsed.path + (parsed.query != "" ? "?#{parsed.query}": "")
+	end
+
+	def process_pdf url
+		url = url_chomp url
+		return unless @processed_pdfs[url] == nil
+		@log.info "Process pdf #{url}"
+		dest = get_dest_path url
+		if not @conf["pdfs"]["download_once"]
+			header_date = get_file_header(url)["Last-Modified"]
+			header_time = header_date != nil ? Time.parse(header_date).to_i : Time.now.to_i
+			file_time = File.exists?(dest) ? File.mtime(dest).to_i : 0
+			@log.info "Process pdf #{url} with mtime #{header_time}, file mtime #{file_time}"
+			if file_time >= header_time
+				@log.info "Destination file #{dest} isn't younger => no download"
+				return
+			end
+		elsif File.exists? dest
+			@log.info "Destination file exists => no download"
+			return
+		end
+		`mkdir -p "#{File.dirname(dest)}"` unless File.exists? File.dirname(dest)
+		@log.info "Destination file #{dest} is older => download"
+		begin
+			@log.debug(fetch_url url, dest)
+		rescue => ex
+			@log.error "Downloading #{url} failed"
+			@log.error ex
+		end
+		@processed_pdfs[url] = dest
+		access_pause_sleep
+	end
+	
+	def get_dest_path url
+		url_path = get_uri(url).path
+		src_path = @conf["pdfs"]["src_path"]
+		dest_folder = @conf["pdfs"]["dest_folder"]
+		dest_folder + "/" + url_path.slice(src_path.length, url_path.length - src_path.length)
+	end
+
+	def self.add_service_class name, description, service_class, needs_auth = true, url_regex = nil
+		@@service_classes[name] = { 
+			"class" => service_class,
+			"url_regex" => url_regex,
+			"description" => description,
+			"needs_auth" => needs_auth
+		}
+	end
+	
+	def get_uri url
+		if @uri_cache[url] == nil
+			@uri_cache[url] = URI.parse url
+		end
+		return @uri_cache[url]
+	end
+	
+	def url_chomp url
+		uri = get_uri url
+		uri.scheme + "://" + uri.host + uri.path + (uri.query != nil ? "?#{uri.query}" : "")
+	end
+	
+	def self.get_services
+		@@service_classes.clone
+	end
+	
+	def self.get_service_for_url url
+		@@service_classes.each do |name, service|
+			unless service["url_regex"] == nil && service["url_regex"] =~ url
+				return name 
+			end
+		end
+		return "base"
+	end	
+	
+	self.add_service_class "base", "without any authentication", self, false, nil
+
+end
+
+class SecuredService < BaseService
+
+	def authenticate
+		unless @conf["auth"] != nil && @conf["auth"]["user"] != nil && @conf["auth"]["pass"]
+			raise "No authentication (user name and password) given"
+		end
+		_authenticate
+	end
+	
+	def _authenticate
+	end
+
+end
+
+class StudiumKITService < SecuredService
+	
+	def _authenticate
+		login_url = "https://studium.kit.edu/_layouts/login.aspx?ReturnUrl=%2f"
+		html = nil
+		post_html = nil
+		begin 
+			html = fetch_url login_url
+			params = {
+				"ctl00$PlaceHolderMain$Login$UserName" => @conf["auth"]["user"],
+				"ctl00$PlaceHolderMain$Login$password" => @conf["auth"]["pass"],
+				"ctl00$PlaceHolderMain$Login$loginbutton" => "Anmelden",
+				"__VIEWSTATE" => get_field_value(html, "__VIEWSTATE"),
+				"__EVENTVALIDATION" => get_field_value(html, "__EVENTVALIDATION"),
+				"__spDummyText1" => "",
+				"__spDummyText2" => ""
+			}
+		rescue => ex
+			@log.fatal ex
+			raise "Fetching and parsing login page failed"
+		end
+		begin
+			post_html = post login_url, params
+		rescue => ex
+			@log.fatal ex
+			raise "POST request to login page failed"
+		end
+		if post_html == html
+			raise "Authentication failed, wrong user name or password"
+		end
+	end
+	
+	self.add_service_class "studium_kit", "studium.kit.edu service", self, true, /studium\.kit\.edu/
+	
+end
+
+class HTTPAuthService < SecuredService
+	
+	def _authenticate
+		@auth_app = "-u #{URI::escape @conf["auth"]["user"]}:#{URI::escape @conf["auth"]["pass"]}"
+	end
+	
+	self.add_service_class "http_auth", "http authenticated service", self, true, nil
+	
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: kitcrawler
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+  prerelease: 
+platform: ruby
+authors:
+- Johannes Bechberger
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-07-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.6.1
+- !ruby/object:Gem::Dependency
+  name: highline
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.19.0
+- !ruby/object:Gem::Dependency
+  name: damerau-levenshtein
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: ! "\tCrawl lecture websites and fetch the PDFs automatically. \n\tIt
+  currently supports the studium.kit.edu and other HTTP password protected sites.\n"
+email: me@mostlynerdless.de
+executables:
+- kitcrawler
+extensions: []
+extra_rdoc_files:
+- README.md
+files:
+- lib/kitcrawler.rb
+- lib/services.rb
+- lib/cli.rb
+- lib/cli_add.rb
+- README.md
+- bin/kitcrawler
+homepage: https://github.com/parttimenerd/KITCrawler
+licenses:
+- GPL v3
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: 1.8.6
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- Linux (other UNIXes might also work)
+- curl
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Fetch lecture PDFs with ease
+test_files: []