brandeins-dl 0.0.1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,26 @@
+# About BrandEins Downloader
+
+BrandEins Downloader is a command line tool to download former volumes
+of the german oeconimic magazine "Brand Eins". The articles of former
+are available through there website and BrandEins Downloader takes all
+these fragmented PDFs, downloads and merges them into a single pdf.
+
+
+## Requirements
+BrandEins Downloader uses *pdftk* and depends on *ruby*, *rubygems*, and
+several ruby libraries (that you can get through rubygems)
+
+
+## Install
+`gem install brandeins-dl`
+
+
+## Usage
+
+Download just one magazine
+
+`brandeins download --path=/Path/where/to/download/the/files --year=2011 --volume=5`
+
+Download the whole collecion of a certain year
+
+`brandeins download_all --path=/Path/where/to/download/the/files --year=2011`

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/bin/brandeins

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'brandeins-dl'
+BrandEins::CLI.start

data/brandeins-dl.gemspec

@@ -0,0 +1,23 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "brandeins-dl/version"
+
+Gem::Specification.new do |s|
+  s.name        = "brandeins-dl"
+  s.version     = BrandEins::VERSION
+  s.authors     = ["Gregory Igelmund"]
+  s.email       = ["gregory.igelmund@gmail.com"]
+  s.homepage    = "http://www.grekko.de"
+  s.summary     = %q{BrandEins Downloader allows you to download past volumes of the Brand Eins magazine}
+  s.description = %q{BrandEins Downloader offers two commands: 'brandeins download YEAR' and 'brandeins download YEAR --volume=NUMBER'}
+
+  #s.rubyforge_project = "brandeins-dl"
+  s.add_dependency "rake"
+  s.add_dependency "thor"
+  s.add_dependency "nokogiri"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/brandeins-dl/version.rb

@@ -0,0 +1,3 @@
+module BrandEins
+  VERSION = "0.0.1"
+end

data/lib/brandeins-dl.rb

@@ -0,0 +1,228 @@
+require "brandeins-dl/version"
+require 'nokogiri'
+require 'open-uri'
+require 'uri'
+require 'fileutils'
+require 'thor'
+
+module BrandEins
+
+  class CLI < Thor
+    desc "download_all", "Download all magazines of the defined year"
+    method_option :year, :type => :numeric, :required => true
+    method_option :path, :type => :string, :required => true
+    def download_all
+      b1 = BrandEins::Downloader.new(options.path)
+      b1.get_magazines_of_year(options.year)
+    end
+
+    desc "download", "Download all magazines of the defined year"
+    method_option :path, :type => :string, :required => true
+    method_option :volume, :type => :numeric, :required => true
+    method_option :year, :type => :numeric, :required => true
+    def download
+      b1 = BrandEins::Downloader.new(options.path)
+      b1.get_magazine(options.year, options.volume)
+    end
+  end
+
+  class Downloader
+    attr_reader :archive
+  
+    def initialize(path)
+      @url     = "http://www.brandeins.de"
+      @archive = ArchiveSite.new
+      @dl_dir  = path
+
+      check_download_path
+    end
+
+    def check_download_path
+      Dir.mkdir(@dl_dir) unless File.exists?(@dl_dir)
+    end
+  
+    def get_magazines_of_year(year = 2000)
+      puts "Getting all brand eins magazines of a #{year}. This could take a while..."
+      magazine_links_per_year = @archive.magazine_links_by_year(year)
+      magazine_links_per_year.each_with_index do |magazine_link, volume|
+        puts "Parsing Volume #{volume} of #{year}"
+        target_pdf = get_target_pdf(year, volume)
+        get_magazine_by_link(magazine_link, target_pdf)
+      end
+    end
+  
+    def get_magazine(year = 2000, volume = 1)
+      puts "Parsing Volume #{volume} of #{year}"
+      target_pdf = get_target_pdf(year, volume)
+  
+      magazine_links = @archive.magazine_links_by_year(year)
+      target_magazine_link = magazine_links[volume-1]
+  
+      get_magazine_by_link(target_magazine_link, target_pdf)
+    end
+  
+    def get_magazine_by_link(target_magazine_link, target_pdf)
+      pdf_links = @archive.magazine_pdf_links(target_magazine_link)
+      process_pdf_links(pdf_links, target_pdf)
+      cleanup
+    end
+  
+  
+    def get_target_pdf(year, volume)
+      "Brand-Eins-#{year}-#{volume}.pdf"
+    end
+  
+    def process_pdf_links(pdf_links, target_pdf)
+      pdf_downloader = PDFDownloader.new(pdf_links, @dl_dir)
+      pdf_files = pdf_downloader.download_all
+      merge_pdfs(pdf_files, target_pdf)
+    end
+  
+    def merge_pdfs(pdf_files, target_pdf)
+      puts "Merging single PDFs now"
+      pdf_sources = pdf_files.join(" ")
+      system "pdftk #{pdf_sources} output #{@dl_dir}/#{target_pdf}"
+    end
+  
+    def cleanup
+      FileUtils.rm_r Dir.glob("#{@dl_dir}/*")
+    end
+  
+    class PDFDownloader
+  
+      def initialize(pdf_links, dl_dir)
+        @dl_dir    = dl_dir
+        @pdf_links = pdf_links
+      end
+  
+      def download_all
+        pdf_files = Array.new
+        @pdf_links.each do |pdf_link|
+          pdf_name = @dl_dir + '/' + File.basename(pdf_link)
+          pdf_url = pdf_link
+          download_pdf(pdf_url, pdf_name)
+  
+          pdf_files << pdf_name
+        end
+        pdf_files
+      end
+  
+      private 
+  
+      def download_pdf(pdf_url, filename)
+        puts "Downloading PDF from #{pdf_url} to #{filename}"
+        File.open(filename,'w') do |f|
+          uri = URI.parse(pdf_url)
+          Net::HTTP.start(uri.host,uri.port) do |http| 
+            http.request_get(uri.path) do |res| 
+              res.read_body do |seg|
+                f << seg
+        #hack -- adjust to suit:
+                sleep 0.005
+              end
+            end
+          end
+        end
+      end
+  
+    end
+  
+    class ArchiveSite
+  
+      attr_accessor :doc
+  
+      def initialize
+        @base_url = "http://www.brandeins.de"
+        @archive_url = @base_url + "/archiv.html"
+        @doc = Nokogiri::HTML(open(@archive_url))
+      end
+  
+      def magazine_links_by_year(year = 2000)
+        puts "Loading Magazine from year #{year}"
+        magazine_nodes_with_meta = @doc.css(".jahrgang-#{year} ul li")
+        magazine_links = Array.new
+        magazine_nodes_with_meta.each_with_index do |node, index|
+          if node['id'].nil? then
+            link = node.css('a')
+            if link[0].nil? then
+              next
+            end
+            href = link[0]['href']
+            magazine_links << @base_url + '/' + href
+          end
+        end
+        magazine_links
+      end
+  
+      def magazine_pdf_links(url)
+        magazine = ArchiveMagazine.new(url)
+        magazine.get_magazine_pdf_links
+      end
+  
+      class ArchiveMagazine
+        attr_accessor :url, :doc
+  
+        def initialize(url)
+          puts "Parsing #{url}"
+          @url = url
+          @base_url = "http://www.brandeins.de"
+          @doc = Nokogiri::HTML(open(url))
+        end
+  
+        def get_magazine_pdf_links
+          [get_editorial_article_links, get_schwerpunkt_article_links].flatten
+  
+        end
+  
+        def get_schwerpunkt_article_links
+          get_links("div.articleList ul h4 a")
+        end
+  
+        def get_editorial_article_links
+          get_links(".editorial-links li a")
+        end
+  
+        def get_links(css_selector)
+          pdf_links = Array.new
+          link_nodes = @doc.css(css_selector)
+          link_nodes.each do |node|
+            article_link = @base_url + '/' + node['href']
+            article = MagazineArticle.new(article_link)
+            pdf_link = article.get_pdf_link
+            if pdf_link.nil? then
+              puts "------------------------------"
+              puts "No Content for: #{article_link}"
+              puts "------------------------------"
+            else
+              pdf_links << @base_url + '/' + pdf_link
+            end
+          end
+          pdf_links
+        end
+  
+        class MagazineArticle
+          attr_accessor :url, :doc
+  
+          def initialize(url)
+            puts "Parsing Article: #{url}"
+            @url = url
+            @doc = Nokogiri::HTML(open(url))
+          end
+  
+          def get_pdf_link
+            link = @doc.css("div#sidebar ul li#downloaden a")
+            if link[0].nil? then
+              return nil
+            else
+              href = link[0]['href']
+            end
+          end
+
+        end
+
+      end
+
+    end
+
+  end  
+end

metadata

@@ -0,0 +1,94 @@
+--- !ruby/object:Gem::Specification 
+name: brandeins-dl
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Gregory Igelmund
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-11-04 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: rake
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: thor
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+description: "BrandEins Downloader offers two commands: 'brandeins download YEAR' and 'brandeins download YEAR --volume=NUMBER'"
+email: 
+- gregory.igelmund@gmail.com
+executables: 
+- brandeins
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- Gemfile
+- README.md
+- Rakefile
+- bin/brandeins
+- brandeins-dl.gemspec
+- lib/brandeins-dl.rb
+- lib/brandeins-dl/version.rb
+homepage: http://www.grekko.de
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: BrandEins Downloader allows you to download past volumes of the Brand Eins magazine
+test_files: []
+