brandeins 0.0.9

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/.rvmrc

@@ -0,0 +1,48 @@
+#!/usr/bin/env bash
+
+# This is an RVM Project .rvmrc file, used to automatically load the ruby
+# development environment upon cd'ing into the directory
+
+# First we specify our desired <ruby>[@<gemset>], the @gemset name is optional,
+# Only full ruby name is supported here, for short names use:
+#     echo "rvm use 1.9.3" > .rvmrc
+environment_id="ruby-1.9.3-p286@brandeins"
+
+# Uncomment the following lines if you want to verify rvm version per project
+# rvmrc_rvm_version="1.16.19 (stable)" # 1.10.1 seams as a safe start
+# eval "$(echo ${rvm_version}.${rvmrc_rvm_version} | awk -F. '{print "[[ "$1*65536+$2*256+$3" -ge "$4*65536+$5*256+$6" ]]"}' )" || {
+#   echo "This .rvmrc file requires at least RVM ${rvmrc_rvm_version}, aborting loading."
+#   return 1
+# }
+
+# First we attempt to load the desired environment directly from the environment
+# file. This is very fast and efficient compared to running through the entire
+# CLI and selector. If you want feedback on which environment was used then
+# insert the word 'use' after --create as this triggers verbose mode.
+if [[ -d "${rvm_path:-$HOME/.rvm}/environments"
+  && -s "${rvm_path:-$HOME/.rvm}/environments/$environment_id" ]]
+then
+  \. "${rvm_path:-$HOME/.rvm}/environments/$environment_id"
+  [[ -s "${rvm_path:-$HOME/.rvm}/hooks/after_use" ]] &&
+    \. "${rvm_path:-$HOME/.rvm}/hooks/after_use" || true
+else
+  # If the environment file has not yet been created, use the RVM CLI to select.
+  rvm --create  "$environment_id" || {
+    echo "Failed to create RVM environment '${environment_id}'."
+    return 1
+  }
+fi
+
+# If you use bundler, this might be useful to you:
+# if [[ -s Gemfile ]] && {
+#   ! builtin command -v bundle >/dev/null ||
+#   builtin command -v bundle | GREP_OPTIONS= \grep $rvm_path/bin/bundle >/dev/null
+# }
+# then
+#   printf "%b" "The rubygem 'bundler' is not installed. Installing it now.\n"
+#   gem install bundler
+# fi
+# if [[ -s Gemfile ]] && builtin command -v bundle >/dev/null
+# then
+#   bundle install | GREP_OPTIONS= \grep -vE '^Using|Your bundle is complete'
+# fi

data/Gemfile

@@ -0,0 +1,11 @@
+source "http://rubygems.org"
+
+group :test do
+  if ENV['RUBY_VERSION'][5,3] == '1.8'
+    gem 'minitest'
+  end
+  gem 'fakefs'
+end
+
+
+gemspec

data/README.md

@@ -0,0 +1,26 @@
+# About BrandEins Downloader
+
+BrandEins Downloader is a command line tool to download former volumes
+of the german oeconimic magazine "Brand Eins". The articles of former
+are available through there website and BrandEins Downloader takes all
+these fragmented PDFs, downloads and merges them into a single pdf.
+
+
+## Requirements
+BrandEins Downloader uses *pdftk* and depends on *ruby*, *rubygems*, and
+several ruby libraries (that you can get through rubygems)
+
+
+## Install
+`gem install brandeins-dl`
+
+
+## Usage
+
+Download just one magazine
+
+`brandeins download --path=/Path/where/to/download/the/files --year=2011 --volume=5`
+
+Download the whole collecion of a certain year
+
+`brandeins download_all --path=/Path/where/to/download/the/files --year=2011`

data/Rakefile

@@ -0,0 +1,18 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+require './lib/brandeins/version'
+
+Rake::TestTask.new do |t|
+  t.pattern = 'test/*_test.rb'
+end
+
+task :build do
+  sh 'gem build brandeins.gemspec'
+end
+
+task :publish do
+  version = BrandEins::VERSION
+  sh "gem push brandeins-#{version}.gem"
+end
+
+task :default => :test

data/TODOS.md

@@ -0,0 +1,3 @@
+# ToDos
+gem build brandeins-dl.gemspec
+gem push brandeins-dl-*.*.*.gem

data/bin/brandeins

@@ -0,0 +1,4 @@
+#!/usr/bin/env ruby
+
+require 'brandeins'
+BrandEins::CLI.start

data/brandeins.gemspec

@@ -0,0 +1,29 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "brandeins/version"
+
+Gem::Specification.new do |s|
+  s.name        = "brandeins"
+  s.version     = BrandEins::VERSION
+  s.authors     = ["Gregory Igelmund"]
+  s.email       = ["gregory.igelmund@gmail.com"]
+  s.homepage    = "http://www.grekko.de"
+  s.summary     = %q{BrandEins gem allows you to download past volumes of the Brand Eins magazine}
+  s.description = %q{BrandEins gem offers a download command to download a specific or all volumes. Use `brandeins help` to find out more about it}
+
+  s.add_dependency "rake"
+  s.add_dependency "thor"
+  s.add_dependency "nokogiri"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.post_install_message =<<-EOT
+BrandEins gem currently runs on unix systems only and depends on pdftk to merge downloaded pdfs.
+Run `brandeins setup` to check if all requirements are met and for informations on how to meet them.
+
+
+EOT
+end

data/lib/brandeins/helper.rb

@@ -0,0 +1,9 @@
+module BrandEinsHelper
+  def self.windows?
+    RUBY_PLATFORM.include? 'w32'
+  end
+
+  def self.osx?
+    RUBY_PLATFORM.include? 'darwin'
+  end
+end

data/lib/brandeins/setup.rb

@@ -0,0 +1,32 @@
+class BrandEinsSetup
+  def initialize
+    p 'Checking requirements for your system'
+    if pdftk?
+      p 'It seems you have pdftk installed on your system.'
+    else
+      p 'It seems you are missing pdfk on your system.'
+      p pdfk_install_instructions
+    end
+  end
+
+  def pdfk_install_instructions
+    'Visit http://www.pdflabs.com/docs/install-pdftk/ to install pdftk on your system'
+  end
+
+  def pdftk?
+    cmd? 'pdftk --version', 'pdftk.com'
+  end
+
+  private
+  def cmd?(cmd, hint)
+    f = IO.popen cmd
+    f.readlines.each do |line|
+      if line.include? hint
+        return true
+      end
+    end
+    return false
+  end
+end
+
+BrandEinsSetup.new

data/lib/brandeins/version.rb

@@ -0,0 +1,3 @@
+module BrandEins
+  VERSION = '0.0.9'
+end

data/lib/brandeins.rb

@@ -0,0 +1,256 @@
+%w(
+  brandeins/version
+  brandeins/helper
+  nokogiri
+  open-uri
+  uri
+  fileutils
+  thor
+).each do |lib|
+    require lib
+end
+
+module BrandEins
+
+  class CLI < Thor
+    map '--version' => :version
+
+    desc '--version', 'Displays current version'
+    def version
+      p BrandEins::VERSION
+    end
+
+    desc 'download', 'Download past brand eins magazines (use `brandeins help download` to learn more about options)'
+    method_option :path, :type => :string
+    method_option :volume, :type => :numeric
+    method_option :year, :type => :numeric
+    def download
+      path = options.path || File.expand_path('../..', __FILE__)
+      p "path: #{path}"
+      return false
+      b1 = BrandEins::Downloader.new(options.path)
+      b1.get_magazine(options.year, options.volume)
+    end
+
+    desc 'setup', 'Checks if all requirements to use brandeins are met and gives instructions how to meet them'
+    def setup
+      if BrandEinsHelper.windows? || BrandEinsHelper.osx?
+        require 'brandeins/setup'
+      else
+        p 'Unknown/unsupported operating system. Please contact the gem author.'
+      end
+    end
+  end
+
+  class Downloader
+    attr_reader :archive
+
+    def initialize(path)
+      @url     = 'http://www.brandeins.de'
+      @archive = false
+      @dl_dir  = path
+      @tmp_dir = path + '/tmp'
+      create_tmp_dirs
+    end
+
+    def setup
+      @archive = ArchiveSite.new @url
+    end
+
+    def get_magazines_of_year(year = 2000)
+      setup
+      puts "Getting all brand eins magazines of a #{year}. This could take a while..."
+      magazine_links_per_year = @archive.get_magazine_links_by_year(year)
+      magazine_links_per_year.each_with_index do |magazine_link, volume|
+        puts "Parsing Volume #{volume} of #{year}"
+        target_pdf = get_target_pdf(year, volume)
+        get_magazine_by_link(magazine_link, target_pdf)
+      end
+    end
+
+    def get_magazine(year = 2000, volume = 1)
+      setup
+      puts "Parsing Volume #{volume} of #{year}"
+      target_pdf = get_target_pdf(year, volume)
+
+      magazine_links = @archive.get_magazine_links_by_year(year)
+      target_magazine_link = magazine_links[volume-1]
+
+      get_magazine_by_link(target_magazine_link, target_pdf)
+    end
+
+    private
+    def create_tmp_dirs
+      FileUtils.mkdir_p @tmp_dir unless File.directory?(@tmp_dir)
+    end
+
+    def get_magazine_by_link(target_magazine_link, target_pdf)
+      pdf_links = @archive.magazine_pdf_links(target_magazine_link)
+      process_pdf_links(pdf_links, target_pdf)
+      cleanup
+    end
+
+    def get_target_pdf(year, volume)
+      "Brand-Eins-#{year}-#{volume}.pdf"
+    end
+
+    def process_pdf_links(pdf_links, target_pdf)
+      pdf_downloader = PDFDownloader.new(pdf_links, @tmp_dir)
+      pdf_files = pdf_downloader.download_all
+      merge_pdfs(pdf_files, target_pdf)
+    end
+
+    def merge_pdfs(pdf_files, target_pdf)
+      puts "Merging single PDFs now"
+      pdf_sources = pdf_files.join(" ")
+      system "pdftk #{pdf_sources} output #{@dl_dir}/#{target_pdf}"
+    end
+
+    def cleanup
+      FileUtils.rm_r @tmp_dir
+    end
+
+    class PDFDownloader
+
+      def initialize(pdf_links, dl_dir)
+        @dl_dir    = dl_dir
+        @pdf_links = pdf_links
+      end
+
+      def download_all
+        pdf_files = Array.new
+        @pdf_links.each do |pdf_link|
+          pdf_name = @dl_dir + '/' + File.basename(pdf_link)
+          pdf_url = pdf_link
+          download_pdf(pdf_url, pdf_name)
+
+          pdf_files << pdf_name
+        end
+        pdf_files
+      end
+
+      private
+
+      def download_pdf(pdf_url, filename)
+        puts "Downloading PDF from #{pdf_url} to #{filename}"
+        File.open(filename,'w') do |f|
+          uri = URI.parse(pdf_url)
+          Net::HTTP.start(uri.host,uri.port) do |http|
+            http.request_get(uri.path) do |res|
+              res.read_body do |seg|
+                f << seg
+        #hack -- adjust to suit:
+                sleep 0.005
+              end
+            end
+          end
+        end
+      end
+
+    end
+
+    class ArchiveSite
+      attr_accessor :doc
+
+      def initialize(base_url, html = false)
+        @base_url = base_url
+        @archive_url = @base_url + "/archiv.html"
+        if html
+          @doc = Nokogiri::HTML(html)
+        end
+      end
+
+      def setup
+        return if defined?(@doc) != nil
+        @doc = Nokogiri::HTML(open(@archive_url))
+      end
+
+      def get_magazine_links_by_year(year = 2000)
+        setup
+        puts "Loading Magazine from year #{year}"
+        magazine_nodes_with_meta = @doc.css(".jahrgang-#{year} ul li")
+        magazine_links = Array.new
+        magazine_nodes_with_meta.each_with_index do |node, index|
+          if node['id'].nil? then
+            link = node.css('a')
+            if link[0].nil? then
+              next
+            end
+            href = link[0]['href']
+            magazine_links << @base_url + '/' + href
+          end
+        end
+        magazine_links
+      end
+
+      def magazine_pdf_links(url)
+        magazine = ArchiveMagazine.new(url, @base_url)
+        magazine.get_magazine_pdf_links
+      end
+
+      class ArchiveMagazine
+        attr_accessor :url, :doc
+
+        def initialize(url, base_url, html = false)
+          puts "Parsing #{url}"
+          @url = url
+          @base_url = base_url
+          @doc = Nokogiri::HTML(open(url))
+        end
+
+        def get_magazine_pdf_links
+          [get_editorial_article_links, get_schwerpunkt_article_links].flatten
+
+        end
+
+        def get_schwerpunkt_article_links
+          get_links("div.articleList ul h4 a")
+        end
+
+        def get_editorial_article_links
+          get_links(".editorial-links li a")
+        end
+
+        def get_links(css_selector)
+          pdf_links = Array.new
+          link_nodes = @doc.css(css_selector)
+          link_nodes.each do |node|
+            article_link = @base_url + '/' + node['href']
+            article = MagazineArticle.new(article_link)
+            pdf_link = article.get_pdf_link
+            if pdf_link.nil? then
+              puts "------------------------------"
+              puts "No Content for: #{article_link}"
+              puts "------------------------------"
+            else
+              pdf_links << @base_url + '/' + pdf_link
+            end
+          end
+          pdf_links
+        end
+
+        class MagazineArticle
+          attr_accessor :url, :doc
+
+          def initialize(url)
+            puts "Parsing Article: #{url}"
+            @url = url
+            @doc = Nokogiri::HTML(open(url))
+          end
+
+          def get_pdf_link
+            link = @doc.css("div#sidebar ul li#downloaden a")
+            if link[0].nil? then
+              return nil
+            else
+              href = link[0]['href']
+            end
+          end
+
+        end
+
+      end
+    end
+
+  end
+end

data/test/brandeins_test.rb

@@ -0,0 +1,38 @@
+require File.expand_path('../../lib/brandeins' , __FILE__)
+require 'minitest/autorun'
+require 'fakefs/safe'
+
+class TestBrandEinsDownload < MiniTest::Unit::TestCase
+  def setup
+    @base_url = 'http://www.brandeins.de'
+    @dir      = 'bdl'
+    # Es muss eine Moeglichkeit geben beim Testen zu verhindern, dass
+    # die Existenz des Pfades geprueft wird.
+    # testing im allgemeinen: http://holmwood.id.au/~lindsay/2008/04/26/hints-for-testing-your-evolving-ruby-scripts/
+  end
+
+  def test_tmp_directories_get_created
+    FakeFS do
+      bdl = BrandEins::Downloader.new @dir
+      assert File.directory?(File.expand_path("./#{@dir}/tmp"))
+    end
+  end
+
+  def test_magazine_url_scraping
+    html = <<-EOF
+      <div class="jahrgang jahrgang-2012 jahrgang-latest">
+        <h4>2012</h4>
+        <ul>
+          <li><a href="magazin/nein-sagen.html" title="Zum Magazin brand eins Online 1 2012" onmouseover="switch_magazine(2012, 1)" onfocus="switch_magazine(2012, 1)"><img src="typo3temp/pics/b9d755e0d1.jpg" width="55" height="73" alt="Ausgabe 01/2012 SCHWERPUNKT NEIN SAGEN"></a> 1</li>
+          <li><a href="magazin/markenkommunikation.html" title="Zum Magazin brand eins Online 2 2012" onmouseover="switch_magazine(2012, 2)" onfocus="switch_magazine(2012, 2)"><img src="typo3temp/pics/1dccfc2c74.jpg" width="55" height="73" alt="Ausgabe 02/2012 SCHWERPUNKT Markenkommunikation"></a> 2</li>
+        </ul>
+      </div>
+    EOF
+
+    archive_site = BrandEins::Downloader::ArchiveSite.new @base_url, html
+    magazine_links = archive_site.get_magazine_links_by_year(2012)
+    assert_equal magazine_links.length, 2
+    assert_equal magazine_links[0], (@base_url + '/magazin/nein-sagen.html')
+    assert_equal magazine_links[1], (@base_url + '/magazin/markenkommunikation.html')
+  end
+end

metadata

@@ -0,0 +1,123 @@
+--- !ruby/object:Gem::Specification
+name: brandeins
+version: !ruby/object:Gem::Version
+  version: 0.0.9
+  prerelease: 
+platform: ruby
+authors:
+- Gregory Igelmund
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: BrandEins gem offers a download command to download a specific or all
+  volumes. Use `brandeins help` to find out more about it
+email:
+- gregory.igelmund@gmail.com
+executables:
+- brandeins
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rvmrc
+- Gemfile
+- README.md
+- Rakefile
+- TODOS.md
+- bin/brandeins
+- brandeins.gemspec
+- lib/brandeins.rb
+- lib/brandeins/helper.rb
+- lib/brandeins/setup.rb
+- lib/brandeins/version.rb
+- test/brandeins_test.rb
+homepage: http://www.grekko.de
+licenses: []
+post_install_message: ! 'BrandEins gem currently runs on unix systems only and depends
+  on pdftk to merge downloaded pdfs.
+
+  Run `brandeins setup` to check if all requirements are met and for informations
+  on how to meet them.
+
+
+
+'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 78102480951930165
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 78102480951930165
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.24
+signing_key: 
+specification_version: 3
+summary: BrandEins gem allows you to download past volumes of the Brand Eins magazine
+test_files:
+- test/brandeins_test.rb