brandeins 0.1.6 → 0.2.0

data/lib/brandeins/downloader.rb

@@ -0,0 +1,112 @@
+# encoding: utf-8
+require 'brandeins/parser/article_site'
+require 'brandeins/parser/magazine_site'
+require 'brandeins/parser/archive_site'
+require 'brandeins/merger/pdf_tools'
+require 'net/http'
+require 'prawn'
+
+module BrandEins
+  class Downloader
+
+    def initialize(path, opts = {})
+      $BE_VERBOSE = !!opts[:verbose]
+
+      @url     = 'http://www.brandeins.de'
+      @dl_dir  = path
+      @tmp_dir = @dl_dir + '/brand-eins-tmp'
+      @pdftool = BrandEins::Merger::PDFTools.get_pdf_tool
+      @archive = BrandEins::Parser::ArchiveSite.new(@url)
+      create_tmp_dirs
+    end
+
+    def get_magazines_of_year(year = 2000)
+      puts "Getting all brand eins magazines of #{year}. This may take a while..." if $BE_VERBOSE
+      magazine_links_per_year = @archive.get_magazine_links_by_year(year)
+      magazine_links_per_year.each_with_index do |magazine_link, index|
+        volume = index+1
+        puts "Parsing Volume #{volume} of #{year}" if $BE_VERBOSE
+        target_pdf = pdf_filename(year, volume)
+        get_magazine_by_link(magazine_link, target_pdf, year, volume)
+      end
+    end
+
+    def get_magazine(year = 2000, volume = 1)
+      puts "Parsing Volume #{volume} of #{year}" if $BE_VERBOSE
+      target_pdf = pdf_filename(year, volume)
+      magazine_links = @archive.get_magazine_links_by_year(year)
+      magazine_link  = magazine_links[volume-1]
+      get_magazine_by_link(magazine_link, target_pdf, year, volume)
+    end
+
+    private
+    def create_tmp_dirs
+      FileUtils.mkdir_p @tmp_dir unless File.directory?(@tmp_dir)
+    end
+
+    def create_cover_pdf(year, volume)
+      puts "Creating cover for Volume #{volume} of #{year}" if $BE_VERBOSE
+      cover = @archive.get_magazine_cover(year, volume)
+      cover_title    = cover[:title]
+      cover_img_url  = cover[:img_url]
+      cover_img_file = @tmp_dir + "/cover-#{year}-#{volume}.jpg"
+      cover_pdf_file = @tmp_dir + "/cover-#{year}-#{volume}.pdf"
+
+      puts "Downloading cover image from #{cover_img_url} to #{cover_img_file}" if $BE_VERBOSE
+      IO.binwrite(cover_img_file, Net::HTTP.get(URI(cover_img_url)))
+
+      puts "Creating cover pdf #{cover_pdf_file} from #{cover_img_file}" if $BE_VERBOSE
+      Prawn::Document.generate(cover_pdf_file) do |pdf|
+        pdf.text "<font size='18'><b>" + cover_title + "</b></font>", :align => :center, :inline_format => true
+        pdf.image cover_img_file, :position => :center, :vposition => :center
+      end
+
+      return cover_pdf_file
+    end
+
+    def get_magazine_by_link(target_magazine_link, target_pdf, year, volume)
+      pdf_links = @archive.magazine_pdf_links(target_magazine_link)
+      pdf_files = download(pdf_links)
+
+      pdf_cover = create_cover_pdf(year, volume)
+      pdf_files = pdf_files.reverse.push(pdf_cover).reverse
+
+      if !@pdftool.nil?
+        target_pdf_path = "#{@dl_dir}/#{target_pdf}"
+        @pdftool.merge_pdf_files(pdf_files, target_pdf_path)
+        cleanup
+      else
+        puts 'brandeins wont merge the single pdf files since it didnt find an appropriate pdf tool' if $BE_VERBOSE
+      end
+    end
+
+    def pdf_filename(year, volume)
+      "Brand-Eins-#{year}-#{volume}.pdf"
+    end
+
+    def download(pdf_links)
+      pdf_links.each_with_object([]) do |pdf_link, pdf_files|
+        pdf_filename = @tmp_dir + '/' + File.basename(pdf_link)
+        pdf_url = pdf_link
+        download_pdf(pdf_url, pdf_filename)
+        pdf_files << pdf_filename
+      end
+    end
+
+    def download_pdf(pdf_url, pdf_filename)
+      if File.exists? pdf_filename
+        puts "File #{pdf_filename} seems to be already downloaded" if $BE_VERBOSE
+        return true
+      end
+
+      puts "Downloading PDF from #{pdf_url} to #{pdf_filename}" if $BE_VERBOSE
+      IO.binwrite(pdf_filename, Net::HTTP.get(URI(pdf_url)))
+    end
+
+    def cleanup
+      FileUtils.rm_r @tmp_dir
+    end
+
+  end
+
+end

data/lib/brandeins/merger/extensions.rb

@@ -0,0 +1,9 @@
+# encoding: utf-8
+
+class Class
+
+  def subclasses
+    ObjectSpace.each_object(Class).select { |klass| klass < self }
+  end
+
+end

data/lib/brandeins/merger/pdf_tools.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+require 'brandeins/merger/extensions'
+require 'brandeins/merger/templates/base'
+require 'brandeins/merger/templates/osx'
+require 'brandeins/merger/templates/pdftk_osx'
+require 'brandeins/merger/templates/windows'
+require 'brandeins/merger/templates/ghostscript_windows'
+
+module BrandEins
+  module Merger
+    class PDFTools
+
+      class << self
+        def get_pdf_tool(env = nil)
+          @env = env || {}
+          @env[:os] ||= RUBY_PLATFORM
+
+          init_pdf_tools and return_pdf_tool
+        end
+
+        private
+        def init_pdf_tools
+          template_group = choose_template_group
+          @pdf_tools = get_subclasses template_group
+        end
+
+        def choose_template_group
+          return BrandEins::Merger::Templates::Windows if @env[:os].include? 'w32'
+          return BrandEins::Merger::Templates::OSX     if @env[:os].include? 'darwin'
+        end
+
+        def get_subclasses(klass)
+          classes = []
+          klass.subclasses.each do |sklass|
+            classes << sklass.new
+          end
+        end
+
+        def return_pdf_tool
+          @pdf_tools.first.new if @pdf_tools.length > 0
+        end
+
+      end
+
+    end
+  end
+end

data/lib/brandeins/merger/templates/base.rb

@@ -0,0 +1,47 @@
+# encoding: utf-8
+
+module BrandEins
+  module Merger
+    module Templates
+
+      class Base
+
+        def available?
+          _cmd_available? cmd, noop
+        end
+
+        def cmd;  raise "Must be implemtented by the subclasses"; end
+        def noop; raise "Must be implemtented by the subclasses"; end
+        def args; raise "Must be implemtented by the subclasses"; end
+
+        def merge_pdf_files(pdf_files, target_pdf)
+          begin
+            pdf_files_arg = pdf_files.map {|pdf_file| "'#{pdf_file}'" }.join ' '
+            args = self.args.join(' ').gsub(/__pdf_files__/, pdf_files_arg).gsub(/__target_pdf__/, target_pdf)
+            puts "executing: #{cmd} #{args}"
+            _exec("#{cmd} #{args}")
+          rescue Exception => e
+            puts "error: #{e.inspect}"
+            return false
+          end
+          return true
+        end
+
+        private
+        def _exec (cmd)
+          IO.popen(cmd)
+        end
+
+        def _cmd_available? (cmd, args)
+          begin
+            open("|#{cmd} #{args}").close
+          rescue Exception
+            return false
+          end
+          return true
+        end
+      end
+
+    end
+  end
+end

data/lib/brandeins/merger/templates/ghostscript_windows.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+module BrandEins
+  module Merger
+    module Templates
+
+      class GhostscriptWin < BrandEins::Merger::Templates::Windows
+
+        def cmd
+          'gswin64c.exe'
+        end
+
+        def args
+          ['-dNOPAUSE', '-dBATCH', '-sDEVICE=pdfwrite', '-sOutputFile=__target_pdf__', '__pdf_files__']
+        end
+
+        def noop
+          ['--version']
+        end
+
+      end
+
+    end
+  end
+end

data/lib/brandeins/merger/templates/osx.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+module BrandEins
+  module Merger
+    module Templates
+
+      class OSX < BrandEins::Merger::Templates::Base; end
+
+    end
+  end
+end

data/lib/brandeins/merger/templates/pdftk_osx.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+
+module BrandEins
+  module Merger
+    module Templates
+
+      class PdftkOSX < BrandEins::Merger::Templates::OSX
+
+        def cmd
+          'pdftk'
+        end
+
+        def args
+          ['__pdf_files__', 'output', '__target_pdf__']
+        end
+
+        def noop
+          ['--version']
+        end
+
+      end
+
+    end
+  end
+end

data/lib/brandeins/merger/templates/windows.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+
+module BrandEins
+  module Merger
+    module Templates
+
+      class Windows < BrandEins::Merger::Templates::Base; end
+
+    end
+  end
+end

data/lib/brandeins/parser/archive_site.rb

@@ -0,0 +1,54 @@
+# encoding: utf-8
+require 'nokogiri'
+require 'net/http'
+
+module BrandEins
+  module Parser
+    class ArchiveSite
+
+      def initialize(base_url, opts = {})
+        @base_url    = base_url
+        @archive_url = @base_url + "/archiv.html"
+        if html = opts[:html]
+          @doc = Nokogiri::HTML(html)
+        end
+      end
+
+      def doc
+        @doc || @doc = Nokogiri::HTML(Net::HTTP.get(URI(@archive_url)))
+      end
+
+      def get_magazine_links_by_year(year = 2000)
+        puts "Loading Magazine from year #{year}" if $BE_VERBOSE
+        magazine_nodes_with_meta = doc.css(".jahrgang-#{year} ul li")
+        magazine_nodes_with_meta.each_with_object([]) do |node, links|
+          if node['id'].nil? then
+            link = node.css('a')
+            if link[0].nil? then
+              next
+            end
+            href = link[0]['href']
+            links << @base_url + '/' + href
+          end
+        end
+      end
+
+      def get_magazine_cover(year, volume)
+        title   = doc.css("#month_detail_#{year}_#{volume} .titel").children[0].to_s
+        img_url = ''
+        doc.css("#month_detail_#{year}_#{volume} .cover a img").each do |node|
+          img_url = node['src']
+        end
+        return { :title => title, :img_url => @base_url + '/' + img_url }
+      end
+
+      def magazine_pdf_links(url)
+        magazine = BrandEins::Parser::MagazineSite.new(url, @base_url)
+        magazine.get_magazine_pdf_links
+      end
+
+    end
+
+  end
+
+end

data/lib/brandeins/parser/article_site.rb

@@ -0,0 +1,26 @@
+# encoding: utf-8
+require 'nokogiri'
+require 'net/http'
+
+module BrandEins
+  module Parser
+    class ArticleSite
+
+      def initialize(url)
+        @url = url
+      end
+
+      def doc
+        @doc || @doc = Nokogiri::HTML(Net::HTTP.get(URI(@url)))
+      end
+
+      def get_pdf_link
+        puts "Parsing Article: #{@url}" if $BE_VERBOSE
+        link = doc.css("div#sidebar ul li#downloaden a")
+        link[0] and link[0]['href']
+      end
+
+    end
+
+  end
+end

data/lib/brandeins/parser/magazine_site.rb

@@ -0,0 +1,49 @@
+# encoding: utf-8
+require 'nokogiri'
+require 'net/http'
+
+module BrandEins
+  module Parser
+    class MagazineSite
+
+      def initialize(url, base_url)
+        @url, @base_url = url, base_url
+      end
+
+      def doc
+        @doc || @doc = Nokogiri::HTML(Net::HTTP.get(URI(@url)))
+      end
+
+      def get_magazine_pdf_links
+        puts "Parsing #{@url}" if $BE_VERBOSE
+        [get_editorial_article_links, get_schwerpunkt_article_links].flatten
+      end
+
+      def get_schwerpunkt_article_links
+        get_links("div.articleList ul h4 a")
+      end
+
+      def get_editorial_article_links
+        get_links(".editorial-links li a")
+      end
+
+      def get_links(css_selector)
+        link_nodes = doc.css(css_selector)
+        link_nodes.each_with_object([]) do |node, links|
+          article_link = @base_url + '/' + node['href']
+          article  = BrandEins::Parser::ArticleSite.new(article_link)
+          pdf_link = article.get_pdf_link
+          if pdf_link.nil? and $BE_VERBOSE then
+            puts "-------------------------------"
+            puts "No Content for: #{article_link}"
+            puts "-------------------------------"
+          else
+            links << @base_url + '/' + pdf_link
+          end
+        end
+      end
+    end
+
+  end
+
+end

data/lib/brandeins/setup.rb

@@ -1,13 +1,11 @@
-require File.expand_path '../pdf-tools', __FILE__
+# encoding: utf-8
 
 module BrandEins
   class Setup
-    attr_reader :pdf_tool
 
-    def initialize(env = nil)
-      env = Hash.new if env.nil?
+    def initialize(env = {})
       @os = env[:os] || RUBY_PLATFORM
-      @pdf_tool = env[:pdf_tool] || BrandEins::PdfTools.get_pdf_tool(env)
+      @pdf_tool = env[:pdf_tool] || BrandEins::Merging::PdfTools.get_pdf_tool(env)
     end
 
     def run
@@ -37,4 +35,4 @@ module BrandEins
     end
 
   end
-end
+end