webpage-archivist 0.0.1

data/lib/webpage-archivist/fetcher/stylesheet_request.rb

@@ -0,0 +1,112 @@
+require 'andand'
+
+module WebpageArchivist::Fetcher
+# A request specific for stylesheet as a stylesheet can reference images or other stylesheets
+  class StyleSheetRequest < ElementRequest
+
+
+    def initialize uri, plumber
+      super WebpageArchivist::Stylesheet, uri, plumber
+      @parsed_uri = Addressable::URI.parse(uri)
+      @waiting_requests = 0
+    end
+
+    def content_not_changed http
+      @modified = false
+      if element.andand.last_content
+        create_sub_requests http
+      else
+        self.status = :over
+        notify_requesters
+      end
+    end
+
+    def save_content_end_request http
+      @modified = true
+      create_sub_requests http
+    end
+
+    # Don't directly save the content but first retrieve any possible other stylesheet and image
+    def create_sub_requests http
+      if @modified
+        charset = @plumber.response_charset(http)
+        @stylesheet = WebpageArchivist::StylesheetDocument.new(http.response, @uri, charset)
+      else
+        @stylesheet = WebpageArchivist::StylesheetDocument.new(element.last_content, @uri, element.last_charset)
+      end
+
+      already = Set.new
+      @stylesheet.each_import do |i|
+        unless already.include? i
+          already.add i
+          i = @parsed_uri.absolutize(i)
+          @plumber.request_element self, WebpageArchivist::Stylesheet, i
+          @waiting_requests += 1
+          i
+        end
+      end
+
+      already = Set.new
+      @stylesheet.each_image do |i|
+        unless already.include? i
+          already.add i
+          i = @parsed_uri.absolutize(i)
+          @plumber.request_element self, WebpageArchivist::Image, i
+          @waiting_requests += 1
+          i
+        end
+      end
+
+      if @modified
+        element.update(:last_content => @stylesheet.to_css, :last_charset => @stylesheet.charset)
+      end
+
+      self.status = :fetching_requests
+
+      # No external resource -> end here
+      if @waiting_requests == 0
+        after_requests
+      end
+
+    end
+
+    # Called by a request when it is over
+    # uri:: the request uri
+    def request_over uri
+      @waiting_requests -= 1
+      ::WebpageArchivist.debug "Request over for [#{@uri}] on [#{uri}], missing #{@waiting_requests}" if ::WebpageArchivist.log
+
+      if (@waiting_requests == 0) && (status == :fetching_requests)
+        after_requests
+      end
+    end
+
+    # Process the response once all the elements have been fetched
+    def after_requests
+      ::WebpageArchivist.debug "After requests [#{@uri}]" if ::WebpageArchivist.log
+      if @modified
+        @stylesheet.each_import do |i|
+          if e = @plumber[i].andand.element
+            e.file_name
+          else
+            nil
+          end
+        end
+
+        @stylesheet.each_image do |i|
+          if e = @plumber[i].andand.element
+            e.file_name
+          else
+            nil
+          end
+        end
+
+        element.save_content @stylesheet.to_css
+      end
+      self.status = :over
+      notify_requesters
+    end
+
+  end
+
+end

data/lib/webpage-archivist/fetcher/thread-pool.rb

@@ -0,0 +1,101 @@
+# Ruby Thread Pool
+# ================
+# A thread pool is useful when you wish to do some work in a thread, but do
+# not know how much work you will be doing in advance. Spawning one thread
+# for each task is potentially expensive, as threads are not free.
+# 
+# In this case, it might be more beneficial to start a predefined set of
+# threads and then hand off work to them as it becomes available. This is
+# the pure essence of what a thread pool is: an array of threads, all just
+# waiting to do some work for you!
+#
+# Prerequisites
+# -------------
+
+# We need the [Queue](http://rdoc.info/stdlib/thread/1.9.2/Queue), as our
+# thread pool is largely dependent on it. Thanks to this, the implementation
+# becomes very simple!
+require 'thread'
+
+# Public Interface
+# ----------------
+
+# `Pool` is our thread pool class. It will allow us to do three operations:
+# 
+# - `.new(size)` creates a thread pool of a given size
+# - `#schedule(*args, &job)` schedules a new job to be executed
+# - `#shutdown` shuts down all threads (after letting them finish working, of course)
+class Pool
+
+  # ### initialization, or `Pool.new(size)`
+  # Creating a new `Pool` involves a certain amount of work. First, however,
+  # we need to define its’ `size`. It defines how many threads we will have
+  # working internally.
+  # 
+  # Which size is best for you is hard to answer. You do not want it to be
+  # too low, as then you won’t be able to do as many things concurrently.
+  # However, if you make it too high Ruby will spend too much time switching
+  # between threads, and that will also degrade performance!
+  def initialize(size)
+    # Before we do anything else, we need to store some information about
+    # our pool. `@size` is useful later, when we want to shut our pool down,
+    # and `@jobs` is the heart of our pool that allows us to schedule work.
+    @size = size
+    @jobs = Queue.new
+    
+    # #### Creating our pool of threads
+    # Once preparation is done, it’s time to create our pool of threads.
+    # Each thread store its’ index in a thread-local variable, in case we
+    # need to know which thread a job is executing in later on.
+    @pool = Array.new(@size) do |i|
+      Thread.new do
+        Thread.current[:id] = i
+
+        # We start off by defining a `catch` around our worker loop. This
+        # way we’ve provided a method for graceful shutdown of our threads.
+        # Shutting down is merely a `#schedule { throw :exit }` away!
+        catch(:exit) do
+          # The worker thread life-cycle is very simple. We continuously wait
+          # for tasks to be put into our job `Queue`. If the `Queue` is empty,
+          # we will wait until it’s not.
+          loop do
+            # Once we have a piece of work to be done, we will pull out the
+            # information we need and get to work.
+            job, args = @jobs.pop
+            job.call(*args)
+          end
+        end
+      end
+    end
+  end
+  
+  # ### Work scheduling
+  
+  # To schedule a piece of work to be done is to say to the `Pool` that you
+  # want something done.
+  def schedule(*args, &block)
+    # Your given task will not be run immediately; rather, it will be put
+    # into the work `Queue` and executed once a thread is ready to work.
+    @jobs << [block, args]
+  end
+  
+  # ### Graceful shutdown
+  
+  # If you ever wish to close down your application, I took the liberty of
+  # making it easy for you to wait for any currently executing jobs to finish
+  # before you exit.
+  def shutdown
+    # A graceful shutdown involves threads exiting cleanly themselves, and
+    # since we’ve defined a `catch`-handler around the threads’ worker loop
+    # it is simply a matter of throwing `:exit`. Thus, if we throw one `:exit`
+    # for each thread in our pool, they will all exit eventually!
+    @size.times do
+      schedule { throw :exit }
+    end
+    
+    # And now one final thing: wait for our `throw :exit` jobs to be run on
+    # all our worker threads. This call will not return until all worker threads
+    # have exited.
+    @pool.map(&:join)
+  end
+end

data/lib/webpage-archivist/fetcher/webpage_request.rb

@@ -0,0 +1,197 @@
+require 'addressable/uri'
+require 'em-http'
+require 'andand'
+
+module WebpageArchivist::Fetcher
+
+  # Requesting a webpage.
+  class WebpageRequest
+
+    attr_reader :webpage, :result_code, :instance, :uri, :status
+
+    # Create a request
+    # webpage:: the Webpage we want to fetch
+    # fetcher_watcher:: to be notified when the request is over
+    def initialize webpage, fetcher_watcher
+      @webpage = webpage
+      @fetcher_watcher = fetcher_watcher
+
+      @uri = Addressable::URI.parse(webpage.uri)
+
+      @waiting_requests = 0
+
+      @status = :fetching
+
+      @plumber = RequestsPlumber.new self
+    end
+
+    # Start the request
+    # Not in initialize so we can throttle the number of connection
+    # retries:: number of retries in case of error
+    def start retries = 3
+      @http = EventMachine::HttpRequest.new(@uri).get :redirects => 5, :timeout => 30, :head => {'If-Modified-Since' => webpage.last_modified, 'accept-encoding' => 'gzip, compressed'}
+      @http.callback do
+        if ([500, 503].include? @http.response_header.status) && (retries > 0)
+          start(retries - 1)
+        else
+          begin
+            process_response
+          rescue Exception => e
+            ::WebpageArchivist.error e if ::WebpageArchivist.log
+          end
+        end
+      end
+      @http.errback do
+        if retries > 0
+          start(retries - 1)
+        else
+          begin
+            process_response
+          rescue Exception => e
+            ::WebpageArchivist.error e if ::WebpageArchivist.log
+          end
+        end
+      end
+    end
+
+    # Process the response
+    def process_response
+      @result_code = @http.response_header.status
+      ::WebpageArchivist.debug "[#{@uri}] returned #{@result_code}" if ::WebpageArchivist.log
+      if [304, 408, 0].include? @result_code
+        # Not changed
+        ::WebpageArchivist.debug "[#{@uri}] not modified" if ::WebpageArchivist.log
+        @modified = false
+        process_content
+      elsif result_code == 200
+        ::WebpageArchivist.debug "[#{@uri}] modified" if ::WebpageArchivist.log
+        @modified = true
+        process_content
+      else
+        ::WebpageArchivist.debug "Error #{@uri} #{@result_code}" if ::WebpageArchivist.log
+        @fetcher_watcher.end_request self, false
+      end
+    end
+
+    # Process the content
+    def process_content
+      if @modified
+        charset = @plumber.response_charset @http
+        @content = WebpageArchivist::HtmlDocument.new(@http.response, @uri, charset)
+      else
+        @content = WebpageArchivist::HtmlDocument.new(webpage.last_content, @uri, webpage.last_encoding)
+      end
+
+      # Elements from the stylesheets
+      already = Set.new
+      @content.each_stylesheet do |stylesheet|
+        uri = make_absolute_if_modified(stylesheet, 'href')
+        unless already.include? uri
+          already.add uri
+          @plumber.request_element self, WebpageArchivist::Stylesheet, uri
+          @waiting_requests += 1
+        end
+      end
+
+      # Elements from the scripts
+      already = Set.new
+      @content.each_script do |script|
+        uri = make_absolute_if_modified(script, 'src')
+        unless already.include? uri
+          already.add uri
+          @plumber.request_element self, WebpageArchivist::Script, uri
+          @waiting_requests += 1
+        end
+      end
+
+      # Elements from the images
+      already = Set.new
+      @content.each_image do |img|
+        uri = make_absolute_if_modified(img, 'src')
+        unless already.include? uri
+          already.add uri
+          @plumber.request_element self, WebpageArchivist::Image, uri
+          @waiting_requests += 1
+        end
+      end
+
+      if @modified
+        # Make links absolutes
+        @content.each_link do |link|
+          make_absolute_if_modified(link, 'href')
+        end
+
+        @webpage.update(
+            :last_modified => @plumber.last_modified(@http),
+            :last_content => @content.to_html,
+            :last_charset => @content.charset)
+      end
+
+      @status = :fetching_requests
+
+      # No external resource -> end here
+      if @waiting_requests == 0
+        after_requests
+      end
+
+    end
+
+    # Make an element's uri absolute
+    # element:: the element
+    # link_property:: the property holding the uri
+    def make_absolute_if_modified element, link_property
+      if @modified
+        element[link_property] = uri.absolutize(element[link_property])
+      else
+        element[link_property]
+      end
+    end
+
+    # Called by a request when it is over
+    # uri:: the request uri
+    def request_over uri
+      @waiting_requests -= 1
+
+      ::WebpageArchivist.debug "Request over for [#{@uri}] on [#{uri}], missing #{@waiting_requests}" if ::WebpageArchivist.log
+      if (@status == :fetching_requests) && (@waiting_requests <= 0)
+        after_requests
+      end
+    end
+
+    # Process the response once all the elements have been fetched
+    def after_requests
+      ::WebpageArchivist.debug "After requests #{@uri}" if ::WebpageArchivist.log
+      if @modified
+
+        # replace elements with the local uris of the elements
+        @content.each_stylesheet do |stylesheet|
+          if e = @plumber[stylesheet['href']].andand.element
+            stylesheet['href'] = e.file_name
+          end
+        end
+
+        @content.each_script do |script|
+          if e = @plumber[script['src']].andand.element
+            script['src'] = e.file_name
+          end
+        end
+
+        @content.each_image do |img|
+          if e = @plumber[img['src']].andand.element
+            img['src'] = e.file_name
+          end
+        end
+        webpage.save_content @content.to_html
+      end
+
+      commit_timestamp = DateTime.now.strftime('%Y/%m/%d %H:%M:%S')
+      # store the content, clean the repo and commit
+      webpage.update_repo_commit_changes @plumber.requests_files, commit_timestamp
+      @instance = WebpageArchivist::Instance.create(:webpage => webpage, :commit_timestamp => commit_timestamp)
+
+      @fetcher_watcher.end_request self, true
+    end
+
+  end
+
+end

data/lib/webpage-archivist/html_document.rb

@@ -0,0 +1,66 @@
+require 'nokogiri'
+require 'iconv'
+
+module WebpageArchivist
+
+  # API around Nokogiri
+  class HtmlDocument
+
+    attr_reader :content, :charset
+
+    ENCODING_REGEXP = /<meta http-equiv="content-type" content="text\/html; charset=([^"]+)"/i
+
+    CONVERTER = Iconv.new('UTF-8//IGNORE//TRANSLIT', 'ASCII//IGNORE//TRANSLIT')
+
+    # Create document
+    # content:: the content
+    def initialize content, uri = nil, charset = nil
+      @charset = charset
+      unless @charset
+        @charset = ENCODING_REGEXP.match(CONVERTER.iconv(content)).andand[1].andand.upcase
+      end
+      @content = Nokogiri::HTML(content, uri, @charset)
+    end
+
+    # Convert document to html
+    def to_html
+      content.to_html
+    end
+
+    # Call a block for each stylesheet
+    # Block call parameter will be the stylesheet node
+    def each_stylesheet &block
+      content.search('link[@type="text/css"]').each do |link|
+        block.yield link
+      end
+    end
+
+    # Call a block for each script
+    # Block call parameter will be the script node
+    def each_script &block
+      content.search('script[@src]').each do |script|
+        block.yield script
+      end
+    end
+
+    # Call a block for each image
+    # Block call parameter will be the image node
+    def each_image &block
+      content.search('img[@src]').each do |img|
+        block.yield img
+      end
+    end
+
+    # Call a block for each link
+    # Block call parameter will be the link node
+    def each_link &block
+      content.search('a[@href]').each do |link|
+        block.yield link
+      end
+    end
+
+
+  end
+
+
+end