webpage-archivist 0.0.1

data/.gitignore

@@ -0,0 +1,7 @@
+.DS_Store
+*.gem
+*.sqlite3
+.bundle
+Gemfile.lock
+pkg/*
+webpages/*

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://www.rubygems.org"
+
+gemspec

data/README.rdoc

@@ -0,0 +1,50 @@
+An utility to archive webpages through time.
+
+Takes snapshots and make incremental backups of webpages assets so you can follow the pages' evolutions through time.
+
+* Assets are stored in a git respository to simplify incremental storage and easy retrieval
+* Snapshots and thumbails are stored in a plain repository so they can easily be served by a webserver
+* List of webpages and archives instances are stored in an SQL database
+* Some caching data are stored in the same databse
+
+= Required tools:
+
+* An SQL database supported by Sequel[http://sequel.rubyforge.org/]
+* Git[http://git-scm.com/]
+* ImageMagick[http://www.imagemagick.org/script/index.php]
+* wkhtmltoimage[http://code.google.com/p/wkhtmltopdf/] (work but not so well on Mac OS, prefer Linux for real usage)
+
+= Installation
+
+* Install the required tools
+* Install the gem
+* All configuration items have default value, have a look bellow if you want to customize them (default database configuration require the sqlite3 gem)
+* Use it !: all the required files and database structure will be created at first call
+
+= API
+
+The public API is provided by WebpageArchivist::WebpageArchivist, example:
+
+  require 'webpage-archivist'
+  archivist = WebpageArchivist::WebpageArchivist.new
+  webpage = archivist.add_webpage('http://www.nytimes.com/' , 'The New York Times')
+  archivist.fetch_webpages [webpage.id]
+
+Models are available in the lib/webpag-archivist/models.rb file, have a look at the Sequel[http://sequel.rubyforge.org/] API if you want to querry them.
+
+= Configuration
+
+Basic configuration is done through environment variables:
+
+* +DATABASE_URL+ : database url, default to <tt>sqlite://#{Dir.pwd}/webpage-archivist.sqlite3</tt> syntax is described here[http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html], remember to add the required database gem
+* +ARCHIVIST_ASSETS_PATH+ : path to store the assets, default to +./archivist_assets+
+* +ARCHIVIST_SNAPSHOTS_PATH+  : path to store the thumbnail, default to +./archivist_snapshots+
+* +ARCHIVIST_MAX_RUNNING_REQUESTS+ : number of elements requests running in parallel (not so important since requests are run using EventMachine[http://rubyeventmachine.com/], default to 20
+* +IMAGE_MAGICK_PATH+ : path to ImageMagick executables if they aren't in the path
+
+Configuration for snapshoting is done through the WebpageArchivist::Snapshoter class.
+
+To enable debugging use
+
+  WebpageArchivist.log= true
+

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rake'
+require 'rake/testtask'
+
+desc "Run all tests"
+Rake::TestTask.new do |t|
+  t.test_files = FileList['test/*_test.rb']
+  t.verbose = true
+end
+
+task :default  => :test

data/lib/webpage-archivist.rb

@@ -0,0 +1,20 @@
+require_relative 'webpage-archivist/migrations'
+
+require_relative 'webpage-archivist/patches'
+
+require_relative 'webpage-archivist/html_document'
+require_relative 'webpage-archivist/stylesheet_document'
+
+require_relative 'webpage-archivist/webpage-archivist'
+
+require_relative 'webpage-archivist/models'
+
+require_relative 'webpage-archivist/extracter'
+
+require_relative 'webpage-archivist/snapshoter'
+
+require_relative 'webpage-archivist/fetcher/fetcher'
+require_relative 'webpage-archivist/fetcher/requests_plumber'
+require_relative 'webpage-archivist/fetcher/element_request'
+require_relative 'webpage-archivist/fetcher/stylesheet_request'
+require_relative 'webpage-archivist/fetcher/webpage_request'

data/lib/webpage-archivist/extracter.rb

@@ -0,0 +1,31 @@
+require 'addressable/uri'
+
+module WebpageArchivist
+
+  # Fetch webpages' content
+  module Extracter
+
+    # Write the full content of a webpage instance into a zip file
+    # id:: the instance id
+    # file:: the file to write to
+    def self.instance_content id, file
+      @instance = Instance.filter(:id => id).first
+      unless @instance
+        raise "Instance [#{id}] not found"
+      end
+      repository = @instance.webpage.repository
+      commit = repository.log.find{ |l| l.message == @instance.commit_timestamp }
+
+      unless file.end_with? '.zip'
+        file << '.zip'
+      end
+
+      if commit
+        repository.archive_zip commit.id, file
+      else
+        raise "Version [#{@instance.commit_timestamp}] not found"
+      end
+    end
+
+  end
+end

data/lib/webpage-archivist/fetcher/element_request.rb

@@ -0,0 +1,126 @@
+module WebpageArchivist::Fetcher
+
+  # Requesting a webpage
+  class ElementRequest
+
+    attr_reader :element, :uri
+
+    attr_accessor :status
+
+    def initialize request_element_type, uri, plumber
+      ::WebpageArchivist.debug "Creating request [#{uri}]" if ::WebpageArchivist.log
+
+      @requesters = []
+      @request_element_type = request_element_type
+      @uri = uri
+      @element = request_element_type.filter(:uri => uri).first
+      @status = :fetching
+      @plumber = plumber
+
+      @requesters_notified = false
+    end
+
+    # Start the request
+    # Not in initialize so we can register the request before executing it and throttle the number of connections
+    # retries:: number of retries in case of error
+    def start retries = 3
+      ::WebpageArchivist.debug "Starting request [#{uri}]" if ::WebpageArchivist.log
+      head = {'accept-encoding' => 'gzip, compressed'}
+      if element
+        head['If-Modified-Since'] = element.last_modified
+        @plumber.register_file_name element.file_name
+      end
+
+      # Don't overflow the servers or they will kick us out
+      http = EventMachine::HttpRequest.new(uri).get :redirects => 5, :timeout => 30, :head => head
+      http.callback do
+        if ([500, 503].include? http.response_header.status) && (retries > 0)
+          start(retries - 1)
+        else
+          will_process_response http
+        end
+      end
+      http.errback do
+        ::WebpageArchivist.debug "[#{@uri}] errback" if ::WebpageArchivist.log
+        if retries > 0
+          start(retries - 1)
+        else
+          will_process_response http
+        end
+      end
+    end
+
+    # Call process_response and ensure managment is done
+    def will_process_response http
+      begin
+        process_response http
+      rescue Exception => e
+        ::WebpageArchivist.error e if ::WebpageArchivist.log
+        notify_requesters
+      ensure
+        @plumber.request_ended
+      end
+    end
+
+    # Add a requester to be notified when the request is over
+    def add_requester requester
+      @requesters << requester
+    end
+
+    # Process the response
+    def process_response http
+      result_code = http.response_header.status
+      ::WebpageArchivist.debug "[#{@uri}] returned #{result_code}" if ::WebpageArchivist.log
+
+      if [304, 408, 0].include? result_code
+        # Not changed or connection error
+        if element
+          element.update(:last_fetched => DateTime.now)
+        end
+        content_not_changed http
+      elsif result_code == 200
+        if element
+          element.update(:last_fetched => DateTime.now,
+                         :last_modified => http.response_header.last_modified || DateTime.now.rfc2822)
+        else
+          extension = @request_element_type.extention(@uri, http.response_header[EventMachine::HttpClient::CONTENT_TYPE])
+          @element = @request_element_type.create(:webpage => @plumber.webpage,
+                                                  :uri => @uri,
+                                                  :file_hash => @plumber.get_hash(@request_element_type, @uri, extension),
+                                                  :extension => extension,
+                                                  :last_fetched => DateTime.now,
+                                                  :last_modified => @plumber.last_modified(http))
+        end
+        save_content_end_request http
+      else
+        self.status= :over
+        notify_requesters
+      end
+    end
+
+    # Content has not changed since last fetch
+    def content_not_changed http
+      self.status= :over
+      notify_requesters
+    end
+
+    # Content has changed: save the content and end the request
+    def save_content_end_request http
+      ::WebpageArchivist.debug "[#{@uri}] writing content to #{element.file_name}" if ::WebpageArchivist.log
+
+      element.save_content http.response
+      self.status= :over
+      notify_requesters
+    end
+
+    def notify_requesters
+      unless @requesters_notified
+        ::WebpageArchivist.debug "[#{@uri}] notify #{@requesters.length}" if ::WebpageArchivist.log
+        @requesters.each { |r| r.request_over(@uri) }
+        @requester_notified = true
+      end
+    end
+
+  end
+
+end

data/lib/webpage-archivist/fetcher/fetcher.rb

@@ -0,0 +1,83 @@
+require 'eventmachine'
+require_relative 'thread-pool'
+
+# Module in charge of fetching pages
+module WebpageArchivist::Fetcher
+
+  SEMAPHORE = Mutex.new
+
+  # Fetch several webpages, return an hash indexed by the ids holding the corresponding Instances or http result codes
+  # (may be existing instances if the pages haven't changed)
+  def self.fetch_webpages ids
+    if ids.empty?
+      []
+    else
+      SEMAPHORE.synchronize do
+        @fetcher_watcher = FetcherWatcher.new
+        EventMachine.run do
+          WebpageArchivist::Webpage.filter(:id => ids).each do |webpage|
+            @fetcher_watcher.add_request WebpageRequest.new(webpage, @fetcher_watcher)
+          end
+          @fetcher_watcher.wait
+        end
+
+        result = {}
+        @fetcher_watcher.requests.each do |webpage_request|
+          result[webpage_request.webpage.id] = webpage_request.instance ? webpage_request.instance : webpage_request.result_code
+        end
+        result
+      end
+    end
+  end
+
+  # Wait for callbacks for webpages' requests, stop event machine when all requests are over and manage the snapshot generation
+  class FetcherWatcher
+
+    attr_reader :requests
+
+    def initialize
+      @waiting_requests = 0
+      @status = :starting
+      @requests = []
+      @thread_pool = Pool.new 1
+    end
+
+    # Start to wait
+    def wait
+      @status = :waiting
+      if @waiting_requests == 0
+        end_watcher
+      end
+    end
+
+    # Add a request to wait for
+    def add_request request
+      @waiting_requests += 1
+      @requests << request
+    end
+
+    # A request is over
+    # request:: the request
+    # ok:: indicates if the request went ok, in this case ask for a snapshot
+    def end_request request, ok
+      @waiting_requests -= 1
+      if ok && request.instance
+        @thread_pool.schedule do
+          ::WebpageArchivist::Snapshoter.snapshot_instance request.instance
+        end
+      end
+      if (@status == :waiting) && (@waiting_requests <= 0)
+        end_watcher
+      end
+    end
+
+    # End the watch
+    def end_watcher
+      EM.stop
+      @thread_pool.shutdown
+    end
+
+
+  end
+
+end

data/lib/webpage-archivist/fetcher/requests_plumber.rb

@@ -0,0 +1,140 @@
+require 'digest/sha1'
+require 'andand'
+
+module WebpageArchivist::Fetcher
+# Contains the plumbing for the fetching code
+  class RequestsPlumber
+
+    MAX_RUNNING_REQUESTS = (ENV['ARCHIVIST_MAX_RUNNING_REQUESTS'].andand.to_i || 20)
+
+    attr_reader :requests_files
+
+    @@next_tick = false
+    @@waiting_requests = []
+    @@running_requests = 0
+
+    def initialize webpage_request
+      @requests = {}
+      @requests_hashes = Set.new
+      @requests_files = Set.new
+      @webpage_request = webpage_request
+      @@waiting_requests << webpage_request
+      RequestsPlumber.new_request
+    end
+
+    # The page being fetched
+    def webpage
+      @webpage_request.webpage
+    end
+
+    # Access an element request by its uri
+    def [] key
+      @requests[key]
+    end
+
+    # Request an element to be fetched
+    # When the fetch is called, request_over will be called on the requester with the uri
+    # requester:: the element doing the request
+    # type:: the type the type of requested element
+    # uri:: the requested uri
+    def request_element requester, request_element_type, uri
+      ::WebpageArchivist.debug "Requesting [#{uri}] for [#{requester.uri}]" if ::WebpageArchivist.log
+
+      if request = @requests[uri]
+        if request.status == :over
+          ::WebpageArchivist.debug "Request already done" if ::WebpageArchivist.log
+          requester.request_over uri
+        else
+          ::WebpageArchivist.debug "Adding to requesters" if ::WebpageArchivist.log
+          request.add_requester requester
+        end
+      else
+        ::WebpageArchivist.debug "Creating new request" if ::WebpageArchivist.log
+        if request_element_type == WebpageArchivist::Stylesheet
+          request = StyleSheetRequest.new(uri, self)
+        else
+          request = ElementRequest.new(request_element_type, uri, self)
+        end
+        @requests[uri] = request
+        request.add_requester requester
+
+        @@waiting_requests << request
+
+        # try registering for the next tick
+        RequestsPlumber.new_request
+      end
+    end
+
+    # Notify that a requst as ended so it can start another one
+    def request_ended
+      @@running_requests -= 1
+      unless @@waiting_requests.empty?
+        RequestsPlumber.new_request
+      end
+    end
+
+    # Get the has corresponding to an uri and make sure there is no collision
+    def get_hash type, uri, extension
+      file_hash = Digest::SHA1.hexdigest(uri)
+      if @requests_hashes.include?(file_hash) || (type.filter(:file_hash => file_hash).count > 0)
+        i = 0
+        begin
+          file_hash = Digest::SHA1.hexdigest("#{uri}#{i}")
+          i += 1
+        end while @requests_hashes.include?(file_hash) || (type.filter(:file_hash => file_hash).count > 0)
+      end
+
+      @requests_files << "#{file_hash}#{extension}"
+      file_hash
+    end
+
+    # Register a filename so it is considered part of the webpage
+    def register_file_name file_name
+      @requests_files << file_name
+    end
+
+    # Get the charset of a response, may be nil
+    def response_charset http
+      type = http.response_header[EventMachine::HttpClient::CONTENT_TYPE]
+      if type
+        match = /.+;\s*charset=(.+)/i.match(type)
+        if match
+          match[1].upcase
+        end
+      end
+    end
+
+    def last_modified http
+      http.response_header.last_modified || DateTime.now.rfc2822
+    end
+
+    private
+
+    # Called when a new request has been added
+    # Register a callback for the next tick
+    def self.new_request
+      if (!@@next_tick) && (@@running_requests < MAX_RUNNING_REQUESTS)
+        EventMachine.next_tick do
+          RequestsPlumber.next_tick
+        end
+        @@next_tick = true
+      end
+    end
+
+    # Start the first waiting request
+    def self.next_tick
+      unless @@waiting_requests.empty?
+        @@waiting_requests.shift.start
+        @@running_requests += 1
+      end
+
+      @@next_tick = false
+
+      unless @@waiting_requests.empty?
+        RequestsPlumber.new_request
+      end
+    end
+
+  end
+
+end