webpage-archivist 0.0.1

data/lib/webpage-archivist/version.rb

@@ -0,0 +1,3 @@
+module WebpageArchivist
+    VERSION = "0.0.1"
+end

data/lib/webpage-archivist/webpage-archivist.rb

@@ -0,0 +1,79 @@
+require 'logger'
+
+module WebpageArchivist
+
+  # Entry point for the Web Archivist features.
+  # Database configuration will rely on the DATABASE_uri environment variable
+  # see http://sequel.rubyforge.org/rdoc/files/doc/opening_databases_rdoc.html for the syntax detail
+  class WebpageArchivist
+
+    # Add a webpage for future fetching, return the corresponding Webpage
+    # uri:: page uri
+    # name:: page name
+    def add_webpage uri, name
+      Webpage.create(:name => name, :uri => uri)
+    end
+
+    # Fetch several webpages, return an hash indexed by the ids holding the corresponding instances or http result codes
+    def fetch_webpages ids
+      Fetcher.fetch_webpages ids
+    end
+
+    # List the webpages
+    def list_webpages
+      Webpage.all
+    end
+
+    # List the instances of a webpage
+    # id:: the webpage id
+    def list_instances webpage_id
+      Instance.where(:webpage_id => webpage_id)
+    end
+
+    # Write the full content of a webpage instance into a zip file
+    # id:: the instance id
+    # file:: the file to write to
+    def extract_instance_content id, file
+      Extracter.instance_content id, file
+    end
+
+    # Purge cached elements from the database, they are *not* deleted from the disk
+    # retention_period:: number of days after which the purge should start
+    def purge_cache retention_period
+      purge_starting_date = DateTime.now - retention_period
+      Stylesheet.filter('last_fetched < ?', purge_starting_date).delete
+      Script.filter('last_fetched < ?', purge_starting_date).delete
+      Image.filter('last_fetched < ?', purge_starting_date).delete
+    end
+
+    # Create a snapshot of a web page
+    # See Snapshoter class for configuration
+    # uri:: the uri to snapshot
+    # snapshot_path:: path to the snapshot file
+    # thumbnail_path:: path to the thumbnail (can be nil for no thumbnail)
+    def snapshot instance, thumbnail
+      Snapshoter.snapshot instance, thumbnail
+    end
+
+  end
+
+  @@log = false
+  @@logger = ::Logger.new(STDOUT)
+
+  def self.log
+    @@log
+  end
+
+  def self.log= value
+    @@log = value
+  end
+
+  def self.debug(str)
+    @@logger.debug { str }
+  end
+
+  def self.error(str)
+    @@logger.error { str }
+  end
+
+end

data/test/crud_test.rb

@@ -0,0 +1,28 @@
+require_relative 'helper'
+
+describe 'crud' do
+
+  it 'has no webpage by default' do
+    WebpageArchivist::DATABASE.transaction do
+
+      WebpageArchivist::Webpage.count.must_equal 0
+      @@archivist.list_webpages.count.must_equal 0
+
+      raise(Sequel::Rollback)
+    end
+  end
+
+  it 'can add a webpage' do
+    WebpageArchivist::DATABASE.transaction do
+
+      webpage = @@archivist.add_webpage 'http://example.com', 'example'
+      webpage.must_be_instance_of WebpageArchivist::Webpage
+      webpage.name.must_equal 'example'
+      webpage.uri.must_equal 'http://example.com'
+      @@archivist.list_webpages.count.must_equal 1
+
+      raise(Sequel::Rollback)
+    end
+  end
+
+end

data/test/files/stylesheet.css

@@ -0,0 +1,14 @@
+@import url("relative.css");
+@import url("http://absolute.net/stylesheet.css");
+
+#something1 {
+    background-image: url("relative1.jpg")
+}
+
+#something2 {
+    background:#ffffff url("relative2.jpg") no-repeat right top;
+}
+
+#something3 {
+    background-image: url("http://absolute.net/absolute.jpg")
+}

data/test/helper.rb

@@ -0,0 +1,15 @@
+database_path = "sqlite://#{Dir.pwd}/webpage-archivist-test.sqlite3"
+
+if File.exist? database_path
+  File.delete database_path
+end
+
+ENV['DATABASE_URL'] = database_path
+
+require 'bundler'
+Bundler.setup
+require 'test/unit'
+require 'minitest/spec'
+
+require_relative '../lib/webpage-archivist'
+@@archivist = WebpageArchivist::WebpageArchivist.new

data/test/stylesheet_test.rb

@@ -0,0 +1,48 @@
+require_relative 'helper'
+
+describe 'stylesheet' do
+
+  def stylesheet_file_path
+    current_file = File.expand_path(File.dirname(__FILE__))
+    File.join(current_file, 'files', 'stylesheet.css')
+  end
+
+  def base_url
+    Addressable::URI.parse('http://my.example.com')
+  end
+
+  def parse_stylesheet
+    WebpageArchivist::StylesheetDocument.new(IO.read(stylesheet_file_path), base_url)
+  end
+
+  it 'lists imports' do
+    imports = []
+    stylesheet = parse_stylesheet
+    stylesheet.each_import do |i|
+      imports << i
+      nil
+    end
+    imports.length.must_equal 2
+    imports[0].must_equal 'relative.css'
+    imports[1].must_equal 'http://absolute.net/stylesheet.css'
+    stylesheet.to_css.must_include '@import url("relative.css");'
+    stylesheet.to_css.must_include '@import url("http://absolute.net/stylesheet.css");'
+  end
+
+  it 'list images' do
+    images = []
+    stylesheet = parse_stylesheet
+    stylesheet.each_image do |image|
+      images << image
+      nil
+    end
+    images.length.must_equal 3
+    images[0].must_equal 'relative1.jpg'
+    images[1].must_equal 'relative2.jpg'
+    images[2].must_equal 'http://absolute.net/absolute.jpg'
+    stylesheet.to_css.must_include 'url("relative1.jpg");'
+    stylesheet.to_css.must_include 'url("relative2.jpg");'
+    stylesheet.to_css.must_include 'url("http://absolute.net/absolute.jpg");'
+  end
+
+end

data/webpage-archivist.gemspec

@@ -0,0 +1,38 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "webpage-archivist/version"
+
+Gem::Specification.new do |s|
+  s.name        = 'webpage-archivist'
+  s.version     = WebpageArchivist::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Julien Kirch"]
+  s.homepage    = 'https://github.com/archiloque/webpage-archivist'
+  s.summary     = 'An utility to archive webpages through time'
+  s.description = s.summary
+
+  s.rubyforge_project = s.name
+
+  s.has_rdoc = true
+  s.extra_rdoc_files = ['README.rdoc']
+  s.rdoc_options = ['--main', 'README.rdoc']
+
+  s.add_runtime_dependency 'andand', '~> 1.3.1'
+  s.add_runtime_dependency 'sequel', '~> 3.25'
+  s.add_runtime_dependency 'eventmachine', '~> 1.0.0.beta.3'
+  s.add_runtime_dependency 'em-http-request', '~> 1.0.0.beta.4'
+  s.add_runtime_dependency 'nokogiri', '~> 1.5'
+  s.add_runtime_dependency 'addressable', '~> 2.2.6'
+  s.add_runtime_dependency 'css_parser', '~> 1.1.9'
+  s.add_runtime_dependency 'grit', '~> 2.4.1'
+  s.add_runtime_dependency 'websnap', '~> 0.1.3'
+  s.add_runtime_dependency 'mini_magick', '~> 3.3'
+  s.add_runtime_dependency 'mime-types', '~> 1.16'
+
+  s.add_development_dependency 'sqlite3', '~> 1.3.3'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- test/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

metadata

@@ -0,0 +1,284 @@
+--- !ruby/object:Gem::Specification 
+name: webpage-archivist
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Julien Kirch
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-08-10 00:00:00 Z
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: andand
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 25
+        segments: 
+        - 1
+        - 3
+        - 1
+        version: 1.3.1
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: sequel
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 53
+        segments: 
+        - 3
+        - 25
+        version: "3.25"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: eventmachine
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 62196357
+        segments: 
+        - 1
+        - 0
+        - 0
+        - beta
+        - 3
+        version: 1.0.0.beta.3
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: em-http-request
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 62196363
+        segments: 
+        - 1
+        - 0
+        - 0
+        - beta
+        - 4
+        version: 1.0.0.beta.4
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: nokogiri
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 5
+        segments: 
+        - 1
+        - 5
+        version: "1.5"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: addressable
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 11
+        segments: 
+        - 2
+        - 2
+        - 6
+        version: 2.2.6
+  type: :runtime
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: css_parser
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 1
+        - 1
+        - 9
+        version: 1.1.9
+  type: :runtime
+  version_requirements: *id007
+- !ruby/object:Gem::Dependency 
+  name: grit
+  prerelease: false
+  requirement: &id008 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 2
+        - 4
+        - 1
+        version: 2.4.1
+  type: :runtime
+  version_requirements: *id008
+- !ruby/object:Gem::Dependency 
+  name: websnap
+  prerelease: false
+  requirement: &id009 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 0
+        - 1
+        - 3
+        version: 0.1.3
+  type: :runtime
+  version_requirements: *id009
+- !ruby/object:Gem::Dependency 
+  name: mini_magick
+  prerelease: false
+  requirement: &id010 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 1
+        segments: 
+        - 3
+        - 3
+        version: "3.3"
+  type: :runtime
+  version_requirements: *id010
+- !ruby/object:Gem::Dependency 
+  name: mime-types
+  prerelease: false
+  requirement: &id011 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 47
+        segments: 
+        - 1
+        - 16
+        version: "1.16"
+  type: :runtime
+  version_requirements: *id011
+- !ruby/object:Gem::Dependency 
+  name: sqlite3
+  prerelease: false
+  requirement: &id012 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ~>
+      - !ruby/object:Gem::Version 
+        hash: 29
+        segments: 
+        - 1
+        - 3
+        - 3
+        version: 1.3.3
+  type: :development
+  version_requirements: *id012
+description: An utility to archive webpages through time
+email: 
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- .gitignore
+- Gemfile
+- README.rdoc
+- Rakefile
+- lib/webpage-archivist.rb
+- lib/webpage-archivist/extracter.rb
+- lib/webpage-archivist/fetcher/element_request.rb
+- lib/webpage-archivist/fetcher/fetcher.rb
+- lib/webpage-archivist/fetcher/requests_plumber.rb
+- lib/webpage-archivist/fetcher/stylesheet_request.rb
+- lib/webpage-archivist/fetcher/thread-pool.rb
+- lib/webpage-archivist/fetcher/webpage_request.rb
+- lib/webpage-archivist/html_document.rb
+- lib/webpage-archivist/migrations.rb
+- lib/webpage-archivist/models.rb
+- lib/webpage-archivist/patches.rb
+- lib/webpage-archivist/snapshoter.rb
+- lib/webpage-archivist/stylesheet_document.rb
+- lib/webpage-archivist/version.rb
+- lib/webpage-archivist/webpage-archivist.rb
+- test/crud_test.rb
+- test/files/stylesheet.css
+- test/helper.rb
+- test/stylesheet_test.rb
+- webpage-archivist.gemspec
+homepage: https://github.com/archiloque/webpage-archivist
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --main
+- README.rdoc
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: webpage-archivist
+rubygems_version: 1.8.5
+signing_key: 
+specification_version: 3
+summary: An utility to archive webpages through time
+test_files: 
+- test/crud_test.rb
+- test/files/stylesheet.css
+- test/helper.rb
+- test/stylesheet_test.rb