webpage-archivist 0.0.1 → 0.0.2

data/CHANGELOG.md

@@ -0,0 +1,6 @@
+# 0.0.2
+
+- replace websnap + wkhtmltoimage by PhantomJS
+- replace mini_magick by custom code
+- WebpageArchivist#fetch_webpages no takes pages instead of ids
+

data/README.rdoc

@@ -11,8 +11,8 @@ Takes snapshots and make incremental backups of webpages assets so you can follo
 
 * An SQL database supported by Sequel[http://sequel.rubyforge.org/]
 * Git[http://git-scm.com/]
-* ImageMagick[http://www.imagemagick.org/script/index.php]
-* wkhtmltoimage[http://code.google.com/p/wkhtmltopdf/] (work but not so well on Mac OS, prefer Linux for real usage)
+* GraphicsMagick[http://www.graphicsmagick.org/]
+* PhantomJS [http://code.google.com/p/phantomjs]
 
 = Installation
 
@@ -40,7 +40,8 @@ Basic configuration is done through environment variables:
 * +ARCHIVIST_ASSETS_PATH+ : path to store the assets, default to +./archivist_assets+
 * +ARCHIVIST_SNAPSHOTS_PATH+  : path to store the thumbnail, default to +./archivist_snapshots+
 * +ARCHIVIST_MAX_RUNNING_REQUESTS+ : number of elements requests running in parallel (not so important since requests are run using EventMachine[http://rubyeventmachine.com/], default to 20
-* +IMAGE_MAGICK_PATH+ : path to ImageMagick executables if they aren't in the path
+* +PHANTOMJS_PATH+: path to PhantomJS executable if they aren't in the path
+* +GRAPHICS_MAGICK_PATH+ : path to GraphicsMagick executable if it isn't in the path
 
 Configuration for snapshoting is done through the WebpageArchivist::Snapshoter class.
 

data/lib/webpage-archivist/fetcher/fetcher.rb

@@ -6,16 +6,16 @@ module WebpageArchivist::Fetcher
 
   SEMAPHORE = Mutex.new
 
-  # Fetch several webpages, return an hash indexed by the ids holding the corresponding Instances or http result codes
+  # Fetch several webpages, return an hash indexed by the webpages holding the corresponding Instances or http result codes
   # (may be existing instances if the pages haven't changed)
-  def self.fetch_webpages ids
-    if ids.empty?
+  def self.fetch_webpages webpages
+    if webpages.empty?
       []
     else
       SEMAPHORE.synchronize do
         @fetcher_watcher = FetcherWatcher.new
         EventMachine.run do
-          WebpageArchivist::Webpage.filter(:id => ids).each do |webpage|
+          webpages.each do |webpage|
             @fetcher_watcher.add_request WebpageRequest.new(webpage, @fetcher_watcher)
           end
           @fetcher_watcher.wait
@@ -23,7 +23,7 @@ module WebpageArchivist::Fetcher
 
         result = {}
         @fetcher_watcher.requests.each do |webpage_request|
-          result[webpage_request.webpage.id] = webpage_request.instance ? webpage_request.instance : webpage_request.result_code
+          result[webpage_request.webpage] = webpage_request.instance ? webpage_request.instance : webpage_request.result_code
         end
         result
       end

data/lib/webpage-archivist/patches.rb

@@ -1,7 +1,6 @@
 # Contains monkey patches
 require 'addressable/uri'
 require 'grit'
-require 'mini_magick'
 
 module Addressable
 
@@ -37,20 +36,6 @@ class Grit::Repo
 
 end
 
-# Patching mini magic so we can specify the path to the image magick installation
-module MiniMagick
-
-  class CommandBuilder
-
-    alias :parent_command :command
-
-    def command
-      "#{ENV['IMAGE_MAGICK_PATH'] ? "#{ENV['IMAGE_MAGICK_PATH']}/" : ''}#{parent_command}"
-    end
-
-  end
-end
-
 class Dir
 
   # Create a dit if it does not exist

data/lib/webpage-archivist/rasterize.js

@@ -0,0 +1,22 @@
+var page = new WebPage(),
+    address, output, size;
+
+if (phantom.args.length != 4) {
+    console.log('Usage: rasterize.js URL filename width height');
+    phantom.exit();
+} else {
+    address = phantom.args[0];
+    output = phantom.args[1];
+    page.viewportSize = { width: phantom.args[2], height: phantom.args[3]};
+    page.clipRect = { top: 0, left: 0, width: phantom.args[2], height: phantom.args[3]};
+    page.open(address, function (status) {
+        if (status !== 'success') {
+            console.log('Unable to load the address!');
+        } else {
+            window.setTimeout(function () {
+                page.render(output);
+                phantom.exit();
+            }, 200);
+        }
+    });
+}

data/lib/webpage-archivist/snapshoter.rb

@@ -1,17 +1,19 @@
-require 'websnap'
-require 'mini_magick'
-
 module WebpageArchivist
 
   # Snapshot the pages and create thumbnails
   class Snapshoter
 
     SNAPSHOTS_PATH = File.expand_path(ENV['ARCHIVIST_SNAPSHOTS_PATH'] || './archivist_snapshots')
+
+    GRAPHICS_MAGICK_PATH = "#{ENV['GRAPHICS_MAGICK_PATH'] ? "#{ENV['GRAPHICS_MAGICK_PATH']}/" : ''}gm"
+
+    PHANTOMJS_PATH = ENV['PHANTOMJS_PATH'] || 'phantomjs'
+
     p "Archivist snapshots path is [#{SNAPSHOTS_PATH}]"
     Dir.mkdir_if_not_exist SNAPSHOTS_PATH
 
     class << self
-      attr_accessor :width, :height, :format, :thumbnail_scale, :thumbnail_crop_width, :thumbnail_crop_height
+      attr_accessor :width, :height, :format, :thumbnail_scale, :thumbnail_crop_width, :thumbnail_crop_height, :quality
     end
 
     Snapshoter.width = 1024
@@ -36,8 +38,9 @@ module WebpageArchivist
 
       snapshot_path = File.join(dir_path, "#{instance.id}.#{Snapshoter.format}")
       thumbnail_path = File.join(dir_path, "#{instance.id.to_s}-small.#{Snapshoter.format}")
-      snapshot File.new(instance.webpage.index_path), snapshot_path, thumbnail_path
-      instance.update(:snapshot => true)
+      if snapshot(instance.webpage.index_path, snapshot_path, thumbnail_path)
+        instance.update(:snapshot => true)
+      end
     end
 
     # Create a snapshot of a web page
@@ -45,15 +48,25 @@ module WebpageArchivist
     # snapshot_path:: path to the snapshot file
     # thumbnail_path: path to the thumbnail (can be nil for no thumbnail)
     def self.snapshot uri_or_file, snapshot_path, thumbnail_path = nil
-      ::WebpageArchivist.debug "Snapshot for [#{uri_or_file.kind_of?(File) ? uri_or_file.path : uri_or_file}] on [#{snapshot_path}]" if ::WebpageArchivist.log
+      ::WebpageArchivist.debug "Snapshot for [#{uri_or_file}] on [#{snapshot_path}]" if ::WebpageArchivist.log
 
       if File.exists? snapshot_path
         File.delete snapshot_path
       end
 
-      snapper = WebSnap::Snapper.new(uri_or_file)
-      snapper.options.clear.merge!({'--height' => Snapshoter.height, '--width' => Snapshoter.width, '--format' => Snapshoter.format})
-      snapper.to_file(snapshot_path)
+      # if the result is not a png we use an intermediate image as PhantomJS makes crappy jpeg images
+      intermediate_image = (Snapshoter.format != 'png')
+      real_snapshot_path = intermediate_image ? "#{snapshot_path[0..(-(File.extname(snapshot_path).length + 1))]}.png" : snapshot_path
+
+      `#{PHANTOMJS_PATH} #{File.dirname(__FILE__)}/rasterize.js '#{uri_or_file}' #{real_snapshot_path} #{Snapshoter.width} #{Snapshoter.height}`
+
+      unless File.exists?(real_snapshot_path)
+        return false
+      end
+
+      if intermediate_image
+        `#{GRAPHICS_MAGICK_PATH} convert -background white #{real_snapshot_path} #{snapshot_path}`
+      end
 
       if thumbnail_path
         ::WebpageArchivist.debug "Thumbnail of [#{snapshot_path}] on [#{thumbnail_path}]" if ::WebpageArchivist.log
@@ -62,14 +75,14 @@ module WebpageArchivist
           File.delete thumbnail_path
         end
 
-        img = MiniMagick::Image::open(snapshot_path)
-        img.combine_options do |c|
-          c.crop "#{Snapshoter.thumbnail_crop_width}x#{Snapshoter.thumbnail_crop_height}+0+0"
-          c.scale "#{Snapshoter.thumbnail_scale}%"
-        end
-        img.write thumbnail_path
+        `#{GRAPHICS_MAGICK_PATH} convert -background white -scale #{Snapshoter.thumbnail_scale}% -crop #{Snapshoter.thumbnail_crop_width}x#{Snapshoter.thumbnail_crop_height}+0+0 #{real_snapshot_path} #{thumbnail_path}`
+      end
+
+      if intermediate_image
+        File.delete real_snapshot_path
       end
 
+      return true
     end
 
   end

data/lib/webpage-archivist/stylesheet_document.rb

@@ -4,15 +4,6 @@ require 'iconv'
 
 module CssParser
 
-  # Replace original implementation with one using addressable
-  def self.convert_uris(css, base_uri)
-    return css.gsub(URI_RX) do
-      uri = $1.to_s
-      uri.gsub!(/["']+/, '')
-      "url('#{base_uri.absolutize(uri)}')"
-    end
-  end
-
   # Make the declaration accessible
   class RuleSet
 
@@ -87,6 +78,7 @@ module WebpageArchivist
       unless @expanded
         parser.each_rule_set do |rs|
           rs.expand_background_shorthand!
+          rs.expand_list_style_shorthand!
         end
         @expanded = true
       end

data/lib/webpage-archivist/version.rb

@@ -1,3 +1,3 @@
 module WebpageArchivist
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
 end

data/lib/webpage-archivist/webpage-archivist.rb

@@ -14,9 +14,14 @@ module WebpageArchivist
       Webpage.create(:name => name, :uri => uri)
     end
 
-    # Fetch several webpages, return an hash indexed by the ids holding the corresponding instances or http result codes
-    def fetch_webpages ids
-      Fetcher.fetch_webpages ids
+    # Fetch all webpages
+    def fetch_all
+      Fetcher.fetch_webpages list_webpages
+    end
+
+    # Fetch several webpages, return an hash indexed by the webpages holding the corresponding instances or http result codes
+    def fetch_webpages webpages
+      Fetcher.fetch_webpages webpages
     end
 
     # List the webpages
@@ -24,12 +29,6 @@ module WebpageArchivist
       Webpage.all
     end
 
-    # List the instances of a webpage
-    # id:: the webpage id
-    def list_instances webpage_id
-      Instance.where(:webpage_id => webpage_id)
-    end
-
     # Write the full content of a webpage instance into a zip file
     # id:: the instance id
     # file:: the file to write to

data/webpage-archivist.gemspec

@@ -23,10 +23,8 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'em-http-request', '~> 1.0.0.beta.4'
   s.add_runtime_dependency 'nokogiri', '~> 1.5'
   s.add_runtime_dependency 'addressable', '~> 2.2.6'
-  s.add_runtime_dependency 'css_parser', '~> 1.1.9'
+  s.add_runtime_dependency 'css_parser', '~> 1.2.3'
   s.add_runtime_dependency 'grit', '~> 2.4.1'
-  s.add_runtime_dependency 'websnap', '~> 0.1.3'
-  s.add_runtime_dependency 'mini_magick', '~> 3.3'
   s.add_runtime_dependency 'mime-types', '~> 1.16'
 
   s.add_development_dependency 'sqlite3', '~> 1.3.3'

metadata

@@ -1,224 +1,135 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: webpage-archivist
-version: !ruby/object:Gem::Version 
-  hash: 29
+version: !ruby/object:Gem::Version
+  version: 0.0.2
   prerelease: 
-  segments: 
-  - 0
-  - 0
-  - 1
-  version: 0.0.1
 platform: ruby
-authors: 
+authors:
 - Julien Kirch
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2011-08-10 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2011-09-08 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: andand
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
+  requirement: &2160802540 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 25
-        segments: 
-        - 1
-        - 3
-        - 1
+      - !ruby/object:Gem::Version
         version: 1.3.1
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: sequel
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
+  version_requirements: *2160802540
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: &2160802040 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 53
-        segments: 
-        - 3
-        - 25
-        version: "3.25"
+      - !ruby/object:Gem::Version
+        version: '3.25'
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: eventmachine
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
+  version_requirements: *2160802040
+- !ruby/object:Gem::Dependency
+  name: eventmachine
+  requirement: &2160801580 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 62196357
-        segments: 
-        - 1
-        - 0
-        - 0
-        - beta
-        - 3
+      - !ruby/object:Gem::Version
         version: 1.0.0.beta.3
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: em-http-request
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
+  version_requirements: *2160801580
+- !ruby/object:Gem::Dependency
+  name: em-http-request
+  requirement: &2160801120 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 62196363
-        segments: 
-        - 1
-        - 0
-        - 0
-        - beta
-        - 4
+      - !ruby/object:Gem::Version
         version: 1.0.0.beta.4
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: nokogiri
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  version_requirements: *2160801120
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &2160800660 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 5
-        segments: 
-        - 1
-        - 5
-        version: "1.5"
+      - !ruby/object:Gem::Version
+        version: '1.5'
   type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: addressable
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  version_requirements: *2160800660
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: &2160800200 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 11
-        segments: 
-        - 2
-        - 2
-        - 6
+      - !ruby/object:Gem::Version
         version: 2.2.6
   type: :runtime
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: css_parser
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
+  version_requirements: *2160800200
+- !ruby/object:Gem::Dependency
+  name: css_parser
+  requirement: &2160799740 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 1
-        - 1
-        - 9
-        version: 1.1.9
+      - !ruby/object:Gem::Version
+        version: 1.2.3
   type: :runtime
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: grit
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  version_requirements: *2160799740
+- !ruby/object:Gem::Dependency
+  name: grit
+  requirement: &2160799280 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 29
-        segments: 
-        - 2
-        - 4
-        - 1
+      - !ruby/object:Gem::Version
         version: 2.4.1
   type: :runtime
-  version_requirements: *id008
-- !ruby/object:Gem::Dependency 
-  name: websnap
   prerelease: false
-  requirement: &id009 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 29
-        segments: 
-        - 0
-        - 1
-        - 3
-        version: 0.1.3
-  type: :runtime
-  version_requirements: *id009
-- !ruby/object:Gem::Dependency 
-  name: mini_magick
-  prerelease: false
-  requirement: &id010 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 1
-        segments: 
-        - 3
-        - 3
-        version: "3.3"
-  type: :runtime
-  version_requirements: *id010
-- !ruby/object:Gem::Dependency 
+  version_requirements: *2160799280
+- !ruby/object:Gem::Dependency
   name: mime-types
-  prerelease: false
-  requirement: &id011 !ruby/object:Gem::Requirement 
+  requirement: &2160798820 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 47
-        segments: 
-        - 1
-        - 16
-        version: "1.16"
+      - !ruby/object:Gem::Version
+        version: '1.16'
   type: :runtime
-  version_requirements: *id011
-- !ruby/object:Gem::Dependency 
-  name: sqlite3
   prerelease: false
-  requirement: &id012 !ruby/object:Gem::Requirement 
+  version_requirements: *2160798820
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: &2160798360 !ruby/object:Gem::Requirement
     none: false
-    requirements: 
+    requirements:
     - - ~>
-      - !ruby/object:Gem::Version 
-        hash: 29
-        segments: 
-        - 1
-        - 3
-        - 3
+      - !ruby/object:Gem::Version
         version: 1.3.3
   type: :development
-  version_requirements: *id012
+  prerelease: false
+  version_requirements: *2160798360
 description: An utility to archive webpages through time
 email: 
 executables: []
-
 extensions: []
-
-extra_rdoc_files: 
+extra_rdoc_files:
 - README.rdoc
-files: 
+files:
 - .gitignore
+- CHANGELOG.md
 - Gemfile
 - README.rdoc
 - Rakefile
@@ -234,6 +145,7 @@ files:
 - lib/webpage-archivist/migrations.rb
 - lib/webpage-archivist/models.rb
 - lib/webpage-archivist/patches.rb
+- lib/webpage-archivist/rasterize.js
 - lib/webpage-archivist/snapshoter.rb
 - lib/webpage-archivist/stylesheet_document.rb
 - lib/webpage-archivist/version.rb
@@ -245,39 +157,31 @@ files:
 - webpage-archivist.gemspec
 homepage: https://github.com/archiloque/webpage-archivist
 licenses: []
-
 post_install_message: 
-rdoc_options: 
+rdoc_options:
 - --main
 - README.rdoc
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: webpage-archivist
-rubygems_version: 1.8.5
+rubygems_version: 1.8.8
 signing_key: 
 specification_version: 3
 summary: An utility to archive webpages through time
-test_files: 
+test_files:
 - test/crud_test.rb
 - test/files/stylesheet.css
 - test/helper.rb