webpage-archivist 0.0.1

data/lib/webpage-archivist/migrations.rb

@@ -0,0 +1,93 @@
+require 'time'
+require 'sequel'
+
+module WebpageArchivist
+
+  DATABASE = ::Sequel.connect(ENV['DATABASE_URL'] || "sqlite://#{Dir.pwd}/webpage-archivist.sqlite3", :encoding => 'utf-8')
+
+  # Define the sequel migrations and run them at startup, code adapted from sinatra-sequel gem
+  class Migrations
+
+    @@migrations = []
+
+    def run
+      create_migrations_table
+      @@migrations.each do |migration|
+        if DATABASE[:migrations].filter(:name => migration[:name]).count == 0
+          p "Running migration: #{migration[:name]}"
+          DATABASE.transaction do
+            migration[:block].yield
+            DATABASE[:migrations] << {:name => migration[:name], :ran_at => Time.now}
+          end
+        end
+      end
+    end
+
+    def self.migration(name, &block)
+      @@migrations << {:name => name, :block => block}
+    end
+
+    migration 'create initial tables' do
+      DATABASE.create_table :webpages do
+        primary_key :id
+        String :name, :size => 250, :null => false, :index => true, :unique => true
+        String :uri, :size => 5000, :null => false
+        String :last_modified, :size => 50
+        File :last_content, :null => true
+        String :last_charset, :size => 20, :null => true
+        DateTime :created_at, :null => false, :index => true
+      end
+
+      DATABASE.create_table :instances do
+        primary_key :id
+        foreign_key :webpage_id, :webpages
+        String :commit_timestamp, :size => 25, :null => false
+        Boolean :snapshot, :default => false, :index => true
+        DateTime :created_at, :null => false, :index => true
+      end
+
+      DATABASE.create_table :images do
+        primary_key :id
+        foreign_key :webpage_id, :webpages
+        String :uri, :size => 5000, :null => false
+        String :last_modified, :size => 50, :null => false
+        String :extension, :size => 10, :null => false
+        String :file_hash, :size => 50, :null => false
+        DateTime :last_fetched, :null => false, :index => true
+      end
+
+      DATABASE.create_table :scripts do
+        primary_key :id
+        foreign_key :webpage_id, :webpages
+        String :uri, :size => 5000, :null => false
+        String :last_modified, :size => 50, :null => false
+        String :file_hash, :size => 50, :null => false
+        DateTime :last_fetched, :null => false, :index => true
+      end
+
+      DATABASE.create_table :stylesheets do
+        primary_key :id
+        foreign_key :webpage_id, :webpages
+        String :uri, :size => 5000, :null => false
+        String :last_modified, :size => 50, :null => false
+        String :file_hash, :size => 50, :null => false
+        DateTime :last_fetched, :null => false, :index => true
+        File :last_content, :null => true
+        String :last_charset, :size => 20, :null => true
+      end
+
+    end
+
+    def create_migrations_table
+      DATABASE.create_table? :migrations do
+        primary_key :id
+        String :name, :null => false, :index => true
+        timestamp :ran_at, :null => false
+      end
+    end
+
+  end
+
+  Migrations.new.run
+
+end

data/lib/webpage-archivist/models.rb

@@ -0,0 +1,190 @@
+require 'addressable/uri'
+require 'grit'
+require 'andand'
+require 'mime/types'
+
+Sequel::Model.plugin :validation_helpers
+Sequel::Model.plugin :timestamps
+
+module WebpageArchivist
+
+  class Webpage < Sequel::Model
+
+    ASSETS_PATH = File.expand_path(ENV['ARCHIVIST_ASSETS_PATH'] || './archivist_assets')
+    p "Archivist assets path is [#{ASSETS_PATH}]"
+    Dir.mkdir_if_not_exist ASSETS_PATH
+
+    one_to_many :instances
+    one_to_many :stylesheets
+    one_to_many :scripts
+    one_to_many :images
+
+    def index_path
+      File.join(repository_dir, 'index.html')
+    end
+
+    def save_content content
+      File.open(index_path, 'w') { |f| f.write(content) }
+    end
+
+    # The directory containing the git repository
+    def repository_dir
+      unless @repository_dir
+        @repository_dir = File.expand_path(File.join(ASSETS_PATH, self.id.to_s))
+        Dir.mkdir_if_not_exist @repository_dir
+      end
+      @repository_dir
+    end
+
+    def repository
+      unless @repository
+        if File.exist? "#{repository_dir}.git"
+          @repository = Grit::Repo.new(repository_dir)
+        else
+          Dir.mkdir_if_not_exist repository_dir
+          @repository = Grit::Repo.init(repository_dir)
+        end
+      end
+      @repository
+    end
+
+    def validate
+      super
+      validates_presence [:uri, :name]
+      validates_max_length 5000, :uri
+      validates_max_length 250, :name
+      validates_unique :name, :message => "[#{self.name}] is already taken"
+      if self.uri
+        begin
+          URI.parse self.uri
+        rescue URI::InvalidURIError
+          errors.add('uri', "[#{self.uri}] is not a valid uri")
+        end
+      end
+    end
+
+    def after_create
+      super
+      repository
+    end
+
+    # Update the repo and commit the changes
+    # files:: the files that should be in the repository
+    # message:: the commit message
+    def update_repo_commit_changes files, message
+      Dir.foreach(repository_dir) do |file|
+        unless file.start_with?('.') || ('index.html' == file) || files.include?(file)
+          File.delete File.join(repository_dir, file)
+        end
+      end
+      status = repository.status
+      repository.add status.untracked.keys
+      repository.add status.changed.keys
+      repository.remove status.deleted.keys
+      repository.commit_index message
+    end
+
+  end
+
+  class Instance < Sequel::Model
+
+    many_to_one :webpage
+
+    def validate
+      super
+      validates_presence [:webpage_id, :commit_timestamp]
+    end
+
+  end
+
+  module ElementWithContent
+
+    def save_content content
+      File.open(File.join(webpage.repository_dir, file_name), 'w') { |f| f.write(content) }
+    end
+
+  end
+
+  module WebpageElement
+    include ElementWithContent
+
+    def validate
+      super
+      validates_presence [:webpage_id, :uri, :last_fetched, :last_modified, :file_hash]
+      validates_max_length 5000, :uri
+      if self.uri
+        begin
+          URI.parse self.uri
+        rescue URI::InvalidURIError
+          errors.add('uri', "[#{self.uri}] is not a valid uri")
+        end
+      end
+    end
+
+  end
+
+  class Image < Sequel::Model
+    many_to_one :webpage
+
+    include WebpageElement
+
+    def self.compress
+      false
+    end
+
+    def validate
+      super
+      validates_max_length 10, :extension
+    end
+
+    def file_name
+      "#{file_hash}#{extension}"
+    end
+
+    def self.extention uri, content_type
+      extension = MIME::Types[content_type].andand[0].extensions.andand[0]
+      extension ? ".#{extension}" : File.extname(Addressable::URI.parse(uri).normalize.path)[0...10]
+    end
+
+  end
+
+  class Script < Sequel::Model
+    many_to_one :webpage
+
+    include WebpageElement
+
+    def file_name
+      "#{file_hash}.js"
+    end
+
+    def self.extention uri, content_type
+      '.js'
+    end
+
+    def extension= extension
+      # do nothing
+    end
+
+  end
+
+  class Stylesheet < Sequel::Model
+    many_to_one :webpage
+
+    include WebpageElement
+
+    def file_name
+      "#{file_hash}.css"
+    end
+
+    def self.extention uri, content_type
+      '.css'
+    end
+
+    def extension= extension
+      # do nothing
+    end
+
+  end
+
+end
+

data/lib/webpage-archivist/patches.rb

@@ -0,0 +1,63 @@
+# Contains monkey patches
+require 'addressable/uri'
+require 'grit'
+require 'mini_magick'
+
+module Addressable
+
+  class URI
+
+    # Make an URI absolute
+    def absolutize uri
+      join(uri).normalize.to_s
+    end
+
+  end
+
+end
+
+#Patching the repo grit class to specify the work tree
+class Grit::Repo
+
+  def add(*files)
+    self.git.add({:chdir => self.git.work_tree}, *files.flatten)
+  end
+
+  def remove(*files)
+    self.git.rm({:chdir => self.git.work_tree}, *files.flatten)
+  end
+
+  def commit_index(message)
+    self.git.commit({:chdir => self.git.work_tree}, '--allow-empty', '-m', message)
+  end
+
+  def archive_zip(id, file)
+    self.git.archive({:chdir => self.git.work_tree}, "--output=#{file}", id)
+  end
+
+end
+
+# Patching mini magic so we can specify the path to the image magick installation
+module MiniMagick
+
+  class CommandBuilder
+
+    alias :parent_command :command
+
+    def command
+      "#{ENV['IMAGE_MAGICK_PATH'] ? "#{ENV['IMAGE_MAGICK_PATH']}/" : ''}#{parent_command}"
+    end
+
+  end
+end
+
+class Dir
+
+  # Create a dit if it does not exist
+  def self.mkdir_if_not_exist p1
+    unless Dir.exist? p1
+      Dir.mkdir p1
+    end
+  end
+
+end

data/lib/webpage-archivist/snapshoter.rb

@@ -0,0 +1,77 @@
+require 'websnap'
+require 'mini_magick'
+
+module WebpageArchivist
+
+  # Snapshot the pages and create thumbnails
+  class Snapshoter
+
+    SNAPSHOTS_PATH = File.expand_path(ENV['ARCHIVIST_SNAPSHOTS_PATH'] || './archivist_snapshots')
+    p "Archivist snapshots path is [#{SNAPSHOTS_PATH}]"
+    Dir.mkdir_if_not_exist SNAPSHOTS_PATH
+
+    class << self
+      attr_accessor :width, :height, :format, :thumbnail_scale, :thumbnail_crop_width, :thumbnail_crop_height
+    end
+
+    Snapshoter.width = 1024
+
+    Snapshoter.height = 1536
+
+    Snapshoter.format = 'jpeg'
+
+    Snapshoter.thumbnail_crop_width = 1024
+
+    Snapshoter.thumbnail_crop_height = 768
+
+    Snapshoter.thumbnail_scale = 25
+
+    # Create a snapshot corresponding to an instance
+    # instance:: the instance
+    def self.snapshot_instance instance
+      dir_path = File.join(SNAPSHOTS_PATH, instance.webpage.id.to_s)
+      unless Dir.exist? dir_path
+        Dir.mkdir dir_path
+      end
+
+      snapshot_path = File.join(dir_path, "#{instance.id}.#{Snapshoter.format}")
+      thumbnail_path = File.join(dir_path, "#{instance.id.to_s}-small.#{Snapshoter.format}")
+      snapshot File.new(instance.webpage.index_path), snapshot_path, thumbnail_path
+      instance.update(:snapshot => true)
+    end
+
+    # Create a snapshot of a web page
+    # uri_or_file:: the uri of the file to snapshot
+    # snapshot_path:: path to the snapshot file
+    # thumbnail_path: path to the thumbnail (can be nil for no thumbnail)
+    def self.snapshot uri_or_file, snapshot_path, thumbnail_path = nil
+      ::WebpageArchivist.debug "Snapshot for [#{uri_or_file.kind_of?(File) ? uri_or_file.path : uri_or_file}] on [#{snapshot_path}]" if ::WebpageArchivist.log
+
+      if File.exists? snapshot_path
+        File.delete snapshot_path
+      end
+
+      snapper = WebSnap::Snapper.new(uri_or_file)
+      snapper.options.clear.merge!({'--height' => Snapshoter.height, '--width' => Snapshoter.width, '--format' => Snapshoter.format})
+      snapper.to_file(snapshot_path)
+
+      if thumbnail_path
+        ::WebpageArchivist.debug "Thumbnail of [#{snapshot_path}] on [#{thumbnail_path}]" if ::WebpageArchivist.log
+
+        if File.exists? thumbnail_path
+          File.delete thumbnail_path
+        end
+
+        img = MiniMagick::Image::open(snapshot_path)
+        img.combine_options do |c|
+          c.crop "#{Snapshoter.thumbnail_crop_width}x#{Snapshoter.thumbnail_crop_height}+0+0"
+          c.scale "#{Snapshoter.thumbnail_scale}%"
+        end
+        img.write thumbnail_path
+      end
+
+    end
+
+  end
+
+end

data/lib/webpage-archivist/stylesheet_document.rb

@@ -0,0 +1,129 @@
+require 'addressable/uri'
+require 'css_parser'
+require 'iconv'
+
+module CssParser
+
+  # Replace original implementation with one using addressable
+  def self.convert_uris(css, base_uri)
+    return css.gsub(URI_RX) do
+      uri = $1.to_s
+      uri.gsub!(/["']+/, '')
+      "url('#{base_uri.absolutize(uri)}')"
+    end
+  end
+
+  # Make the declaration accessible
+  class RuleSet
+
+    attr_reader :declarations
+
+  end
+
+  class Parser
+
+    attr_reader :imports
+
+    # Add the imports first
+    def to_s(media_types = :all)
+      out = imports ? "#{imports.collect { |i| "@import url(\"#{i}\");" }.join("\n")}\n\n" : ''
+
+      each_selector(media_types) do |selectors, declarations, specificity|
+        out << "#{selectors} {\n#{declarations}\n}\n"
+      end
+      out
+    end
+
+    # Adapted from add_block! so the imports are listed instead of fetched
+    def add_block_archivist(block, options = {})
+      @imports ||= []
+
+      options = {:base_uri => nil, :base_dir => nil, :charset => nil, :media_types => :all, :only_media_types => :all}.merge(options)
+      options[:media_types] = [options[:media_types]].flatten
+      options[:only_media_types] = [options[:only_media_types]].flatten
+
+      block = cleanup_block(block)
+
+      if options[:base_uri] and @options[:absolute_paths]
+        block = CssParser.convert_uris(block, options[:base_uri])
+      end
+
+      # List @imported CSS
+      block.scan(RE_AT_IMPORT_RULE).each do |import_rule|
+        imports << import_rule[0].to_s.gsub(/['"]*/, '').strip
+      end
+
+      block.gsub!(RE_AT_IMPORT_RULE, '')
+
+      parse_block_into_rule_sets!(block, options)
+    end
+  end
+end
+
+module WebpageArchivist
+
+  # Wrapper around css_parser
+  class StylesheetDocument
+
+    CONVERTER = Iconv.new('UTF-8//IGNORE//TRANSLIT', 'ASCII//IGNORE//TRANSLIT')
+
+    attr_reader :parser, :charset
+
+    def initialize content, base_uri, charset = nil
+      @base_uri = base_uri
+      @parser = CssParser::Parser.new :import => false
+
+      @charset = charset
+      unless @charset
+        content = CONVERTER.iconv(content)
+        @charset = 'ASCII-8BIT'
+      end
+
+      parser.add_block_archivist(content, {:base_uri => base_uri, :charset => @charset})
+      @expanded = false
+    end
+
+    def expand_if_needed
+      unless @expanded
+        parser.each_rule_set do |rs|
+          rs.expand_background_shorthand!
+        end
+        @expanded = true
+      end
+    end
+
+    # Call a block for each import
+    # Block call parameter will be the import's uri
+    # if the block return something it will replace the uri
+    def each_import &block
+      parser.imports.collect! do |uri|
+        block.yield(uri) || uri
+      end
+    end
+
+    # Call a block for each image
+    # Block call parameter will be the image uri,
+    # if the block return something it will replace the uri
+    def each_image &block
+      expand_if_needed
+      parser.each_rule_set do |rs|
+        rs.declarations.each do |r|
+          value = r[1][:value]
+          if uri = /url\(['|"]([^']+)['|"]\)/i.match(value)
+            uri = uri[1]
+            if (uri = block.yield(uri))
+              r[1][:value] = "url(\"#{uri}\")"
+            end
+          end
+        end
+      end
+    end
+
+    # Get the css content
+    def to_css
+      parser.to_s
+    end
+
+  end
+
+end