snapurl 0.0.3

data/History

@@ -0,0 +1,21 @@
+=== 0.0.3 / 2009-10-19
+
+* Added install-cocoa script.
+* Moved to jeweler for package/project management
+  (http://github.com/technicalpickles/jeweler)
+* Added support for file:// urls - thanks to Joel Chippindale
+  (http://blog.monkeysthumb.org) 
+* Added dependency to thoughtbot-shoulda 
+  (http://thoughtbot.com/projects/shoulda)
+* Gem name is now jurisgalang-snapurl
+* No longer uses Hoe
+
+=== 0.0.2 / 2009-04-04
+
+* Light house-keeping.
+* Merged changes by Michael Kohl (http://citizen428.net)
+* Added RubyCocoa source package and installation task.
+
+=== 0.0.1 / 2009-02-08
+
+* Birthday!

data/LICENSE

@@ -0,0 +1,27 @@
+Copyright (c) 2008, 2009 Juris Galang
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+Except as contained in this notice, the name(s) of the above 
+copyright holders shall not be used in advertising or otherwise to 
+promote the sale, use or other dealings in this Software without 
+prior written authorization.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,106 @@
+= snapurl
+
+Take snapshots of web pages. This is a Ruby/RubyCocoa port of the webkit2png.py script by Paul Hammond (http://www.paulhammond.org/webkit2png/) with some minor modifications.
+
+See also: http://www.jurisgalang.com/articles/show/SnapUrl
+
+  
+== Requirements:
+
+  * OS X only.
+  * RubyCocoa (Leopard and Snow Leopard will already have this installed)
+  * Ruby 1.8.7 (Ruby 1.9.x wont play with RubyCocoa)
+
+If you intend to work with the source and do your own builds, the project expects that you have the following gems installed:
+
+  * gemcutter
+  * jeweler
+  * rcov
+  * thoughtbot-shoulda
+ 
+
+== Installation
+
+  $ sudo gem install snapurl
+
+This gem is published at gemcutter (http://gemcutter.org), so you'll need to add it as a gem source (unless you already have):
+  
+  $ gem sources -a http://gemcutter.org 
+  
+Alternatively you can build the gem from source and install:
+
+  $ git clone git://github.com/jurisgalang/snapurl.git
+  $ cd snapurl
+  $ rake gemspec install
+
+To build and install a specific tag:
+
+  $ git clone git://github.com/jurisgalang/snapurl.git
+  $ cd snapurl
+  $ git checkout -f -b `whoami`-snapurl v0.0.3
+  $ rake gemspec install
+
+
+== Install RubyCocoa
+
+If you have a custom Ruby installation then you'll also need to do a custom build and install of RubyCocoa. I've bundled the latest release source and a script to do this for you:
+
+  $ cd snapurl
+  $ ./thirdparty/install-cocoa
+
+NOTE: Your custom Ruby installation must have had --enable-shared flag on when you built it.
+
+
+== Usage
+
+Just pass a list of URLs to the snapurl script and it will take a snapshot of the pages found in those URLs:
+
+   $ snapurl http://artists.gawker.com/                  # takes a snapshot of gawker artists
+   $ snapurl http://io9.com/ http://artists.gawker.com/  # ... and io9 websites.
+
+By default snapurl generates a set of files per URL that it processes: thumbnail, clipped, and full-sized view of the web pages in PNG format. 
+
+
+== Note on Patches/Pull Requests
+
+  * Fork the project.
+  * Make your feature addition or bug fix.
+  * Add tests for it. This is important so I don't break it in a
+    future version unintentionally.
+  * Commit, do not mess with rakefile, version, or history.
+    (if you want to have your own version, that is fine but
+    bump version in a commit by itself I can ignore when I pull)
+  * Send me a pull request. Bonus points for topic branches.
+
+
+== License
+
+(The MIT License)
+
+Copyright (c) 2008, 2009 Juris Galang
+
+Permission is hereby granted, free of charge, to any person
+obtaining a copy of this software and associated documentation
+files (the "Software"), to deal in the Software without
+restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following
+conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+Except as contained in this notice, the name(s) of the above 
+copyright holders shall not be used in advertising or otherwise to 
+promote the sale, use or other dealings in this Software without 
+prior written authorization.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.

data/bin/snapurl

@@ -0,0 +1,7 @@
+#!/usr/bin/env ruby
+
+$:.unshift File::join(File::dirname(__FILE__), '..', 'lib')
+require 'rubygems'
+require 'snapurl'
+
+SnapUrl::Application::run(ARGV)

data/lib/core/application.rb

@@ -0,0 +1,96 @@
+=begin
+* Name: SnapUrl
+* Description:
+*   This is a Ruby/RubyCocoa port of the webkit2png.py script by Paul Hammond
+*   (http://www.paulhammond.org/webkit2png/) with some minor modifications.
+* Author: Juris Galang (http://www.JurisGalang.com)
+* Date: 09/15/08
+* License:
+=end
+module SnapUrl
+  class Application
+    def Application::run(args)
+      config = Application.parse(args)
+      Camera::snap(config[:urls], config[:options])
+    end
+
+    # TODO: add option to control log-level
+    # TODO: add implementation for verbose run
+    # TODO: set --include-format = true if more than one format
+    def Application::parse(args)
+      options = Camera::DEFAULT_OPTIONS
+      begin
+        opts = OptionParser.new do |opts|
+          opts.banner = <<-EOF
+          Usage: #{opts.program_name} url0 [url1] [url2]... [options]
+
+          #{opts.program_name} is a command line tool that creates PNG screenshots of webpages. 
+          With tall or wide pages that would normally require scrolling, it takes
+          screenshots of the whole webpage, not just the area that would be visible
+          in a browser window.
+
+          It is a Ruby/RubyCocoa port of the webkit2png.py script by Paul Hammond
+          (http://www.paulhammond.org/webkit2png/)
+          EOF
+          opts.banner.gsub!(/  /, '')
+          
+          opts.separator ' '
+          opts.separator 'Specific Options:'
+          opts.on('--width [WIDTH]', Float, "initial and minimum browser width (default: #{options[:browser][:width].to_i})")     { |n| options[:browser][:width] = n }
+          opts.on('--height [HEIGHT]', Float, "initial and minimum browser height (default: #{options[:browser][:height].to_i})") { |n| options[:browser][:height] = n }
+
+          opts.separator ' '
+          opts.on('--[no-]fullsize', "create fullsize screenshot")  { |n| n ? options[:snapFormat] << :fullsize  : options[:snapFormat].delete(:fullsize) }
+          opts.on('--[no-]thumbnail', "create thumbnail")           { |n| n ? options[:snapFormat] << :thumbnail : options[:snapFormat].delete(:thumbnail) }
+          opts.on('--[no-]clip', "create clip of thumbnail")        { |n| n ? options[:snapFormat] << :clip      : options[:snapFormat].delete(:clip) }
+
+          opts.separator ' '
+          opts.on('--scale [SCALE]', Float, "scale factor for thumbnails (default: #{options[:scaleFactor]})") { |n| options[:scaleFactor] = n }
+
+          opts.separator ' '
+          opts.on('--clipwidth [WIDTH]', Float, "width of clipped thumbnail (default: #{options[:clip][:width].to_i})")     { |n| options[:clip][:width] = n }
+          opts.on('--clipheight [HEIGHT]', Float, "height of clipped thumbnail (default: #{options[:clip][:height].to_i})") { |n| options[:clip][:height] = n }
+
+          opts.separator ' '
+          opts.on('-f', '--filename [NAME]', "use NAME as part of filename (derived from url if not specified)") { |n| options[:filename] = n }
+          
+          opts.on('--use-md5hash', "use md5 hash for filename (overriden by --filename)") { |n| options[:useHashForFilename?] = n }
+          opts.on('--include-datestamp', "include date in filename")                  { |n| options[:datestampInFilename?] = n }
+          opts.on('--include-timestamp', "include time in filename")                  { |n| options[:timestampInFilename?] = n }
+          opts.on('--include-size', "include image size in filename")                 { |n| options[:sizeInFilename?] = n }
+          opts.on('--include-format', "include image format in filename")             { |n| options[:snapFormatInFilename?] = n }
+
+          opts.separator ' '
+          opts.on('-d', '--output-dir [DIRNAME]', "directory to place images into (default: #{options[:outputDirectory]})") { |n| options[:outputDirectory] = n }
+
+          opts.separator ' '
+          opts.separator 'Common Options:'
+          opts.on('-v', "--[no-]verbose", "Run verbosely")  { |n| options[:verbose?] = n }
+          opts.on_tail("-h", "--help", "Show this message") do
+            puts opts.help
+            exit
+          end
+
+          opts.on_tail("--version", "Show version") do
+            puts opts.ver
+            exit
+          end   
+        end
+
+        urls = opts.permute!(args)
+
+        raise OptionParser::MissingArgument.new('you need to supply at least one URL')                           if urls.empty?
+        raise OptionParser::InvalidArgument.new('you can only use the filename option if there is only one URL') if (urls.size > 1) && !options[:filename].empty?
+        raise OptionParser::InvalidArgument.new('scale cannot be zero')                                          if options[:scaleFactor] < 0.0
+        # width and height cannot be zero or less than minimum
+        # clipwidth and clipheight cannot be zero or less than minimum
+        raise OptionParser::AmbiguousArgument.new('you need to allow at least one format')                       if options[:snapFormat].empty?
+
+        return { :urls => urls, :options => options }
+      rescue OptionParser::ParseError => e
+        puts "#{e.message.capitalize}"
+        exit(1)
+      end
+    end
+  end
+end

data/lib/core/camera.rb

@@ -0,0 +1,225 @@
+=begin
+* Name: SnapUrl
+* Description:
+*   This is a Ruby/RubyCocoa port of the webkit2png.py script by Paul Hammond
+*   (http://www.paulhammond.org/webkit2png/) with some minor modifications.
+* Author: Juris Galang (http://www.JurisGalang.com)
+* Date: 09/15/08
+* License:
+=end
+
+module SnapUrl
+  module Camera
+    DEFAULT_OPTIONS = {
+      :browser               => {:width => 800.0, :height => 600.0},
+      :clip                  => {:width => 200.0, :height => 150.0},
+      :snapFormat            => [:fullsize, :thumbnail, :clip],
+      :scaleFactor           => 0.25,
+      :filename              => '',
+      :datestampInFilename?  => false,
+      :timestampInFilename?  => false,
+      :sizeInFilename?       => false,
+      :snapFormatInFilename? => false,
+      :useHashForFilename?   => false,
+      :outputDirectory       => Dir.pwd,
+      :verbose?              => false
+    }
+
+    def Camera::snap(urls, options = {})
+      # create context
+      OSX::NSApplication.sharedApplication
+
+      # create an app delegate
+      delegate = AppDelegate.alloc.init
+      OSX::NSApp.setDelegate(delegate)
+
+      # create a webview object
+      rect    = OSX::NSMakeRect(-16000, -16000, 100, 100)
+      webview = OSX::WebView.alloc.initWithFrame(rect)
+      webview.setMediaStyle('screen')
+      webview.mainFrame.frameView.setAllowsScrolling(false)
+
+      # make sure we don't save any of the prefs that we change.
+      webview.preferences.setAutosaves(false)
+      webview.preferences.setShouldPrintBackgrounds(true)
+      webview.preferences.setJavaScriptCanOpenWindowsAutomatically(false)
+      webview.preferences.setAllowsAnimatedImages(false)
+
+      # create a window and set view
+      win = OSX::NSWindow.alloc
+      win.initWithContentRect_styleMask_backing_defer(rect, OSX::NSBorderlessWindowMask, OSX::NSBackingStoreBuffered, false)
+      win.setContentView(webview)
+
+      # create load delegate
+      loader         = LoadDelegate.alloc.init
+      loader.urls    = urls
+      loader.options = DEFAULT_OPTIONS.merge(options)
+      webview.setFrameLoadDelegate(loader)
+
+      # now run
+      OSX::NSApp.run
+    end
+
+    class AppDelegate < OSX::NSObject
+      def applicationDidFinishLaunching(aNotification)
+        webview = aNotification.object.windows.objectAtIndex(0).contentView
+        webview.frameLoadDelegate.fetchUrl(webview)
+      end
+    end
+
+    class LoadDelegate < OSX::NSObject
+      attr_accessor :options, :urls, :logger
+
+      def urls=(urls)
+        @urls = urls.to_a
+      end
+
+      def options=(options)
+        # adjust configs
+        options[:snapFormatInFilename?] = true if (options[:snapFormat].size > 1) && !options[:snapFormatInFilename?]
+
+        w = options[:clip][:width]  / options[:scaleFactor]
+        h = options[:clip][:height] / options[:scaleFactor]
+        options[:browser][:width]  = w unless w < options[:browser][:width]
+        options[:browser][:height] = h unless h < options[:browser][:height]
+
+        @options = options
+      end
+
+      def init
+        @logger = Logger.new(STDOUT)
+        self
+      end
+
+      def webView_didFailLoadWithError_forFrame(webview, error, frame)
+        @logger.warn "#{error.localizedDescription}"
+        fetchUrl(webview)
+      end
+
+      def webView_didFailProvisionalLoadWithError_forFrame(webview, error, frame)
+        @logger.warn "#{error.localizedDescription}"
+        fetchUrl(webview)
+      end
+
+      def webView_didFinishLoadForFrame(webview, frame)
+        # don't care about subframes
+        return unless (frame == webview.mainFrame)
+        docview = frame.frameView.documentView
+        resizeWebView(docview)
+
+        @logger.info "Capturing..."
+        url = frame.dataSource.initialRequest.URL.absoluteString
+        bitmap = captureView(docview)
+        saveImages(url, bitmap)
+
+        @logger.info "Done!"
+        fetchUrl(webview)
+      end
+
+      def resetWebView(view)
+        rect = OSX::NSMakeRect(0, 0, browser[:width], browser[:height])
+        view.window.setContentSize([browser[:width], browser[:height]])
+        view.setFrame(rect)
+      end
+
+      def resizeWebView(view)
+        view.setNeedsDisplay(true)
+        view.displayIfNeeded
+        view.window.setContentSize(view.bounds.size)
+        view.setFrame(view.bounds)
+      end
+
+      def captureView(view)
+        view.lockFocus
+        bitmap = OSX::NSBitmapImageRep.alloc
+        bitmap.initWithFocusedViewRect(view.bounds)
+        view.unlockFocus
+        bitmap
+      end
+
+      def fetchUrl(webview)
+        OSX::NSApplication.sharedApplication.terminate(self) if @urls.empty?
+
+        url = String.new(@urls.shift)
+
+        url.gsub!(/^/, "http:\/\/") unless url =~ /^(http(s)?|file):\/\//
+        @logger.info "Fetching #{url}..."
+
+        resetWebView(webview)
+        webview.mainFrame.loadRequest(OSX::NSURLRequest.requestWithURL(OSX::NSURL.URLWithString(url)))
+
+        return if webview.mainFrame.provisionalDataSource
+        fetchUrl(webview)
+      end
+
+      def makeFilename(url, bitmap, format)
+        if !filename.empty?
+          name = filename
+
+        elsif useHashForFilename?
+          name = Digest::MD5.new.hexdigest(url)
+
+        else
+          name = String.new(url)
+          name.gsub!(/\W/, '');
+          name.gsub!(/^http(s)?/i, '');
+          name.gsub!(/\//, '.');
+        end
+
+        name = "#{Time.now.strftime('%H%M%S')}-#{name}"            if timestampInFilename?
+        name = "#{Time.now.strftime('%Y%m%d')}-#{name}"            if datestampInFilename?
+        name = "#{name}-#{bitmap.pixelsWide}x#{bitmap.pixelsHigh}" if sizeInFilename?
+        name = "#{name}-#{format}"                                 if snapFormatInFilename?
+
+        name = "#{name}.png"
+
+        name = File.join(File.expand_path(outputDirectory), name)  unless outputDirectory.empty?
+        name
+      end
+
+      def saveImages(url, bitmap)
+        writeToFile(url, bitmap, :fullsize) if snapFormat.include?(:fullsize)
+
+        if snapFormat.include?(:thumbnail) || snapFormat.include?(:clip)
+          # work out how big the thumbnail is
+          thumbWidth  = bitmap.pixelsWide * scaleFactor
+          thumbHeight = bitmap.pixelsHigh * scaleFactor
+
+          # make the thumbnails in a scratch image
+          scratch = OSX::NSImage.alloc.initWithSize(OSX::NSMakeSize(thumbWidth, thumbHeight))
+          scratch.lockFocus
+          OSX::NSGraphicsContext.currentContext.setImageInterpolation(OSX::NSImageInterpolationHigh)
+          thumbRect = OSX::NSMakeRect(0.0, 0.0, thumbWidth, thumbHeight)
+          clipRect  = OSX::NSMakeRect(0.0, thumbHeight - clip[:height], clip[:width], clip[:height])
+          bitmap.drawInRect(thumbRect)
+          thumbnail = OSX::NSBitmapImageRep.alloc.initWithFocusedViewRect(thumbRect)
+          clipping  = OSX::NSBitmapImageRep.alloc.initWithFocusedViewRect(clipRect)
+          scratch.unlockFocus
+
+          # save the thumbnails as pngs
+          writeToFile(url, thumbnail, :thumbnail) if snapFormat.include?(:thumbnail)
+          writeToFile(url, clipping, :clip)   if snapFormat.include?(:clip)
+        end
+      end
+
+      def writeToFile(url, bitmap, format)
+        name = makeFilename(url, bitmap, format)
+        FileUtils.mkdir_p File.dirname(name)
+        bitmap.representationUsingType_properties(OSX::NSPNGFileType, nil).writeToFile_atomically(name, true)
+      end
+
+      def ping(url)
+        open(url) rescue nil
+      end
+
+      def method_missing(name, *args)
+        if args.empty?
+          k = name.to_sym
+          super(name, args) unless @options.keys.include?(k)
+          return @options[k]
+        end
+        super(name, args)
+      end
+    end
+  end
+end

data/lib/snapurl.rb

@@ -0,0 +1,28 @@
+=begin
+* Name: SnapUrl
+* Description:
+*   This is a Ruby/RubyCocoa port of the webkit2png.py script by Paul Hammond
+*   (http://www.paulhammond.org/webkit2png/) with some minor modifications.
+* Author: Juris Galang (http://www.JurisGalang.com)
+* Date: 09/15/08
+* License:
+=end
+require 'digest/md5'
+require 'fileutils'
+require 'logger'
+require 'optparse'
+require 'ostruct'
+require 'open-uri'
+
+require 'osx/cocoa'
+OSX.require_framework 'WebKit'
+
+require 'core/camera'
+require 'core/application'
+
+Version = "0.0.2"
+Release = "alpha"
+
+module SnapUrl
+  VERSION = Version
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification 
+name: snapurl
+version: !ruby/object:Gem::Version 
+  version: 0.0.3
+platform: ruby
+authors: 
+- Juris Galang
+- Michael Kohl
+- Joel Chippindale
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-20 00:00:00 -07:00
+default_executable: snapurl
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: thoughtbot-shoulda
+  type: :development
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: "\n      Take snapshots of websites. This is a Ruby/RubyCocoa port of the \n      webkit2png.py script by Paul Hammond \n      (http://www.paulhammond.org/webkit2png/) with some minor modifications -\n      Generates a set of image files representing the thumbnail, clipped, \n      and full-sized view of a web page in PNG format. "
+email: jurisgalang@gmail.com
+executables: 
+- snapurl
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- History
+- README.rdoc
+- bin/snapurl
+- lib/core/application.rb
+- lib/core/camera.rb
+- lib/snapurl.rb
+- test/test_snapurl.rb
+- LICENSE
+has_rdoc: true
+homepage: http://github.com/jurisgalang/snapurl
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: Take snapshots of websites.
+test_files: 
+- test/test_snapurl.rb