seep 0.0.2

data/lib/seep.rb

@@ -0,0 +1,19 @@
+require 'forwardable'
+require 'uri'
+require 'curb'
+require 'nokogiri'
+require 'gd2-ffij'
+
+module Seep
+  require 'seep/fetcher'
+  require 'seep/doc'
+  require 'seep/image'
+
+  def self.fetch(*args)
+    Fetcher.open(*args)
+  end
+
+  def self.doc(*args)
+    fetch(*args).to_doc
+  end
+end

data/lib/seep/doc.rb

@@ -0,0 +1,44 @@
+class Seep::Doc
+  attr_reader :dom, :url
+
+  def initialize(url, html)
+    @url = url
+    @dom = Nokogiri::HTML.parse(html)
+  end
+
+  def links
+    @links ||= [].tap do |links|
+      @dom.search("a").each do |anchor|
+        link = compute_url( anchor.get_attribute("href") )
+        links << link unless link.nil? or link == url
+      end
+    end
+  end
+
+  def images
+    @images ||= [].tap do |images|
+      @dom.search("img").each do |image|
+        link = compute_url( image.get_attribute("src") )
+        images << link unless link.nil?
+      end
+    end
+  end
+
+  def urls
+    links + images
+  end
+
+  def compute_url( relative_url )
+    return nil if relative_url.nil?
+    url = begin
+      URI.join( @url, relative_url ).to_s
+    rescue
+      URI.join( @url, URI.escape(relative_url) ).to_s
+    end
+    url =~ /^http/i ? url : nil
+  end
+
+  def inspect
+    "#<Seep::Doc #{url} links: #{links.count}, images: #{images.count}>"
+  end
+end

data/lib/seep/fetcher.rb

@@ -0,0 +1,107 @@
+class Seep::Fetcher
+  AGENT = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2'
+  
+  attr_reader :curb, :size, :body
+  attr_accessor :url, :max_file_size
+  attr_accessor :request_headers, :response_headers
+
+  def initialize(url, options = {})
+    self.url              = url
+    self.request_headers  = {}.tap do |head|
+      head['User-Agent']  = options[:user_agent] || AGENT
+    end
+    self.response_headers = {}
+    self.max_file_size    = options[:max_file_size] || 1_048_576 # 1MB
+    
+    @curb                 = Curl::Easy.new(url)
+    @curb.follow_location = true
+    @curb.max_redirects   = options[:max_redirects] || 5
+
+    register_on_header!
+    register_on_body!
+  end
+
+  def content_type
+    response_headers['Content-Type']
+  end
+
+  def dest_url
+    curb.last_effective_url || url
+  end
+
+  def open(redirect = 0)
+    @body = ""; @size = 0
+    curb.headers = request_headers
+    curb.perform
+    self
+  end
+
+  def inspect
+    "#<Seep::Fetcher #{ content_type.nil? ? dest_url : content_type + ' ' + dest_url }>"
+  end
+
+  def export(path)
+    File.open(path, 'w') do |file|
+      file.write(@body)
+    end
+  end
+
+  def ext
+    case content_type
+    when "image/jpeg"; ".jpg"
+    when "image/png" ; ".png"
+    when "image/gif" ; ".gif"
+    when "text/html" ; ".html"
+    when "text/plain"; ".txt"
+    else; ""
+    end
+  end
+
+  def image?
+    (!! content_type =~ /^image/) and to_image.valid?
+  end
+
+  def to_image
+    @image ||= Seep::Image.new(body)
+  end
+
+  def doc?
+    content_type == "text/html"
+  end
+
+  def to_doc
+    @doc ||= Seep::Doc.new(url, body)
+  end
+
+  def self.open(url, options = {})
+    self.new(url, options).open
+  end
+
+  def register_on_header!
+    curb.on_header do |header|
+      key, value = header.split(":", 2)
+      unless key.nil? or value.nil?
+        key.strip!; value.strip!
+        @size = value.to_i if key == "Content-Length"
+        response_headers[key] = value
+      end
+      if @size > max_file_size
+        @size = -1
+      else
+        header.length
+      end
+    end
+  end
+
+  def register_on_body!
+    curb.on_body do |body|
+      @body += body
+      @size  = @body.length
+      if @size > max_file_size
+        @size = -1
+      else
+        body.length
+      end
+    end
+  end
+end

data/lib/seep/image.rb

@@ -0,0 +1,18 @@
+class Seep::Image
+  extend Forwardable
+
+  attr_reader :image
+  def_delegators :image, :size, :width, :height
+  alias_method   :dimensions, :size
+
+  def initialize(data)
+    @valid = begin
+      @image = GD2::Image.load( data ); true
+    rescue
+      false
+    end
+  end
+
+  def valid?; @valid; end
+
+end

data/seep.gemspec

@@ -0,0 +1,85 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "seep"
+  s.version = "0.0.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Carl Zulauf"]
+  s.date = "2011-12-27"
+  s.description = "Collection of web spidering and downloading tools using redis, curl, and gd."
+  s.email = "carl@linkleaf.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README"
+  ]
+  s.files = [
+    ".document",
+    ".rspec",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README",
+    "Rakefile",
+    "doc/small.jpg",
+    "doc/test_a.html",
+    "doc/test_b.html",
+    "lib/seep.rb",
+    "lib/seep/doc.rb",
+    "lib/seep/fetcher.rb",
+    "lib/seep/image.rb",
+    "seep.gemspec",
+    "spec/a_spec.rb",
+    "spec/doc_spec.rb",
+    "spec/fetcher_spec.rb",
+    "spec/image_spec.rb",
+    "spec/spec_helper.rb",
+    "spider.rb",
+    "spidr_test.rb"
+  ]
+  s.homepage = "http://github.com/carlzulauf/seep"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "1.8.10"
+  s.summary = "web spidering/downloading tools"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<redis-native_hash>, [">= 0"])
+      s.add_runtime_dependency(%q<gd2-ffij>, [">= 0"])
+      s.add_runtime_dependency(%q<curb>, [">= 0"])
+      s.add_runtime_dependency(%q<nokogiri>, [">= 0"])
+      s.add_development_dependency(%q<ruby-debug19>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_development_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_development_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_development_dependency(%q<rcov>, [">= 0"])
+    else
+      s.add_dependency(%q<redis-native_hash>, [">= 0"])
+      s.add_dependency(%q<gd2-ffij>, [">= 0"])
+      s.add_dependency(%q<curb>, [">= 0"])
+      s.add_dependency(%q<nokogiri>, [">= 0"])
+      s.add_dependency(%q<ruby-debug19>, [">= 0"])
+      s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+      s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+      s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+      s.add_dependency(%q<rcov>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<redis-native_hash>, [">= 0"])
+    s.add_dependency(%q<gd2-ffij>, [">= 0"])
+    s.add_dependency(%q<curb>, [">= 0"])
+    s.add_dependency(%q<nokogiri>, [">= 0"])
+    s.add_dependency(%q<ruby-debug19>, [">= 0"])
+    s.add_dependency(%q<rspec>, ["~> 2.3.0"])
+    s.add_dependency(%q<bundler>, ["~> 1.0.0"])
+    s.add_dependency(%q<jeweler>, ["~> 1.6.4"])
+    s.add_dependency(%q<rcov>, [">= 0"])
+  end
+end
+

data/spec/a_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "stuff" do
+  it "should pass" do
+    true.should be_true
+  end
+end

data/spec/doc_spec.rb

@@ -0,0 +1,42 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Seep::Doc" do
+  before :all do
+    @doc_a = Seep::Doc.new(
+      "http://linkleaf.com/test_a.html",
+      File.read("doc/test_a.html")
+    )
+    @doc_b = Seep::Doc.new(
+      "http://examancer.com/gallery/",
+      File.read("doc/test_b.html")
+    )
+    @link_a   = "https://exanotes.com/"
+    @link_b   = "http://linkleaf.com/index.php?leaf=212"
+    @image_a  = "http://l.yimg.com/a/i/us/we/52/34.gif"
+    @image_b  = "http://examancer.com/pictures/polished/bolGallery/thumbnail_Good%20Catch%20Missy.jpg"
+  end
+
+  describe "#links" do
+    it "should contain absolute links" do
+      @doc_a.links.member?(@link_a).should be_true
+    end
+
+    it "should contain relative links, converted to absolute format" do
+      @doc_a.links.member?(@link_b).should be_true
+    end
+  end
+
+  describe "#images" do
+    it "should contain images from the test pages" do
+      @doc_a.images.member?(@image_a).should be_true
+      @doc_b.images.member?(@image_b).should be_true
+    end
+  end
+
+  describe "#urls" do
+    it "should contain both links and images" do
+      @doc_a.urls.member?(@link_b).should be_true
+      @doc_a.urls.member?(@image_a).should be_true
+    end
+  end
+end

data/spec/fetcher_spec.rb

@@ -0,0 +1,66 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Seep::Fetcher" do
+  before :all do
+    @link = Seep::Fetcher.open( "http://linkleaf.com/test_a.html" )
+    @jpg  = Seep::Fetcher.open("http://examancer.com/pictures/funny/funny-pictures-ooooooo.jpg")
+    @png  = Seep::Fetcher.open("http://examancer.com/pictures/funny/cnnpoll.png")
+    @gif  = Seep::Fetcher.open("http://examancer.com/pictures/funny/lassie-shreds.gif")
+    @txt  = Seep::Fetcher.open("http://examancer.com/remoteip.txt")
+  end
+
+  describe "#open" do
+    it "populates #size" do
+      @link.size.should == 56_607
+    end
+    it "populates #response_headers" do
+      @link.response_headers["Server"].should match(/Apache/)
+    end
+    it "throws exception when :max_file_size is exceeded" do
+      lambda do
+        Seep::Fetcher.open( @link.url, max_file_size: 25_000 )
+      end.should( raise_exception(Curl::Err::WriteError) )
+    end
+  end
+
+  describe "#size" do
+    it "should be the same as #body.length" do
+      @link.size.should == @link.body.length
+    end
+  end
+
+  describe "#dest_url" do
+    it "should show the url resulting from a redirect" do
+      Seep::Fetcher.open("http://reddit.com").dest_url.should == "http://www.reddit.com/"
+    end
+  end
+
+  describe "#content_type" do
+    it "should be nil before #open is called" do
+      link = Seep::Fetcher.new("http://linkleaf.com/")
+      link.content_type.should be_nil
+    end
+    it "returns the correct type for html" do
+      @link.content_type.should == "text/html"
+    end
+    it "returns the correct type for images" do
+      @jpg.content_type.should == "image/jpeg"
+      @png.content_type.should == "image/png"
+      @gif.content_type.should == "image/gif"
+    end
+    it "returns the correct type for text" do
+      @txt.content_type.should == "text/plain"
+    end
+  end
+
+  describe "#ext" do
+    it "should be blank when #open has not been called" do
+      Seep::Fetcher.new(@link.url)
+    end
+    it "should give proper extensions for images" do
+      @jpg.ext.should == ".jpg"
+      @png.ext.should == ".png"
+      @gif.ext.should == ".gif"
+    end
+  end
+end

data/spec/image_spec.rb

@@ -0,0 +1,35 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "Seep::Image" do
+  before :all do
+    @small = Seep::Image.new( File.read("doc/small.jpg") )
+    @bad = Seep::Image.new("Not an image")
+  end
+
+  describe "#dimensions" do
+    it "should provide the dimensions of an image" do
+      @small.dimensions.should == [320, 240]
+    end
+  end
+
+  describe "#width" do
+    it "should provide the width of an image" do
+      @small.width.should == 320
+    end
+  end
+
+  describe "#height" do
+    it "should provide the height of an image" do
+      @small.height.should == 240
+    end
+  end
+
+  describe "#valid?" do
+    it "should return true if the image is good" do
+      @small.valid?.should be_true
+    end
+    it "should return false if the image is bad" do
+      @bad.valid?.should be_false
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,13 @@
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'rspec'
+require 'seep'
+require 'ruby-debug'
+
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+
+RSpec.configure do |config|
+
+end