seep 0.0.2

data/spider.rb

@@ -0,0 +1,226 @@
+require 'rubygems'
+require 'open-uri'
+require 'nokogiri'
+require 'fileutils'
+require 'pp'
+require 'singleton'
+require 'net/http'
+
+def grab_data_from_uri (link)
+  #uri = link.uri
+  headers = {
+    'User-Agent' => 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.2 (KHTML, like Gecko) Chrome/15.0.874.121 Safari/535.2'
+  }
+  headers['Referer'] = link.prev_uri.to_s if not link.prev_uri.nil?
+  req_path = link.uri.path
+  req_path = req_path + "?#{link.uri.query}" if not link.uri.query.nil?
+  req_path = "/" if req_path == "" or req_path.nil?
+  begin
+    res = Net::HTTP.start(link.uri.host) { |http|
+      http.get(req_path, headers)
+    }
+  rescue Net::HTTPExceptions => e
+    p "something is going wrong"
+    pp e
+    return nil
+  rescue Errno::ETIMEDOUT
+    p "timeout error"
+    return nil
+  rescue Errno::ECONNRESET
+    p "connection reset"
+    return nil
+  rescue EOFError
+    p "end of file error"
+    return nil
+  rescue NoMethodError
+    p "weird error"
+    return nil
+  rescue SystemExit
+    exit
+  rescue
+    p "some other error"
+    return nil
+  end
+  #pp res
+  return res.body if (res.is_a? Net::HTTPSuccess)
+  if (res.is_a? Net::HTTPRedirection)
+    p "Redirection detected."
+    if ( not res.header['location'].nil? )
+      begin
+        new_link = SpiderLink.new(URI::join(link.uri.to_s,res.header['location']),link.uri,link.depth)
+      rescue
+        return nil
+      end
+      p "Redirected to: #{new_link.uri.to_s}"
+      SpiderConfig.instance.queued_links << new_link
+    end
+  end
+  return nil
+end
+
+def open_link (link)
+  grab_data_from_uri(link)
+end
+
+#url = "http://galleries.nextdoor-models.com/static/clean/149.html?ccbill_id=1757428&site_link=http://www.nextdoor-models.com/"
+#p open_link(url)
+
+def get_links(uri,html)
+  return [],[] if html.nil?
+  #begin
+    doc = Nokogiri::HTML.parse(html)
+  #rescue NoMethodError #capture parsing errors
+  #  return [],[]
+  #end
+  images = []; links = []
+  doc.search("a").each do |e|
+    link = e.get_attribute("href")
+    #p link
+    begin
+      new_uri = URI::join(uri.to_s,link) if not link.nil?
+    rescue
+      p "Bad URI. Skipping. #{link}"
+      next
+    end
+    if (new_uri.to_s =~ /jpe?g$/i) then
+      images << new_uri
+    else
+      links << new_uri
+    end
+  end
+  return links.uniq, images.uniq
+end
+
+#url = "http://galleries.nextdoor-models.com/static/clean/149.html?ccbill_id=1757428&site_link=http://www.nextdoor-models.com/"
+#html = open_link(url)
+#links, images = get_links(url,html)
+#p "links: "
+#pp links
+#p "images: "
+#pp images
+
+$base = "/mnt/media/stuff/porn/"
+def save_image(uri,name,parent)
+  dir = $base + uri.host
+  path = "#{dir}/#{name}.jpg"
+  FileUtils.mkdir_p(dir)
+  res = grab_data_from_uri(SpiderLink.new(uri,parent,0))
+  p "Saving #{uri.to_s} to #{path}"
+  File.open(path,"w").write(res)
+end
+
+def random_string(length=10)
+  chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
+  password = ''
+  length.times { password << chars[rand(chars.size)] }
+  password
+end
+
+#url = "http://galleries.nextdoor-models.com/content/149/thm/01.jpg"
+#uri = URI::parse(url)
+#save_image(uri,1001)
+
+#url = "http://galleries.nextdoor-models.com/static/clean/149.html?ccbill_id=1757428&site_link=http://www.nextdoor-models.com/"
+#html = open_link(url)
+#links, images = get_links(url,html)
+#batch_id = 1
+#image_count = 0
+#images.each do |img|
+#  image_count += 1
+#  save_image(img,"%08s.%08d.jpg" % [ batch_id, image_count ] )
+#end
+
+def proc_url(link)
+  html = open_link(link)
+  #p html
+  links, images = get_links(link.uri,html)
+  batch = Time.now.strftime("%Y%m%d.%H%M.%S.") + random_string(4)
+  image_count = 0
+  images.each do |img|
+    image_count += 1
+    save_image(img,"#{batch}.%04d" % image_count, link.uri )
+  end
+  links
+end
+
+class SpiderLink
+  attr_accessor :uri, :prev_uri, :depth
+  def initialize ( uri, prev_uri, depth )
+    @uri = uri
+    @prev_uri = prev_uri
+    @depth = depth
+  end
+  def == (other)
+    ret = @uri.to_s == other.to_s
+    p "Compare #{@uri.to_s} to #{other.uri.to_s}: #{ret}" if ret
+    return @uri.to_s == other.uri.to_s
+  end
+end
+
+class SpiderConfig
+  include Singleton
+  attr_accessor :max_depth, :root_hosts, :queued_links, :seen_links, :max_queue_size
+  def initialize
+    @max_depth = 5
+    @max_queue_size = 100000
+    @root_hosts = []
+    @queued_links = SpiderQueue.new
+    @seen_links = []
+  end
+end
+
+class SpiderQueue < Hash
+  def initialize
+    super
+    @ordered_queue = []
+  end
+
+  def << (link)
+    if self.has_key?(link.uri)
+      @ordered_queue.delete(link.uri)
+    else
+      self[link.uri] = link
+    end
+    @ordered_queue << link.uri
+  end
+
+  def shift
+    self.delete(@ordered_queue.shift)
+  end
+end
+
+def link_loop(start_url)
+  c = SpiderConfig.instance
+
+  start_uri = URI::parse(start_url)
+  c.queued_links << SpiderLink.new(start_uri,nil,0)
+  p c.queued_links
+  c.root_hosts << start_uri.host
+
+  loop do
+    link = c.queued_links.shift
+    if link.nil? then
+      p "Queue is empty. Quitting"
+      break
+    end
+    #if c.seen_links.include?(link.uri) then
+    #  p "Already seen #{link.uri.to_s}. Skipping."
+    #  next
+    #end
+    #c.seen_links << link.uri
+    new_links = proc_url(link) - (c.seen_links - c.queued_links.keys)
+    p "Q:#{c.queued_links.size} Seen:#{c.seen_links.size} #{link.uri.to_s} - #{new_links.size}"
+    new_links.each do |found_link|
+      next if found_link.nil?
+      #p "Found: #{found_link.to_s}"
+      depth = c.root_hosts.include?(found_link.host) ? link.depth : link.depth + 1
+      unless (depth > c.max_depth or c.queued_links.size > c.max_queue_size)
+        #q_link = SpiderLink.new(found_link,link.uri,depth)
+        c.queued_links << SpiderLink.new(found_link,link.uri,depth)
+        c.seen_links << found_link
+        #p "Added #{found_link.to_s} to queue"
+      end
+    end
+  end
+end
+

data/spidr_test.rb

@@ -0,0 +1,11 @@
+require 'rubygems'
+require 'spidr'
+
+p "Spider test!"
+
+Spidr.site('http://www.definebabe.com/gallery/kme/jana-cova/') do |spider|
+  spider.every_url { |url| puts "url: #{url}"; sleep 0.2 }
+  spider.every_link { |origin,link| puts "link: #{link} (from #{origin})"; sleep 0.2 }
+  spider.every_failed_url { |url| puts "failed: #{url}"; sleep 0.2 }
+end
+

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: seep
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+  prerelease: 
+platform: ruby
+authors:
+- Carl Zulauf
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2011-12-27 00:00:00.000000000Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis-native_hash
+  requirement: &12752080 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *12752080
+- !ruby/object:Gem::Dependency
+  name: gd2-ffij
+  requirement: &12751600 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *12751600
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: &12751060 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *12751060
+- !ruby/object:Gem::Dependency
+  name: nokogiri
+  requirement: &12750520 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *12750520
+- !ruby/object:Gem::Dependency
+  name: ruby-debug19
+  requirement: &12750040 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *12750040
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &12749500 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.3.0
+  type: :development
+  prerelease: false
+  version_requirements: *12749500
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: &12749000 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *12749000
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: &12611560 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+  type: :development
+  prerelease: false
+  version_requirements: *12611560
+- !ruby/object:Gem::Dependency
+  name: rcov
+  requirement: &12610980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *12610980
+description: Collection of web spidering and downloading tools using redis, curl,
+  and gd.
+email: carl@linkleaf.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README
+files:
+- .document
+- .rspec
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README
+- Rakefile
+- doc/small.jpg
+- doc/test_a.html
+- doc/test_b.html
+- lib/seep.rb
+- lib/seep/doc.rb
+- lib/seep/fetcher.rb
+- lib/seep/image.rb
+- seep.gemspec
+- spec/a_spec.rb
+- spec/doc_spec.rb
+- spec/fetcher_spec.rb
+- spec/image_spec.rb
+- spec/spec_helper.rb
+- spider.rb
+- spidr_test.rb
+homepage: http://github.com/carlzulauf/seep
+licenses:
+- MIT
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1561364847905091588
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.10
+signing_key: 
+specification_version: 3
+summary: web spidering/downloading tools
+test_files: []