url-vi0lence 0.0.1

data/README

@@ -0,0 +1 @@
+This is for ripping shit off.

data/bin/url-vi0lence

@@ -0,0 +1,2 @@
+#! /usr/bin/env ruby
+puts "wee"

data/lib/url-vi0lence.rb

@@ -0,0 +1,24 @@
+require "rubygems"
+require "mechanize"
+require "fileutils"
+require "uri"
+
+begin
+  require "ruby-debug"
+  Debugger.start
+  Debugger.settings[:autoeval] = true if Debugger.respond_to?(:settings)
+rescue LoadError
+  puts "You need to install ruby-debug to run the server in debugging mode. With gems, use 'gem install ruby-debug'"
+  exit
+end
+
+Dir.glob("./lib/url_vi0lence/**/*.rb").each do |file|
+  require file
+end
+
+activate_samples = {
+  "sg" => false,
+  "gmail" => false
+}.each do |file,do_start|
+  require "./scrapers/#{file}_scraper.rb" if do_start
+end

data/lib/url_vi0lence/page_scraper.rb

@@ -0,0 +1,152 @@
+class PageScraper
+
+=begin
+  # initialize
+  # 
+  # parameters
+  # === user_agent    <String>        User agent alias (Google it.) (Default: Mac Firefox)
+  # === redirect      <Boolean>       Should redirects be followed  (Default: true)
+  # === use_cookies   <Boolean>       Should cookies be used        (Default: true)
+  # === cookie_jar    <String>        Path to YAML cookie jar       (Default: ./dump/cookies.yml)
+=end
+  def initialize(config={})
+    config = {
+      :user_agent => "Mac FireFox",
+      :redirect   => true,
+      :use_cookies=> true,
+      :cookie_jar => "#{PageScraper.dump}/cookies.yml"
+    }.merge(config)
+    
+    @agent = WWW::Mechanize.new
+    @agent.user_agent_alias = config[:user_agent]
+    @agent.redirect_ok = config[:redirect]
+    
+    if config[:use_cookies]
+      @agent.cookie_jar.save_as(config[:cookie_jar])
+    end
+  end
+  
+=begin
+  # login
+  # 
+  # parameters
+  # = username    <String>        Username
+  # = password    <String>        Password
+  # = url         <String>        Login page URL
+  # = *args       <Array>
+  # === un_field  <String>        Username HTML field name (Default: username)
+  # === pw_field  <String>        Password HTML field name (Default: password)
+  # === form_id   <String|Fixnum> Form HTML name or the sequential number of the form in the page
+  # = block       <Proc>          Additional actions to do after login
+  # =>                            Block should return Page or nil
+=end
+  def login(username,password,url,*args,&block)
+    un_field = args[0] || "username"
+    pw_field = args[1] || "password"
+    form_id  = args[2] || 0           #First form
+    
+    current_page(@agent.get(url))
+    
+    if form_id.is_a? Fixnum
+      form = @page.forms[form_id]
+    else #form_id.is_a? String
+      form = @page.forms.find{|f| f.name == form_id.to_s}
+    end
+    
+    form.fields.find {|f| f.name == un_field}.value = username
+    form.fields.find {|f| f.name == pw_field}.value = password
+        
+    current_page(@agent.submit(form))
+    
+    current_page(block.call({
+      :username => username,
+      :password => password,
+      :un_field => un_field,
+      :pw_field => pw_field,
+      :form_id  => form_id,
+      :agent    => @agent,
+      :page     => @page
+    })) if block_given?
+    
+    return current_page
+  rescue Exception => e
+    puts "There was an exception during #login:\n#{e}"
+    Kernel.exit
+  end
+  
+=begin
+  # current_page  - Current page Mechanize is on
+  #
+  # parameters
+  # new_page  <Mechanize::Page> Page to set current page to (Default nil)
+=end
+  def current_page(new_page=nil)
+    @page = new_page unless new_page.nil?
+    @page
+  end
+  
+=begin
+  # parse - Executes a block against the site
+  #
+  # parameters
+  # = pages   <Array>     List of urls to execute the block against (Default = current_page)
+  # = block   <Proc>      Action that should be performed
+  # =>                    Access will be available to agent and page
+  # =>                    Block should return Page || nil
+=end
+  def parse(pages=nil,&block)
+    pages = [current_page] if pages.nil?
+    
+    #Turn strings into Mechanize::Page
+    pages.collect! do |page|
+      if page.is_a? String
+        page = @agent.get(page)
+      end
+    end
+    debugger
+    pages.collect {|page|
+      current_page(block.call(page,@agent))
+    }
+  end
+  
+=begin
+  # filter_links - Filters links based on a regex match
+  #
+  # parameters
+  # = match_pattern <Regex>       Pattern of links to keep
+  #TODO, Im sure there is a cleaner implementation
+=end
+  def filter_links(match_pattern)
+    links = []
+    current_page.links.each do |link|
+      if link.uri.to_s =~ match_pattern
+        links.push(link)
+      end
+    end
+    
+    @links.uniq!
+  rescue Exception => e
+    #Nothing!
+  ensure
+    links || []
+  end
+  
+  class << self
+    def dump
+      @@dump = "./dump"
+      FileUtils.mkdir_p(@@dump)
+      @@dump
+    end
+    
+    private
+    def children
+      @@children ||= []
+    end
+  
+    def inherited(klass)
+      super
+    ensure
+      children << klass
+    end
+  end
+end

data/scrapers/gmail_scraper.rb

@@ -0,0 +1,32 @@
+class GmailScraper < PageScraper
+  NewInboxMessages = Proc.new {|page, agent|
+    page.search("//tr[@bgcolor='#ffffff']")  do |row|
+      from, subject = *row.search("//b/text()")
+      url = page.uri.to_s.sub(/ui.*$/, row.search("//a").first.attributes["href"])
+      puts "From: #{from}\nSubject: #{subject}\nLink: #{url}\n\n"
+    end
+  }
+end
+
+gmail = GmailScraper.new
+
+gmail_u = ""
+gmail_p = ""
+
+gmail.login gmail_u,gmail_p,"http://www.gmail.com","Email","Passwd" do |info|
+  #This tells Gmail that we want to use the basic, no-js version
+  info[:agent].get info[:page].uri.to_s.sub(/\?.*$/, "?ui=html&zy=n")
+end
+
+
+#you could pass it a block or a proc
+
+#gmail.parse do |page, agent|
+#  page.search("//tr[@bgcolor='#ffffff']")  do |row|
+#    from, subject = *row.search("//b/text()")
+#    url = page.uri.to_s.sub(/ui.*$/, row.search("//a").first.attributes["href"])
+#    puts "From: #{from}\nSubject: #{subject}\nLink: #{url}\n\n"
+#  end
+#end
+
+gmail.parse &GmailScraper::NewInboxMessages

data/scrapers/sg_scraper.rb

@@ -0,0 +1,53 @@
+class SgScraper < PageScraper
+  ImageGanker = lambda { |page, agent|
+    begin
+      hrefs = page.links.map { |m| 
+        m.href
+      }.select { |u| 
+        u =~ /[0-9]+.jpg/ 
+      } #just jpgs with numbers in the name
+
+      create_folder = true
+
+      hrefs.each { |image|
+        image_name = image.split("/")
+        
+        #Folder name = GirlName/PhotoSet
+        folder = "#{SgScraper.dump}/sg/#{image_name[-4]}/#{image_name[-2]}"
+        
+        if create_folder
+          FileUtils.mkdir_p(folder)
+        end
+        
+        filename = "#{folder}/#{image_name[-1]}"
+        puts "Saving #{image} as #{filename}"
+        agent.get(image).save_as(filename)
+        
+        create_folder = false
+      }
+    rescue Exception => e
+      puts e
+      puts 'Failed to get a file boo hoo'
+    end
+  }
+  
+end
+
+
+sg = SgScraper.new
+
+sg_u = ""
+sg_p = ""
+
+sg.login sg_u,sg_p,"http://www.suicidegirls.com" do |info|
+  #Go to a particular page after login
+  info[:agent].get "http://suicidegirls.com/girls/Ren/photos/Mars+Attacks/"
+end
+
+urls=[
+  "http://suicidegirls.com/girls/Ren/photos/Forest+Tea/",
+  "http://suicidegirls.com/girls/Gatsby/photos/Trinkets/",
+  "http://suicidegirls.com/members/Gatsby/albums/site/3578/"
+]
+sg.parse(urls, &SgScraper::ImageGanker)
+

metadata

@@ -0,0 +1,59 @@
+--- !ruby/object:Gem::Specification 
+name: url-vi0lence
+version: !ruby/object:Gem::Version 
+  version: 0.0.1
+platform: ruby
+authors: 
+- Cory ODaniel
+autorequire: url_vi0lence
+bindir: bin
+cert_chain: []
+
+date: 2008-04-29 00:00:00 -07:00
+default_executable: 
+dependencies: []
+
+description: 
+email: urlviolence@coryodaniel.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README
+files: 
+- bin/url-vi0lence
+- lib/url-vi0lence.rb
+- lib/url_vi0lence
+- lib/url_vi0lence/page_scraper.rb
+- scrapers/gmail_scraper.rb
+- scrapers/sg_scraper.rb
+- README
+has_rdoc: true
+homepage: url-vi0lence.rubyforge.com
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.0.1
+signing_key: 
+specification_version: 2
+summary: Ripping off other peoples hard work, or automating testing whatev.  Built on Mechanize
+test_files: []
+