gotascii-crags 1.0.2

data/.gitignore

@@ -0,0 +1 @@
+pkg/

data/Manifest.txt

@@ -0,0 +1,17 @@
+.gitignore
+History.txt
+Manifest.txt
+README.txt
+Rakefile
+crags.gemspec
+lib/crags.rb
+lib/crags/fetch.rb
+lib/crags/proxy.rb
+lib/crags/runner.rb
+lib/crags/searcher.rb
+lib/js/client.html
+test/crags/fetch_test.rb
+test/crags/proxy_test.rb
+test/crags/runner_test.rb
+test/crags/searcher_test.rb
+test/test_helper.rb

data/README.txt

@@ -0,0 +1,56 @@
+craigs
+    by Justin Marney
+    http://github.com/gotascii/crags
+
+== DESCRIPTION:
+
+A library to help search across multiple craigslist locations.
+
+== FEATURES/PROBLEMS:
+
+Allows you to search across multiple craigslist locations.  Automatically
+delays multiple searches so craigslist doesn't ban your ip.
+
+== SYNOPSIS:
+
+# searches all available craigslist sites for a keyword in "for sale".
+search(keyword)
+
+# search also takes a category
+search(keyword, 'sss')
+
+There are a bunch of helpful api methods, check the tests for more info.
+
+== REQUIREMENTS:
+
+hpricot
+curb
+
+== INSTALL:
+
+sudo gem install gotascii-crags -s http://gems.github.com
+
+== LICENSE:
+
+(The MIT License)
+
+Copyright (c) 2008 FIXME (different license?)
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+'Software'), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,20 @@
+begin
+  require 'bones'
+  Bones.setup
+rescue LoadError
+  load 'tasks/setup.rb'
+end
+
+ensure_in_path 'lib'
+require 'crags'
+
+task :default => 'test'
+
+PROJ.name = 'crags'
+PROJ.authors = 'Justin Marney'
+PROJ.email = 'justin.marney@viget.com'
+PROJ.url = 'http://github.com/vigetlabs/crags'
+PROJ.version = Crags::VERSION
+PROJ.rubyforge.name = 'crags'
+PROJ.test.files = FileList['test/**/*_test.rb']
+PROJ.spec.opts << '--color'

data/crags.gemspec

@@ -0,0 +1,33 @@
+Gem::Specification.new do |s|
+  s.name = %q{crags}
+  s.version = "1.0.2"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Justin Marney"]
+  s.date = %q{2009-01-30}
+  s.description = %q{A library to help search across multiple craigslist locations.}
+  s.email = %q{gotascii@gmail.com}
+  s.extra_rdoc_files = ["History.txt", "README.txt", "lib/js/client.html"]
+  s.files = [".gitignore", "History.txt", "Manifest.txt", "README.txt", "Rakefile", "crags.gemspec", "lib/crags.rb", "lib/crags/fetch.rb", "lib/crags/proxy.rb", "lib/crags/runner.rb", "lib/crags/searcher.rb", "lib/js/client.html", "tasks/ann.rake", "tasks/bones.rake", "tasks/gem.rake", "tasks/git.rake", "tasks/manifest.rake", "tasks/notes.rake", "tasks/post_load.rake", "tasks/rdoc.rake", "tasks/rubyforge.rake", "tasks/setup.rb", "tasks/spec.rake", "tasks/svn.rake", "tasks/test.rake", "test/crags/fetch_test.rb", "test/crags/proxy_test.rb", "test/crags/runner_test.rb", "test/crags/searcher_test.rb", "test/test_helper.rb"]
+  s.has_rdoc = true
+  s.homepage = %q{http://github.com/gotascii/crags}
+  s.rdoc_options = ["--main", "README.txt"]
+  s.require_paths = ["lib"]
+  s.rubyforge_project = %q{crags}
+  s.rubygems_version = %q{1.3.1}
+  s.summary = %q{A library to help search across multiple craigslist locations}
+  s.test_files = ["test/crags/fetch_test.rb", "test/crags/proxy_test.rb", "test/crags/runner_test.rb", "test/crags/searcher_test.rb"]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 2
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<bones>, [">= 2.1.1"])
+    else
+      s.add_dependency(%q<bones>, [">= 2.1.1"])
+    end
+  else
+    s.add_dependency(%q<bones>, [">= 2.1.1"])
+  end
+end

data/lib/crags/fetch.rb

@@ -0,0 +1,11 @@
+module Crags
+  module Fetch
+    def fetch_doc(url)
+      Hpricot.parse(fetch_html(url))
+    end
+
+    def fetch_html(url)
+      Curl::Easy.perform(url).body_str
+    end
+  end
+end

data/lib/crags/proxy.rb

@@ -0,0 +1,21 @@
+module Crags
+  module Proxy
+    include Fetch
+
+    def lists
+      ["http://www.proxy4free.com/page1.html","http://www.proxy4free.com/page3.html"]
+    end
+
+    def fetch_lists
+      lists.collect {|url| fetch_html(url)}
+    end
+
+    def scan(text)
+      text.scan(/\d+\.\d+\.\d+\.\d+/)
+    end
+
+    def proxies
+      fetch_lists.collect{|html|scan(html)}.flatten.uniq
+    end
+  end
+end

data/lib/crags/runner.rb

@@ -0,0 +1,10 @@
+module Crags
+  class Runner
+    include Searcher
+
+    def search_location(keyword, loc, category = 'sss')
+      puts "Searching #{loc}..."
+      super
+    end
+  end
+end

data/lib/crags/searcher.rb

@@ -0,0 +1,53 @@
+module Crags
+  module Searcher
+    include Fetch
+
+    def location_doc
+      fetch_doc("http://geo.craigslist.org/iso/us")
+    end
+
+    def location_links
+      location_doc.search("#list a")
+    end
+
+    def locations
+      location_links.collect{|link| link["href"] }
+    end
+
+    def categories
+      doc = fetch_doc("http://sfbay.craigslist.org/")
+      links = doc.search("table[@summary=\"for sale\"] a")
+      categories = {}
+      links.each do |link|
+        categories[link.inner_html] = link["href"]
+      end
+      categories
+    end
+
+    def search(keyword, category = 'sss', &block)
+      locations.collect do |loc|
+        sleep(1 + rand(3))
+        search_location(keyword, loc, category, &block)
+      end.flatten
+    end
+
+    def items(doc)
+      doc.search("item")
+    end
+
+    def search_location(keyword, loc, category = 'sss', &block)
+      doc = fetch_doc("#{loc}search/#{category}?query=#{keyword}&format=rss")
+      items(doc).collect do |item|
+        link = create_link(item)
+        yield link if block_given?
+        link
+      end
+    end
+
+    def create_link(item)
+      link = item["rdf:about"]
+      title = item.at("title").inner_text
+      "<a href=\"#{link}\">#{title}</a>"
+    end
+  end
+end

data/lib/crags.rb

@@ -0,0 +1,26 @@
+require 'rubygems'
+require 'curb'
+require 'hpricot'
+
+module Crags
+  VERSION = '1.0.2'
+  LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
+  PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
+
+  def self.version
+    VERSION
+  end
+
+  def self.libpath( *args )
+    args.empty? ? LIBPATH : ::File.join(LIBPATH, *args)
+  end
+
+  def self.path( *args )
+    args.empty? ? PATH : ::File.join(PATH, *args)
+  end
+end
+
+require 'crags/fetch'
+require 'crags/proxy'
+require 'crags/searcher'
+require 'crags/runner'

data/lib/js/client.html

@@ -0,0 +1,81 @@
+<html>
+<head>
+<script language="javascript">
+var IFrameObj; // our IFrame object
+var IFrameDoc;
+function callToServer() {
+  if (!document.createElement) {return true};
+  var URL = 'http://washingtondc.craigslist.org/bik/index.rss';
+  if (!IFrameObj && document.createElement) {
+    // create the IFrame and assign a reference to the
+    // object to our global variable IFrameObj.
+    // this will only happen the first time 
+    // callToServer() is called
+   try {
+      var tempIFrame=document.createElement('iframe');
+      tempIFrame.setAttribute('id','RSIFrame');
+      tempIFrame.style.border='0px';
+      tempIFrame.style.width='0px';
+      tempIFrame.style.height='0px';
+      IFrameObj = document.body.appendChild(tempIFrame);
+      
+      if (document.frames) {
+        // this is for IE5 Mac, because it will only
+        // allow access to the document object
+        // of the IFrame if we access it through
+        // the document.frames array
+        IFrameObj = document.frames['RSIFrame'];
+      }
+    } catch(exception) {
+      // This is for IE5 PC, which does not allow dynamic creation
+      // and manipulation of an iframe object. Instead, we'll fake
+      // it up by creating our own objects.
+      iframeHTML='\<iframe id="RSIFrame" style="';
+      iframeHTML+='border:0px;';
+      iframeHTML+='width:0px;';
+      iframeHTML+='height:0px;';
+      iframeHTML+='"><\/iframe>';
+      document.body.innerHTML+=iframeHTML;
+      IFrameObj = new Object();
+      IFrameObj.document = new Object();
+      IFrameObj.document.location = new Object();
+      IFrameObj.document.location.iframe = document.getElementById('RSIFrame');
+      IFrameObj.document.location.replace = function(location) {
+        this.iframe.src = location;
+      }
+    }
+  }
+  
+  if (navigator.userAgent.indexOf('Gecko') !=-1 && !IFrameObj.contentDocument) {
+    // we have to give NS6 a fraction of a second
+    // to recognize the new IFrame
+    setTimeout('callToServer()',10);
+    return false;
+  }
+  
+  if (IFrameObj.contentDocument) {
+    // For NS6
+    IFrameDoc = IFrameObj.contentDocument; 
+  } else if (IFrameObj.contentWindow) {
+    // For IE5.5 and IE6
+    IFrameDoc = IFrameObj.contentWindow.document;
+  } else if (IFrameObj.document) {
+    // For IE5
+    IFrameDoc = IFrameObj.document;
+  } else {
+    return true;
+  }
+  
+  IFrameDoc.location.replace(URL);
+  return false;
+}
+
+</script>
+</head>
+<body>
+  <script>
+    callToServer();
+    alert(IFrameDoc.innerHTML);
+  </script>
+</body>
+</html>

data/test/crags/fetch_test.rb

@@ -0,0 +1,20 @@
+require '../test_helper'
+
+context "Fetch" do
+  setup do
+    extend Crags::Fetch
+  end
+
+  specify "fetch doc should hpricot fetched html" do
+    stubs(:fetch_html).with("url").returns("html")
+    Hpricot.expects(:parse).with("html").returns("doc")
+    fetch_doc("url").should == "doc"
+  end
+
+  specify "fetch html should curl a url" do
+    curb = stub(:body_str => "uhh")
+    Curl::Easy.expects(:perform).with("url").returns(curb)
+    fetch_html("url").should == "uhh"
+  end
+end
+

data/test/crags/proxy_test.rb

@@ -0,0 +1,24 @@
+require '../test_helper'
+
+context "Proxy" do
+  setup do
+    extend Crags::Proxy
+  end
+
+  specify "lists should return a list of proxy list websites" do
+    lists.should == ["http://www.proxy4free.com/page1.html"]
+  end
+
+  specify "fetch lists should fetch html for each site in lists" do
+    stubs(:lists).returns(["1", "2"])
+    expects(:fetch_html).with("1").returns("html_1")
+    expects(:fetch_html).with("2").returns("html_2")
+    fetch_lists.should == ["html_1", "html_2"]
+  end
+  
+  specify "scan should return all ips in a text blizoc" do
+    text = "192.168.1.2 omg dude!! wtf.f.f.asdasd9.8.9 78.900.42.32"
+    scan(text).should == ["192.168.1.2", "78.900.42.32"]
+  end
+end
+

data/test/crags/runner_test.rb

@@ -0,0 +1,26 @@
+require '../test_helper'
+
+context "Runner" do
+  setup do
+    @runner = Crags::Runner.new
+    @runner.stubs(:fetch_doc)
+    @runner.stubs(:items).returns([])
+  end
+
+  specify "runner should include searcher" do
+    Crags::Runner.ancestors.should.include Crags::Searcher
+  end
+  
+  specify "search location should puts message with loc" do
+    @runner.expects(:puts).with { |val| val =~ /location/ }
+    @runner.search_location("", "location", "category")
+  end
+  
+  specify "search location should take a category" do
+    @runner.search_location("", "location", "category")
+  end
+
+  specify "search location should have default category sss" do
+    @runner.search_location("", "location")
+  end
+end

data/test/crags/searcher_test.rb

@@ -0,0 +1,123 @@
+require '../test_helper'
+
+context "Searcher with stubbed fetch doc" do
+  setup do
+    extend Crags::Searcher
+    stubs(:sleep)
+    stubs(:fetch_doc)
+  end
+
+  specify "location doc should fetch doc at location url" do
+    expects(:fetch_doc).with("http://geo.craigslist.org/iso/us").returns("doc")
+    location_doc.should == "doc"
+  end
+
+  specify "location links should get all a tags from div with id list" do
+    doc = mock { expects(:search).with("#list a").returns("links") }
+    stubs(:location_doc).returns(doc)
+    location_links.should == "links"
+  end
+
+  specify "locations should return array of urls using a location link's href" do
+    links = []
+    2.times do |i|
+      links << mock {|m| m.expects(:[]).with("href").returns("url#{i}") }
+    end
+    stubs(:location_links).returns(links)
+    locations.should == ["url0", "url1"]
+  end
+
+  specify "search should search location for each location with keyword and return list" do
+    locations = ["url0", "url1"]
+
+    locations.each do |loc|
+      expects(:search_location).with("omg", loc, 'sss').returns(["1#{loc}", "2#{loc}"])
+    end
+
+    stubs(:locations).returns(locations)
+    search("omg").should == ["1url0", "2url0", "1url1", "2url1"]
+  end
+
+  specify "search should call sleep for each location" do
+    expects(:sleep).times(2)
+    stubs(:locations).returns([1,2])
+    stubs(:search_location)
+    search("")
+  end
+
+  specify "search location should fetch doc for search url" do
+    expects(:fetch_doc).with("urlsearch/sss?query=keyword&format=rss")
+    stubs(:items).returns([])
+    search_location("keyword", "url")
+  end
+
+  specify "search location should create link with each item in doc items and return list" do
+    items = [1,2,3]
+    expects(:items).returns(items)
+    items.each do |i|
+      expects(:create_link).with(i).returns("omg#{i}")
+    end
+    search_location("keyword", "url").should == ['omg1','omg2','omg3']
+  end
+
+  specify "create link should return an a href based on item element" do
+    inner_text = mock(:inner_text=>"text")
+    item = mock do |l|
+      expects(:[]).with("rdf:about").returns("link")
+      expects(:at).with("title").returns(inner_text)
+    end
+
+    create_link(item).should == "<a href=\"link\">text</a>"
+  end
+
+  specify "items should get all item elements from doc" do
+    doc = mock { expects(:search).with("item").returns(1) }
+    items(doc).should == 1
+  end
+
+  specify "categories should fetch doc the main sfbay page" do
+    doc = stub(:search => [])
+    expects(:fetch_doc).with("http://sfbay.craigslist.org/").returns(doc)
+    categories
+  end
+
+  specify "categories should search for all links in the table with property summary equal to for sale" do
+    doc = mock { expects(:search).with("table[@summary=\"for sale\"] a").returns([]) }
+    stubs(:fetch_doc).returns(doc)
+    categories
+  end
+
+  specify "categories should return a hash with link inner html keys and link href values" do
+    link = stub(:inner_html => "inner_html") do
+      stubs(:[]).with("href").returns("href")
+    end
+
+    doc = stub(:search => [link, link])
+    stubs(:fetch_doc).returns(doc)
+    categories.should == {'inner_html' => 'href', 'inner_html' => 'href'}
+  end
+
+  specify "search location should accept a category parameter" do
+    expects(:fetch_doc).with("locsearch/scram?query=keyword&format=rss")
+    stubs(:items).returns([])
+    search_location('keyword', 'loc', 'scram')
+  end
+
+  specify "search location default category is sss" do
+    expects(:fetch_doc).with("locsearch/sss?query=keyword&format=rss")
+    stubs(:items).returns([])
+    search_location('keyword', 'loc')
+  end
+
+  specify "search should pass parameter to search location" do
+    stubs(:locations).returns([0])
+    expects(:search_location).with('keyword', 0, 'chum')
+    search('keyword', 'chum')
+  end
+  
+  specify "search should have default category of sss" do
+    stubs(:locations).returns([0])
+    expects(:search_location).with('keyword', 0, 'sss')
+    search('keyword')
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,4 @@
+require 'rubygems'
+require 'test/spec'
+require 'mocha'
+require 'crags'

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification 
+name: gotascii-crags
+version: !ruby/object:Gem::Version 
+  version: 1.0.2
+platform: ruby
+authors: 
+- Justin Marney
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-01-30 00:00:00 -08:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: bones
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 2.1.1
+    version: 
+description: A library to help search across multiple craigslist locations.
+email: gotascii@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- History.txt
+- README.txt
+- lib/js/client.html
+files: 
+- .gitignore
+- History.txt
+- Manifest.txt
+- README.txt
+- Rakefile
+- crags.gemspec
+- lib/crags.rb
+- lib/crags/fetch.rb
+- lib/crags/proxy.rb
+- lib/crags/runner.rb
+- lib/crags/searcher.rb
+- lib/js/client.html
+- tasks/ann.rake
+- tasks/bones.rake
+- tasks/gem.rake
+- tasks/git.rake
+- tasks/manifest.rake
+- tasks/notes.rake
+- tasks/post_load.rake
+- tasks/rdoc.rake
+- tasks/rubyforge.rake
+- tasks/setup.rb
+- tasks/spec.rake
+- tasks/svn.rake
+- tasks/test.rake
+- test/crags/fetch_test.rb
+- test/crags/proxy_test.rb
+- test/crags/runner_test.rb
+- test/crags/searcher_test.rb
+- test/test_helper.rb
+has_rdoc: true
+homepage: http://github.com/gotascii/crags
+post_install_message: 
+rdoc_options: 
+- --main
+- README.txt
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: crags
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: A library to help search across multiple craigslist locations
+test_files: 
+- test/crags/fetch_test.rb
+- test/crags/proxy_test.rb
+- test/crags/runner_test.rb
+- test/crags/searcher_test.rb