gotascii-crags 1.2.7 → 1.4.2

data/Manifest.txt

@@ -9,7 +9,6 @@ lib/crags/fetch.rb
 lib/crags/proxy.rb
 lib/crags/runner.rb
 lib/crags/searcher.rb
-lib/js/client.html
 test/crags/fetch_test.rb
 test/crags/proxy_test.rb
 test/crags/runner_test.rb

data/crags.gemspec

@@ -1,20 +1,22 @@
+# -*- encoding: utf-8 -*-
+
 Gem::Specification.new do |s|
   s.name = %q{crags}
-  s.version = "1.2.7"
+  s.version = "1.4.2"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Justin Marney"]
-  s.date = %q{2009-02-05}
+  s.date = %q{2009-06-16}
   s.description = %q{A library to help search across multiple craigslist locations.}
-  s.email = %q{gotascii@gmail.com}
-  s.extra_rdoc_files = ["History.txt", "README.txt", "lib/js/client.html"]
-  s.files = [".gitignore", "History.txt", "Manifest.txt", "README.txt", "Rakefile", "crags.gemspec", "lib/crags.rb", "lib/crags/fetch.rb", "lib/crags/proxy.rb", "lib/crags/runner.rb", "lib/crags/searcher.rb", "lib/js/client.html", "tasks/ann.rake", "tasks/bones.rake", "tasks/gem.rake", "tasks/git.rake", "tasks/manifest.rake", "tasks/notes.rake", "tasks/post_load.rake", "tasks/rdoc.rake", "tasks/rubyforge.rake", "tasks/setup.rb", "tasks/spec.rake", "tasks/svn.rake", "tasks/test.rake", "test/crags/fetch_test.rb", "test/crags/proxy_test.rb", "test/crags/runner_test.rb", "test/crags/searcher_test.rb", "test/test_helper.rb"]
+  s.email = %q{justin.marney@viget.com}
+  s.extra_rdoc_files = ["History.txt", "README.txt"]
+  s.files = [".gitignore", "History.txt", "Manifest.txt", "README.txt", "Rakefile", "crags.gemspec", "lib/crags.rb", "lib/crags/fetch.rb", "lib/crags/proxy.rb", "lib/crags/runner.rb", "lib/crags/searcher.rb", "test/crags/fetch_test.rb", "test/crags/proxy_test.rb", "test/crags/runner_test.rb", "test/crags/searcher_test.rb", "test/test_helper.rb"]
   s.has_rdoc = true
-  s.homepage = %q{http://github.com/gotascii/crags}
+  s.homepage = %q{http://github.com/vigetlabs/crags}
   s.rdoc_options = ["--main", "README.txt"]
   s.require_paths = ["lib"]
   s.rubyforge_project = %q{crags}
   s.rubygems_version = %q{1.3.1}
   s.summary = %q{A library to help search across multiple craigslist locations}
   s.test_files = ["test/crags/fetch_test.rb", "test/crags/proxy_test.rb", "test/crags/runner_test.rb", "test/crags/searcher_test.rb"]
-end
+end

data/lib/crags/searcher.rb

@@ -7,16 +7,16 @@ module Crags
       url.gsub(/http\:\/\/(.*)(\/|(.html))/,'\1\3')
     end
 
-    def location_doc
-      fetch_doc("http://geo.craigslist.org/iso/us")
+    def location_doc(country)
+      fetch_doc("http://geo.craigslist.org/iso/#{country}")
     end
 
-    def location_links
-      location_doc.search("#list a")
+    def location_links(country)
+      location_doc(country).search("#list a")
     end
 
-    def locations
-      location_links.collect{|link| strip_http(link["href"]) }
+    def locations(country)
+      location_links(country).collect{|link| strip_http(link["href"]) }
     end
 
     def categories
@@ -29,8 +29,8 @@ module Crags
       categories
     end
 
-    def search(keyword, category = 'sss', &block)
-      locations.collect do |loc|
+    def search(keyword, country = 'us', category = 'sss', &block)
+      locations(country).collect do |loc|
         sleep(1 + rand(3))
         search_location(keyword, loc, category, &block)
       end.flatten

data/lib/crags.rb

@@ -1,10 +1,10 @@
 require 'rubygems'
 require 'curb'
 require 'hpricot'
-require "erb"
+require 'erb'
 
 module Crags
-  VERSION = '1.2.7'
+  VERSION = '1.4.2'
   LIBPATH = ::File.expand_path(::File.dirname(__FILE__)) + ::File::SEPARATOR
   PATH = ::File.dirname(LIBPATH) + ::File::SEPARATOR
 
@@ -19,6 +19,36 @@ module Crags
   def self.path( *args )
     args.empty? ? PATH : ::File.join(PATH, *args)
   end
+
+  COUNTRIES = [
+    'jp',
+    'ar',
+    'bd',
+    'br',
+    'ca',
+    'cl',
+    'co',
+    'cr',
+    'cz',
+    'eg',
+    'hu',
+    'id',
+    'ie',
+    'il',
+    'lb',
+    'my',
+    'nl',
+    'nz',
+    'no',
+    'pk',
+    'pa',
+    'ru',
+    'th',
+    'ae',
+    'us',
+    've',
+    'vn'
+  ]
 end
 
 require 'crags/fetch'

data/test/crags/fetch_test.rb

@@ -1,20 +1,21 @@
-require '../test_helper'
+require File.dirname(__FILE__) + '/../test_helper'
 
-context "Fetch" do
-  setup do
-    extend Crags::Fetch
-  end
+class Crags::FetchTest < Test::Unit::TestCase
+  context "Fetch" do
+    setup do
+      extend Crags::Fetch
+    end
 
-  specify "fetch doc should hpricot fetched html" do
-    stubs(:fetch_html).with("url").returns("html")
-    Hpricot.expects(:parse).with("html").returns("doc")
-    fetch_doc("url").should == "doc"
-  end
+    should "fetch doc should hpricot fetched html" do
+      stubs(:fetch_html).with("url").returns("html")
+      Hpricot.expects(:parse).with("html").returns("doc")
+      fetch_doc("url").should == "doc"
+    end
 
-  specify "fetch html should curl a url" do
-    curb = stub(:body_str => "uhh")
-    Curl::Easy.expects(:perform).with("url").returns(curb)
-    fetch_html("url").should == "uhh"
+    should "fetch html should curl a url" do
+      curb = stub(:body_str => "uhh")
+      Curl::Easy.expects(:perform).with("url").returns(curb)
+      fetch_html("url").should == "uhh"
+    end
   end
-end
-
+end

data/test/crags/proxy_test.rb

@@ -1,24 +1,25 @@
-require '../test_helper'
+require File.dirname(__FILE__) + '/../test_helper'
 
-context "Proxy" do
-  setup do
-    extend Crags::Proxy
-  end
+class Crags::ProxyTest < Test::Unit::TestCase
+  context "Proxy" do
+    setup do
+      extend Crags::Proxy
+    end
 
-  specify "lists should return a list of proxy list websites" do
-    lists.should == ["http://www.proxy4free.com/page1.html"]
-  end
+    should "lists should return a list of proxy list websites" do
+      lists.should == ["http://www.proxy4free.com/page1.html", "http://www.proxy4free.com/page3.html"]
+    end
 
-  specify "fetch lists should fetch html for each site in lists" do
-    stubs(:lists).returns(["1", "2"])
-    expects(:fetch_html).with("1").returns("html_1")
-    expects(:fetch_html).with("2").returns("html_2")
-    fetch_lists.should == ["html_1", "html_2"]
-  end
-  
-  specify "scan should return all ips in a text blizoc" do
-    text = "192.168.1.2 omg dude!! wtf.f.f.asdasd9.8.9 78.900.42.32"
-    scan(text).should == ["192.168.1.2", "78.900.42.32"]
-  end
-end
+    should "fetch lists should fetch html for each site in lists" do
+      stubs(:lists).returns(["1", "2"])
+      expects(:fetch_html).with("1").returns("html_1")
+      expects(:fetch_html).with("2").returns("html_2")
+      fetch_lists.should == ["html_1", "html_2"]
+    end
 
+    should "scan should return all ips in a text blizoc" do
+      text = "192.168.1.2 omg dude!! wtf.f.f.asdasd9.8.9 78.900.42.32"
+      scan(text).should == ["192.168.1.2", "78.900.42.32"]
+    end
+  end
+end

data/test/crags/runner_test.rb

@@ -1,26 +1,28 @@
-require '../test_helper'
+require File.dirname(__FILE__) + '/../test_helper'
 
-context "Runner" do
-  setup do
-    @runner = Crags::Runner.new
-    @runner.stubs(:fetch_doc)
-    @runner.stubs(:items).returns([])
-  end
+class Crags::RunnerTest < Test::Unit::TestCase
+  context "instance of Runner" do
+    setup do
+      @runner = Crags::Runner.new
+      @runner.stubs(:fetch_doc)
+      @runner.stubs(:items).returns([])
+    end
 
-  specify "runner should include searcher" do
-    Crags::Runner.ancestors.should.include Crags::Searcher
-  end
-  
-  specify "search location should puts message with loc" do
-    @runner.expects(:puts).with { |val| val =~ /location/ }
-    @runner.search_location("", "location", "category")
-  end
-  
-  specify "search location should take a category" do
-    @runner.search_location("", "location", "category")
-  end
+    should "runner should include searcher" do
+      Crags::Runner.ancestors.include?(Crags::Searcher).should == true
+    end
+
+    should "search location should puts message with loc" do
+      @runner.expects(:puts).with { |val| val =~ /location/ }
+      @runner.search_location("", "location", "category")
+    end
+
+    should "search location should take a category" do
+      @runner.search_location("", "location", "category")
+    end
 
-  specify "search location should have default category sss" do
-    @runner.search_location("", "location")
+    should "search location should have default category sss" do
+      @runner.search_location("", "location")
+    end
   end
 end

data/test/crags/searcher_test.rb

@@ -1,124 +1,127 @@
-require '../test_helper'
+require File.dirname(__FILE__) + '/../test_helper'
 
-context "Searcher with stubbed fetch doc" do
-  setup do
-    extend Crags::Searcher
-    stubs(:sleep)
-    stubs(:fetch_doc)
-  end
+class Crags::SearcherTest < Test::Unit::TestCase
 
-  specify "strip_http should remove http:// and trailing /" do
-    url = "http://omg/"
-    strip_http(url).should == "omg"
-  end
+  context "Searcher with stubbed fetch doc" do
+    setup do
+      extend Crags::Searcher
+      stubs(:sleep)
+      stubs(:fetch_doc)
+    end
 
-  specify "location doc should fetch doc at location url" do
-    expects(:fetch_doc).with("http://geo.craigslist.org/iso/us").returns("doc")
-    location_doc.should == "doc"
-  end
+    should "strip_http should remove http:// and trailing /" do
+      url = "http://omg/"
+      strip_http(url).should == "omg"
+    end
 
-  specify "location links should get all a tags from div with id list" do
-    doc = mock { expects(:search).with("#list a").returns("links") }
-    stubs(:location_doc).returns(doc)
-    location_links.should == "links"
-  end
+    should "location doc should fetch doc at location url" do
+      expects(:fetch_doc).with("http://geo.craigslist.org/iso/us").returns("doc")
+      location_doc('us').should == "doc"
+    end
 
-  specify "locations should return array of urls using a location link's href" do
-    links = []
-    2.times do |i|
-      links << mock {|m| m.expects(:[]).with("href").returns("http://url#{i}/") }
+    should "location links should get all a tags from div with id list" do
+      doc = mock { expects(:search).with("#list a").returns("links") }
+      stubs(:location_doc).returns(doc)
+      location_links('us').should == "links"
     end
-    stubs(:location_links).returns(links)
-    locations.should == ["url0", "url1"]
-  end
 
-  specify "search should search location for each location with keyword and return list" do
-    locations = ["url0", "url1"]
+    should "locations should return array of urls using a location link's href" do
+      links = []
+      2.times do |i|
+        links << mock {|m| m.expects(:[]).with("href").returns("http://url#{i}/") }
+      end
+      stubs(:location_links).returns(links)
+      locations('us').should == ["url0", "url1"]
+    end
 
-    locations.each do |loc|
-      expects(:search_location).with("omg", loc, 'sss').returns(["1#{loc}", "2#{loc}"])
+    should "search should search location for each location with keyword and return list" do
+      locations = ["url0", "url1"]
+
+      locations.each do |loc|
+        expects(:search_location).with("omg", loc, 'sss').returns(["1#{loc}", "2#{loc}"])
+      end
+
+      stubs(:locations).returns(locations)
+      search("omg").should == ["1url0", "2url0", "1url1", "2url1"]
     end
 
-    stubs(:locations).returns(locations)
-    search("omg").should == ["1url0", "2url0", "1url1", "2url1"]
-  end
+    should "search should call sleep for each location" do
+      expects(:sleep).times(2)
+      stubs(:locations).returns([1,2])
+      stubs(:search_location)
+      search("")
+    end
 
-  specify "search should call sleep for each location" do
-    expects(:sleep).times(2)
-    stubs(:locations).returns([1,2])
-    stubs(:search_location)
-    search("")
-  end
+    should "search location should fetch doc for search url" do
+      expects(:fetch_doc).with("http://url/search/sss?query=keyword&format=rss")
+      stubs(:items).returns([])
+      search_location("keyword", "url")
+    end
 
-  specify "search location should fetch doc for search url" do
-    expects(:fetch_doc).with("http://url/search/sss?query=keyword&format=rss")
-    stubs(:items).returns([])
-    search_location("keyword", "url")
-  end
+    should "search location should create return items" do
+      items = [1,2,3]
+      expects(:items).returns(items)
+      search_location("keyword", "url").should == items
+    end
 
-  specify "search location should create return items" do
-    items = [1,2,3]
-    expects(:items).returns(items)
-    search_location("keyword", "url").should == items
-  end
+    should "items should get all item elements from doc" do
+      item = stub
+      stubs(:hashify).with(item).returns(1)
+      doc = mock { expects(:search).with("item").returns([item]) }
+      items(doc).should == [1]
+    end
 
-  specify "items should get all item elements from doc" do
-    item = stub
-    stubs(:hashify).with(item).returns(1)
-    doc = mock { expects(:search).with("item").returns([item]) }
-    items(doc).should == [1]
-  end
+    should "items should hashify all item elements from doc" do
+      item = stub
+      expects(:hashify).with(item).returns(1)
+      doc = stub { stubs(:search).returns([item]) }
+      items(doc).should == [1]
+    end
 
-  specify "items should hashify all item elements from doc" do
-    item = stub
-    expects(:hashify).with(item).returns(1)
-    doc = stub { stubs(:search).returns([item]) }
-    items(doc).should == [1]
-  end
+    should "categories should fetch doc the main sfbay page" do
+      doc = stub(:search => [])
+      expects(:fetch_doc).with("http://sfbay.craigslist.org/").returns(doc)
+      categories
+    end
 
-  specify "categories should fetch doc the main sfbay page" do
-    doc = stub(:search => [])
-    expects(:fetch_doc).with("http://sfbay.craigslist.org/").returns(doc)
-    categories
-  end
+    should "categories should search for all links in the table with property summary equal to for sale" do
+      doc = mock { expects(:search).with("table[@summary=\"for sale\"] a").returns([]) }
+      stubs(:fetch_doc).returns(doc)
+      categories
+    end
 
-  specify "categories should search for all links in the table with property summary equal to for sale" do
-    doc = mock { expects(:search).with("table[@summary=\"for sale\"] a").returns([]) }
-    stubs(:fetch_doc).returns(doc)
-    categories
-  end
+    should "categories should return a hash with link inner html keys and link href values" do
+      link = stub(:inner_html => "inner_html") do
+        stubs(:[]).with("href").returns("href")
+      end
 
-  specify "categories should return a hash with link inner html keys and link href values" do
-    link = stub(:inner_html => "inner_html") do
-      stubs(:[]).with("href").returns("href")
+      doc = stub(:search => [link, link])
+      stubs(:fetch_doc).returns(doc)
+      categories.should == {'inner_html' => 'href', 'inner_html' => 'href'}
     end
 
-    doc = stub(:search => [link, link])
-    stubs(:fetch_doc).returns(doc)
-    categories.should == {'inner_html' => 'href', 'inner_html' => 'href'}
-  end
+    should "search location should accept a category parameter" do
+      expects(:fetch_doc).with("http://loc/search/scram?query=keyword&format=rss")
+      stubs(:items).returns([])
+      search_location('keyword', 'loc', 'scram')
+    end
 
-  specify "search location should accept a category parameter" do
-    expects(:fetch_doc).with("http://loc/search/scram?query=keyword&format=rss")
-    stubs(:items).returns([])
-    search_location('keyword', 'loc', 'scram')
-  end
+    should "search location default category is sss" do
+      expects(:fetch_doc).with("http://loc/search/sss?query=keyword&format=rss")
+      stubs(:items).returns([])
+      search_location('keyword', 'loc')
+    end
 
-  specify "search location default category is sss" do
-    expects(:fetch_doc).with("http://loc/search/sss?query=keyword&format=rss")
-    stubs(:items).returns([])
-    search_location('keyword', 'loc')
-  end
+    should "search should pass parameter to search location" do
+      stubs(:locations).returns([0])
+      expects(:search_location).with('keyword', 0, 'chum')
+      search('keyword', 'us', 'chum')
+    end
 
-  specify "search should pass parameter to search location" do
-    stubs(:locations).returns([0])
-    expects(:search_location).with('keyword', 0, 'chum')
-    search('keyword', 'chum')
-  end
-  
-  specify "search should have default category of sss" do
-    stubs(:locations).returns([0])
-    expects(:search_location).with('keyword', 0, 'sss')
-    search('keyword')
+    should "search should have default category of sss" do
+      stubs(:locations).returns([0])
+      expects(:search_location).with('keyword', 0, 'sss')
+      search('keyword')
+    end
   end
-end
+end

data/test/test_helper.rb

@@ -1,4 +1,5 @@
 require 'rubygems'
-require 'test/spec'
+require 'shoulda'
+require 'matchy'
 require 'mocha'
 require 'crags'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: gotascii-crags
 version: !ruby/object:Gem::Version 
-  version: 1.2.7
+  version: 1.4.2
 platform: ruby
 authors: 
 - Justin Marney
@@ -9,12 +9,12 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-02-05 00:00:00 -08:00
+date: 2009-06-16 00:00:00 -07:00
 default_executable: 
 dependencies: []
 
 description: A library to help search across multiple craigslist locations.
-email: gotascii@gmail.com
+email: justin.marney@viget.com
 executables: []
 
 extensions: []
@@ -22,7 +22,6 @@ extensions: []
 extra_rdoc_files: 
 - History.txt
 - README.txt
-- lib/js/client.html
 files: 
 - .gitignore
 - History.txt
@@ -35,27 +34,13 @@ files:
 - lib/crags/proxy.rb
 - lib/crags/runner.rb
 - lib/crags/searcher.rb
-- lib/js/client.html
-- tasks/ann.rake
-- tasks/bones.rake
-- tasks/gem.rake
-- tasks/git.rake
-- tasks/manifest.rake
-- tasks/notes.rake
-- tasks/post_load.rake
-- tasks/rdoc.rake
-- tasks/rubyforge.rake
-- tasks/setup.rb
-- tasks/spec.rake
-- tasks/svn.rake
-- tasks/test.rake
 - test/crags/fetch_test.rb
 - test/crags/proxy_test.rb
 - test/crags/runner_test.rb
 - test/crags/searcher_test.rb
 - test/test_helper.rb
 has_rdoc: true
-homepage: http://github.com/gotascii/crags
+homepage: http://github.com/vigetlabs/crags
 post_install_message: 
 rdoc_options: 
 - --main

data/lib/js/client.html

@@ -1,81 +0,0 @@
-<html>
-<head>
-<script language="javascript">
-var IFrameObj; // our IFrame object
-var IFrameDoc;
-function callToServer() {
-  if (!document.createElement) {return true};
-  var URL = 'http://washingtondc.craigslist.org/bik/index.rss';
-  if (!IFrameObj && document.createElement) {
-    // create the IFrame and assign a reference to the
-    // object to our global variable IFrameObj.
-    // this will only happen the first time 
-    // callToServer() is called
-   try {
-      var tempIFrame=document.createElement('iframe');
-      tempIFrame.setAttribute('id','RSIFrame');
-      tempIFrame.style.border='0px';
-      tempIFrame.style.width='0px';
-      tempIFrame.style.height='0px';
-      IFrameObj = document.body.appendChild(tempIFrame);
-      
-      if (document.frames) {
-        // this is for IE5 Mac, because it will only
-        // allow access to the document object
-        // of the IFrame if we access it through
-        // the document.frames array
-        IFrameObj = document.frames['RSIFrame'];
-      }
-    } catch(exception) {
-      // This is for IE5 PC, which does not allow dynamic creation
-      // and manipulation of an iframe object. Instead, we'll fake
-      // it up by creating our own objects.
-      iframeHTML='\<iframe id="RSIFrame" style="';
-      iframeHTML+='border:0px;';
-      iframeHTML+='width:0px;';
-      iframeHTML+='height:0px;';
-      iframeHTML+='"><\/iframe>';
-      document.body.innerHTML+=iframeHTML;
-      IFrameObj = new Object();
-      IFrameObj.document = new Object();
-      IFrameObj.document.location = new Object();
-      IFrameObj.document.location.iframe = document.getElementById('RSIFrame');
-      IFrameObj.document.location.replace = function(location) {
-        this.iframe.src = location;
-      }
-    }
-  }
-  
-  if (navigator.userAgent.indexOf('Gecko') !=-1 && !IFrameObj.contentDocument) {
-    // we have to give NS6 a fraction of a second
-    // to recognize the new IFrame
-    setTimeout('callToServer()',10);
-    return false;
-  }
-  
-  if (IFrameObj.contentDocument) {
-    // For NS6
-    IFrameDoc = IFrameObj.contentDocument; 
-  } else if (IFrameObj.contentWindow) {
-    // For IE5.5 and IE6
-    IFrameDoc = IFrameObj.contentWindow.document;
-  } else if (IFrameObj.document) {
-    // For IE5
-    IFrameDoc = IFrameObj.document;
-  } else {
-    return true;
-  }
-  
-  IFrameDoc.location.replace(URL);
-  return false;
-}
-
-</script>
-</head>
-<body>
-  <script>
-    callToServer();
-    alert(IFrameDoc.innerHTML);
-  </script>
-</body>
-</html>