crags 1.5.9

data/.gitignore

@@ -0,0 +1,2 @@
+pkg/
+.DS_Store

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2009 Justin Marney
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,33 @@
+= crags
+
+A library to help search across multiple craigslist locations.
+
+== Usage
+
+searches all available craigslist sites for a keyword in "for sale".
+ search(keyword)
+
+search also takes a category
+ search(keyword, 'sss')
+
+There are a bunch of helpful api methods, check the tests for more info.
+
+== Requirements
+
+* hpricot
+* curb
+
+== Note on Patches/Pull Requests 
+
+* Fork the project.
+* Make your feature addition or bug fix.
+* Add tests for it. This is important so I don't break it in a
+  future version unintentionally.
+* Commit, do not mess with rakefile, version, or history.
+  (if you want to have your own version, that is fine but
+  bump version in a commit by itself I can ignore when I pull)
+* Send me a pull request. Bonus points for topic branches.
+
+== Copyright
+
+Copyright (c) 2009 Justin Marney. See LICENSE for details.

data/Rakefile

@@ -0,0 +1,55 @@
+require 'rubygems'
+require 'rake'
+
+begin
+  require 'jeweler'
+  Jeweler::Tasks.new do |gem|
+    gem.name = "crags"
+    gem.summary = %Q{A library to help search across multiple craigslist locations.}
+    gem.description = %Q{A library to help search across multiple craigslist locations.}
+    gem.email = "gotascii@gmail.com"
+    gem.homepage = "http://github.com/gotascii/crags"
+    gem.authors = ["Justin Marney"]
+  end
+  Jeweler::GemcutterTasks.new
+rescue LoadError
+  puts "Jeweler (or a dependency) not available. Install it with: sudo gem install jeweler"
+end
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/*_test.rb'
+  test.verbose = true
+end
+
+begin
+  require 'rcov/rcovtask'
+  Rcov::RcovTask.new do |test|
+    test.libs << 'test'
+    test.pattern = 'test/**/*_test.rb'
+    test.verbose = true
+  end
+rescue LoadError
+  task :rcov do
+    abort "RCov is not available. In order to run rcov, you must: sudo gem install spicycode-rcov"
+  end
+end
+
+task :test => :check_dependencies
+
+task :default => :test
+
+require 'rake/rdoctask'
+Rake::RDocTask.new do |rdoc|
+  if File.exist?('VERSION')
+    version = File.read('VERSION')
+  else
+    version = ""
+  end
+
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title = "crags #{version}"
+  rdoc.rdoc_files.include('README*')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/VERSION

@@ -0,0 +1 @@
+1.5.9

data/crags.gemspec

@@ -0,0 +1,59 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{crags}
+  s.version = "1.5.9"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Justin Marney"]
+  s.date = %q{2009-10-13}
+  s.description = %q{A library to help search across multiple craigslist locations.}
+  s.email = %q{gotascii@gmail.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".gitignore",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "crags.gemspec",
+     "lib/crags.rb",
+     "lib/crags/fetch.rb",
+     "lib/crags/proxy.rb",
+     "lib/crags/runner.rb",
+     "lib/crags/searcher.rb",
+     "test/crags/fetch_test.rb",
+     "test/crags/proxy_test.rb",
+     "test/crags/runner_test.rb",
+     "test/crags/searcher_test.rb",
+     "test/test_helper.rb"
+  ]
+  s.homepage = %q{http://github.com/gotascii/crags}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{A library to help search across multiple craigslist locations.}
+  s.test_files = [
+    "test/crags/fetch_test.rb",
+     "test/crags/proxy_test.rb",
+     "test/crags/runner_test.rb",
+     "test/crags/searcher_test.rb",
+     "test/test_helper.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+    else
+    end
+  else
+  end
+end

data/lib/crags.rb

@@ -0,0 +1,41 @@
+require 'curb'
+require 'hpricot'
+require 'erb'
+
+module Crags
+  COUNTRIES = [
+    'jp',
+    'ar',
+    'bd',
+    'br',
+    'ca',
+    'cl',
+    'co',
+    'cr',
+    'cz',
+    'de',
+    'eg',
+    'hu',
+    'id',
+    'ie',
+    'il',
+    'lb',
+    'my',
+    'nl',
+    'nz',
+    'no',
+    'pk',
+    'pa',
+    'ru',
+    'th',
+    'ae',
+    'us',
+    've',
+    'vn'
+  ]
+end
+
+require 'crags/fetch'
+require 'crags/proxy'
+require 'crags/searcher'
+require 'crags/runner'

data/lib/crags/fetch.rb

@@ -0,0 +1,19 @@
+module Crags
+  module Fetch
+    def fetch_doc(url)
+      Hpricot.parse(fetch_html(url))
+    end
+
+    def fetch_html(url)
+      req = fetch_request(url)
+      req.body_str
+    end
+
+    def fetch_request(url)
+      req = Curl::Easy.new(url)
+      req.follow_location = true
+      req.perform
+      req
+    end
+  end
+end

data/lib/crags/proxy.rb

@@ -0,0 +1,21 @@
+module Crags
+  module Proxy
+    include Fetch
+
+    def lists
+      ["http://www.proxy4free.com/page1.html","http://www.proxy4free.com/page3.html"]
+    end
+
+    def fetch_lists
+      lists.collect {|url| fetch_html(url)}
+    end
+
+    def scan(text)
+      text.scan(/\d+\.\d+\.\d+\.\d+/)
+    end
+
+    def proxies
+      fetch_lists.collect{|html|scan(html)}.flatten.uniq
+    end
+  end
+end

data/lib/crags/runner.rb

@@ -0,0 +1,10 @@
+module Crags
+  class Runner
+    include Searcher
+
+    def search_location(keyword, loc, category = 'sss')
+      puts "Searching #{loc}..."
+      super
+    end
+  end
+end

data/lib/crags/searcher.rb

@@ -0,0 +1,77 @@
+module Crags
+  module Searcher
+    include Fetch
+    include ERB::Util
+
+    def strip_http(url)
+      url.gsub(/^http\:\/\//,'').gsub(/\/$/,'')
+    end
+
+    def location_link(country)
+      "http://geo.craigslist.org/iso/#{country}"
+    end
+
+    def location_doc(country)
+      fetch_doc(location_link(country))
+    end
+
+    def location_request(country)
+      fetch_request(location_link(country))
+    end
+
+    def location_links(country)
+      location_doc(country).search("#list a")
+    end
+
+    def locations(country)
+      linkz = location_links(country)
+      if linkz.empty?
+        [strip_http(location_request(country).last_effective_url)]
+      else
+        linkz.collect{|link| strip_http(link["href"]) }
+      end
+    end
+
+    def categories
+      doc = fetch_doc("http://sfbay.craigslist.org/")
+      links = doc.search("table[@summary=\"for sale\"] a")
+      categories = {}
+      links.each do |link|
+        categories[link.inner_html] = link["href"]
+      end
+      categories
+    end
+
+    def search(keyword, country = 'us', category = 'sss', &block)
+      locations(country).collect do |loc|
+        sleep(1 + rand(3))
+        search_location(keyword, loc, category, &block)
+      end.flatten
+    end
+
+    def items(doc)
+      doc.search("item").collect do |item|
+        hashify(item)
+      end
+    end
+
+    def hashify(item)
+      title = item.at("title").inner_text
+      url = strip_http(item["rdf:about"])
+      date = DateTime.parse(item.at("dc:date").inner_text)
+      {:title => title, :url => url, :date => date}
+    end
+
+    def search_location_link(keyword, loc, category = 'sss')
+      "http://#{loc}/search/#{category}?query=#{url_encode(keyword)}"
+    end
+
+    def search_location(keyword, loc, category = 'sss', &block)
+      doc = fetch_doc("#{search_location_link(keyword, loc, category)}&format=rss")
+      items(doc).collect do |item|
+        yield item if block_given?
+        item
+      end
+    end
+  end
+end

data/test/crags/fetch_test.rb

@@ -0,0 +1,33 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class Crags::FetchTest < Test::Unit::TestCase
+  context "Fetch" do
+    setup do
+      extend Crags::Fetch
+    end
+
+    should "fetch doc should hpricot fetched html" do
+      stubs(:fetch_html).with("url").returns("html")
+      Hpricot.expects(:parse).with("html").returns("doc")
+      fetch_doc("url").should == "doc"
+    end
+
+    should "fetch html should fetch_request a url" do
+      curb = stub(:body_str => "uhh")
+      expects(:fetch_request).with("url").returns(curb)
+      fetch_html("url").should == "uhh"
+    end
+
+    should "create a new request" do
+      req = stub(:follow_location= => nil, :perform => nil)
+      Curl::Easy.expects(:new).with("url").returns(req)
+      fetch_request("url").should == req
+    end
+    
+    should "follow redirects for fetched requests" do
+      req = mock(:follow_location= => nil, :perform => nil)
+      Curl::Easy.stubs(:new).returns(req)
+      fetch_request("url")
+    end
+  end
+end

data/test/crags/proxy_test.rb

@@ -0,0 +1,25 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class Crags::ProxyTest < Test::Unit::TestCase
+  context "Proxy" do
+    setup do
+      extend Crags::Proxy
+    end
+
+    should "lists should return a list of proxy list websites" do
+      lists.should == ["http://www.proxy4free.com/page1.html", "http://www.proxy4free.com/page3.html"]
+    end
+
+    should "fetch lists should fetch html for each site in lists" do
+      stubs(:lists).returns(["1", "2"])
+      expects(:fetch_html).with("1").returns("html_1")
+      expects(:fetch_html).with("2").returns("html_2")
+      fetch_lists.should == ["html_1", "html_2"]
+    end
+
+    should "scan should return all ips in a text blizoc" do
+      text = "192.168.1.2 omg dude!! wtf.f.f.asdasd9.8.9 78.900.42.32"
+      scan(text).should == ["192.168.1.2", "78.900.42.32"]
+    end
+  end
+end

data/test/crags/runner_test.rb

@@ -0,0 +1,28 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class Crags::RunnerTest < Test::Unit::TestCase
+  context "instance of Runner" do
+    setup do
+      @runner = Crags::Runner.new
+      @runner.stubs(:fetch_doc)
+      @runner.stubs(:items).returns([])
+    end
+
+    should "runner should include searcher" do
+      Crags::Runner.ancestors.include?(Crags::Searcher).should == true
+    end
+
+    should "search location should puts message with loc" do
+      @runner.expects(:puts).with { |val| val =~ /location/ }
+      @runner.search_location("", "location", "category")
+    end
+
+    should "search location should take a category" do
+      @runner.search_location("", "location", "category")
+    end
+
+    should "search location should have default category sss" do
+      @runner.search_location("", "location")
+    end
+  end
+end

data/test/crags/searcher_test.rb

@@ -0,0 +1,139 @@
+require File.dirname(__FILE__) + '/../test_helper'
+
+class Crags::SearcherTest < Test::Unit::TestCase
+
+  context "Searcher with stubbed fetch doc" do
+    setup do
+      extend Crags::Searcher
+      stubs(:sleep)
+      stubs(:fetch_doc)
+    end
+
+    should "strip_http should remove http:// and trailing /" do
+      url = "http://omg/"
+      strip_http(url).should == "omg"
+    end
+
+    should "strip_http should remove http:// when there is no trailing slash" do
+      url = "http://omg"
+      strip_http(url).should == "omg"
+    end
+
+    should "location doc should fetch doc at location url" do
+      expects(:fetch_doc).with("http://geo.craigslist.org/iso/us").returns("doc")
+      location_doc('us').should == "doc"
+    end
+
+    should "location links should get all a tags from div with id list" do
+      doc = mock { expects(:search).with("#list a").returns("links") }
+      stubs(:location_doc).returns(doc)
+      location_links('us').should == "links"
+    end
+
+    should "locations should return array of urls using a location link's href" do
+      links = []
+      2.times do |i|
+        links << mock {|m| m.expects(:[]).with("href").returns("http://url#{i}/") }
+      end
+      stubs(:location_links).returns(links)
+      locations('us').should == ["url0", "url1"]
+    end
+
+    should "locations should return array of one url using location_urls last_effective_url when no links are present on location_url page" do
+      stubs(:location_links).returns([])
+      req = mock(:last_effective_url => 'http://url.org/')
+      stubs(:location_request).with('us').returns(req)
+      locations('us').should == ["url.org"]
+    end
+
+    should "search should search location for each location with keyword and return list" do
+      locations = ["url0", "url1"]
+
+      locations.each do |loc|
+        expects(:search_location).with("omg", loc, 'sss').returns(["1#{loc}", "2#{loc}"])
+      end
+
+      stubs(:locations).returns(locations)
+      search("omg").should == ["1url0", "2url0", "1url1", "2url1"]
+    end
+
+    should "search should call sleep for each location" do
+      expects(:sleep).times(2)
+      stubs(:locations).returns([1,2])
+      stubs(:search_location)
+      search("")
+    end
+
+    should "search location should fetch doc for search url" do
+      expects(:fetch_doc).with("http://url/search/sss?query=keyword&format=rss")
+      stubs(:items).returns([])
+      search_location("keyword", "url")
+    end
+
+    should "search location should create return items" do
+      items = [1,2,3]
+      expects(:items).returns(items)
+      search_location("keyword", "url").should == items
+    end
+
+    should "items should get all item elements from doc" do
+      item = stub
+      stubs(:hashify).with(item).returns(1)
+      doc = mock { expects(:search).with("item").returns([item]) }
+      items(doc).should == [1]
+    end
+
+    should "items should hashify all item elements from doc" do
+      item = stub
+      expects(:hashify).with(item).returns(1)
+      doc = stub { stubs(:search).returns([item]) }
+      items(doc).should == [1]
+    end
+
+    should "categories should fetch doc the main sfbay page" do
+      doc = stub(:search => [])
+      expects(:fetch_doc).with("http://sfbay.craigslist.org/").returns(doc)
+      categories
+    end
+
+    should "categories should search for all links in the table with property summary equal to for sale" do
+      doc = mock { expects(:search).with("table[@summary=\"for sale\"] a").returns([]) }
+      stubs(:fetch_doc).returns(doc)
+      categories
+    end
+
+    should "categories should return a hash with link inner html keys and link href values" do
+      link = stub(:inner_html => "inner_html") do
+        stubs(:[]).with("href").returns("href")
+      end
+
+      doc = stub(:search => [link, link])
+      stubs(:fetch_doc).returns(doc)
+      categories.should == {'inner_html' => 'href', 'inner_html' => 'href'}
+    end
+
+    should "search location should accept a category parameter" do
+      expects(:fetch_doc).with("http://loc/search/scram?query=keyword&format=rss")
+      stubs(:items).returns([])
+      search_location('keyword', 'loc', 'scram')
+    end
+
+    should "search location default category is sss" do
+      expects(:fetch_doc).with("http://loc/search/sss?query=keyword&format=rss")
+      stubs(:items).returns([])
+      search_location('keyword', 'loc')
+    end
+
+    should "search should pass parameter to search location" do
+      stubs(:locations).returns([0])
+      expects(:search_location).with('keyword', 0, 'chum')
+      search('keyword', 'us', 'chum')
+    end
+
+    should "search should have default category of sss" do
+      stubs(:locations).returns([0])
+      expects(:search_location).with('keyword', 0, 'sss')
+      search('keyword')
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,5 @@
+require 'rubygems'
+require 'shoulda'
+require 'matchy'
+require 'mocha'
+require 'crags'

metadata

@@ -0,0 +1,75 @@
+--- !ruby/object:Gem::Specification 
+name: crags
+version: !ruby/object:Gem::Version 
+  version: 1.5.9
+platform: ruby
+authors: 
+- Justin Marney
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2009-10-13 00:00:00 -04:00
+default_executable: 
+dependencies: []
+
+description: A library to help search across multiple craigslist locations.
+email: gotascii@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- LICENSE
+- README.rdoc
+files: 
+- .gitignore
+- LICENSE
+- README.rdoc
+- Rakefile
+- VERSION
+- crags.gemspec
+- lib/crags.rb
+- lib/crags/fetch.rb
+- lib/crags/proxy.rb
+- lib/crags/runner.rb
+- lib/crags/searcher.rb
+- test/crags/fetch_test.rb
+- test/crags/proxy_test.rb
+- test/crags/runner_test.rb
+- test/crags/searcher_test.rb
+- test/test_helper.rb
+has_rdoc: true
+homepage: http://github.com/gotascii/crags
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.5
+signing_key: 
+specification_version: 3
+summary: A library to help search across multiple craigslist locations.
+test_files: 
+- test/crags/fetch_test.rb
+- test/crags/proxy_test.rb
+- test/crags/runner_test.rb
+- test/crags/searcher_test.rb
+- test/test_helper.rb