caule 0.0.1

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format Fivemat
+--order rand

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Rafael Souza
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README

@@ -0,0 +1,29 @@
+# Caule
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'caule'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install caule
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"

data/caule.gemspec

@@ -0,0 +1,22 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/caule/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Rafael Souza"]
+  gem.email         = ["me@rafaelss.com"]
+  gem.description   = %q{DSL to build crawlers easily}
+  gem.summary       = %q{DSL to build crawlers easily}
+  gem.homepage      = "http://github.com/rafaelss/caule"
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "caule"
+  gem.require_paths = ["lib"]
+  gem.version       = Caule::VERSION
+
+  gem.add_dependency "mechanize", "~> 2.3"
+  gem.add_development_dependency "rspec", "~> 2.9.0"
+  gem.add_development_dependency "fivemat", "~> 1.0.0"
+  gem.add_development_dependency "webmock", "~> 1.8.6"
+end

data/crawler_test.rb

@@ -0,0 +1,10 @@
+require ""
+
+crawler = Caule::Bot.new("http://sfbay.craigslist.org/")
+crawler.on_pages_like(/forums/) do |page|
+  page.
+end
+crawler.on_every_page do
+
+end
+crawler.run

data/lib/caule.rb

@@ -0,0 +1,11 @@
+require "caule/version"
+
+module Caule
+  autoload :Bot, "caule/bot"
+
+  def self.start(*urls)
+    bot = Bot.new(*urls)
+    yield bot
+    bot.run
+  end
+end

data/lib/caule/bot.rb

@@ -0,0 +1,79 @@
+require "mechanize"
+
+module Caule
+  class Bot
+    attr_reader :urls, :agent
+
+    def initialize(*urls)
+      @urls = urls.flatten
+      @agent = Mechanize.new do |a|
+        a.user_agent_alias = "Windows IE 7"
+      end
+
+      @on_pages_like_blocks = Hash.new { |hash,key| hash[key] = [] }
+    end
+
+    def on_every_page(&block)
+      on_pages_like(/.+/, &block)
+    end
+
+    def on_pages_like(*patterns, &block)
+      if patterns && !patterns.empty?
+        patterns.each do |pattern|
+          @on_pages_like_blocks[pattern] << block
+        end
+      end
+    end
+
+    def run
+      urls.each do |url|
+        page = agent.get(url)
+        do_page_blocks(page)
+
+        stack = links_to_follow(page)
+        while l = stack.shift
+          begin
+            page = l.click
+            next unless Mechanize::Page === page
+            do_page_blocks(page)
+            stack.push(*links_to_follow(page))
+          rescue Mechanize::ResponseCodeError => ex
+            puts ex.message.red
+          end
+        end
+      end
+    end
+
+    def focus_crawl(&block)
+      @focus_crawl_block = block
+    end
+
+    protected
+
+    def do_page_blocks(page)
+      @on_pages_like_blocks.each do |pattern, blocks|
+        blocks.each do |block|
+          if page.uri.to_s =~ pattern
+            block.call(page)
+          end
+        end
+      end
+    end
+
+    def links_to_follow(page)
+      if @focus_crawl_block
+        links = @focus_crawl_block.call(page) || []
+        links.compact! unless links.empty?
+      else
+        links = page.links
+      end
+
+      links.select { |link| visit_link?(link) }.map { |link| link.dup }
+    end
+
+    def visit_link?(link)
+      return false if agent.visited?(link.href)
+      true
+    end
+  end
+end

data/lib/caule/version.rb

@@ -0,0 +1,3 @@
+module Caule
+  VERSION = "0.0.1"
+end

data/spec/caule/bot_spec.rb

@@ -0,0 +1,59 @@
+require "spec_helper"
+
+describe Caule::Bot do
+  context "hooks" do
+    describe "on_every_page" do
+      subject { described_class.new("http://sfbay.craigslist.org/", "http://sfbay.craigslist.org/about") }
+
+      it "runs on every page" do
+        stub_request(:get, "http://sfbay.craigslist.org/").
+          to_return(:status => 200, :body => "", :headers => { "Content-Type" => "text/html" })
+
+        stub_request(:get, "http://sfbay.craigslist.org/about").
+          to_return(:status => 200, :body => "", :headers => { "Content-Type" => "text/html" })
+
+        subject.on_every_page do |page|
+          page.uri.to_s.should match(/(\/|about)$/)
+        end
+
+        subject.run
+      end
+    end
+
+    describe "on_pages_like" do
+      subject { described_class.new("http://sfbay.craigslist.org/", "http://sfbay.craigslist.org/about") }
+
+      it "runs on pages that match the regular expression passed" do
+        stub_request(:get, "http://sfbay.craigslist.org/").
+          to_return(:status => 200, :body => "", :headers => { "Content-Type" => "text/html" })
+
+        stub_request(:get, "http://sfbay.craigslist.org/about").
+          to_return(:status => 200, :body => "", :headers => { "Content-Type" => "text/html" })
+
+        subject.on_pages_like(/about/) do |page|
+          page.uri.to_s.should match(/about$/)
+        end
+
+        subject.run
+      end
+    end
+
+    describe "focus_crawl" do
+      subject { described_class.new("http://sfbay.craigslist.org/") }
+
+      it "crawls only links returned by the block" do
+        stub_request(:get, "http://sfbay.craigslist.org/").
+          to_return(:status => 200, :body => "<a href='/about'>about</a><a href='/forums'></a>", :headers => { "Content-Type" => "text/html" })
+
+        stub_request(:get, "http://sfbay.craigslist.org/forums").
+          to_return(:status => 200, :body => "", :headers => { "Content-Type" => "text/html" })
+
+        subject.focus_crawl do |page|
+          page.links_with(:href => /forums$/)
+        end
+
+        subject.run
+      end
+    end
+  end
+end

data/spec/caule_spec.rb

@@ -0,0 +1,9 @@
+require "spec_helper"
+
+describe Caule do
+  it "instantiates bot, yield the instance and run the crawler" do
+    bot = nil
+    Caule.start { |b| bot = b }
+    bot.should be_instance_of(Caule::Bot)
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,8 @@
+require "caule"
+require "webmock/rspec"
+
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: caule
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Rafael Souza
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-04-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: mechanize
+  requirement: &70237564894140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70237564894140
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70237564893620 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.9.0
+  type: :development
+  prerelease: false
+  version_requirements: *70237564893620
+- !ruby/object:Gem::Dependency
+  name: fivemat
+  requirement: &70237564893140 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :development
+  prerelease: false
+  version_requirements: *70237564893140
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: &70237564892680 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 1.8.6
+  type: :development
+  prerelease: false
+  version_requirements: *70237564892680
+description: DSL to build crawlers easily
+email:
+- me@rafaelss.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE
+- README
+- Rakefile
+- caule.gemspec
+- crawler_test.rb
+- lib/caule.rb
+- lib/caule/bot.rb
+- lib/caule/version.rb
+- spec/caule/bot_spec.rb
+- spec/caule_spec.rb
+- spec/spec_helper.rb
+homepage: http://github.com/rafaelss/caule
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: DSL to build crawlers easily
+test_files:
+- spec/caule/bot_spec.rb
+- spec/caule_spec.rb
+- spec/spec_helper.rb