klepto 0.1.0

data/.gitignore

@@ -0,0 +1,21 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+log
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+spec/orm/database.yml
+.DS_Store
+**/.DS_Store

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.3-p194@klepto

data/Gemfile

@@ -0,0 +1,18 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in klepto.gemspec
+gemspec
+gem 'rspec', '2.12.0'
+gem 'rb-fsevent'
+gem "guard", '1.6.1'
+gem "guard-bundler"
+gem "guard-rspec"
+gem 'debugger', '1.2.0'
+gem 'vcr'
+gem 'fakeweb'
+#gem 'activerecord', "~>3.2.0"
+gem 'activerecord'
+gem 'mysql2'
+gem 'rb-fsevent'
+gem 'ruby_gntp'
+gem 'simplecov', :require => false

data/Guardfile

@@ -0,0 +1,11 @@
+guard 'bundler' do
+  watch('Gemfile')
+  # Uncomment next line if Gemfile contain `gemspec' command
+  # watch(/^.+\.gemspec/)
+end
+
+guard 'rspec', :version => 2 do
+  watch(%r{^spec/.+_spec\.rb$})
+  watch(%r{^lib/(.+)\.rb$})     { |m| "spec/lib/#{m[1]}_spec.rb" }
+  watch('spec/spec_helper.rb')  { "spec" }
+end

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Cory O'Daniel
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,129 @@
+# Klepto
+
+A mean little DSL'd capybara (poltergeist) based web crawler that stuffs data into your Rails app.
+
+## Features 
+
+* CSS or XPath Syntax
+* Full javascript processing via phantomjs / poltergeist
+* All the fun of capybara
+* Scrape multiple pages with a single bot
+* Scrape individuals pages with multiple 'crawlers', see Bieber example.
+* Test coverage!
+
+## Usage
+Say you want a bunch of Bieb tweets! How is there not profit in that?
+
+```ruby
+# Make a bot
+@bot = Klepto::Bot.new do
+  # Set your selector syntax. You can change to :xpath if you are 40+ or love C#.
+  syntax :css
+  
+  # Send some headers, confuse everyone.
+  headers 'Referer' => 'http://www.twitter.com'
+
+  # The more the merrier. It takes a *splat.
+  urls  'https://twitter.com/justinbieber'
+
+  # Crawl the body of the page to get the user info
+  crawl 'body' do
+    # The default handler is to call .text on the scraped node.
+    scrape "h1.fullname", :name
+
+    # Scrape finds the first matching element for the given selector within
+    # the scope above (here: 'body')
+    scrape '.username span.screen-name', :username
+
+    # Scrape all matching elements with #scrape_all
+    scrape_all 'span.url a' do |nodes|
+      {
+        links: nodes.map{|n| n[:href]}
+      }
+    end
+
+    # Each 'match' of the crawlers selector (here: 'body') will have the 
+    #   content from 'scrape' passed in as a hash
+    save do |params|
+      user = User.find_by_name(params[:username]) || User.new
+      user.update_attributes params
+    end
+  end
+
+  # Get dem tweets
+  crawl 'li.stream-item' do
+    # Passing no parameters to scrape will set the context to be the
+    #  outer matched crawled element (here: 'li.stream-item')
+    scrape do |node|
+      {:twitter_id => node['data-item-id']}
+    end
+    
+    # Put '.content p' into params[:content]
+    scrape '.content p', :content
+    
+    # Pass a block for more control
+    scrape '._timestamp' do |node|
+      {timestamp: node['data-time']}
+    end
+
+    scrape '.time a' do |node|
+      {permalink: node[:href]}
+    end
+        
+    # Each 'match' of the crawlers selector (here: 'li.stream-item') will have the 
+    #   content from 'scrape' passed in as a hash        
+    save do |params|
+      tweet = Tweet.find_by_twitter_id(params[:twitter_id]) || Tweet.new
+      tweet.update_attributes params
+    end
+  end  
+end
+
+#start that bot for some sweet victory, heart throb.
+@bot.start! 
+```
+
+All your content are belong to us.
+
+
+
+## Stuff I'm going to add.
+
+event handlers...
+--------------------
+
+    on_http_status(500,404) do |response, bot|
+      email('admin@example.com', bot.status, bot.summary)
+    end
+    on_assertion_failure{ |response, bot| }
+    on_invalid_resource{ |resource, bot| }
+
+Pre-req Steps
+--------------------  
+
+    prepare [
+      [:GET, 'http://example.com'],
+      [:POST, 'http://example.com/login', {username: 'cory', password: '123456'}],
+    ]
+
+Page Assertions
+--------------------
+
+    assertions do
+      present 'li.offer'
+      present 'h3 a', :present => [:href]
+      within 'li.offer' do
+        present 'h3'
+      end
+
+      scrape 'h3 a' do |node|
+        node.is_a_link_to_someplace_we_like
+      end    
+    end
+
+Cookie Stufing
+-------------------
+
+    cookies({
+      'Has Fun' => true
+    })  

data/Rakefile

@@ -0,0 +1,7 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new('spec')
+
+# If you want to make this the default task
+task :default => :spec

data/klepto.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'klepto/version'
+
+Gem::Specification.new do |gem|
+  gem.name          = "klepto"
+  gem.version       = Klepto::VERSION
+  gem.authors       = ["Cory O'Daniel"]
+  gem.email         = ["github@coryodaniel.com"]
+  gem.description   = "Tearing up web pages into ActiveRecord resources"
+  gem.summary       = "Tearing up web pages into ActiveRecord resources"
+  gem.homepage      = "http://github.com/coryodaniel/klepto"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+  gem.add_dependency "docile"
+  gem.add_dependency "poltergeist", '1.1.0'
+  gem.add_dependency "capybara", '2.0.2'
+  #gem.add_dependency "thor"
+  gem.add_dependency "nokogiri", '~> 1.5.6'
+  gem.add_dependency "activesupport"
+  gem.add_dependency 'multi_json', '~> 1.0'
+end

data/lib/klepto.rb

@@ -0,0 +1,26 @@
+require 'docile'
+require 'open-uri'
+require 'logger'
+require "capybara"
+require "capybara/dsl"
+require 'capybara/poltergeist'
+require 'pp'
+
+Capybara.run_server = false
+
+Capybara.register_driver :poltergeist do |app|
+  Capybara::Poltergeist::Driver.new(app, {
+    js_errors: false
+  })
+end
+Capybara.current_driver = :poltergeist
+
+module Klepto
+  LOG = Logger.new(STDOUT)
+  LOG.level = Logger::WARN
+end
+
+require 'klepto/version'
+require 'klepto/crawler'
+require 'klepto/browser'
+require 'klepto/bot'

data/lib/klepto/bot.rb

@@ -0,0 +1,59 @@
+module Klepto
+  class Bot
+    def initialize(*args, &block)
+      @syntax     = :css
+      @is_dry_run = false
+      @urls       = []
+      @crawlers   = []
+      @browser    = Klepto::Browser.new
+      Docile.dsl_eval(self, &block) if block_given?
+    end
+
+    attr_reader :browser, :crawlers
+
+    def dry_run!
+      @is_dry_run = true
+    end
+
+    def dry_run?
+      !!@is_dry_run
+    end
+
+    def syntax(kind=nil)
+      @syntax = kind unless kind.nil?
+      @syntax
+    end
+
+    def headers(_headers)
+      @browser.set_headers(_headers)
+    end
+
+    def url(*args)
+      @urls += args
+    end
+    alias :urls :url
+
+    def crawl(scope, options={}, &block)
+      options[:syntax] = @syntax
+      @crawlers << Klepto::Crawler.new(scope, options, &block)
+    end
+
+    def start!
+      @urls.each do |url|
+        browser.fetch!(url)
+        @crawlers.each do |crawler|
+          crawler.crawl browser.page
+        end
+      end
+
+      @crawlers.each do |crawler|
+        if dry_run?
+          pp crawler.resources
+        else
+          crawler.persist!
+        end
+      end
+    end
+
+  end
+end

data/lib/klepto/browser.rb

@@ -0,0 +1,18 @@
+module Klepto
+  class Browser
+    include Capybara::DSL
+
+    def initialize(*args)
+      super
+    end
+
+    def set_headers(headers)
+      page.driver.headers = headers
+    end
+    
+    def fetch!(url)
+      visit url
+      page
+    end
+  end
+end

data/lib/klepto/crawler.rb

@@ -0,0 +1,72 @@
+require 'docile'
+module Klepto
+  class Crawler
+
+    def initialize(scope,options={},&block)
+      @resources = []
+      @limit  = options[:limit] 
+      @skip   = options[:skip]
+      @syntax = options[:syntax]
+      @scope  = scope
+      @designations = []
+      
+      Docile.dsl_eval(self, &block) if block_given?
+    end   
+    attr_accessor :resources
+    attr_reader :scope, :syntax
+
+    def scrape(selector=nil, assignee=nil, &block)
+      raise Exception if assignee.nil? && !block_given?
+      raise Exception if !assignee.nil? && block_given?
+      designate(:first, selector, assignee, &block)
+    end
+
+    def scrape_all(selector, assignee=nil, &block)
+      raise Exception if assignee.nil? && !block_given?
+      raise Exception if !assignee.nil? && block_given?
+      designate(:all, selector, assignee, &block)
+    end
+
+    def save(&block)
+      @resource_handler = block
+    end
+
+    def crawl(page)
+      page.all(syntax, scope).each do |selection|
+        params = {}
+        @designations.each do |first_or_all, selector, assignee, handler|
+          if selector.nil?
+            attribs = handler.call selection
+            params.merge!( attribs )            
+          elsif first_or_all == :first
+            node = selection.first(syntax, selector)
+            if assignee
+              params[assignee] = node.try(:text)
+            else
+              attribs = handler.call node
+              params.merge!( attribs )
+            end
+          else
+            nodes = selection.all(syntax, selector)
+            attribs = handler.call nodes
+            params.merge!( attribs )
+          end
+        end
+        @resources << params
+      end
+
+      @resources
+    end
+
+    def persist!
+      if @resource_handler
+        @resources.each {|resource| @resource_handler.call(resource)}
+      end
+    end
+
+    protected
+      def designate(count, selector, assignee, &block)
+        @designations << [count, selector, assignee, block]
+      end
+  end
+end

data/lib/klepto/tasks.rb

@@ -0,0 +1,15 @@
+# require this file to load the tasks
+require 'rake'
+
+# noop 
+=begin
+This is here as a start point for adding rake tasks that can be 'required' by another project
+Just add: require 'klepto/tasks' to your Rakefile
+=end
+
+namespace :klepto do
+  desc "Example task"
+  task :example do
+    puts "I'm a task"
+  end
+end