klepto 0.6.9 → 0.7.0

data/klepto.gemspec

@@ -16,6 +16,7 @@ Gem::Specification.new do |gem|
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
+  gem.license = 'MIT'
   
   gem.add_dependency "poltergeist", '~> 1.3.0'
   gem.add_dependency "capybara", '~> 2.1.0'

data/lib/klepto/browser.rb

@@ -57,10 +57,9 @@ module Klepto
       @url_to_structure = _url
       Klepto.logger.debug("Fetching #{@url_to_structure}")
 
-      Capybara.using_driver use_driver do
-        visit @url_to_structure
-        page
-      end
+      Capybara.current_driver = Capybara.javascript_driver = use_driver
+      visit @url_to_structure
+      page
     end
   end
 end

data/lib/klepto/version.rb

@@ -1,3 +1,3 @@
 module Klepto
-  VERSION = "0.6.9"
+  VERSION = "0.7.0"
 end

data/phantom/test.js

@@ -0,0 +1,100 @@
+/*
+  CoffeeScript
+  JS Lint
+  PhantomJS
+
+  Ruby 'configuration' gem
+  Ruby blocks -> Javascript -> Ruby OR Javascript post processors
+  Ruby blocks -> Assertion? Auto generate cucumbers? OR callbacks on node not found?
+  https://github.com/ariya/phantomjs/wiki/API-Reference-WebPage
+
+  Config.defaults {
+    on(200,'2xx', :redirect){}
+    on('4xx'){}
+    on('5xx'){}
+    on(:timeout){}
+    on(:abort){}
+    headers({})
+    cookies({})  
+    agent "Mozilla/5.0 (Macintosh; Intel Mac OS X) AppleWebKit/534.34 (KHTML, like Gecko) Klepto/#{Klepto::Version} Safari/534.34"
+  }
+  Bot.new("http://google.com")do 
+    config{
+      # merges with Defaults, creates a Configuration
+      url "http://google.com"
+      auto_structure false # stops it from running structure (@bot.process! will run it)
+      abort_on_failure true
+      agent "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.51.22 (KHTML, like Gecko) Version/5.1.1 Safari/534.51.22";
+      
+      headers({})
+      cookies({})
+
+      on(200,'2xx', :redirect){}
+      on('4xx'){}
+      on('5xx'){}
+      on(:timeout){}
+      on(:abort){}
+
+      before(:get){}
+      after(:get){}
+      before(:structure){}    
+      after(:structure){}
+    }
+
+    structure{
+      # Should yield against Proxy so method_missing and queueing isn't in Bot
+    }
+  end
+
+*/
+var page = require('webpage').create(),
+  system = require('system'),
+  lt, pt, t, currentAddress, requestedAddress;
+
+page.settings.userAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_2) AppleWebKit/534.51.22 (KHTML, like Gecko) Version/5.1.1 Safari/534.51.22";
+page.settings.loadImages = false;
+
+page.onUrlChanged = function(targetUrl){
+  currentAddress = targetUrl;
+  console.log("Redirecting to: " + currentAddress);
+}
+
+page.onResourceReceived = function(resource) {
+  if (resource.stage === 'end' && resource.status == 200 && resource.url == currentAddress) {
+    lt = Date.now() - t;
+    console.log("Crawling: " + resource.url);
+    page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
+      var title = page.evaluate(function(){
+        return $("title").text();
+      });
+
+      pt = Date.now() - t;
+      var structure = JSON.stringify({
+        title: title,
+        _meta: {
+          loadTime:           lt,
+          parseTime:          pt,
+          redirectOccurred:   (requestedAddress != resource.url),
+          requestedAddress:   requestedAddress,
+          currentAddress:     resource.url,
+          httpCode:    resource.status
+        }
+      });
+      system.stdout.write(structure);
+      
+      phantom.exit();
+    });
+  } else if(resource.stage === 'end' && resource.status != 200 && resource.url == currentAddress){
+    console.log("Oops: " + resource.status);
+    phantom.exit();
+  } else {/* NOOP*/}
+}
+
+if (system.args.length === 1) {
+  console.log('Usage: test.js <some URL>');
+  phantom.exit(1);
+} else {
+  t = Date.now();
+  currentAddress = requestedAddress = system.args[1];
+  page.open(requestedAddress);
+}

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: klepto
 version: !ruby/object:Gem::Version
-  version: 0.6.9
+  version: 0.7.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-10 00:00:00.000000000 Z
+date: 2013-07-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: poltergeist
@@ -115,6 +115,7 @@ files:
 - lib/klepto/structure.rb
 - lib/klepto/tasks.rb
 - lib/klepto/version.rb
+- phantom/test.js
 - samples/bieber.html
 - samples/concept.rb
 - spec/cassettes/Klepto_Crawler/dsl_interaction/should_crawl_the_resource.yml
@@ -133,7 +134,8 @@ files:
 - spec/orm/database.example.yml
 - spec/spec_helper.rb
 homepage: http://github.com/coryodaniel/klepto
-licenses: []
+licenses:
+- MIT
 post_install_message: 
 rdoc_options: []
 require_paths: