snapsearch-client-ruby 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f2959c7bd59e817f2eef37602a4abdc8ae1f1be8
+  data.tar.gz: 3ddaec20074f02943df30adf39de5e0ad60b0ccd
+SHA512:
+  metadata.gz: 1482fc4b66a72119cb2e26637429d3c48b9561e65ebcb4029fd1b173a60d0b5491aeb23719e7f0666298491e338708c3110ecdee1b0fbbe053b6ff529b520ad6
+  data.tar.gz: 1b847ec7a8e27533a0cde5f39bf0445ac923b3232dd55f85a1676814a73b8ba9dcb2637c48aed327c4cc84474b2b815274572de2ef81f56d5e2b1b9ab64a6387

data/Gemfile

@@ -0,0 +1,9 @@
+source 'https://rubygems.org'
+
+platforms :rbx do
+  gem 'racc'
+  gem 'rubysl', '~> 2.0'
+  gem 'psych'
+end
+
+gemspec

data/Gemfile.lock

@@ -0,0 +1,83 @@
+PATH
+  remote: .
+  specs:
+    snapsearch-client-ruby (0.1.0)
+      addressable (~> 2.0.0)
+      httpi (~> 2.1.0)
+      rack (~> 1.5.0)
+      version (~> 1.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.0.2)
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    ffi (1.9.3-x86-mingw32)
+    formatador (0.2.4)
+    fuubar (1.3.2)
+      rspec (>= 2.14.0, < 3.1.0)
+      ruby-progressbar (~> 1.3)
+    guard (2.4.0)
+      formatador (>= 0.2.4)
+      listen (~> 2.1)
+      lumberjack (~> 1.0)
+      pry (>= 0.9.12)
+      thor (>= 0.18.1)
+    guard-rspec (4.2.5)
+      guard (~> 2.1)
+      rspec (>= 2.14, < 4.0)
+    guard-yard (2.1.0)
+      guard (>= 1.1.0)
+      yard (>= 0.7.0)
+    httpi (2.1.0)
+      rack
+      rubyntlm (~> 0.3.2)
+    listen (2.4.0)
+      celluloid (>= 0.15.2)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    lumberjack (1.0.4)
+    method_source (0.8.2)
+    pry (0.9.12.6-x86-mingw32)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+      win32console (~> 1.3)
+    rack (1.5.2)
+    rake (10.1.1)
+    rb-fsevent (0.9.4)
+    rb-inotify (0.9.3)
+      ffi (>= 0.5.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.5)
+    ruby-progressbar (1.4.1)
+    rubyntlm (0.3.4)
+    slop (3.4.7)
+    thor (0.18.1)
+    timers (1.1.0)
+    version (1.0.0)
+    win32console (1.3.2-x86-mingw32)
+    yard (0.8.7.3)
+
+PLATFORMS
+  x86-mingw32
+
+DEPENDENCIES
+  fuubar (~> 1.3.2)
+  guard-rspec (~> 4.2.5)
+  guard-yard (~> 2.1.0)
+  psych
+  racc
+  rake (~> 10.1.1)
+  rspec (~> 2.14.1)
+  rubysl (~> 2.0)
+  snapsearch-client-ruby!

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 SnapSearch
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,109 @@
+SnapSearch-Client-Ruby
+======================
+
+[![Build Status](https://travis-ci.org/SnapSearch/SnapSearch-Client-Ruby.png?branch=master)](https://travis-ci.org/SnapSearch/SnapSearch-Client-Ruby)
+
+Snapsearch Client Ruby is Ruby based framework agnostic HTTP client library for SnapSearch (https://snapsearch.io/).
+
+SnapSearch provides similar libraries in other languages: https://github.com/SnapSearch/Snapsearch-Clients
+
+Installation
+------------
+
+Usage
+-----
+
+Development
+---------
+
+Get the bundler dependency management tool.
+
+```
+gem install bundler
+```
+
+Install/update all dependencies:
+
+```
+bundle install
+```
+
+See all build tasks:
+
+```
+bundle exec rake -T
+```
+
+Make your changes. Release a new version tag with (see the other `rake version:bump:... etc` tasks):
+
+```
+bundle exec rake version:bump
+```
+
+Synchronise and push the tag to Github:
+
+```
+git push
+git push --tags
+```
+
+Create the gem package:
+
+```
+bundle exec rake gem
+```
+
+Push the gem to Ruby Gems:
+
+```
+gem push pkg/snapsearch-client-ruby-MAJOR.MINOR.PATCH.gem
+```
+
+Setting Up the Detector
+-----------------------
+
+The `Detector` class detects if the incoming request is coming from a robot or not.
+
+Detects if the request came from a search engine robot. It will intercept in cascading order:
+
+1. on a GET request
+2. on an HTTP or HTTPS protocol
+3. not on any ignored robot user agents
+4. not on any route not matching the whitelist
+5. not on any route matching the blacklist
+6. not on any static files that is not a PHP file if it is detected
+7. on requests with _escaped_fragment_ query parameter
+8. on any matched robot user agents
+
+You can customize a few aspects of this process:
+
+#### User Agents
+
+Most robots send a unique `user-agent` HTTP header that we match against to confirm if it indeed a request from a robot.  
+We also ignore certain user agents, such as the SnapSearch robot.
+
+The list of user agents to match and ignore is contained in `resources/robots.json`. You can customize this list through the Detector instance
+you are working with:
+
+```
+# Retrieve the list of user agents to match and ignore:
+detector.robots # => { 'match' => ['SomeRobot', 'AnotherRobot'], 'ignore' => ['SnapSearch'] }
+
+# Add a user agent to match against:
+detector.robots['match'] << 'NewRobot'
+
+# Add a user agent to ignore:
+detector.robots['ignore'] << 'MyRobot'
+
+# Set a new list of user agents to match and ignore:
+detector.robots = { 'match' => ['WebScraper', 'SillyBot'], 'ignore' => ['MyBotToIgnore'] }
+
+# Load from a custom JSON file:
+detector.robots_json = './my_robots.json'
+detector.robots # => { 'match' => ['MyCustomBot', 'AnotherRobot'], 'ignore' => ['MyLoadedBotFromJSON'] }
+```
+
+Tests
+----
+
+Tests are written with RSpec. Run tests with `bundle exec rspec spec/`

data/Rakefile

@@ -0,0 +1,20 @@
+require 'bundler/setup'
+require 'pathname'
+require 'rake/version_task'
+require 'rubygems/package_task'
+require 'rspec/core/rake_task'
+
+gemspec = Pathname.glob( Pathname.new(__FILE__).join('..', '*.gemspec') ).first
+$spec = Gem::Specification.load( gemspec.to_s )
+
+Gem::PackageTask.new($spec) do |task|
+  task.need_zip = false
+end
+
+Rake::VersionTask.new do |task|
+  task.with_git_tag = true
+end
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/examples/rack/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'rack'
+
+gemspec :path => '../..'

data/examples/rack/config.ru

@@ -0,0 +1,88 @@
+# Notes to run:
+#     gem install bundler
+#     bundle install
+#     rackup
+# 
+# Testing:
+#    Visit http://localhost:9292/
+#    Visit http://localhost:9292/?_escaped_fragment_
+
+require 'bundler/setup'
+require 'rack/snap_search'
+
+use Rack::Static, urls: ['/img', '/js', '/css'], root: 'public'
+
+use Rack::SnapSearch do |config|
+    
+    # Required: The email to authenticate with.
+    config.email = 'user@example.com'
+    
+    # Required: The key to authenticate with.
+    config.key = 'API_KEY_HERE'
+    
+    # Optional: The API URL to send requests to.
+    config.api_url = 'https://snapsearch.io/api/v1/robot' # Default
+    
+    # Optional: The CA Cert file to use when sending HTTPS requests to the API.
+    config.ca_cert_file = SnapSearch.root.join('resources', 'cacert.pem') # Default
+    
+    # Optional: Check X-Forwarded-Proto because Heroku SSL Support terminates at the load balancer.
+    config.x_forwarded_proto = true # Default
+    
+    # Optional: Extra parameters to send to the API.
+    config.parameters = {} # Default
+    
+    # Optional: Whitelisted routes. Should be an Array of Regexp instances.
+    config.matched_routes = [] # Default
+    
+    # Optional: Blacklisted routes. Should be an Array of Regexp instances.
+    config.ignored_routes = [] # Default
+    
+    # Optional: A path of the JSON file containing the user agent whitelist & blacklist.
+    config.robots_json = SnapSearch.root.join('resources', 'robots.json') # Default
+    
+    # Optional: A path to the JSON file containing a single Hash with the keys `ignore` and `match`. These keys contain Arrays of Strings (user agents)
+    config.extensions_json = SnapSearch.root.join('resources', 'extensions.json') # Default
+    
+    # Optional: Set to `true` to ignore direct requests to files.
+    config.check_static_files = false # Default
+    
+    # Optional: A block to run when an exception occurs when making requests to the API.
+    config.on_exception do |exception|
+        p exception
+    end
+    
+    # Optional: A block to run before the interception of a bot.
+    config.before_intercept do |url|
+        puts "Before interception\n  URL: #{url}"
+    end
+    
+    # Optional: A block to run after the interception of a bot.
+    config.after_intercept do |url, response|
+        puts "After interception\n  URL: #{url}\n  Response: #{response}"
+    end
+    
+    # Optional: A block to manipulate the response from the SnapSearch API if a bit is intercepted.
+    config.response_callback do |status, headers, body|
+        puts "Response callback\n  Status: #{status}\n  Headers: #{headers}\n  Body: #{body}"
+        
+        [ status, headers, body ]
+    end
+    
+end
+
+class Application
+    
+    def call(env)
+        headers = {
+            'Content-Type'    => 'text/html',
+            'Cache-Control' => 'public, max-age=86400'
+        }
+        body = File.read('public/index.html')
+        
+        [ 200, headers, [body] ]
+    end
+    
+end
+
+run Application.new

data/examples/rack/public/index.html

@@ -0,0 +1,15 @@
+<!DOCTYPE html>
+<html lang='en'>
+  <head>
+    <meta charset='utf-8'>
+    <title>SnapSearch Example</title>
+    <link href="/css/blah.css" media="all" rel="stylesheet" />
+    <!--[if lt IE 9]>
+      <script src="http://html5shim.googlecode.com/svn/trunk/html5.js"></script>
+    <![endif]-->
+    <!-- <script src="/js/blah.js"></script> -->
+  </head>
+  <body>
+    <h1>SnapSearch Example</h1>
+  </body>
+</html>

data/examples/sinatra/Gemfile

@@ -0,0 +1,5 @@
+source 'https://rubygems.org'
+
+gem 'sinatra'
+
+gemspec :path => '../..'

data/examples/sinatra/Gemfile.lock

@@ -0,0 +1,90 @@
+PATH
+  remote: /Users/ryguy/Drive/Code/Ruby/Work/Polycadamy/SnapSearch-Client-Ruby
+  specs:
+    snapsearch-client-ruby (0.0.3)
+      addressable (~> 2.0.0)
+      httpi (~> 2.1.0)
+      version (~> 1.0.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    addressable (2.0.2)
+    celluloid (0.15.2)
+      timers (~> 1.1.0)
+    celluloid-io (0.15.0)
+      celluloid (>= 0.15.0)
+      nio4r (>= 0.5.0)
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    ffi (1.9.3)
+    formatador (0.2.4)
+    fuubar (1.3.2)
+      rspec (>= 2.14.0, < 3.1.0)
+      ruby-progressbar (~> 1.3)
+    guard (2.4.0)
+      formatador (>= 0.2.4)
+      listen (~> 2.1)
+      lumberjack (~> 1.0)
+      pry (>= 0.9.12)
+      thor (>= 0.18.1)
+    guard-rspec (4.2.5)
+      guard (~> 2.1)
+      rspec (>= 2.14, < 4.0)
+    guard-yard (2.1.0)
+      guard (>= 1.1.0)
+      yard (>= 0.7.0)
+    httpi (2.1.0)
+      rack
+      rubyntlm (~> 0.3.2)
+    listen (2.5.0)
+      celluloid (>= 0.15.2)
+      celluloid-io (>= 0.15.0)
+      rb-fsevent (>= 0.9.3)
+      rb-inotify (>= 0.9)
+    lumberjack (1.0.4)
+    method_source (0.8.2)
+    nio4r (1.0.0)
+    pry (0.9.12.6)
+      coderay (~> 1.0)
+      method_source (~> 0.8)
+      slop (~> 3.4)
+    rack (1.5.2)
+    rack-protection (1.5.2)
+      rack
+    rake (10.1.1)
+    rb-fsevent (0.9.4)
+    rb-inotify (0.9.3)
+      ffi (>= 0.5.0)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.7)
+    rspec-expectations (2.14.5)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.5)
+    ruby-progressbar (1.4.1)
+    rubyntlm (0.3.4)
+    sinatra (1.4.4)
+      rack (~> 1.4)
+      rack-protection (~> 1.4)
+      tilt (~> 1.3, >= 1.3.4)
+    slop (3.4.7)
+    thor (0.18.1)
+    tilt (1.4.1)
+    timers (1.1.0)
+    version (1.0.0)
+    yard (0.8.7.3)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fuubar (~> 1.3.2)
+  guard-rspec (~> 4.2.5)
+  guard-yard (~> 2.1.0)
+  rake (~> 10.1.1)
+  rspec (~> 2.14.1)
+  sinatra
+  snapsearch-client-ruby!