diffbot_simple 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 25486e768597bc81f971624d2c26575f8c4a787e
+  data.tar.gz: 57f2933c6f76d5c2e1c61144d86be56f1d4645fd
+SHA512:
+  metadata.gz: 6effb80f0a1d27ded4133eb17287335f0a4e0203b63453745bd48cde2ba5c50a9a47805d015a96279cfd280a7b56d9131dfa15a7cc7c23cf7cbcfa39ea972ea7
+  data.tar.gz: 205b68c3ef70977c1570c8511324390edfe021d24e2cbcdf723fa65e77298e2055a22e8ad3645a118118cbd9600beb6506c1918d071b44b54ee67cdd8022ff2f

data/.autotest

@@ -0,0 +1,3 @@
+Autotest.add_hook :initialize do |at|
+  %w{.git coverage}.each {|exception| at.add_exception(exception)}
+end

data/.gitignore

@@ -0,0 +1,18 @@
+*.gem
+*.rbc
+.bundle
+.config
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+Gemfile.lock
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/.rspec

@@ -0,0 +1,2 @@
+--color
+--format documentation

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.0
+  - 2.0.0

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+gem 'coveralls', require: false
+# Specify your gem's dependencies in diffbot_simple.gemspec
+gemspec

data/LICENSE

@@ -0,0 +1,20 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Lars Krantz
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,109 @@
+[![Coverage Status](https://coveralls.io/repos/larskrantz/diffbot_simple/badge.png)](https://coveralls.io/r/larskrantz/diffbot_simple)
+[![Build Status](https://travis-ci.org/larskrantz/diffbot_simple.png?branch=master)](https://travis-ci.org/larskrantz/diffbot_simple)
+[![Code Climate](https://codeclimate.com/github/larskrantz/diffbot_simple.png)](https://codeclimate.com/github/larskrantz/diffbot_simple)
+
+DiffbotSimple
+=============
+
+A simple, nothing-fancy, helper for the [Diffbot API](http://www.diffbot.com/).
+
+Will not objectify any responses, just pass on the json data as hash with symbolized keys.
+One exception to that rule, when using CrawlBot and requesting a single_crawl, it will return the single item in the :jobs-array, and when requesting all, it will return the array in :jobs.
+Send options to the api as named args, se usage below with article and fields-argument.
+
+## Installation
+```ruby
+gem 'diffbot-simple'
+```
+
+## Dependencies
+* Ruby 2.0 or 2.1
+* [rest-core](https://github.com/cardinalblue/rest-core)
+* [multi_json](https://github.com/intridea/multi_json)
+
+
+## Usage
+```ruby
+require 'diffbot_simple'
+
+token = "my_diffbot_assigned_token"
+client = DiffbotSimple::V2::Client.new token: token
+
+article = client.article
+url = "http://www.xconomy.com/san-francisco/2012/07/25/diffbot-is-using-computer-vision-to-reinvent-the-semantic-web/"
+# Pass on diffbot parameters as options to the call
+diffbot_response_as_symbolized_hash = article.single_article url: url, fields: "icon,title"
+# =>
+{
+  icon: "http://www.xconomy.com/wordpress/wp-content/themes/xconomy/images/favicon.ico",
+  author: "Wade Roush",
+  date: "7/25/12",
+  text: "...",
+# and more, see http://www.diffbot.com/products/automatic/article/
+}
+```
+
+### Supports these Diffbot apis
+Please see [Diffbot Help and Documentation](http://www.diffbot.com/dev/docs/) for details and arguments.
+Check the spec-directory too.
+
+```ruby
+require 'diffbot_simple'
+
+token = "my_diffbot_assigned_token"
+client = DiffbotSimple::V2::Client.new token: token
+url = "http://some_url_to_check"
+
+# Custom API
+custom = client.custom name: "my_custom_api_name" 
+response = custom.single_custom url: url 
+
+# Analyze API (beta)
+analysis = client.analyze 
+response = analyze.single_analysis url: url
+
+# Article API
+article = client.article
+response = article.single_article url: url
+
+# Image API
+image = client.image
+response = image.single_image url: url
+
+# Product API
+product = client.product
+response = product.single_product url: url
+
+# Crawlbot API
+crawlbot = client.crawlbot
+all_my_crawls = crawlbot.all
+current_settings = crawlbot.single_crawl name: "my_crawl"
+# shorthand for using apiUrl, use the api object from client, 
+# it will create a correct value for you 
+# (custom, image, article, product or analyze for automatic)
+# A call to single_crawl will create if not exists or update settings
+settings = crawlbot.single_crawl name: "my_new_crawl", onlyProcessIfNew: 0, seeds: "http://www.upptec.se", apiUrl: custom
+crawlbot.pause name: "my_new_crawl"
+crawlbot.unpause name: "my_new_crawl"
+crawlbot.restart name: "my_new_crawl"
+result = crawlbot.result "my_new_crawl" # shorthand for downloading the json that are specifed in :downloadJson
+crawlbot.delete name: "my_new_crawl" 
+```
+
+### On error
+If Diffbot returns an error, it will raise and fill `DiffbotSimple::V2::DiffbotError` with passed on info, as stated in [http://www.diffbot.com/dev/docs/error/](http://www.diffbot.com/dev/docs/error/) and put errorCode in `:error_code` and error in `:message` .
+
+## TODO
+* Frontpage API
+* Bulk API
+* Async http fetching
+* Batch API
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/diffbot_simple.gemspec

@@ -0,0 +1,32 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'diffbot_simple/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "diffbot_simple"
+  spec.version       = DiffbotSimple::VERSION
+  spec.authors       = ["Lars Krantz"]
+  spec.email         = ["lars.krantz@alaz.se"]
+  spec.summary       = %q{A simple, nothing-fancy, helper for the Diffbot API}
+  # spec.description   = %q{}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.required_ruby_version = "~> 2.0"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "autotest-standalone"
+  spec.add_development_dependency "webmock", "~> 1.17"
+
+  spec.add_runtime_dependency "rest-core", "~> 2.1"
+  spec.add_runtime_dependency "multi_json"
+
+end

data/lib/diffbot_simple.rb

@@ -0,0 +1,14 @@
+require 'diffbot_simple/version'
+require 'diffbot_simple/symbolize'
+require 'diffbot_simple/v2/diffbot_error'
+require 'diffbot_simple/v2/api_helper'
+require 'diffbot_simple/v2/client'
+require 'diffbot_simple/v2/crawlbot'
+require 'diffbot_simple/v2/article'
+require 'diffbot_simple/v2/custom'
+require 'diffbot_simple/v2/product'
+require 'diffbot_simple/v2/image'
+require 'diffbot_simple/v2/analyze'
+
+module DiffbotSimple
+end

data/lib/diffbot_simple/symbolize.rb

@@ -0,0 +1,29 @@
+module DiffbotSimple
+	module Symbolize
+		private
+    def y_combinator(&f)
+      lambda do |g|
+        f.call {|*args| g[g][*args]}
+      end.tap {|g| break g[g]}
+    end
+
+    def symbolize hash
+    	return hash unless hash.kind_of? Hash or hash.kind_of? Array
+      sym_hash = y_combinator do |&f| 
+      	lambda do |h|
+	      	if h.kind_of? Array
+	      		h.map {|r| f.call(r)}
+	      	else
+	        	h.reduce({}) do |memo,(k,v)|
+	          	v = f.call(v) if v.kind_of? Hash
+	        		v = v.map {|u| f.call(u)} if v.kind_of? Array
+	        		memo[k.to_sym] = v
+	        		memo
+	        	end
+		    	end
+	    	end
+	    end
+  	  sym_hash.call hash
+  	end
+	end
+end

data/lib/diffbot_simple/v2/analyze.rb

@@ -0,0 +1,18 @@
+module DiffbotSimple::V2
+	# Complies to http://www.diffbot.com/dev/docs/analyze
+	class Analyze
+		include ApiHelper
+		def post_initialize
+			@api = :analyze
+		end
+		def to_crawl_api_url
+			default = super
+			"#{default}?mode=auto"
+		end
+		def single_analysis url: nil, **options
+			raise ArgumentError.new "Must pass an url to fetch" unless url
+			execute_call options.merge(url: url)
+		end
+		alias :single_analyze :single_analysis
+	end
+end

data/lib/diffbot_simple/v2/api_client.rb

@@ -0,0 +1,10 @@
+require 'multi_json'
+require 'rest-core'
+require 'rest-client'
+module DiffbotSimple::V2
+	ApiClient = RestCore::Builder.client do
+		use RestCore::DefaultSite , 'http://api.diffbot.com/v2/'
+  	use RestCore::JsonResponse, true
+  	use RestCore::DefaultHeaders, {}
+	end
+end

data/lib/diffbot_simple/v2/api_helper.rb

@@ -0,0 +1,44 @@
+module DiffbotSimple::V2
+	# Complies to http://www.diffbot.com/dev/docs/crawl/
+	module ApiHelper
+		include DiffbotSimple::Symbolize
+		def initialize api_client: nil, token: nil
+			@api_client = api_client
+			@token = token
+			post_initialize
+		end
+		def post_initialize
+			raise "Must overload to set api path"
+		end
+		def to_crawl_api_url
+			"#{api_client.site}#{api}"
+		end
+		private
+		attr_reader :token, :api_client, :api
+		def execute_call custom_headers: nil, method: :get, payload: nil, **options
+			args = create_from_options options
+			opts = {}
+			opts[:headers] = custom_headers if custom_headers
+			response = api_client.get(api, args, opts) if method == :get
+			response = api_client.post(api, payload, args, opts) if method == :post
+			cleanup response
+		end
+		def cleanup response
+			result_hash = symbolize response
+			raise_if_error_response result_hash
+			result_hash
+		end
+		def create_from_options options
+			merged = options.merge({token: token})
+			merged[:apiUrl] = expand_api_url merged[:apiUrl] if merged[:apiUrl]
+			merged
+		end
+		def expand_api_url api_url
+			api_url.to_crawl_api_url if api_url.respond_to?(:to_crawl_api_url)
+		end
+		def raise_if_error_response result_from_diffbot
+			return unless result_from_diffbot[:error]
+			raise DiffbotError.new(result_from_diffbot[:error], result_from_diffbot[:errorCode])
+		end
+	end
+end

data/lib/diffbot_simple/v2/article.rb

@@ -0,0 +1,18 @@
+module DiffbotSimple::V2
+	# Complies to http://www.diffbot.com/dev/docs/article
+	class Article
+		include ApiHelper
+		def post_initialize
+			@api = :article
+		end
+		def single_article url: nil, custom_headers: nil, body: nil, **options
+			raise ArgumentError.new "Must pass an url for the article api to fetch" unless url
+			if body
+				custom_headers ||= {}
+				custom_headers['Content-Type'] = 'text/html'
+				options[:method] = :post
+			end
+			execute_call options.merge(url: url, custom_headers: custom_headers, payload: body)
+		end
+	end
+end

data/lib/diffbot_simple/v2/client.rb

@@ -0,0 +1,30 @@
+require_relative 'api_client'
+module DiffbotSimple::V2
+	class Client
+		def initialize token: nil
+			raise ArgumentError.new("Must supply developer token") if token.to_s.empty?
+			@token = token
+			@api_client = ApiClient.new
+		end
+		def crawlbot
+			Crawlbot.new api_client: api_client, token: token
+		end
+		def article
+			Article.new api_client: api_client, token: token
+		end
+		def custom name: nil
+			Custom.new api_client: api_client, token: token, name: name
+		end
+		def product
+			Product.new api_client: api_client, token: token
+		end
+		def image
+			Image.new api_client: api_client, token: token
+		end
+		def analyze
+			Analyze.new api_client: api_client, token: token
+		end
+		private
+		attr_reader :token, :api_client
+	end
+end

data/lib/diffbot_simple/v2/crawlbot.rb

@@ -0,0 +1,75 @@
+module DiffbotSimple::V2
+	# Complies to http://www.diffbot.com/dev/docs/crawl/
+	class Crawlbot
+		include ApiHelper
+		def post_initialize
+			@api = :crawl
+		end
+		# Get all your crawls as an array
+		# The "jobs" parameter is stripped and only the array is returned
+		#
+		# @return [Array] your jobs from the "jobs"-array in api response
+		def all
+			execute_call()[:jobs]
+		end
+		
+		# Gets, creates or updates a named crawl
+		#
+		# @name [String] name of the crawl to get/create/update
+		# @**options options from http://www.diffbot.com/dev/docs/crawl/ when updating or creating a crawl
+		# @return [Hash] with current parameters for the single crawl
+		def single_crawl name: nil, **options
+			raise ArgumentError.new "Must pass a name for the crawl" unless name
+			response = execute_call options.merge(name: name)
+			jobs = response[:jobs]
+			jobs.first
+		end
+
+		# Deletes a crawl
+		#
+		# @name [String] name of crawl to delete
+		# @return [Hash] statusmessage from diffbot, for example: {response: "Successfully deleted job."}
+		def delete name: nil
+			raise ArgumentError.new "Must pass a name for the crawl to delete" unless name
+			execute_call name: name, delete: 1
+		end
+
+		# Pauses a crawl
+		#
+		# @name [String] name of the crawl to pause
+		# @return [Hash] with current parameters for the single crawl
+		def pause name: nil
+			single_crawl name: name, pause: 1
+		end
+
+		# Unpauses/ resumes a crawl
+		#
+		# @name [String] name of the crawl to unpause
+		# @return [Hash] with current parameters for the single crawl
+		def unpause name: nil
+			single_crawl name: name, pause: 0
+		end
+
+		# Restarts a crawl
+		#
+		# @name [String] name of the crawl to restart
+		# @return [Hash] with current parameters for the crawl in jobs-key (as an array), and a response-text
+		def restart name: nil
+			raise ArgumentError.new "Must pass a name for the crawl to restart" unless name
+			execute_call name: name, restart: 1
+		end
+
+		# Get the crawl-result (downloadJson from diffbot crawl)
+		#
+		# @name [String] name of the crawl to restart
+		# @return [Array] of results (hashes)
+		def result name: name
+			crawl = single_crawl name: name
+			download_url = crawl[:downloadJson]
+			response = api_client.get download_url
+			symbolize response
+		end
+
+
+	end
+end

data/lib/diffbot_simple/v2/custom.rb

@@ -0,0 +1,19 @@
+module DiffbotSimple::V2
+	# Complies to http://www.diffbot.com/dev/docs/custom
+	class Custom
+		include ApiHelper
+		attr_reader :name
+		def initialize name: nil, **options
+			raise ArgumentError.new "Must pass a name for the custom api" unless name
+			@name = name
+			super options
+		end
+		def post_initialize
+			@api = "api/#{CGI::escape(name)}"
+		end
+		def single_custom url: nil, **options
+			raise ArgumentError.new "Must pass an url for the custom api to fetch" unless url
+			execute_call options.merge(url: url)
+		end
+	end
+end

data/lib/diffbot_simple/v2/diffbot_error.rb

@@ -0,0 +1,9 @@
+module DiffbotSimple::V2
+	class DiffbotError < StandardError
+		attr_reader :error_code
+		def initialize error_message, error_code = nil
+			@error_code = error_code
+			super error_message
+		end
+	end
+end

data/lib/diffbot_simple/v2/image.rb

@@ -0,0 +1,13 @@
+module DiffbotSimple::V2
+	# Complies to http://www.diffbot.com/dev/docs/image
+	class Image
+		include ApiHelper
+		def post_initialize
+			@api = :image
+		end
+		def single_image url: nil, **options
+			raise ArgumentError.new "Must pass an url to fetch" unless url
+			execute_call options.merge(url: url)
+		end
+	end
+end

data/lib/diffbot_simple/v2/product.rb

@@ -0,0 +1,13 @@
+module DiffbotSimple::V2
+	# Complies to http://www.diffbot.com/dev/docs/product
+	class Product
+		include ApiHelper
+		def post_initialize
+			@api = :product
+		end
+		def single_product url: nil, **options
+			raise ArgumentError.new "Must pass an url to fetch" unless url
+			execute_call options.merge(url: url)
+		end
+	end
+end

data/lib/diffbot_simple/version.rb

@@ -0,0 +1,3 @@
+module DiffbotSimple
+  VERSION = "0.0.3"
+end

data/spec/analyze_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+module DiffbotSimple::V2
+	describe Analyze do
+		let(:client) { Client.new token: token }
+		let(:url) { "http://foo.bar" }
+		let(:single_response) { { body: {url: url, foo: "bar"}.to_json} }
+		let(:analyze) { client.analyze }
+		let(:api_url) { "#{base_url}/analyze" }
+		shared_examples_for "an analyze request" do
+			before(:each) { stubbed_request }
+			it "should make a valid request to the analyze api" do
+				subject
+				expect(stubbed_request).to have_been_requested
+			end
+			it "should return the response body as an symbolized hash" do
+				expect(subject).to eql JSON.parse(single_response[:body], symbolize_names: true)
+			end
+			it "should respond and return the apis url in to_crawl_api_url" do
+				expect(analyze.to_crawl_api_url).to eql "#{api_url}?mode=auto"
+			end
+		end
+		context "when asking for an analyze with no options" do
+			let(:subject) { analyze.single_analysis url: url}
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url}).to_return(single_response) }
+			it_should_behave_like "an analyze request"
+		end
+		context "when asking for an analyze with analyze options" do
+			let(:fields) {"a,b,c"}
+			let(:mode) { "article" }
+			let(:stats) { true }
+			let(:subject) { analyze.single_analyze url: url, stats: stats, mode: mode, fields: fields }
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url, stats: stats.to_s, mode: mode, fields: fields}).to_return(single_response) }
+			it_should_behave_like "an analyze request"
+		end
+	end
+end

data/spec/image_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+module DiffbotSimple::V2
+	describe Image do
+		let(:client) { Client.new token: token }
+		let(:url) { "http://foo.bar" }
+		let(:single_response) { { body: {url: url, foo: "bar"}.to_json} }
+		let(:image) { client.image }
+		let(:api_url) { "#{base_url}/image" }
+		shared_examples_for "an image request" do
+			before(:each) { stubbed_request }
+			it "should make a valid request to the image api" do
+				subject
+				expect(stubbed_request).to have_been_requested
+			end
+			it "should return the response body as an symbolized hash" do
+				expect(subject).to eql JSON.parse(single_response[:body], symbolize_names: true)
+			end
+			it "should respond and return the apis url in to_crawl_api_url" do
+				expect(image.to_crawl_api_url).to eql api_url
+			end
+		end
+		context "when asking for an image with no options" do
+			let(:subject) { image.single_image url: url}
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url}).to_return(single_response) }
+			it_should_behave_like "an image request"
+		end
+		context "when asking for an image with image options" do
+			let(:fields) {"a,b,c"}
+			let(:callback) { "my_callback" }
+			let(:timeout) { 4200 }
+			let(:subject) { image.single_image url: url, timeout: timeout, callback: callback, fields: fields }
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url, timeout: timeout, callback: callback, fields: fields}).to_return(single_response) }
+			it_should_behave_like "an image request"
+		end
+	end
+end

data/spec/product_spec.rb

@@ -0,0 +1,37 @@
+require 'spec_helper'
+
+module DiffbotSimple::V2
+	describe Product do
+		let(:client) { Client.new token: token }
+		let(:url) { "http://foo.bar" }
+		let(:single_response) { { body: {url: url, foo: "bar"}.to_json} }
+		let(:product) { client.product }
+		let(:api_url) { "#{base_url}/product" }
+		shared_examples_for "a product request" do
+			before(:each) { stubbed_request }
+			it "should make a valid request to the product api" do
+				subject
+				expect(stubbed_request).to have_been_requested
+			end
+			it "should return the response body as an symbolized hash" do
+				expect(subject).to eql JSON.parse(single_response[:body], symbolize_names: true)
+			end
+			it "should respond and return the apis url in to_crawl_api_url" do
+				expect(product.to_crawl_api_url).to eql api_url
+			end
+		end
+		context "when asking for a product with no options" do
+			let(:subject) { product.single_product url: url}
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url}).to_return(single_response) }
+			it_should_behave_like "a product request"
+		end
+		context "when asking for a product with product options" do
+			let(:fields) {"a,b,c"}
+			let(:callback) { "my_callback" }
+			let(:timeout) { 4200 }
+			let(:subject) { product.single_product url: url, timeout: timeout, callback: callback, fields: fields }
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url, timeout: timeout, callback: callback, fields: fields}).to_return(single_response) }
+			it_should_behave_like "a product request"
+		end
+	end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,28 @@
+require 'diffbot_simple'
+require 'webmock/rspec'
+require 'coveralls'
+Coveralls.wear!
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end
+
+def base_url 
+	"http://api.diffbot.com/v2"
+end
+def token
+	"TestToken"
+end

data/spec/v2/article_spec.rb

@@ -0,0 +1,53 @@
+require 'spec_helper'
+
+module DiffbotSimple::V2
+	describe Article do
+		let(:client) { Client.new token: token }
+		let(:url) { "http://foo.bar" }
+		let(:article) { client.article }
+		let(:api_url) { "#{base_url}/article" }
+		let(:single_article_response) do 
+			{ 
+				body:  
+					{ :type=>"article", :icon=>"http://www.diffbot.com/favicon.ico", :title=>"Diffbot's New Product API Teaches Robots to Shop Online", :author=>"John Davi"}.to_json 
+			} 
+		end
+
+		shared_examples_for "an article request" do
+			before(:each) { stubbed_request }
+			it "should make a valid request to the article api" do
+				subject
+				expect(stubbed_request).to have_been_requested
+			end
+			it "should return the response body as an symbolized hash" do
+				expect(subject).to eql JSON.parse(single_article_response[:body], symbolize_names: true)
+			end
+			it "should respond and return the apis url in to_crawl_api_url" do
+				expect(article.to_crawl_api_url).to eql api_url
+			end
+		end
+		context "when asking for a single article with no additional options" do
+			let(:subject) { article.single_article url: url }
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url }).to_return(single_article_response) }
+			it_should_behave_like "an article request"
+		end
+
+		context "when asking for a single article with some additional options" do
+			let(:subject) { article.single_article url: url, fields: "meta,querystring,images(*)" }
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url, fields: "meta,querystring,images(*)" }).to_return(single_article_response) }
+			it_should_behave_like "an article request"
+		end
+		context "when asking for a single article with custom headers" do
+			let(:subject) { article.single_article url: url, custom_headers: { "X-Forward-User-Agent" => "I AM CHROME" } }
+			let(:stubbed_request) { stub_request(:get, "#{base_url}/article").with(query: {token: token, url: url }, headers: { "X-Forward-User-Agent" => "I AM CHROME" }).to_return(single_article_response) }
+			it_should_behave_like "an article request"
+		end
+
+		context "when posting a body directly to analyze" do
+			let(:body) { "<html><fake><body>" }
+			let(:subject) { article.single_article url: url, body: body }
+			let(:stubbed_request) { stub_request(:post, "#{base_url}/article").with(query: {token: token, url: url }, body: body).to_return(single_article_response) }
+			it_should_behave_like "an article request"
+		end
+	end
+end

data/spec/v2/client_spec.rb

@@ -0,0 +1,20 @@
+require 'spec_helper'
+module DiffbotSimple::V2
+	describe Client do
+		let(:subject) { Client.new token: "FOO_TOKEN" }
+		context "when initializing with a token" do
+			it "should not raise an error" do
+				expect{subject}.to_not raise_error
+			end
+			it "should respond to crawlbot" do
+				expect(subject).to respond_to :crawlbot
+			end
+		end
+		context 'when initializing without token' do
+			let(:subject) { Client.new }
+			it "should raise an ArgumentError" do
+				expect{subject}.to raise_error ArgumentError
+			end
+		end
+	end
+end

data/spec/v2/crawlbot_spec.rb

@@ -0,0 +1,113 @@
+require 'spec_helper'
+module DiffbotSimple::V2
+	describe Crawlbot do
+		let(:client) { Client.new token: token }
+		let(:custom) { client.custom name: "my_custom_api" }
+		let(:single_crawl_response_body) {{body: '{"jobs":[{"foo":"bar"}]}'}}
+		let(:name) { "crawl_name"}
+		let(:subject) { client.crawlbot }
+		context "when retreiving all crawls" do
+			let(:all) { stubbed_request;subject.all; }
+			let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: {token: token}).to_return( single_crawl_response_body) }
+			it "should make a request to /crawl with the token as argument" do
+				all
+				expect(stubbed_request).to have_been_requested
+			end
+			it "should return an crawl array " do
+				expect(all).to eql([{ foo: 'bar' }])
+			end
+		end
+		context "when asking for a named crawl" do
+			let(:named_crawl) { stubbed_request; subject.single_crawl name: name, onlyProcessIfNew: 0, apiUrl: custom }
+			let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, onlyProcessIfNew: 0, apiUrl: custom.to_crawl_api_url}).to_return single_crawl_response_body() }
+			it "should make a request to /crawl with the token and name as arguments" do
+				named_crawl
+				expect(stubbed_request).to have_been_requested
+			end
+			it "should return an crawl hash" do
+				expect(named_crawl).to eql({ foo: 'bar' })
+			end
+		end
+		context "when deleting a named crawl" do
+			let(:delete) { stubbed_request; subject.delete name: name }
+			let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, delete: 1 }).to_return(body: '{"response":"Successfully deleted job." }') }
+			it "should make the request to delete it" do
+				delete
+				expect(stubbed_request).to have_been_requested
+			end
+			it "should return the faked response" do
+				expect(delete).to eql({response: "Successfully deleted job."})
+			end
+		end
+		context "when pausing or unpausing a named crawl" do
+			let(:pause) { stubbed_pause_request; subject.pause name: name }
+			let(:unpause) { stubbed_unpause_request; subject.unpause name: name }
+			let(:stubbed_pause_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, pause: 1 }).to_return(single_crawl_response_body) }
+			let(:stubbed_unpause_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, pause: 0 }).to_return(single_crawl_response_body) }
+			it "should make the request to pause it" do
+				pause
+				expect(stubbed_pause_request).to have_been_requested
+			end
+			it "should make the request to unpause it" do
+				unpause
+				expect(stubbed_unpause_request).to have_been_requested
+			end
+		end
+		context "when restarting a named crawl" do
+			let(:restart) { stubbed_request; subject.restart name: name }
+			let(:stubbed_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token, restart: 1 }).to_return(single_crawl_response_body) }
+			it "should make the request to restart it" do
+				restart
+				expect(stubbed_request).to have_been_requested
+			end
+		end
+		context "when requesting a crawls result" do
+			let(:result) { stubbed_crawl_request;stubbed_result_request; subject.result name: name }
+			let(:test_download_url) { "http://google.com" }
+			let(:stubbed_crawl_request) { stub_request(:get, "#{base_url}/crawl").with(query: { name: name, token: token}).to_return(body: "{\"jobs\":[{\"downloadJson\":\"#{test_download_url}\"}]}") }
+			let(:stubbed_result_request) { stub_request(:get, test_download_url).to_return(body: "[{'f':'b'}]") }
+			it "should make the two requests to get the results" do
+				result
+				expect(stubbed_crawl_request).to have_been_requested
+				expect(stubbed_result_request).to have_been_requested
+			end
+		end
+		describe "if diffbots response is an error" do
+			let(:error_from_diffbot) { { error: "Your token has exceeded the allowed number of calls, or has otherwise been throttled for API abuse.", errorCode: 429 }.to_json }
+			let(:stubbed_request) { stub_request(:get, /#{base_url}\/crawl*/).to_return(body: error_from_diffbot) }
+			shared_examples_for "an error" do
+				it "and raise an DiffbotError" do
+					expect{raiser}.to raise_error DiffbotError
+				end
+			end
+			context "on all" do
+				let(:raiser) { stubbed_request;subject.all; }
+				it_should_behave_like "an error"
+			end
+			context "on single_crawl" do
+				let(:raiser) { stubbed_request;subject.single_crawl name: name; }
+				it_should_behave_like "an error"
+			end
+			context "on delete" do
+				let(:raiser) { stubbed_request;subject.delete name: name; }
+				it_should_behave_like "an error"
+			end
+			context "on pause" do
+				let(:raiser) { stubbed_request;subject.pause name: name; }
+				it_should_behave_like "an error"
+			end
+			context "on unpause" do
+				let(:raiser) { stubbed_request;subject.unpause name: name; }
+				it_should_behave_like "an error"
+			end
+			context "on restart" do
+				let(:raiser) { stubbed_request;subject.restart name: name; }
+				it_should_behave_like "an error"
+			end
+			context "on result" do
+				let(:raiser) { stubbed_request;subject.result name: name; }
+				it_should_behave_like "an error"
+			end
+		end
+	end
+end

data/spec/v2/custom_spec.rb

@@ -0,0 +1,36 @@
+
+require 'spec_helper'
+
+module DiffbotSimple::V2
+	describe Custom do
+		let(:client) { Client.new token: token }
+		let(:url) { "http://foo.bar" }
+		let(:single_response) { { body: {url: url, foo: "bar"}.to_json} }
+		let(:custom_name) { "foobar" }
+		let(:api_url) { "#{base_url}/api/#{CGI::escape(custom_name)}" }
+		let(:custom) { client.custom name: custom_name }
+		shared_examples_for "a custom request" do
+			before(:each) { stubbed_request }
+			it "should make a valid request to the custom api" do
+				subject
+				expect(stubbed_request).to have_been_requested
+			end
+			it "should return the response body as an symbolized hash" do
+				expect(subject).to eql JSON.parse(single_response[:body], symbolize_names: true)
+			end
+			it "should respond and return the apis url in to_crawl_api_url" do
+				expect(custom.to_crawl_api_url).to eql api_url
+			end
+		end
+		context "when asking for a custom api with no options" do
+			let(:subject) { custom.single_custom url: url }
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url }).to_return(single_response) }
+			it_should_behave_like "a custom request"
+		end
+		context "when asking for a custom api with custom options" do
+			let(:subject) { custom.single_custom url: url, timeout: 12000, callback: "my_callback" }
+			let(:stubbed_request) { stub_request(:get, api_url).with(query: {token: token, url: url, timeout: 12000, callback: "my_callback" }).to_return(single_response) }
+			it_should_behave_like "a custom request"
+		end
+	end
+end

metadata

@@ -0,0 +1,180 @@
+--- !ruby/object:Gem::Specification
+name: diffbot_simple
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Lars Krantz
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-02-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: autotest-standalone
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.17'
+- !ruby/object:Gem::Dependency
+  name: rest-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- lars.krantz@alaz.se
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".autotest"
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- diffbot_simple.gemspec
+- lib/diffbot_simple.rb
+- lib/diffbot_simple/symbolize.rb
+- lib/diffbot_simple/v2/analyze.rb
+- lib/diffbot_simple/v2/api_client.rb
+- lib/diffbot_simple/v2/api_helper.rb
+- lib/diffbot_simple/v2/article.rb
+- lib/diffbot_simple/v2/client.rb
+- lib/diffbot_simple/v2/crawlbot.rb
+- lib/diffbot_simple/v2/custom.rb
+- lib/diffbot_simple/v2/diffbot_error.rb
+- lib/diffbot_simple/v2/image.rb
+- lib/diffbot_simple/v2/product.rb
+- lib/diffbot_simple/version.rb
+- spec/analyze_spec.rb
+- spec/image_spec.rb
+- spec/product_spec.rb
+- spec/spec_helper.rb
+- spec/v2/article_spec.rb
+- spec/v2/client_spec.rb
+- spec/v2/crawlbot_spec.rb
+- spec/v2/custom_spec.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - "~>"
+    - !ruby/object:Gem::Version
+      version: '2.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A simple, nothing-fancy, helper for the Diffbot API
+test_files:
+- spec/analyze_spec.rb
+- spec/image_spec.rb
+- spec/product_spec.rb
+- spec/spec_helper.rb
+- spec/v2/article_spec.rb
+- spec/v2/client_spec.rb
+- spec/v2/crawlbot_spec.rb
+- spec/v2/custom_spec.rb