scrapinghub-client 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5388fb17c08f77b3091bdb87b8be779237f21404
+  data.tar.gz: 3756c4dcaefbf715608a9d958beb986f18b6f8da
+SHA512:
+  metadata.gz: 457ceddece99e63f370213e5b17a667d414d315f750cc7ff675dc98e7ce56f179c03fa84b9d73c6951511a6adb2e68c80ef28bfccfe3fa9cc7ea19ec305a05e6
+  data.tar.gz: ed0dcb6729211f0dc44e57ebe047e71ccc8ea87b24b21e1b7d30ca6be223c3f361f61e905ff005c15ce3d9ef9d38f96e60513e97c694db9bad92f3b62407f38d

data/.gitignore

@@ -0,0 +1 @@
+Gemfile.lock

data/.rspec

@@ -0,0 +1,3 @@
+--tty
+--color
+--profile

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - "2.2"
+  - "2.2.0"
+  - "2.1.0"
+  - ruby-head

data/Gemfile

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gemspec
+
+gem "codeclimate-test-reporter", require: nil

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Abe Voelker
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,46 @@
+Scrapinghub client
+==================
+
+[![Build Status](https://travis-ci.org/abevoelker/scrapinghub-client.svg?branch=master)][travis]
+[![Dependency Status](https://gemnasium.com/abevoelker/scrapinghub-client.svg)][gemnasium]
+[![Code Climate](https://codeclimate.com/github/abevoelker/scrapinghub-client/badges/gpa.svg)][codeclimate]
+[![Test Coverage](https://codeclimate.com/github/abevoelker/scrapinghub-client/badges/coverage.svg)][codeclimate]
+
+Ruby client for the [Scrapinghub API][api]. So far it only supports the [Jobs API][jobs-api] (pull requests welcome).
+
+This library tries to take an FP-ish approach. It uses the [contracts gem][contracts] for validating function input and output types (see the [docs][] for the full list of functions and their types) and the [kleisli gem][kleisli] for returning composition-friendly output types. Outputs will be a `Try::Failure` if an exception was raised (e.g. a network timeout), a `Left` if the Scrapinghub API returns failure, or a `Right` if the operation was successful.
+
+The Kleisli gem [introductory blog post][kleisli-blog] gives some great examples on how to work with the output types.
+
+Install
+--------
+
+Add to Gemfile:
+
+```
+gem "scrapinghub-client"
+```
+
+**Note**: although the gem is named `scrapinghub-client`, the gem's namespace is `Scrapinghub`.
+
+Example
+--------
+
+```ruby
+require "scrapinghub-client"
+
+j = Scrapinghub::Jobs.new(api_key: 'abc123')
+j.schedule(project: 123, spider: "foo", add_tag: "bar", extra: { DOWNLOAD_DELAY: "0.5" })
+  .fmap{|r| puts "Job scheduled! Jobid: #{r['jobid']}"}
+  .or{|f| puts "Failed to schedule job! Reason: #{f.inspect}"}
+```
+
+[travis]: https://travis-ci.org/abevoelker/scrapinghub-client
+[gemnasium]: https://gemnasium.com/abevoelker/scrapinghub-client
+[codeclimate]: https://codeclimate.com/github/abevoelker/scrapinghub-client
+[api]: http://doc.scrapinghub.com/api.html
+[jobs-api]: http://doc.scrapinghub.com/jobs.html
+[docs]: http://www.rubydoc.info/github/abevoelker/scrapinghub-client/master/Scrapinghub/Jobs
+[contracts]: https://github.com/egonSchiele/contracts.ruby/blob/master/TUTORIAL.md
+[kleisli]: https://github.com/txus/kleisli
+[kleisli-blog]: http://thoughts.codegram.com/cleaner-safer-ruby-api-clients-with-kleisli/

data/Rakefile

@@ -0,0 +1,22 @@
+require "rspec"
+require "rspec/core/rake_task"
+require "rake/testtask"
+require "bundler"
+
+Rake::TestTask.new("spec:unit") do |t|
+  t.libs << ["lib", "spec"]
+  t.pattern = "spec/unit/**/*spec.rb"
+end
+
+RSpec::Core::RakeTask.new("spec:integration") do |t|
+  t.pattern = "spec/integration/**/*spec.rb"
+end
+
+Rake::TestTask.new("spec") do |t|
+  t.libs << ["lib", "spec"]
+  t.pattern = "spec/**/*spec.rb"
+end
+
+task :default => "spec"
+
+Bundler::GemHelper.install_tasks

data/lib/scrapinghub-client.rb

@@ -0,0 +1,3 @@
+require "scrapinghub/version"
+require "scrapinghub/jobs"
+ScrapingHub = Scrapinghub

data/lib/scrapinghub/jobs.rb

@@ -0,0 +1,175 @@
+require "contracts"
+require "kleisli"
+require "httparty"
+
+module Scrapinghub
+  class Jobs
+    include Contracts
+    include HTTParty
+    disable_rails_query_string_format
+    base_uri "dash.scrapinghub.com"
+
+    Contract ({ :api_key => String }) => Any
+    # Initialize a new Jobs API client
+    #
+    # @param api_key [String] Scrapinghub API key
+    #
+    # @return Object
+    def initialize(api_key:)
+      @api_key = api_key
+    end
+
+    Contract KeywordArgs[:project => Nat,
+                         :job => Optional[Or[String, ArrayOf[String]]],
+                         :spider => Optional[String],
+                         :state => Optional[Or["pending", "running", "finished"]],
+                         :has_tag => Optional[Or[String, ArrayOf[String]]],
+                         :lacks_tag => Optional[Or[String, ArrayOf[String]]],
+                         :count => Optional[Nat] ] => Or[Kleisli::Try, Kleisli::Either]
+    # Retrieve information about jobs.
+    #
+    # @param project [Fixnum] the project's numeric ID
+    # @param job [String, Array<String>] (optional) ID(s) of specific jobs to
+    #   retrieve
+    # @param spider [String] (optional) a spider name (only jobs belonging to
+    #   this spider will be returned)
+    # @param state [String] (optional) return only jobs with this state. Valid
+    #   values: "pending", "running", "finished"
+    # @param has_tag [String, Array<String>] (optional) return only jobs
+    #   containing the given tag(s)
+    # @param lacks_tag [String, Array<String>] (optional) return only jobs not
+    #   containing the given tag(s)
+    # @param count [Fixnum] (optional) maximum number of jobs to return
+    #
+    # @return [Kleisli::Try, Kleisli::Either] a Kleisli::Try if a low-level
+    #   exception is raised (e.g. the host is down), a Kleisli::Either::Left
+    #   if validation fails (e.g. bad authentication), or a
+    #   Kleisli::Either::Right if everything was successful.
+    def list(args)
+      options = { query: args, basic_auth: { username: @api_key } }
+      Try { self.class.get("/api/jobs/list.json", options) } >-> response {
+        if response.code == 200
+          Right(response)
+        else
+          Left(response)
+        end
+      }
+    end
+
+    Contract KeywordArgs[:project => Nat,
+                         :spider => String,
+                         :add_tag => Optional[Or[String, ArrayOf[String]]],
+                         :priority => Optional[Or[0, 1, 2, 3, 4]],
+                         :extra => Optional[HashOf[Symbol => String]] ] => Or[Kleisli::Try, Kleisli::Either]
+    # Schedule a job.
+    #
+    # @param project [Fixnum] the project's numeric ID
+    # @param spider [String] the spider name
+    # @param add_tag [String, Array<String>] (optional) add tag(s) to the job
+    # @param priority [Fixnum] (optional) set the job priority: possible values
+    #   range from 0 (lowest priority) to 4 (highest priority), default is 2
+    # @param extra [Hash] (optional) extra parameters passed as spider
+    #   arguments
+    #
+    # @return [Kleisli::Try, Kleisli::Either] a Kleisli::Try if a low-level
+    #   exception is raised (e.g. the host is down), a Kleisli::Either::Left
+    #   if validation fails (e.g. bad authentication), or a
+    #   Kleisli::Either::Right if everything was successful.
+    def schedule(args)
+      extra = args.delete(:extra) || {}
+      options = { body: args.merge(extra), basic_auth: { username: @api_key } }
+      Try { self.class.post("/api/schedule.json", options) } >-> response {
+        if response.code == 200
+          Right(response)
+        else
+          Left(response)
+        end
+      }
+    end
+
+    Contract KeywordArgs[:project => Nat,
+                         :job => Optional[Or[String, ArrayOf[String]]],
+                         :spider => Optional[String],
+                         :state => Optional[Or["pending", "running", "finished"]],
+                         :has_tag => Optional[Or[String, ArrayOf[String]]],
+                         :lacks_tag => Optional[Or[String, ArrayOf[String]]],
+                         :add_tag => Optional[Or[String, ArrayOf[String]]],
+                         :remove_tag => Optional[Or[String, ArrayOf[String]]] ] => Or[Kleisli::Try, Kleisli::Either]
+    # Update information about jobs.
+    #
+    # @param project [Fixnum] the project's numeric ID
+    # @param job [String, Array<String>] (optional) ID(s) of specific jobs to
+    #   update
+    # @param spider [String] (optional) query on spider name to update
+    # @param state [String] (optional) query on jobs with this state to update.
+    #   Valid values: "pending", "running", "finished"
+    # @param has_tag [String, Array<String>] (optional) query on jobs
+    #   containing the given tag(s) to update
+    # @param lacks_tag [String, Array<String>] (optional) query on jobs not
+    #   containing the given tag(s) to update
+    # @param add_tag [String, Array<String>] (optional) tag(s) to add to the
+    #   queried jobs
+    # @param remove_tag [String, Array<String>] (optional) tag(s) to remove
+    #   from the queried jobs
+    #
+    # @return [Kleisli::Try, Kleisli::Either] a Kleisli::Try if a low-level
+    #   exception is raised (e.g. the host is down), a Kleisli::Either::Left
+    #   if validation fails (e.g. bad authentication), or a
+    #   Kleisli::Either::Right if everything was successful.
+    def update(args)
+      options = { body: args, basic_auth: { username: @api_key } }
+      Try { self.class.post("/api/jobs/update.json", options) } >-> response {
+        if response.code == 200
+          Right(response)
+        else
+          Left(response)
+        end
+      }
+    end
+
+    Contract KeywordArgs[:project => Nat,
+                         :job => Or[String, ArrayOf[String]] ] => Or[Kleisli::Try, Kleisli::Either]
+    # Delete one or more jobs.
+    #
+    # @param project [Fixnum] the project's numeric ID
+    # @param job [String, Array<String>] the ID of a specific job to delete
+    #
+    # @return [Kleisli::Try, Kleisli::Either] a Kleisli::Try if a low-level
+    #   exception is raised (e.g. the host is down), a Kleisli::Either::Left
+    #   if validation fails (e.g. bad authentication), or a
+    #   Kleisli::Either::Right if everything was successful.
+    def delete(args)
+      options = { body: args, basic_auth: { username: @api_key } }
+      Try { self.class.post("/api/jobs/delete.json", options) } >-> response {
+        if response.code == 200
+          Right(response)
+        else
+          Left(response)
+        end
+      }
+    end
+
+    Contract KeywordArgs[:project => Nat,
+                         :job => String ] => Or[Kleisli::Try, Kleisli::Either]
+    # Stop one or more running jobs.
+    #
+    # @param project [Fixnum] the project's numeric ID
+    # @param job [String] the ID of a job to stop
+    #
+    # @return [Kleisli::Try, Kleisli::Either] a Kleisli::Try if a low-level
+    #   exception is raised (e.g. the host is down), a Kleisli::Either::Left
+    #   if validation fails (e.g. bad authentication), or a
+    #   Kleisli::Either::Right if everything was successful.
+    def stop(args)
+      options = { body: args, basic_auth: { username: @api_key } }
+      Try { self.class.post("/api/jobs/stop.json", options) } >-> response {
+        if response.code == 200
+          Right(response)
+        else
+          Left(response)
+        end
+      }
+    end
+
+  end
+end

data/lib/scrapinghub/version.rb

@@ -0,0 +1,3 @@
+module Scrapinghub
+  VERSION = "0.0.1"
+end

data/scrapinghub.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path("../lib/scrapinghub/version", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "scrapinghub-client"
+  s.version     = Scrapinghub::VERSION
+  s.authors     = ["Abe Voelker"]
+  s.email       = "abe@abevoelker.com"
+  s.homepage    = "https://github.com/abevoelker/scrapinghub-client"
+  s.summary     = %q{Ruby client for Scrapinghub API}
+  s.description = s.summary
+  s.license     = "MIT"
+
+  s.require_paths = ["lib"]
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {spec}/*`.split("\n")
+
+  s.add_dependency "contracts", "~> 0.11"
+  s.add_dependency "kleisli"
+  s.add_dependency "httparty"
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "webmock"
+  s.add_development_dependency "vcr"
+end

data/spec/fixtures/vcr_cassettes/jobs/delete/bad_auth.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/jobs/delete.json
+    body:
+      encoding: UTF-8
+      string: job=1%2F1%2F7&project=1
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 403
+      message: FORBIDDEN
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sat, 25 Jul 2015 23:28:52 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "error", "message": "Authentication failed"}'
+    http_version: 
+  recorded_at: Sat, 25 Jul 2015 23:28:52 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/delete/job/invalid.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/jobs/delete.json
+    body:
+      encoding: UTF-8
+      string: job=1%2F1%2F123&project=1
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sat, 25 Jul 2015 23:26:42 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "ok", "count": 0}'
+    http_version: 
+  recorded_at: Sat, 25 Jul 2015 23:26:43 GMT
+recorded_with: VCR 2.9.3

data/spec/fixtures/vcr_cassettes/jobs/delete/job/multiple.yml

@@ -0,0 +1,38 @@
+---
+http_interactions:
+- request:
+    method: post
+    uri: http://XXX:@dash.scrapinghub.com/api/jobs/delete.json
+    body:
+      encoding: US-ASCII
+      string: job=1%2F1%2F7&job=1%2F1%2F8&project=1
+    headers:
+      Accept-Encoding:
+      - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+      Accept:
+      - "*/*"
+      User-Agent:
+      - Ruby
+  response:
+    status:
+      code: 200
+      message: OK
+    headers:
+      Server:
+      - nginx/1.8.0
+      Date:
+      - Sat, 25 Jul 2015 23:24:28 GMT
+      Content-Type:
+      - application/json
+      Transfer-Encoding:
+      - chunked
+      Connection:
+      - keep-alive
+      Vary:
+      - Cookie
+    body:
+      encoding: ASCII-8BIT
+      string: '{"status": "ok", "count": 2}'
+    http_version: 
+  recorded_at: Sat, 25 Jul 2015 23:24:28 GMT
+recorded_with: VCR 2.9.3