scrapinghub 0.0.2 → 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: efb25630cbfa2300e582aa732ee45ede8ce2fd0f
+  data.tar.gz: 62d00953d56f4e03265f7e60b0f2d83cf21101b6
+SHA512:
+  metadata.gz: bc3b458a543a3864daac2de070bae64a056829ee2e369e41de22e4d50404550723c6558ae65c8e82305b5495a4a14c7b2acf95e1b7f5525603c39e573c86ad84
+  data.tar.gz: d2ad323f2381719e169deb53de92d49791a281473b1175aec66f07f8f6059d7f36dab13f6399e95dd7ae068f55e2a56a4a3a6f2a0facdedc903b6518324f3256

data/Gemfile.lock

@@ -0,0 +1,16 @@
+PATH
+  remote: .
+  specs:
+    scrapinghub (0.0.3)
+      json
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    json (1.8.2)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  scrapinghub!

data/README.md

@@ -1,7 +1,5 @@
 # Scrapinghub
 
-TODO: Write a gem description
-
 ## Installation
 
 Add this line to your application's Gemfile:
@@ -16,9 +14,23 @@ Or install it yourself as:
 
     $ gem install scrapinghub
 
+
 ## Usage
 
-TODO: Write usage instructions here
+Get Job Items
+
+```ruby
+  Scrapinghub::Scrapinghub.new("SCRAPING HUB API KEY").job_items(project: project_id, job: job_id)
+```
+
+##### Avaialbe methods:
+
+* projects
+* spiders - required param: { project: project_id }
+* jobs - required param: { project: project_id }
+* job - required param: { project: project_id, job: job_id }
+* job_items - required param: { project: project_id, job: job_id }
+* spider_items - required param: { project: project_id, spider: spider_id }
 
 ## Contributing
 
@@ -27,3 +39,4 @@ TODO: Write usage instructions here
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin feature/my-new-feature`)
 5. Create new Pull Request
+

data/Rakefile

@@ -1 +1,5 @@
 require "bundler/gem_tasks"
+
+task :console do
+  exec "irb -r scrapinghub -I ./lib"
+end

data/lib/scrapinghub/api_method.rb

@@ -1,6 +1,6 @@
 module Scrapinghub
   class ApiMethod
-    def initialize(location, requires={}, format = :json)
+    def initialize(location, requires=[], format = :json)
       @location = "#{location}.#{format.to_s}"
       @requires = requires
     end

data/lib/scrapinghub/api_response.rb

@@ -1,3 +1,5 @@
+require 'json'
+
 module Scrapinghub
   class ApiResponse
     def initialize(net_http_request)

data/lib/scrapinghub/scrapinghub.rb

@@ -3,19 +3,21 @@ require_relative 'api_method'
 module Scrapinghub
   class Scrapinghub
     METHODS = {
-      projects:       ApiMethod.new('scrapyd/listprojects', []),
-      spiders:        ApiMethod.new('spiders/list', [:project]),
-      jobs:           ApiMethod.new('jobs/list', [:project]),
-      job:            ApiMethod.new('jobs/list', [:project, :job]),
-      job_items:      ApiMethod.new('items', [:project, :job]),
-      spider_items:   ApiMethod.new('items', [:project, :spider])
+      projects:       { http_type: :get, method: ApiMethod.new('scrapyd/listprojects', []) },
+      spiders:        { http_type: :get, method: ApiMethod.new('spiders/list', [:project]) },
+      jobs:           { http_type: :get, method: ApiMethod.new('jobs/list', [:project]) },
+      job:            { http_type: :get, method: ApiMethod.new('jobs/list', [:project, :job]) },
+      job_items:      { http_type: :get, method: ApiMethod.new('items', [:project, :job]) },
+      spider_items:   { http_type: :get, method: ApiMethod.new('items', [:project, :spider]) },
+      schedule:   { http_type: :post, method: ApiMethod.new('schedule', [:project, :spider]) },
     }
 
     attr_reader :api_key
 
-    def initialize(api_key, url='http://panel.scrapinghub.com/api/')
+    def initialize(api_key, url='https://dash.scrapinghub.com/api/', debug_mode=false)
       @api_key = api_key
       @base_url = url
+      @debug_mode = debug_mode
     end
 
     def get(method, parameters = {})
@@ -28,12 +30,31 @@ module Scrapinghub
       fetch(uri)
     end
 
+    def post(method, parameters = {})
+      if method.is_a? Symbol
+        uri = get_method_url(method, parameters)
+      else
+        uri = build_url(method, parameters)
+      end
+
+      Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
+        request = Net::HTTP::Post.new(uri.request_uri)
+        request.basic_auth @api_key, ''
+
+        request.add_field('Content-Type', 'application/json')
+        request.set_form_data(parameters)
+        response = http.request(request)
+
+        ApiResponse.new(response)
+      end
+    end
+
     def fetch(uri, redirect_limit = 5)
       if redirect_limit <= 0
         raise "Request redirected too many times."
       end
 
-      Net::HTTP.start(uri.host, uri.port) do |http|
+      Net::HTTP.start(uri.host, uri.port, use_ssl: true) do |http|
         request = Net::HTTP::Get.new(uri.request_uri)
         request.basic_auth @api_key, ''
 
@@ -43,8 +64,10 @@ module Scrapinghub
         when Net::HTTPFound
           new_location = URI(response['location'])
 
-          debug "<- #{uri.request_uri} redirects to",
-                "-> #{new_location.request_uri}"
+          if @debug_mode
+            debug "<- #{uri.request_uri} redirects to",
+                  "-> #{new_location.request_uri}"
+          end
 
           fetch(new_location, redirect_limit-1)
         else
@@ -55,7 +78,7 @@ module Scrapinghub
 
     def method_missing(method, *args, &block)
       if METHODS[method]
-        get(method, args)
+        self.public_send(METHODS[method][:http_type], method, *args)
       else
         super
       end
@@ -67,7 +90,7 @@ module Scrapinghub
     end
 
     def get_method_url(method, parameters)
-      api_method = METHODS[method]
+      api_method = METHODS[method][:method]
 
       return build_url(api_method, parameters)
     end

data/lib/scrapinghub/version.rb

@@ -1,3 +1,3 @@
 module Scrapinghub
-  VERSION = "0.0.2"
+  VERSION = "0.0.3"
 end

data/scrapinghub.gemspec

@@ -6,14 +6,16 @@ require 'scrapinghub/version'
 Gem::Specification.new do |gem|
   gem.name          = "scrapinghub"
   gem.version       = Scrapinghub::VERSION
-  gem.authors       = ["Emil Ahlback"]
-  gem.email         = ["e.ahlback@gmail.com"]
+  gem.authors       = ["Emil Ahlback", "Chien Kuo"]
+  gem.email         = ["e.ahlback@gmail.com", "chien.cc.kuo@gmail.com"]
   gem.description   = %q{Simple interface to Scrapinghub's API}
   gem.summary       = %q{Just a Ruby wrapper for the Scrapinghub API, see docs at: http://help.scrapinghub.com/api.html}
-  gem.homepage      = "https://github.com/mull/scrapinghub"
+  gem.homepage      = "https://github.com/chien/scrapinghub"
 
   gem.files         = `git ls-files`.split($/)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
+
+  gem.add_runtime_dependency 'json'
 end

metadata

@@ -1,25 +1,41 @@
 --- !ruby/object:Gem::Specification
 name: scrapinghub
 version: !ruby/object:Gem::Version
-  version: 0.0.2
-  prerelease: 
+  version: 0.0.3
 platform: ruby
 authors:
 - Emil Ahlback
+- Chien Kuo
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-06-04 00:00:00.000000000 Z
-dependencies: []
+date: 2016-05-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Simple interface to Scrapinghub's API
 email:
 - e.ahlback@gmail.com
+- chien.cc.kuo@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
 - Gemfile
+- Gemfile.lock
 - LICENSE.txt
 - README.md
 - Rakefile
@@ -29,28 +45,28 @@ files:
 - lib/scrapinghub/scrapinghub.rb
 - lib/scrapinghub/version.rb
 - scrapinghub.gemspec
-homepage: https://github.com/mull/scrapinghub
+homepage: https://github.com/chien/scrapinghub
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
-summary: ! 'Just a Ruby wrapper for the Scrapinghub API, see docs at: http://help.scrapinghub.com/api.html'
+specification_version: 4
+summary: 'Just a Ruby wrapper for the Scrapinghub API, see docs at: http://help.scrapinghub.com/api.html'
 test_files: []
+has_rdoc: