serrano 0.1.4.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ced6b8bd574ca8a373c61dc9f239499817e27bf8
-  data.tar.gz: ab478119cc61771107057f55f07bea2653a611e0
+  metadata.gz: ce96cb27ec2070ee612d0b85d29949b441d552a4
+  data.tar.gz: 90791e525778c083fb7e57d445a5596a2ffbf7a4
 SHA512:
-  metadata.gz: 029c66c10b76af77eeeba4a76d58552b8de60eb97b310eb0ee7610a9c47726ee9a6a403a15330ea983b4fed2d6396e50a57709445998d5c6a5cc661b61edba13
-  data.tar.gz: fc940b4c0375aaac5d234aecf5e0939d8c017e8c05629f1f61db97c593ef619cf0b8967c98158b00dd78b029b737fc484e9bac72f2d80c510b468e2dedff5b33
+  metadata.gz: dc0937a45f90e0c3d01581149fa09a21cb0a3f4c669181404b0317a553d184e8d4d0a816cb7a1b30e9a373ff1eefea1082a531308f863b3f09f17644d8dcaa27
+  data.tar.gz: e36b25922967b65c48bd92656c8db58359987dfae0380025043201edb8527e56098f44e3c4f3c78048276780abda2a7b276f4c860580d34774d4b598528a5961

data/.travis.yml

@@ -1,4 +1,4 @@
 language: ruby
 rvm:
-  - 2.1.7
-  - 2.2.3
+  - 2.1.8
+  - 2.2.4

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.2.0 (2016-03-07)
+
+* Added error classes to fail more gracefully, adapted from instagram gem (#4)
+* Added support for the cursor feature in the Crossref API for deep paging (#14)
+* Added disclaimer to docs that full text/abstracts aren't searched (#24)
+* Now passing user agent string with serrano version in each request (#25)
+
 ## 0.1.4 (2015-12-04)
 
 * Added `csl_styles()` method to get CSL styles info (#23)

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    serrano (0.1.4.1)
+    serrano (0.2.0)
       faraday (~> 0.9.1)
       faraday_middleware (~> 0.10.0)
       multi_json (~> 1.0)
@@ -15,7 +15,7 @@ GEM
       simplecov
       url
     docile (1.1.5)
-    faraday (0.9.1)
+    faraday (0.9.2)
       multipart-post (>= 1.2, < 3)
     faraday_middleware (0.10.0)
       faraday (>= 0.7.4, < 0.10)

data/README.md

@@ -41,6 +41,10 @@ Other methods:
 * [Citation count][ccount] - `Serrano.citation_count()`
 * [get CSL styles][csl] -  `Serrano.csl_styles()`
 
+Note about searching:
+
+You are using the Crossref search API described at https://github.com/CrossRef/rest-api-doc/blob/master/rest_api.md. When you search with query terms, on Crossref servers they are not searching full text, or even abstracts of articles, but only what is available in the data that is returned to you. That is, they search article titles, authors, etc. For some discussion on this, see https://github.com/CrossRef/rest-api-doc/issues/101
+
 ## Install
 
 ### Release version

data/lib/serrano/cnrequest.rb

@@ -1,8 +1,9 @@
 require "faraday"
 require "faraday_middleware"
 require "multi_json"
-require "serrano/errors"
+require "serrano/error"
 require "serrano/constants"
+require 'serrano/utils'
 require 'serrano/helpers/configuration'
 
 ##
@@ -53,6 +54,8 @@ def make_request(ids, format, style, locale)
   if format == "citeproc-json"
     endpt = "http://api.crossref.org/works/" + ids + "/" + type
     cr_works = Faraday.new(:url => endpt)
+    cr_works.headers[:user_agent] = make_ua
+    cr_works.headers["X-USER-AGENT"] = make_ua
     res = cr_works.get
   else
     if format == "text"
@@ -62,6 +65,8 @@ def make_request(ids, format, style, locale)
     res = $conn.get do |req|
       req.url ids
       req.headers['Accept'] = type
+      req.headers[:user_agent] = make_ua
+      req.headers["X-USER-AGENT"] = make_ua
     end
   end
 

data/lib/serrano/cursor_testing.rb

@@ -0,0 +1,52 @@
+require 'faraday'
+require "multi_json"
+
+query = "widget"
+cursor = "*"
+limit = 100
+cursor_max = 500
+rows = limit
+
+filter = nil
+offset = nil
+sample = nil
+sort = nil
+order = nil
+facet = nil
+
+args = { query: query, filter: filter, offset: offset,
+  rows: limit, sample: sample, sort: sort,
+  order: order, facet: facet, cursor: cursor }
+opts = args.delete_if { |k, v| v.nil? }
+
+conn = Faraday.new(:url => "http://api.crossref.org/", :request => nil)
+
+def _req(conn, path, opts)
+  res = conn.get path, opts
+  return MultiJson.load(res.body)
+end
+
+def _redo_req(conn, path, js, opts, cu, max_avail, cursor_max)
+  if !cu.nil? and cursor_max > js['message']['items'].length
+    res = [js]
+    total = js['message']['items'].length
+    while (!cu.nil? and cursor_max > total and total < max_avail)
+      opts[:cursor] = cu
+      out = _req(conn, path, opts)
+      cu = out['message']['next-cursor']
+      res << out
+      total = res.collect { |x| x['message']['items'].length }.inject(:+)
+    end
+    return res
+  else
+    return js
+  end
+end
+
+path = 'works'
+
+js = _req(conn, path, opts)
+cu = js['message']['next-cursor']
+max_avail = js['message']['total-results']
+res = _redo_req(conn, path, js, opts, cu, max_avail, cursor_max)
+

data/lib/serrano/error.rb

@@ -0,0 +1,22 @@
+module Serrano
+  # Custom error class for rescuing from all Serrano errors
+  class Error < StandardError; end
+
+  # Raised when Crossref returns the HTTP status code 400
+  class BadRequest < Error; end
+
+  # Raised when Crossref returns the HTTP status code 404
+  class NotFound < Error; end
+
+  # Raised when Crossref returns the HTTP status code 500
+  class InternalServerError < Error; end
+
+  # Raised when Crossref returns the HTTP status code 502
+  class BadGateway < Error; end
+
+  # Raised when Crossref returns the HTTP status code 503
+  class ServiceUnavailable < Error; end
+
+  # Raised when Crossref returns the HTTP status code 504
+  class GatewayTimeout < Error; end
+end

data/lib/serrano/faraday.rb

@@ -0,0 +1,71 @@
+require 'faraday'
+require 'multi_json'
+
+# @private
+module FaradayMiddleware
+  # @private
+  class RaiseHttpException < Faraday::Middleware
+    def call(env)
+      @app.call(env).on_complete do |response|
+        case response[:status].to_i
+        when 400
+          raise Serrano::BadRequest, error_message_400(response)
+        when 404
+          raise Serrano::NotFound, error_message_400(response)
+        when 500
+          raise Serrano::InternalServerError, error_message_500(response, "Something is technically wrong.")
+        when 502
+          raise Serrano::BadGateway, error_message_500(response, "The server returned an invalid or incomplete response.")
+        when 503
+          raise Serrano::ServiceUnavailable, error_message_500(response, "Crossref is rate limiting your requests.")
+        when 504
+          raise Serrano::GatewayTimeout, error_message_500(response, "504 Gateway Time-out")
+        end
+      end
+    end
+
+    def initialize(app)
+      super app
+      @parser = nil
+    end
+
+    private
+
+    def error_message_400(response)
+      "\n   #{response[:method].to_s.upcase} #{response[:url].to_s}\n   Status #{response[:status]}#{error_body(response[:body])}"
+    end
+
+    def error_body(body)
+      if not body.nil? and not body.empty? and body.kind_of?(String)
+        if is_json?(body)
+          body = ::MultiJson.load(body)
+          if body['message'].nil?
+            body = nil
+          elseif body['message'].length == 1
+            body = body['message']
+          else
+            body = body['message'].collect { |x| x['message'] }.join('; ')
+          end
+        end
+      end
+
+      if body.nil?
+        nil
+      else
+        ": #{body}"
+      end
+    end
+
+    def error_message_500(response, body=nil)
+      "#{response[:method].to_s.upcase} #{response[:url].to_s}: #{[response[:status].to_s + ':', body].compact.join(' ')}"
+    end
+
+    def is_json?(string)
+      MultiJson.load(string)
+      return true
+    rescue MultiJson::ParseError => e
+      return false
+    end
+
+  end
+end

data/lib/serrano/filterhandler.rb

@@ -1,58 +1,64 @@
 # helper functions
-module Serrano
-  class Request #:nodoc:
+module Helpers
 
-    private
-
-    $others = ['license_url','license_version','license_delay','full_text_version','full_text_type',
-               'award_number','award_funder']
-
-    def filter_handler(x = nil)
-      if x.nil?
-        nil
-      else
-        x = stringify(x)
-        nn = x.keys.collect{ |x| x.to_s }
-        if nn.collect{ |x| $others.include? x }.any?
-          nn = nn.collect{ |x|
-            if $others.include? x
-              case x
-              when 'license_url'
-                'license.url'
-              when 'license_version'
-                'license.version'
-              when 'license_delay'
-                'license.delay'
-              when 'full_text_version'
-                'full-text.version'
-              when 'full_text_type'
-                'full-text.type'
-              when 'award_number'
-                'award.number'
-              when 'award_funder'
-                'award.funder'
-              end
-            else
-              x
-            end
-          }
-        end
+  $others = ['license_url','license_version','license_delay','full_text_version','full_text_type',
+             'award_number','award_funder']
 
-        newnn = nn.collect{ |x| x.gsub("_", "-") }
-        x = rename_keys(x, newnn)
-        x = x.collect{ |k,v| [k, v].join(":") }.join(',')
-        return x
+  def filter_handler(x = nil)
+    if x.nil?
+      nil
+    else
+      x = stringify(x)
+      nn = x.keys.collect{ |x| x.to_s }
+      if nn.collect{ |x| $others.include? x }.any?
+        nn = nn.collect{ |x|
+          if $others.include? x
+            case x
+            when 'license_url'
+              'license.url'
+            when 'license_version'
+              'license.version'
+            when 'license_delay'
+              'license.delay'
+            when 'full_text_version'
+              'full-text.version'
+            when 'full_text_type'
+              'full-text.type'
+            when 'award_number'
+              'award.number'
+            when 'award_funder'
+              'award.funder'
+            end
+          else
+            x
+          end
+        }
       end
-    end
 
-    def stringify(x)
-      (x.keys.map{ |k,v| k.to_s }.zip x.values).to_h
+      newnn = nn.collect{ |x| x.gsub("_", "-") }
+      x = rename_keys(x, newnn)
+      x = x.collect{ |k,v| [k, v].join(":") }.join(',')
+      return x
     end
+  end
 
-    def rename_keys(x, y)
-      (y.zip x.values).to_h
-    end
+  def stringify(x)
+    (x.keys.map{ |k,v| k.to_s }.zip x.values).to_h
+  end
 
+  def rename_keys(x, y)
+    (y.zip x.values).to_h
   end
 
 end
+
+module Serrano
+  class Request #:nodoc:
+    include Helpers
+  end
+
+  class RequestCursor #:nodoc:
+    include Helpers
+  end
+end
+

data/lib/serrano/request.rb

@@ -1,7 +1,8 @@
 require "faraday"
 require "multi_json"
-require "serrano/errors"
+require "serrano/error"
 require "serrano/constants"
+require 'serrano/utils'
 require 'serrano/helpers/configuration'
 
 ##
@@ -58,18 +59,21 @@ module Serrano
         conn = Faraday.new(:url => Serrano.base_url, :request => options) do |f|
           f.response :logger
           f.adapter  Faraday.default_adapter
+          f.use FaradayMiddleware::RaiseHttpException
         end
       else
-        conn = Faraday.new(:url => Serrano.base_url, :request => options)
+        conn = Faraday.new(:url => Serrano.base_url, :request => options) do |f|
+          f.adapter  Faraday.default_adapter
+          f.use FaradayMiddleware::RaiseHttpException
+        end
       end
 
+      conn.headers[:user_agent] = make_ua
+      conn.headers["X-USER-AGENT"] = make_ua
+
       if self.id.nil?
-        # begin
         res = conn.get self.endpt, opts
         return MultiJson.load(res.body)
-        # rescue *NETWORKABLE_EXCEPTIONS => e
-        #   rescue_faraday_error(endpt, e)
-        # end
       else
         coll = []
         Array(self.id).each do |x|
@@ -85,12 +89,6 @@ module Serrano
 
           res = conn.get endpt, opts
           coll << MultiJson.load(res.body)
-          # begin
-          #   res = conn.get endpt, opts
-          #   coll << MultiJson.load(res.body)
-          # rescue *NETWORKABLE_EXCEPTIONS => e
-          #   rescue_faraday_error(endpt, e)
-          # end
         end
         return coll
       end

data/lib/serrano/request_cursor.rb

@@ -0,0 +1,138 @@
+require "faraday"
+require 'faraday_middleware'
+require "multi_json"
+require "serrano/error"
+require "serrano/constants"
+require 'serrano/helpers/configuration'
+require 'serrano/filterhandler'
+require 'serrano/error'
+require 'serrano/faraday'
+
+##
+# Serrano::RequestCursor
+#
+# Class to perform HTTP requests to the Crossref API
+module Serrano
+  class RequestCursor #:nodoc:
+
+    attr_accessor :endpt
+    attr_accessor :id
+    attr_accessor :query
+    attr_accessor :filter
+    attr_accessor :offset
+    attr_accessor :limit
+    attr_accessor :sample
+    attr_accessor :sort
+    attr_accessor :order
+    attr_accessor :facet
+    attr_accessor :works
+    attr_accessor :agency
+    attr_accessor :options
+    attr_accessor :verbose
+    attr_accessor :cursor
+    attr_accessor :cursor_max
+
+    def initialize(endpt, id, query, filter, offset,
+      limit, sample, sort, order, facet, works, agency,
+      options, verbose, cursor, cursor_max)
+
+      self.endpt = endpt
+      self.id = id
+      self.query = query
+      self.filter = filter
+      self.offset = offset
+      self.limit = limit
+      self.sample = sample
+      self.sort = sort
+      self.order = order
+      self.facet = facet
+      self.works = works
+      self.agency = agency
+      self.options = options
+      self.verbose = verbose
+      self.cursor = cursor
+      self.cursor_max = cursor_max
+    end
+
+    def perform
+      filt = filter_handler(self.filter)
+
+      if self.cursor_max.class != nil
+        if self.cursor_max.class != Fixnum
+          raise "cursor_max must be of class int"
+        end
+      end
+
+      args = { query: self.query, filter: filt, offset: self.offset,
+              rows: self.limit, sample: self.sample, sort: self.sort,
+              order: self.order, facet: self.facet, cursor: self.cursor }
+      opts = args.delete_if { |k, v| v.nil? }
+
+      if verbose
+        $conn = Faraday.new(:url => Serrano.base_url, :request => options) do |f|
+          f.response :logger
+          f.adapter Faraday.default_adapter
+          f.use FaradayMiddleware::RaiseHttpException
+        end
+      else
+        $conn = Faraday.new(:url => Serrano.base_url, :request => options) do |f|
+          f.adapter Faraday.default_adapter
+          f.use FaradayMiddleware::RaiseHttpException
+        end
+      end
+
+      $conn.headers[:user_agent] = make_ua
+      $conn.headers["X-USER-AGENT"] = make_ua
+
+      if self.id.nil?
+        js = self._req(self.endpt, opts)
+        cu = js['message']['next-cursor']
+        max_avail = js['message']['total-results']
+        res = self._redo_req(js, opts, cu, max_avail)
+        return res
+      else
+        coll = []
+        Array(self.id).each do |x|
+          if self.works
+            $endpt2 = self.endpt + '/' + x.to_s + "/works"
+          else
+            if self.agency
+              $endpt2 = self.endpt + '/' + x.to_s + "/agency"
+            else
+              $endpt2 = self.endpt + '/' + x.to_s
+            end
+          end
+
+          js = self._req($endpt2, opts)
+          cu = js['message']['next-cursor']
+          max_avail = js['message']['total-results']
+          coll << self._redo_req(js, opts, cu, max_avail)
+        end
+        return coll
+      end
+    end
+
+    def _redo_req(js, opts, cu, max_avail)
+      if !cu.nil? and self.cursor_max > js['message']['items'].length
+        res = [js]
+        total = js['message']['items'].length
+        while !cu.nil? and self.cursor_max > total and total < max_avail do
+          opts[:cursor] = cu
+          out = self._req($endpt2, opts)
+          cu = out['message']['next-cursor']
+          res << out
+          total = res.collect {|x| x['message']['items'].length}.reduce(0, :+)
+        end
+        return res
+      else
+        return js
+      end
+    end
+
+    def _req(path, opts)
+      res = $conn.get path, opts
+      return MultiJson.load(res.body)
+    end
+
+  end
+end

data/lib/serrano/utils.rb

@@ -0,0 +1,5 @@
+def make_ua
+	requa = 'Faraday/v' + Faraday::VERSION
+  habua = 'Serrano/v' + Serrano::VERSION
+  return requa + ' ' + habua
+end

data/lib/serrano/version.rb

@@ -1,3 +1,3 @@
 module Serrano
-  VERSION = "0.1.4.1"
+  VERSION = "0.2.0"
 end

data/lib/serrano.rb

@@ -1,5 +1,6 @@
 require "serrano/version"
 require "serrano/request"
+require "serrano/request_cursor"
 require "serrano/filterhandler"
 require "serrano/cnrequest"
 require "serrano/filters"
@@ -12,29 +13,40 @@ require 'rexml/xpath'
 #   @param offset [Fixnum] Number of record to start at, from 1 to infinity.
 #   @param limit [Fixnum] Number of results to return. Not relavant when searching with specific dois. Default: 20. Max: 1000
 #   @param sample [Fixnum] Number of random results to return. when you use the sample parameter,
-#   the limit and offset parameters are ignored. This parameter only used when works requested.
+#       the limit and offset parameters are ignored. This parameter only used when works requested.
 #   @param sort [String] Field to sort on, one of score, relevance,
-#   updated (date of most recent change to metadata. Currently the same as deposited),
-#   deposited (time of most recent deposit), indexed (time of most recent index), or
-#   published (publication date). Note: If the API call includes a query, then the sort
-#   order will be by the relevance score. If no query is included, then the sort order
-#   will be by DOI update date.
+#       updated (date of most recent change to metadata - currently the same as deposited),
+#       deposited (time of most recent deposit), indexed (time of most recent index), or
+#       published (publication date). Note: If the API call includes a query, then the sort
+#       order will be by the relevance score. If no query is included, then the sort order
+#       will be by DOI update date.
 #   @param order [String] Sort order, one of 'asc' or 'desc'
 #   @param facet [Boolean] Include facet results. Default: false
 #   @param verbose [Boolean] Print request headers to stdout. Default: false
 
+# @!macro cursor_params
+#   @param cursor [String] Cursor character string to do deep paging. Default is `nil`.
+#       Pass in '*' to start deep paging. Any combination of query, filters and facets may be
+#       used with deep paging cursors. While limit may be specified along with cursor, offset
+#       and sample cannot be used. See
+#       https://github.com/CrossRef/rest-api-doc/blob/master/rest_api.md#deep-paging-with-cursors
+#   @param cursor_max [Fixnum] Max records to retrieve. Only used when cursor
+#       param used. Because deep paging can result in continuous requests until all
+#       are retrieved, use this parameter to set a maximum number of records. Of course,
+#       if there are less records found than this value, you will get only those found.
+
 # @!macro serrano_options
 #   @param options [Hash] Hash of options for configuring the request, passed on to Faraday.new
-#     :timeout      - [Fixnum] open/read timeout Integer in seconds
-#     :open_timeout - [Fixnum] read timeout Integer in seconds
-#     :proxy        - [Hash] hash of proxy options
-#       :uri - [String] Proxy Server URI
-#       :user - [String] Proxy server username
-#       :password - [String] Proxy server password
-#     :params_encoder - [Hash] not sure what this is
-#     :bind           - [Hash] A hash with host and port values
-#     :boundary       - [String] of the boundary value
-#     :oauth          - [Hash] A hash with OAuth details
+#     - timeout [Fixnum] open/read timeout Integer in seconds
+#     - open_timeout [Fixnum] read timeout Integer in seconds
+#     - proxy [Hash] hash of proxy options
+#       - uri [String] Proxy Server URI
+#       - user [String] Proxy server username
+#       - password [String] Proxy server password
+#     - params_encoder [Hash] not sure what this is
+#     - bind [Hash] A hash with host and port values
+#     - boundary [String] of the boundary value
+#     - oauth [Hash] A hash with OAuth details
 
 ##
 # Serrano - The top level module for using methods
@@ -61,6 +73,17 @@ require 'rexml/xpath'
 #
 # @see https://github.com/CrossRef/rest-api-doc/blob/master/rest_api.md for
 # detailed description of the Crossref API
+#
+# What am I actually searching when using the Crossref search API?
+#
+# You are using the Crossref search API described at
+# https://github.com/CrossRef/rest-api-doc/blob/master/rest_api.md.
+# When you search with query terms, on Crossref servers they are not
+# searching full text, or even abstracts of articles, but only what is
+# available in the data that is returned to you. That is, they search
+# article titles, authors, etc. For some discussion on this, see
+# https://github.com/CrossRef/rest-api-doc/issues/101
+
 module Serrano
   extend Configuration
 
@@ -73,6 +96,7 @@ module Serrano
   #
   # @!macro serrano_params
   # @!macro serrano_options
+  # @!macro cursor_params
   # @param ids [Array] DOIs (digital object identifier) or other identifiers
   # @param query [String] A query string
   # @param filter [Hash] Filter options. See ...
@@ -101,12 +125,30 @@ module Serrano
   #      Serrano.works(ids: '10.1371/journal.pone.0033693', options: {timeout: 3, open_timeout: 2})
   #      ## log request details - uses Faraday middleware
   #      Serrano.works(ids: '10.1371/journal.pone.0033693', verbose: true)
+  #
+  #      # facets
+  #      Serrano.works(facet: 'license:*', limit: 0, filter: {has_full_text: true})
+  #
+  #      # sample
+  #      Serrano.works(sample: 2)
+  #
+  #      # cursor for deep paging
+  #      Serrano.works(query: "widget", cursor: "*", limit: 100)
+  #      # another query, more results this time
+  #      res = Serrano.works(query: "science", cursor: "*", limit: 250, cursor_max: 1000);
+  #      res.collect { |x| x['message']['items'].length }.reduce(0, :+)
+  #      # another query
+  #      res = Serrano.works(query: "ecology", cursor: "*", limit: 1000, cursor_max: 10000);
+  #      res.collect { |x| x['message']['items'].length }.reduce(0, :+)
+  #      items = res.collect {|x| x['message']['items']}.flatten
+  #      items.collect { |x| x['DOI'] }[0,20]
   def self.works(ids: nil, query: nil, filter: nil, offset: nil,
     limit: nil, sample: nil, sort: nil, order: nil, facet: nil,
-    options: nil, verbose: false)
+    options: nil, verbose: false, cursor: nil, cursor_max: 5000)
 
-    Request.new('works', ids, query, filter, offset,
-      limit, sample, sort, order, facet, nil, nil, options, verbose).perform
+    RequestCursor.new('works', ids, query, filter, offset,
+      limit, sample, sort, order, facet, nil, nil, options,
+      verbose, cursor, cursor_max).perform
   end
 
   ##
@@ -114,6 +156,7 @@ module Serrano
   #
   # @!macro serrano_params
   # @!macro serrano_options
+  # @!macro cursor_params
   # @param ids [Array] DOIs (digital object identifier) or other identifiers
   # @param query [String] A query string
   # @param filter [Hash] Filter options. See ...
@@ -133,12 +176,26 @@ module Serrano
   #      Serrano.members(query: "ecology", order: "asc")
   #      # Works
   #      Serrano.members(ids: 98, works: true)
+  #
+  #      # cursor - deep paging
+  #      res = Serrano.members(ids: 98, works: true, cursor: "*", cursor_max: 1000);
+  #      res[0].collect { |x| x['message']['items'].length }.reduce(0, :+)
+  #      items = res[0].collect { |x| x['message']['items'] }.flatten
+  #      items.collect{ |z| z['DOI'] }[0,50]
+  #
+  #      # multiple ids with cursor
+  #      res = Serrano.members(ids: [98, 340], works: true, cursor: "*", cursor_max: 300);
+  #      res[0].collect { |x| x['message']['items'].length }.reduce(0, :+)
+  #      items = res[0].collect { |x| x['message']['items'] }.flatten
+  #      items.collect{ |z| z['DOI'] }[0,50]
   def self.members(ids: nil, query: nil, filter: nil, offset: nil,
     limit: nil, sample: nil, sort: nil, order: nil, facet: nil,
-    works: false, options: nil, verbose: false)
+    works: false, options: nil, verbose: false,
+    cursor: nil, cursor_max: 5000)
 
-    Request.new('members', ids, query, filter, offset,
-      limit, sample, sort, order, facet, works, nil, options, verbose).perform
+    RequestCursor.new('members', ids, query, filter, offset,
+      limit, sample, sort, order, facet, works, nil, options,
+      verbose, cursor, cursor_max).perform
   end
 
   ##
@@ -146,6 +203,7 @@ module Serrano
   #
   # @!macro serrano_params
   # @!macro serrano_options
+  # @!macro cursor_params
   # @param ids [Array] DOIs (digital object identifier) or other identifiers
   # @param filter [Hash] Filter options. See ...
   # @param works [Boolean] If true, works returned as well. Default: false
@@ -162,12 +220,20 @@ module Serrano
   #      Serrano.prefixes(ids: "10.1016", works: true, limit: 3)
   #      # Sort and order
   #      Serrano.prefixes(ids: "10.1016", works: true, sort: 'relevance', order: "asc")
+  #
+  #      # cursor - deep paging
+  #      res = Serrano.prefixes(ids: "10.1016", works: true, cursor: "*", cursor_max: 1000);
+  #      res[0].collect { |x| x['message']['items'].length }.reduce(0, :+)
+  #      items = res[0].collect { |x| x['message']['items'] }.flatten;
+  #      items.collect{ |z| z['DOI'] }[0,50]
   def self.prefixes(ids:, filter: nil, offset: nil,
     limit: nil, sample: nil, sort: nil, order: nil, facet: nil,
-    works: false, options: nil, verbose: false)
+    works: false, options: nil, verbose: false,
+    cursor: nil, cursor_max: 5000)
 
-    Request.new('prefixes', ids, nil, filter, offset,
-      limit, sample, sort, order, facet, works, nil, options, verbose).perform
+    RequestCursor.new('prefixes', ids, nil, filter, offset,
+      limit, sample, sort, order, facet, works, nil, options,
+      verbose, cursor, cursor_max).perform
   end
 
   ##
@@ -175,6 +241,7 @@ module Serrano
   #
   # @!macro serrano_params
   # @!macro serrano_options
+  # @!macro cursor_params
   # @param ids [Array] DOIs (digital object identifier) or other identifiers
   # @param query [String] A query string
   # @param filter [Hash] Filter options. See ...
@@ -196,12 +263,20 @@ module Serrano
   #      Serrano.funders(ids: '10.13039/100000001', works: true, limit: 3)
   #      # Sort and order
   #      Serrano.funders(ids: "10.13039/100000001", works: true, sort: 'relevance', order: "asc")
+  #
+  #      # cursor - deep paging
+  #      res = Serrano.funders(ids: '10.13039/100000001', works: true, cursor: "*", cursor_max: 500);
+  #      res[0].collect { |x| x['message']['items'].length }.reduce(0, :+)
+  #      items = res[0].collect { |x| x['message']['items'] }.flatten;
+  #      items.collect{ |z| z['DOI'] }[0,50]
   def self.funders(ids: nil, query: nil, filter: nil, offset: nil,
     limit: nil, sample: nil, sort: nil, order: nil, facet: nil,
-    works: false, options: nil, verbose: false)
+    works: false, options: nil, verbose: false,
+    cursor: nil, cursor_max: 5000)
 
-    Request.new('funders', ids, query, filter, offset,
-      limit, sample, sort, order, facet, works, nil, options, verbose).perform
+    RequestCursor.new('funders', ids, query, filter, offset,
+      limit, sample, sort, order, facet, works, nil, options,
+      verbose, cursor, cursor_max).perform
   end
 
   ##
@@ -209,6 +284,7 @@ module Serrano
   #
   # @!macro serrano_params
   # @!macro serrano_options
+  # @!macro cursor_params
   # @param ids [Array] DOIs (digital object identifier) or other identifiers
   # @param query [String] A query string
   # @param filter [Hash] Filter options. See ...
@@ -230,18 +306,27 @@ module Serrano
   #      Serrano.journals(ids: '1803-2427', works: true)
   #      Serrano.journals(limit: 2)
   #      Serrano.journals(sample: 2)
+  #
+  #      # cursor - deep paging
+  #      res = Serrano.journals(ids: "2167-8359", works: true, cursor: "*", cursor_max: 500);
+  #      res[0].collect { |x| x['message']['items'].length }.reduce(0, :+)
+  #      items = res[0].collect { |x| x['message']['items'] }.flatten;
+  #      items.collect{ |z| z['DOI'] }[0,50]
   def self.journals(ids: nil, query: nil, filter: nil, offset: nil,
     limit: nil, sample: nil, sort: nil, order: nil, facet: nil,
-    works: false, options: nil, verbose: false)
+    works: false, options: nil, verbose: false,
+    cursor: nil, cursor_max: 5000)
 
-    Request.new('journals', ids, query, filter, offset,
-      limit, sample, sort, order, facet, works, nil, options, verbose).perform
+    RequestCursor.new('journals', ids, query, filter, offset,
+      limit, sample, sort, order, facet, works, nil, options,
+      verbose, cursor, cursor_max).perform
   end
 
   ##
   # Search the types route
   #
   # @!macro serrano_options
+  # @!macro cursor_params
   # @param ids [Array] DOIs (digital object identifier) or other identifiers
   # @param works [Boolean] If true, works returned as well. Default: false
   # @return [Array] An array of hashes
@@ -252,11 +337,18 @@ module Serrano
   #      Serrano.types(ids: "journal")
   #      Serrano.types(ids: ["journal", "dissertation"])
   #      Serrano.types(ids: "journal", works: true)
-  def self.types(ids: nil, offset: nil,
-    limit: nil, works: false, options: nil, verbose: false)
+  #
+  #      # cursor - deep paging
+  #      res = Serrano.types(ids: "journal", works: true, cursor: "*", cursor_max: 500);
+  #      res[0].collect { |x| x['message']['items'].length }.reduce(0, :+)
+  #      items = res[0].collect { |x| x['message']['items'] }.flatten;
+  #      items.collect{ |z| z['DOI'] }[0,50]
+  def self.types(ids: nil, offset: nil, limit: nil, works: false,
+    options: nil, verbose: false, cursor: nil, cursor_max: 5000)
 
-    Request.new('types', ids, nil, nil, offset,
-      limit, nil, nil, nil, nil, works, nil, options, verbose).perform
+    RequestCursor.new('types', ids, nil, nil, offset,
+      limit, nil, nil, nil, nil, works, nil, options,
+      verbose, cursor, cursor_max).perform
   end
 
   ##

data/serrano.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |s|
   s.version     = Serrano::VERSION
   s.platform    = Gem::Platform::RUBY
   s.required_ruby_version = '>= 2.0'
-  s.date        = '2015-12-04'
+  s.date        = '2016-03-07'
   s.summary     = "Crossref Client"
   s.description = "Low Level Ruby Client for the Crossref Search API"
   s.authors     = "Scott Chamberlain"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: serrano
 version: !ruby/object:Gem::Version
-  version: 0.1.4.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Scott Chamberlain
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-04 00:00:00.000000000 Z
+date: 2016-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -177,12 +177,16 @@ files:
 - lib/serrano/cn.rb
 - lib/serrano/cnrequest.rb
 - lib/serrano/constants.rb
-- lib/serrano/errors.rb
+- lib/serrano/cursor_testing.rb
+- lib/serrano/error.rb
+- lib/serrano/faraday.rb
 - lib/serrano/filterhandler.rb
 - lib/serrano/filters.rb
 - lib/serrano/helpers/configuration.rb
 - lib/serrano/request.rb
+- lib/serrano/request_cursor.rb
 - lib/serrano/styles.rb
+- lib/serrano/utils.rb
 - lib/serrano/version.rb
 - serrano.gemspec
 homepage: http://github.com/sckott/serrano

data/lib/serrano/errors.rb

@@ -1,67 +0,0 @@
-require 'net/http'
-
-def rescue_faraday_error(url, error, options={})
-  details = nil
-  headers = {}
-
-  if error.is_a?(Faraday::Error::TimeoutError)
-    status = 408
-  elsif error.respond_to?('status')
-    status = error[:status]
-  elsif error.respond_to?('response') && error.response.present?
-    status = error.response[:status]
-    details = error.response[:body]
-    headers = error.response[:headers]
-  else
-    status = 400
-  end
-
-  # Some sources use a different status for rate-limiting errors
-  status = 429 if status == 403 && details.include?("Excessive use detected")
-
-  if error.respond_to?('exception')
-    exception = error.exception
-  else
-    exception = ""
-  end
-
-  class_name = class_name_by_status(status) || error.class
-
-  message = parse_error_response(error.message)
-  message = "#{message} for #{url}"
-  message = "#{message} with rev #{options[:data][:rev]}" if class_name == Net::HTTPConflict
-
-  { error: message, status: status }
-end
-
-def parse_error_response(string)
-  if is_json?(string)
-    string = MultiJson.load(string)
-  end
-  string = string['error'] if string.is_a?(Hash) && string['error']
-  string
-end
-
-def is_json?(string)
-  MultiJson.load(string)
-rescue MultiJson::ParseError => e
-  e.data
-  e.cause
-end
-
-def class_name_by_status(status)
-  { 400 => Net::HTTPBadRequest,
-    401 => Net::HTTPUnauthorized,
-    403 => Net::HTTPForbidden,
-    404 => Net::HTTPNotFound,
-    406 => Net::HTTPNotAcceptable,
-    408 => Net::HTTPRequestTimeOut,
-    409 => Net::HTTPConflict,
-    417 => Net::HTTPExpectationFailed,
-    429 => Net::HTTPTooManyRequests,
-    500 => Net::HTTPInternalServerError,
-    502 => Net::HTTPBadGateway,
-    503 => Net::HTTPServiceUnavailable,
-    504 => Net::HTTPGatewayTimeOut }.fetch(status, nil)
-end
-