shep 0.1.0.pre.alpha0 → 0.2.0.pre.alpha0

data/lib/shep/session.rb

@@ -11,8 +11,9 @@ module Shep
 
   # Represents a connection to a Mastodon (or equivalent) server.
   #
-  # @attr_reader [Logger] logger    The logger object
-  # @attr_reader [String] host      The Server's hostname
+  # @attr_reader [Logger] logger        The logger object
+  # @attr_reader [String] host          The Server's hostname
+  # @attr_reader [String] user_agent    User-Agent string; frozen
   #
   # ## Conventions
   #
@@ -24,29 +25,97 @@ module Shep
   # each item in turn and the block's result is ignored.  Otherwise,
   # it returns an `Enumerator` which can be used in the usual ways.
   #
-  # `each_*` will automatically paginate through the available items;
-  # for example,
+  # Some examples:
   #
-  #         statuses = session.each_public_status.to_a
+  #         # Evaluate a block on each status
+  #         session.each_status(account) { |status| do_thing(status) }
   #
-  # will retrieve the server's entire public timeline and put it in an
-  # array. (Note: don't do this.)  The `limit:` keyword option will
-  # set an upper limit on the number of items retrieved (but note that
-  # the method may retrieve more from the API endpoint).
+  #         # Retrieve the last 100 statuses in an array
+  #         statuses = session.each_status(account, limit: 100).to_a
+  #
+  #         # Retrieve the last 200 statuses via an enumerator and do
+  #         # extra transformation on the result before collecting
+  #         # them in an array.
+  #         statuses = session.each_status(account, limit: 200)
+  #             .select{|status| BESTIES.include? status.account.username }
+  #             .map{|status| status.id}
+  #             .to_a
+  #
+  # The actual web API "paginates" the output.  That is, it returns
+  # the first 40 (or so) items and then provides a link to the next
+  # chunk.  Shep's `each_*` methods handle this for you automatically.
+  # This means that unless you use `limit:`, the `each_*` methods will
+  # retrieve **all** available items, at least until you reach the
+  # rate limit (see below).
+  #
+  # Note that it is safe to leave an `each_*` methods block with
+  # `break`, `return`, an exception, or any other such mechanism.
   #
   # The remaining Mastodon API methods will in some way modify the
   # state of the server and return an Entity subinstance on success.
   #
   # All API calls throw an exception on failure.
+  #
+  # ## Rate Limits
+  #
+  # Mastodon servers restrict the number of times you can use a
+  # specific endpoint within a time period as a way to prevent abuse.
+  # Shep provides several tools for handling these limits gracefully.
+  #
+  # 1. The method {rate_limit} will return a Struct that tells you how
+  #    many requests you have left and when the count is reset.
+  #
+  # 2. If a rate limit is exceeded, the method will throw an
+  #    {Error::RateLimit} exception instead of an ordinary
+  #    {Error::Http} exception.
+  #
+  # 3. If the Session is created with argument `rate_limit_retry:` set
+  #    to true, the Session will instead wait out the reset time and
+  #    try again.
+  #
+  # If you enable the wait-and-retry mechanism, you can also provide a
+  # hook function (i.e. a thing that responds to `call`) via
+  # constructor argument `retry_hook:`.  This is called with one
+  # argument, the result of {rate_limit} for the limited API endpoint,
+  # immediately before Shep starts waiting for the limit to reset.
+  #
+  # The built-in wait time takes the callback's execution time into
+  # account so it's possible to use the callback to do your own
+  # waiting and use that time more productively.
+  #
+  # Alternately, all of the `each_*` methods have a `limit:` parameter
+  # so it's easy to avoid making too many API calls and many have a
+  # `max_id:` parameter that allows you to continue where you left off.
+  #
   class Session
-    attr_reader :logger, :host
+    attr_reader :logger, :host, :user_agent
 
     # Initialize a new {Session}.
     #
-    # @param host       [String]    Hostname of the server
-    # @param token      [String]    Bearer token; optional
-    # @param logger     [Logger]    The logger or mode; optional
-    # @param debug_http [Boolean]   Enable `Net::HTTP` debugging; insecure!
+    # @param host               [String]    Hostname of the server
+    # @param token              [String]    Bearer token; optional
+    #
+    # @param user_agent         [String]    User-Agent string to use
+    # @param ua_comment         [String]    Comment part of User-Agent string
+    #
+    # @param rate_limit_retry   [Boolean]   Handle request limits by waiting
+    #                                       for the count to reset and trying
+    #                                       again.
+    #
+    # @param retry_hook         [Proc]      One-argument hook function to call
+    #                                       before waiting for the rate limit
+    #                                       to reset
+    #
+    # @param logger             [Logger]    The logger or mode; optional
+    # @param debug_http         [Boolean]   Enable `Net::HTTP` debugging;
+    #                                       **insecure!**
+    #
+    # By default, the User-Agent header is set to the gem's
+    # identifier, but may be overridden with the `user_agent`
+    # parameter.  It is your responsibility to make sure it is
+    # formatted correctly.  You can also append comment text to the
+    # given User-Agent string with `ua_comment`; this lets you add a
+    # comment to the default text.
     #
     # Parameter `logger` may be a `Logger` object, `nil`, or a
     # `Symbol` whose value is the name of one of the supported log
@@ -56,17 +125,37 @@ module Shep
     #
     # If `debug_http` is true, compression is disabled and the
     # transactions are sent to `STDERR` via
-    # `Net::HTTP.set_debug_output`.  **WARNING:** this opens a serious
-    # security hole and should not be used in production.
+    # `Net::HTTP.set_debug_output`.
+    #
+    # **WARNING:** this opens a serious security hole and should not
+    # be used in production.
     def initialize(host:,
-                   token: nil,
-                   logger: nil,
-                   debug_http: false)
+                   token:               nil,
+                   user_agent:          "ShepRubyGem/#{Shep::Version}",
+                   ua_comment:          nil,
+
+                   rate_limit_retry:    false,
+                   retry_hook:          nil,
+
+                   logger:              nil,
+                   debug_http:          false)
       @host = host
       @token = token
       @logger = init_logger(logger)
-      @rate_limit = Struct.new(:limit, :remaining, :reset).new
+
+      @user_agent = user_agent
+      @user_agent += " #{ua_comment}" if ua_comment
+      @user_agent.freeze
+
+      @rate_limit_retry = rate_limit_retry
+      @retry_hook = retry_hook
+
       @debug_http = debug_http
+
+      @rate_limit = Struct.new(:limit, :remaining, :reset).new
+
+      raise Error::Caller.new("retry_hook: must a callable or nil") unless
+        @retry_hook == nil || @retry_hook.respond_to?(:call)
     end
 
     private
@@ -88,7 +177,7 @@ module Shep
 
     public
 
-    # Return the rate limit information as of the last operation.
+    # Return the rate limit information from the last REST request.
     #
     # The result is a Struct with the following fields:
     #
@@ -101,6 +190,10 @@ module Shep
     # times within 5 minutes but no more than 30 media uploads are
     # allowed within a 30 minute time period.
     #
+    # Note also that some Shep methods will perform multiple API
+    # requests; this is only ever the rate limit information from the
+    # latest of these.
+    #
     # @see https://docs.joinmastodon.org/api/rate-limits/
     #
     # @return [Struct.new(:limit, :remaining, :reset)]
@@ -150,14 +243,19 @@ module Shep
     #
     # The username must belong to a user on the current server.
     #
-    # @param handle [String] the account's username **with** the
+    # @param handle [String] the account's username with or without the
     #                        leading '@' character (e.g. @benoitmandelbot)
     #
-    # @return [Entity::Account]
+    #
+    # @return [Entity::Account, nil] The Account or nil if it can't be found.
     #
     # @see https://docs.joinmastodon.org/methods/accounts/#get
     def fetch_account_by_username(handle)
       return rest_get("accounts/lookup", Entity::Account, {acct: handle})
+    rescue Error::Http => oopsie
+      # As a special case, return nil if the lookup fails
+      return nil if oopsie.response.is_a?(Net::HTTPNotFound)
+      raise oopsie
     end
 
     # Fetch an individual notification by ID.
@@ -233,18 +331,18 @@ module Shep
 
         if !refetch && File.exist?(outfile)
           @logger.info "Found '#{outfile}'; skipping."
-          success = true
         else
           tmp = File.join(media_dir, SecureRandom.uuid + '.tmp')
-          success = basic_get_binary(ma.url, tmp)
-          if success
+          begin
+            basic_get_binary(ma.url, tmp)
             FileUtils.mv(tmp, outfile)
-          else
+          rescue Error::Http => e
             FileUtils.rm(tmp, force: true)
+            raise e
           end
         end
 
-        media[ma.url.to_s] = success ? outfile : nil
+        media[ma.url.to_s] = outfile
       }
 
       return [status, media]
@@ -300,6 +398,8 @@ module Shep
     # @param limit              [Integer]   Maximum number of accounts to
     #                                       retrieve
     #
+    # @param max_id             [String]    retrieve results older than this ID.
+    #
     # @param only_media         [Boolean]   If true, filter for statuses
     #                                       with media
     #
@@ -321,6 +421,7 @@ module Shep
     # @see https://docs.joinmastodon.org/methods/accounts/#statuses
     def each_status(account_id,
                     limit:              nil,
+                    max_id:             "",
                     only_media:         false,
                     exclude_replies:    false,
                     exclude_reblogs:    false,
@@ -337,6 +438,8 @@ module Shep
     #
     # @param limit      [Integer]   Max. items to retrieve.
     #
+    # @param max_id     [String]    retrieve results older than this ID.
+    #
     # @param local      [Boolean]   Retrieve only local statuses
     #
     # @param remote     [Boolean]   Retrieve only remote statuses
@@ -351,6 +454,7 @@ module Shep
     #
     # @see https://docs.joinmastodon.org/methods/timelines/#public
     def each_public_status(limit:              nil,
+                           max_id:             "",
                            local:              false,
                            remote:             false,
                            only_media:         false,
@@ -373,6 +477,8 @@ module Shep
     #
     # @param limit      [Integer] maximum number of items to retrieve
     #
+    # @param max_id     [String]  return results older than this ID.
+    #
     # @param local      [Boolean] retrieve only local statuses
     #
     # @param remote     [Boolean] retrieve only remote statuses
@@ -392,6 +498,7 @@ module Shep
     # @see https://docs.joinmastodon.org/methods/timelines/#tag
     def each_tag_status(hashtag_s,
                         limit:              nil,
+                        max_id:             "",
                         local:              false,
                         remote:             false,
                         only_media:         false,
@@ -422,10 +529,16 @@ module Shep
     #
     # Requires token.
     #
-    # @param limit [Integer] ,
-    # @param local [Boolean] ,
-    # @param remote [Boolean] ,
-    # @param only_media [Boolean] ,
+    # @param limit      [Integer] maximum number of items to retrieve
+    #
+    # @param max_id     [String]  retrieve results older than this ID.
+    #
+    # @param local      [Boolean] retrieve only local statuses
+    #
+    # @param remote     [Boolean] retrieve only remote statuses
+    #
+    # @param only_media [Boolean] retrieve only media status
+    #
     #
     # @yield [item]
     #
@@ -437,6 +550,7 @@ module Shep
     # @see https://docs.joinmastodon.org/methods/timelines/#home
     def each_home_status(limit:              nil,
                          local:              false,
+                         max_id:             "",
                          remote:             false,
                          only_media:         false,
                          &block)
@@ -488,6 +602,18 @@ module Shep
 
     # Retrieve each notification.
     #
+    # Requires a bearer token.
+    #
+    # Notification types are indicated by of the following symbols:
+    #
+    # `:mention`, `:status`, `:reblog`, `:follow`, `:follow_request`
+    # `:favourite`, `:poll`, `:update`, `:admin.sign_up`, or
+    # `:admin.report`
+    #
+    # This method will throw an `Error::Caller` exception if an
+    # unknown value is used.
+    #
+    #
     # @param types          [Array<String>] list of notifications types to
     #                                       enumerate; others are ignoredn
     #
@@ -514,9 +640,13 @@ module Shep
                                  favourite poll update admin.sign_up
                                  admin.report}
 
-      # Ensure valid filter values
-      types.uniq!
-      exclude_types.uniq!
+      # Remove duplicates and convert strings to symbols
+      [types, exclude_types].each{|param|
+        param.map!{|item| item.intern}
+        param.uniq!
+      }
+
+      # Now, ensure there are no incorrect notification types.
       (types + exclude_types).each{|filter|
         assert("Unknown notification type: #{filter}") {
           allowed_notifications.include?(filter.intern)
@@ -550,6 +680,8 @@ module Shep
     #
     # @param language [String]          ISO language code
     #
+    # @return [Entity::Status]          The new status.
+    #
     # @see https://docs.joinmastodon.org/methods/statuses/#create
     def post_status(text,
                     visibility:       :private,
@@ -669,12 +801,14 @@ module Shep
       return nil
     end
 
-    private
+
 
     #
-    # High-level REST support
+    # High(ish)-level REST support
     #
 
+    private
+
     # Extract all of the keyword arguments and their values from the
     # caller's context.  (The context is 'cbinding', which must be a
     # call to method 'cmethod'.)
@@ -711,7 +845,7 @@ module Shep
 
     def rest_delete(path, result_klass)
       uri = rest_uri(path, {})
-      result_obj, _ = basic_rest_get_or_delete(uri, also_delete: true)
+      result_obj, _ = basic_rest_get_or_delete(uri, is_delete: true)
       return result_klass.from(result_obj)
     end
 
@@ -766,7 +900,7 @@ module Shep
 
 
     #
-    # Low-level REST support
+    # Low(ish)-level REST support
     #
 
     # Given a hash of query arguments, return an array of key-value
@@ -804,51 +938,91 @@ module Shep
       return URI("https://#{@host}/api/#{version}/#{path}#{args}")
     end
 
-    def basic_rest_get_or_delete(uri, also_delete: false)
+    def parse_link_header(hdr)
+      result = {}
+      return result unless hdr      # could be nil
+
+      for link in hdr.split(', ')
+        md = link.match(/^<([^>]+)>; rel="([^"]+)"/)
+        assert{md}
+
+        result[ md[2] ] = md[1]
+      end
+
+      return result
+    end
+
+    # Perform a GET or DELETE operation.  (They have mostly the same
+    # structure, so we combine the functionality here.)
+    def basic_rest_get_or_delete(url, is_delete: false)
       headers = headers_for(:get)
 
-      # Do the thing.
-      response = http_get_or_delete(uri, headers, also_delete)
-      update_rate_limit(response)
+      request = is_delete ?
+        Net::HTTP::Delete.new(url, headers) :
+        Net::HTTP::Get.new(url, headers)
 
-      result = response.body() if response.class.body_permitted?
-      raise Error::Http.new(response) if response.is_a? Net::HTTPClientError
+      response = http_operation(request)
 
-      result_obj = JSON.parse(result)
+      result = JSON.parse(response.body)
 
-      if result.is_a?(Hash) && result_obj.has_key?("error")
-        raise Error::Server.new(result_obj["error"])
+      if result.is_a?(Hash) && result.has_key?("error")
+        raise Error::Server.new(result["error"])
       end
 
       link = parse_link_header(response["link"])
 
-      return [result_obj, link]
+      return [result, link]
     rescue JSON::JSONError => e
       raise Error::Remote.new("Error parsing result JSON: #{e}")
     end
 
-    def http_get_or_delete(uri, headers, is_delete)
-      @logger.debug("#{is_delete ? "Deleting" : "Requesting"} #{uri} " +
-                    "(token: #{!!@token})")
+    # Retrieve the resource (assumed to be binary) at 'uri' and write
+    # it to 'filename'.  For now, returns false if the request
+    # returned an error code and true on success.
+    def basic_get_binary(url, filename)
+      request = Net::HTTP::Get.new(url, headers_for(:get))
 
-      response = Net::HTTP.start(
-        uri.hostname,
-        uri.port,
-        use_ssl: uri.scheme == 'https'
-      ) { |http|
-        next is_delete                      ?
-          http.delete(uri.path, headers)    :
-          http.request_get(uri, headers)
-      }
+      @logger.debug("Output file is #{filename}")
+      File.open(filename, "wb") { |outfile| http_operation(request, outfile) }
+      @logger.debug("Done (#{filename})")
+    end
 
-      @logger.debug("Response: #{response}")
-      return response
+    def basic_rest_post_or_put(url, formdata, is_put: false)
+      headers = headers_for(:post)
+
+      # Select request type
+      request = is_put                          ?
+        Net::HTTP::Put.new(url, headers)        :
+        Net::HTTP::Post.new(url, headers)
+
+      # Set the parameters
+      enctype = 'multipart/form-data' if formdata.is_a?(Array)
+      enctype = 'application/x-www-form-urlencoded' if
+        formdata.is_a?(Hash)
+      enctype or
+        raise Error::Caller.new("Unknown formdate type: #{formdata.class}")
+      request.set_form(formdata, enctype)
+
+      # Do the deed
+      response = http_operation(request)
+
+      result = JSON.parse(response.body)
+
+      if result.is_a?(Hash) && result.has_key?("error")
+        raise Error::Server.new(result["error"])
+      end
+
+      return result
+    rescue JSON::JSONError => e
+      raise Error::Remote.new("Error parsing result JSON: #{e}")
     end
 
     def headers_for(method)
       headers = {}
       headers["Authorization"] = "Bearer #{@token}" if @token
 
+      headers['User-Agent'] = @user_agent
+
       if method == :post
         extras = {
           "Indempotency-Key":       SecureRandom.uuid,
@@ -860,111 +1034,83 @@ module Shep
       return headers
     end
 
-    def update_rate_limit(response)
-      @rate_limit.limit = @rate_limit.remaining = @rate_limit.reset = nil
+    def http_operation(request, output_handle = nil)
+      url = request.uri
 
-      @rate_limit.limit = response['X-RateLimit-Limit'].to_i
-      @rate_limit.remaining = response['X-RateLimit-Remaining'].to_i
+      while true
+        http = Net::HTTP.new(url.hostname, url.port)
+        http.use_ssl = (url.scheme == 'https')
 
-      reset = response['X-RateLimit-Reset']
-      @rate_limit.reset = Time.iso8601(reset) if reset
+        if @debug_http
+          http.set_debug_output(STDERR)
+          request['Accept-Encoding'] = 'identity'
+        end
 
-      @logger.debug "Rate limit: #{rate_limit_desc}"
-    end
+        http.start do |http|
+          @logger.debug("Request #{request}; (token: #{!!@token})")
 
-    # Retrieve the resource (assumed to be binary) at 'uri' and write
-    # it to 'filename'.  For now, returns false if the request
-    # returned an error code and true on success.
-    def basic_get_binary(uri, filename)
-      headers = headers_for(:get)
+          # We have to invoke 'request' with a block argument to get the
+          # response because that's the only way we can get at it before
+          # it's downloaded the entire response into RAM.
+          http.request(request) do |response|
+            @logger.debug("Response: #{response}")
 
-      @logger.debug("Requesting #{uri} (token: #{!!@token})")
-      Net::HTTP.get_response(uri, headers) {|response|
-        @logger.debug("Response: #{response.code}")
+            update_rate_limit(response)
 
-        update_rate_limit(response)
+            #raise_http_exception_if_error(response)
 
-        if response.is_a? Net::HTTPClientError
-          @logger.warn("Got response #{response} for #{uri}.")
-          return false
-        end
+            if response.is_a?(Net::HTTPClientError)
+              # Finish any body reading that may have been in
+              # progress.
+              response.read_body()
 
-        @logger.debug("Writing body to #{filename}")
-        File.open(filename, "wb") { |outfile|
-          response.read_body { |chunk|
-            @logger.debug("  Writing #{chunk.size} bytes...")
-            outfile.write(chunk)
-          }
-        }
-        @logger.debug("Done (#{filename})")
-      }
+              # Special case: too many requests.  We may throw an
+              # exception or wait until it resets and try again.
+              if response.is_a?(Net::HTTPTooManyRequests)
+                handle_rate_limit_reached(response)
+                next  # if we get here, we're trying again
+              end
 
-      return true
-    end
+              raise Error::Http.new(response)
+            end
 
-    def parse_link_header(hdr)
-      result = {}
-      return result unless hdr      # could be nil
+            # read_body will write the response body to output_handle if
+            # it's a handle or keep it internally if it's nil.
+            response.read_body(output_handle)
 
-      for link in hdr.split(', ')
-        md = link.match(/^<([^>]+)>; rel="([^"]+)"/)
-        assert{md}
-
-        result[ md[2] ] = md[1]
+            return response
+          end
+        end
       end
-
-      return result
     end
 
+    def handle_rate_limit_reached(response)
+      raise Error::RateLimit.new(response) unless @rate_limit_retry
 
-    def basic_rest_post_or_put(uri, formdata, is_put: false)
-      headers = headers_for(:post)
-
-      # Do the thing.
-      @logger.debug("Posting #{uri} (token: #{!!@token})")
-      response = http_post_form(uri, headers, formdata, is_put)
-      @logger.debug("Response: #{response}")
-
-      update_rate_limit(response)
+      # Call the retry hook first.
+      @retry_hook.call(self.rate_limit().dup) if @retry_hook
 
-      result = response.body() if response.class.body_permitted?
-      raise Error::Http.new(response) if response.is_a? Net::HTTPClientError
-
-      result_obj = JSON.parse(result)
-
-      if result.is_a?(Hash) && result_obj.has_key?("error")
-        raise Error::Server.new(result_obj["error"])
+      # Now, wait out any remaining elapsed time.
+      while true
+        delay = (@rate_limit.reset - Time.now) + 2
+        break if delay <= 0
+        @logger.info "Sleeping for #{delay.round} seconds."
+        sleep delay
       end
-
-      return result_obj
-    rescue JSON::JSONError => e
-      raise Error::Remote.new("Error parsing result JSON: #{e}")
     end
 
-    def http_post_form(url, headers, formdata, is_put)
-      request = is_put                          ?
-        Net::HTTP::Put.new(url, headers)        :
-        Net::HTTP::Post.new(url, headers)
-
-      enctype = 'multipart/form-data' if formdata.is_a?(Array)
-      enctype = 'application/x-www-form-urlencoded' if
-        formdata.is_a?(Hash)
-      enctype or
-        raise Error::Caller.new("Unknown formdate type: #{formdata.class}")
-      request.set_form(formdata, enctype)
-
+    def update_rate_limit(response)
+      @rate_limit.limit = @rate_limit.remaining = @rate_limit.reset = nil
 
-      http = Net::HTTP.new(url.hostname, url.port)
-      http.use_ssl = (url.scheme == 'https')
+      @rate_limit.limit = response['X-RateLimit-Limit'].to_i
+      @rate_limit.remaining = response['X-RateLimit-Remaining'].to_i
 
-      if @debug_http
-        http.set_debug_output(STDERR)
-        request['Accept-Encoding'] = 'identity'
-      end
+      reset = response['X-RateLimit-Reset']
+      @rate_limit.reset = Time.iso8601(reset) if reset
 
-      return http.start {|http|
-        http.request(request)
-      }
+      @logger.debug "Rate limit: #{rate_limit_desc}"
     end
+
   end
+
 end

data/lib/shep/version.rb

@@ -0,0 +1,4 @@
+
+module Shep
+  Version = '0.2.0-alpha0'
+end