numerousapp 0.9.2 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 26caf302981f15e44e4c555ef9c43562dd7fad63
-  data.tar.gz: df17bba5974e81bf6410969923cd9a3de9c4e2d9
+  metadata.gz: 3bdd91b07bfb89bd80ad1a1ab27e1ca922b50e43
+  data.tar.gz: c94f71b9997951d8ad71f906bd1410905ef90f10
 SHA512:
-  metadata.gz: 0a2051121d9d80a8e65080e86bff2aac1a59988c5afce9a28fe700dca54802b5401c732325c1770d8bd8cc5ffb4503d2b765ab90c0eb6ccf463d71023b19c311
-  data.tar.gz: 5cdef175e815cf56137cd8bc506928a6233c1413e67d94e8b77b20d312dd14dd049e7a295a887cbdc322c30e711350398f7bdaedbc18f27d585c9003881a8650
+  metadata.gz: 6eef3a14e6f0342a39062943b155e7bca3222747ad4d9f15049f3194ce0c464379da64c81a87b76cdf9196cf77c1282dc337996fa5495e962223bd705503a310
+  data.tar.gz: 3222957cbdcea1304bad84c199cf2aa3767066eb4b52f048494bc65207018c78f98a9b92a914ba12aa26bdeb352a88c29c8b97dd94e15c1ece41d1b97050df00

data/lib/numerousapp.rb

@@ -88,9 +88,13 @@ end
 #
 class NumerousClientInternals
 
+
+
     #
     # @param apiKey [String] API authentication key
     # @param server [String] Optional (keyword arg). Server name.
+    # @param throttle [Proc] Optional throttle policy
+    # @param throttleData [Any] Optional data for throttle
     #
     # @!attribute agentString 
     #    @return [String] User agent string sent to the server.
@@ -101,8 +105,12 @@ class NumerousClientInternals
     # @!attribute [r] debugLevel
     #    @return [Fixnum] Current debugging level; use debug() method to change.
     #
-    def initialize(apiKey, server:'api.numerousapp.com')
+    def initialize(apiKey, server:'api.numerousapp.com',
+                           throttle:nil, throttleData:nil)
 
+        if not apiKey
+            apiKey = Numerous.numerousKey()
+        end
 
         @serverName = server
         @auth = { user: apiKey, password: "" }
@@ -113,10 +121,32 @@ class NumerousClientInternals
         @agentString = "NW-Ruby-NumerousClass/" + VersionString +
                        " (Ruby #{RUBY_VERSION}) NumerousAPI/v2"
 
+        @filterDuplicates = true     # see discussion elsewhere
+
+        # throttling.
+        # The arbitraryMaximum is just that: under no circumstances will we retry
+        # any particular request more than that. Tough noogies.
+        #
+        # the throttlePolicy "tuple" is:
+        #     [ 0 ] - Proc
+        #     [ 1 ] - specific data for Proc
+        #     [ 2 ] - "up" tuple for chained policy
+        #
+        # and the default policy uses the "data" (40) as the voluntary backoff point
+        #
+        @arbitraryMaximumTries = 10
+        @throttlePolicy = [ThrottleDefault, 40, nil]
+        if throttle
+            @throttlePolicy = [throttle, throttleData, @throttlePolicy]
+        end
+
+        @statistics = Hash.new { |h, k| h[k] = 0 }  # just useful debug/testing info
         @debugLevel = 0
+
     end
     attr_accessor :agentString
     attr_reader :serverName, :debugLevel
+    attr_reader :statistics
 
     # Set the debug level
     #
@@ -134,9 +164,17 @@ class NumerousClientInternals
         return prev
     end
 
+    # XXX This is primarily for testing; control filtering of bogus duplicates
+    #     If you are calling this you are probably doing something wrong.
+    def setBogusDupFilter(f)
+        prev = @filterDuplicates
+        @filterDuplicates = f
+        return prev
+    end
+
     protected
 
-    VersionString = '20141224.1'
+    VersionString = '20150123-1.0.0'
 
     MethMap = {
         GET: Net::HTTP::Get,
@@ -228,6 +266,8 @@ class NumerousClientInternals
     #
     def simpleAPI(api, jdict:nil, multipart:nil, url:nil)
 
+        @statistics[:simpleAPI] += 1
+
         # take the base url if you didn't give us an override
         url ||= api[:basePath]
 
@@ -279,14 +319,40 @@ class NumerousClientInternals
             end
         end
 
-        resp = @http.request(rq)
-       
-        if @debugLevel > 0
-            puts "Response headers:\n"
-            resp.each do | k, v |
-                puts "k: " + k + " :: " + v + "\n"
+        resp = nil   # ick, is there a better way to get this out of the block?
+        @arbitraryMaximumTries.times do |attempt|
+
+            @statistics[:serverRequests] += 1
+            resp = @http.request(rq)
+
+            if @debugLevel > 0
+                puts "Response headers:\n"
+                resp.each do | k, v |
+                    puts "k: " + k + " :: " + v + "\n"
+                end
+                puts "Code: " + resp.code + "/" + resp.code.class.to_s + "/\n"
+            end
+
+            # invoke the rate-limiting policy
+            rateRemain = resp['x-rate-limit-remaining'].to_i
+            rateReset = resp['x-rate-limit-reset'].to_i
+            @statistics[:rateRemaining] = rateRemain
+            @statistics[:rateReset] = rateReset
+
+            tp = { :debug=> @debug,
+                   :attempt=> attempt,
+                   :rateRemaining=> rateRemain,
+                   :rateReset=> rateReset,
+                   :resultCode=> resp.code.to_i,
+                   :resp=> resp,
+                   :statistics=> @statistics,
+                   :request=> { :httpMethod => api[:httpMethod], :url => path } }
+
+            td = @throttlePolicy[1]
+            up = @throttlePolicy[2]
+            if not @throttlePolicy[0].call(self, tp, td, up)
+                break
             end
-            puts "Code: " + resp.code + "/" + resp.code.class.to_s + "/\n"
         end
 
         goodCodes = api[:successCodes] || [200]
@@ -352,6 +418,14 @@ class NumerousClientInternals
         api = makeAPIcontext(info, :GET, subs)
         list = []
         nextURL = api[:basePath]
+        firstTime = true
+
+        # see discussion about duplicate filtering below
+        if @filterDuplicates and api[:dupFilter]
+            filterInfo = { prev: {}, current: {} }
+        else
+            filterInfo = nil
+        end
 
         while nextURL
             # get a chunk from the server
@@ -362,16 +436,167 @@ class NumerousClientInternals
             #     But here we're just letting the whatever-exceptions filter up
             v = simpleAPI(api, url:nextURL)
 
+            # statistics, helpful for testing/debugging. Algorithmically
+            # we don't really care about first time or not, just for the stats
+            if firstTime
+                @statistics[:firstChunks] += 1
+                firstTime = false
+            else
+                @statistics[:additionalChunks] += 1
+            end
+
+            if filterInfo
+                filterInfo[:prev] = filterInfo[:current]
+                filterInfo[:current] = {}
+            end
+            
             list = v[api[:list]]
             nextURL = v[api[:next]]
 
             # hand them out
             if list             # can be nil for a variety of reasons
-                list.each { |i| block.call i }
+                list.each do |i| 
+
+                    # A note about duplicate filtering
+                    #
+		    # There is a bug in the NumerousApp server which can
+		    # cause collections to show duplicates of certain events
+		    # (or interactions/stream items). Explaining the bug in great
+		    # detail is beyond the scope here; suffice to say it only
+		    # happens for events that were recorded nearly-simultaneously
+		    # and happen to be getting reported right at a chunking boundary.
+                    #
+                    # So we are filtering them out here. For a more involved
+                    # discussion of this, see the python implementation. This
+                    # filtering "works" because it knows pragmatically how/where
+                    # the bug can show up
+                    #
+                    # Turning off duplicate filtering is really meant only for testing.
+                    #
+                    # Not all API's require dupfiltering, hence the APIInfo test
+                    #
+                    if (not filterInfo)    # the easy case, not filtering
+                        block.call i
+                    else
+                        thisId = i[api[:dupFilter]]
+                        if filterInfo[:prev].include? thisId
+                            @statistics[:duplicatesFiltered] += 1
+                        else
+                            filterInfo[:current][thisId] = 1
+                            block.call i
+                        end
+                    end
+                end
             end
         end
         return nil     # the subclasses return (should return) their own self
     end
+
+    #
+    # The default throttle policy.
+    # Invoked after the response has been received and we are supposed to
+    # return true to force a retry or false to accept this response as-is.
+    #
+    # The policy this implements:
+    #    if the server failed with too busy, do backoff based on attempt number
+    #
+    #    if we are "getting close" to our limit, arbitrarily delay ourselves
+    #
+    #    if we truly got spanked with "Too Many Requests"
+    #    then delay the amount of time the server told us to delay.
+    #
+    # The arguments supplied to us are:
+    #     nr is the Numerous (handled explicitly so you can write external funcs too)
+    #     tparams is a Hash containing:
+    #         :attempt         : the attempt number. Zero on the very first try
+    #         :rateRemaining   : X-Rate-Limit-Remaining reported by the server
+    #         :rateReset       : time (in seconds) until fresh rate granted
+    #         :resultCode      : HTTP code from the server (e.g., 409, 200, etc)
+    #         :resp            : the full-on response object if you must have it
+    #         :request         : information about the original request
+    #         :statistics      : place to record stats (purely informational stats)
+    #         :debug           : current debug level
+    #
+    #     td is the data you supplied as "throttleData" to the Numerous() constructor
+    #     up is a tuple useful for calling the original system throttle policy:
+    #          up[0] is the function pointer
+    #          up[1] is the td for *that* function
+    #          up[2] is the "up" for calling *that* function
+    #       ... so after you do your own thing if you then want to defer to the
+    #           built-in throttle policy you can
+    #                     return send(up[0], nr, tparams, up[1], up[2])
+    #
+    # It's really (really really) important to understand the return value and
+    # the fact that we are invoked AFTER each request:
+    #    false : simply means "don't do more retries". It does not imply anything
+    #            about the success or failure of the request; it simply means that
+    #            this most recent request (response) is the one to "take" as
+    #            the final answer
+    #
+    #    true  : means that the response is, indeed, to be interpreted as some
+    #            sort of rate-limit failure and should be discarded. The original
+    #            request will be sent again. Obviously it's a very bad idea to
+    #            return true in cases where the server might have done some
+    #            anything non-idempotent. We assume that a 429 ("Too Many") or
+    #            a 500 ("Too Busy") response from the server means the server didn't
+    #            actually do anything (so a retry, timed appropriately, is ok)
+    #
+    # All of this seems overly general for what basically amounts to "sleep sometimes"
+    #
+
+    ThrottleDefault = Proc.new do |nr, tparams, td, up|
+        rateleft = tparams[:rateRemaining]
+        attempt = tparams[:attempt]    # note: is zero on very first try
+        stats = tparams[:statistics]
+
+        if attempt > 0
+            stats[:throttleMultipleAttempts] += 1
+        end
+
+        backarray = [ 2, 5, 15, 30, 60 ]
+        if attempt < backarray.length
+            backoff = backarray[attempt]
+        else
+            stats[:throttleMaxed] += 1
+            next false               # too many tries
+        end
+
+        # if the server actually has failed with too busy, sleep and try again
+        if tparams[:resultCode] == 500
+            stats[:throttle500] += 1
+            sleep(backoff)
+            next true
+        end
+
+        # if we weren't told to back off, no need to retry
+        if tparams[:resultCode] != 429
+            # but if we are closing in on the limit then slow ourselves down
+            # note that some errors don't communicate rateleft so we have to
+            # check for that as well (will be -1 here if wasn't sent to us)
+            #
+            # at constructor time our "throttle data" (td) was set up as the
+            # voluntary arbitrary limit
+            if rateleft >= 0 and rateleft < td
+                stats[:throttleVoluntaryBackoff] += 1
+                # arbitrary .. 1 second if more than half left, 3 seconds if less
+                if (rateleft*2) > td
+                    sleep(1)
+                else
+                    sleep(3)
+                end
+            end
+            next false               # no retry
+        end
+
+        # decide how long to delay ... we just wait for as long as the
+        # server told us to (plus "backoff" seconds slop to really be sure we
+        # aren't back too soon)
+        stats[:throttle429] += 1
+        sleep(tparams[:rateReset] + backoff)
+        next true
+    end
+    private_constant :ThrottleDefault
+
 end
 
 #
@@ -584,6 +809,145 @@ class Numerous  < NumerousClientInternals
         return NumerousMetric.new(id, self)
     end
 
+
+    #
+    # Version of metric() that accepts a name (label)
+    # instead of an ID, and can even process it as a regexp.
+    #
+    # @param [String] labelspec The name (label) or regexp
+    # @param [String] matchType 'FIRST' or 'BEST' or 'ONE' or 'STRING' (not regexp)
+    # @param [Numerous] nr 
+    #    The {Numerous} object that will be used to access this metric.
+    def metricByLabel(labelspec, matchType:'FIRST')
+        def raiseConflict(s1,s2)
+            raise NumerousMetricConflictError.new("Multiple matches", 409, [s1, s2])
+        end
+
+        bestMatch = [ nil, 0 ]
+
+        if not matchType
+            matchType = 'FIRST'
+        end
+        if not ['FIRST', 'BEST', 'ONE', 'STRING'].include?(matchType)
+            raise ArgumentError
+        end
+
+        self.metrics do |m|
+            if matchType == 'STRING'        # exact full match required
+                if m['label'] == labelspec
+                    if bestMatch[0]
+                        raiseConflict(bestMatch[0]['label'], m['label'])
+                    end
+                    bestMatch = [ m, 1 ]   # the length is irrelevant with STRING
+                end
+            else
+                mm = labelspec.match(m['label'])
+                if mm
+                    if matchType == 'FIRST'
+                        return self.metric(m['id'])
+                    elsif (matchType == 'ONE') and (bestMatch[1] > 0)
+                        raiseConflict(bestMatch[0]['label'], m['label'])
+                    end
+                    if mm[0].length > bestMatch[1]
+                        bestMatch = [ m, mm[0].length ]
+                    end
+                end
+            end
+        end
+        rv = nil
+        if bestMatch[0]
+            rv = self.metric(bestMatch[0]['id'])
+        end
+    end
+
+
+    #
+    # I found this a good way to handle supplying the API key so it's here
+    # as a class method you may find useful. What this function does is
+    # return you an API Key from a supplied string or "readable" object:
+    #
+    #          a "naked" API key (in which case this function is a no-op)
+    #          @-        :: meaning "get it from stdin"
+    #          @blah     :: meaning "get it from the file "blah"
+    #          /blah     :: get it from file /blah
+    #          .blah     :: get it from file .blah (could be ../ etc)
+    #        /readable/  :: if it has a .read method, get it that way
+    #          None      :: get it from environment variable NUMEROUSAPIKEY
+    #
+    # Where the "it" that is being gotten from any of those sources can be:
+    #    a "naked" API key
+    #    a JSON object, from which the credsAPIKey will be used to get the key
+    #
+    # Arguably this doesn't belong here, but it's helpful. Purists are free to
+    # ignore it or delete from their own tree :)
+    #
+
+    def self.numerousKey(s:nil, credsAPIKey:'NumerousAPIKey')
+
+    	if not s
+    	    # try to get from environment
+    	    s = ENV['NUMEROUSAPIKEY']
+    	    if not s
+    	        return nil
+            end
+        end
+
+    	closeThis = nil
+
+    	if s == "@-"             # creds coming from stdin
+    	    s = STDIN
+
+    	# see if they are in a file
+    	else
+    	    begin
+    	        if s.length() > 0         # is it a string or a file object?
+    	            # it's stringy - if it looks like a file open it or fail
+    	            begin
+    	        	if s.length() > 1 and s[0] == '@'
+    	        	    s = open(s[1..-1])
+    	        	    closeThis = s
+    	        	elsif s[0] == '/' or s[0] == '.'
+    	        	    s = open(s)
+    	        	    closeThis = s
+                        end
+    	            rescue
+    	        	return nil
+                    end
+                end
+    	    rescue NoMethodError     # it wasn't stringy, presumably it's a "readable"
+    	    end
+        end
+
+    	# well, see if whatever it is, is readable, and go with that if it is
+    	begin
+    	    v = s.read()
+    	    if closeThis
+    	        closeThis.close()
+            end
+    	    s = v
+    	rescue NoMethodError
+    	end
+
+    	# at this point s is either a JSON or a naked cred (or bogus)
+    	begin
+    	    j = JSON.parse(s)
+    	rescue TypeError, JSON::ParserError
+    	    j = {}
+        end
+
+
+    	#
+    	# This is kind of a hack and might hide some errors on your part
+    	#
+    	if not j.include? credsAPIKey  # this is how the naked case happens
+    	    # replace() bcs there might be a trailing newline on naked creds
+    	    # (usually happens with a file or stdin)
+    	    j[credsAPIKey] = s.sub("\n",'')
+        end 
+
+    	return j[credsAPIKey]
+    end
+
 end
 
 #
@@ -623,12 +987,48 @@ class NumerousMetric < NumerousClientInternals
     # @param [String] id The metric ID string.
     # @param [Numerous] nr 
     #    The {Numerous} object that will be used to access this metric.
+    #
+    # "id" should normally be the naked metric id (as a string).
+    #
+    # It can also be a nmrs: URL, e.g.:
+    #     nmrs://metric/2733614827342384
+    #
+    # Or a 'self' link from the API:
+    #     https://api.numerousapp.com/metrics/2733614827342384
+    #
+    # in either case we get the ID in the obvious syntactic way.
+    #
+    # It can also be a metric's web link, e.g.:
+    #     http://n.numerousapp.com/m/1x8ba7fjg72d
+    #
+    # in which case we "just know" that the tail is a base36
+    # encoding of the ID.
+    #
+    # The decoding logic here makes the specific assumption that
+    # the presence of a '/' indicates a non-naked metric ID. This
+    # seems a reasonable assumption given that IDs have to go into URLs
+    #
+
     def initialize(id, nr)
+        begin
+            if id.include? '/'
+                fields = id.split('/')
+                if fields[-2] == "m"
+                    id = fields[-1].to_i(36)
+                else
+                    id = fields[-1]
+                end
+            end
+        rescue NoMethodError
+            id = id.to_s
+        end
+
         @id = id
         @nr = nr
     end
     attr_reader :id
 
+
     #
     # Obtain the {Numerous} server object associated with a metric.
     # @return [Numerous]
@@ -652,7 +1052,8 @@ class NumerousMetric < NumerousClientInternals
         path: '/v1/metrics/%{metricId}/events',
         GET: {
             next: 'nextURL',
-            list: 'events'
+            list: 'events',
+            dupFilter: 'id'
         },
         POST: {
             successCodes: [ 201 ]
@@ -672,7 +1073,8 @@ class NumerousMetric < NumerousClientInternals
         path: '/v2/metrics/%{metricId}/stream',
         GET: {
             next: 'next',
-            list: 'items'
+            list: 'items',
+            dupFilter: 'id'
         }
       },
 
@@ -681,7 +1083,8 @@ class NumerousMetric < NumerousClientInternals
         path: '/v2/metrics/%{metricId}/interactions',
         GET: {
             next: 'nextURL',
-            list: 'interactions'
+            list: 'interactions',
+            dupFilter: 'id'
         },
         POST: {
             successCodes: [ 201 ]
@@ -1117,5 +1520,3 @@ class NumerousMetric < NumerousClientInternals
 
 end
 
-
-

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: numerousapp
 version: !ruby/object:Gem::Version
-  version: 0.9.2
+  version: 1.0.0
 platform: ruby
 authors:
 - Neil Webber
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-12-24 00:00:00.000000000 Z
+date: 2015-01-23 00:00:00.000000000 Z
 dependencies: []
 description: Classes implementing the NumerousApp REST APIs for metrics. Requires
   Ruby 2.x