numerousapp 1.2.2 → 1.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/numerousapp.rb +83 -27
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 238c5ee878f1b0d248d60726db930695f6179c62
|
4
|
+
data.tar.gz: edde3c80e6ee8396841f11180b18162e44d14f97
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f4eec611d9bfce7c676552229234935f096cb1542a9bee7b233fc73108f02ef571f3596941b833b11d0451cab3ef7d3e862bc3b57b0f943569e333d1b503b55c
|
7
|
+
data.tar.gz: b631ed7da965788038cee88bbbda20c5b9811e34cc86111b1e759f068e2335b9d5607ff7882774bbc72e230416f1ccd982b70ad6d21e33827d31551206dacebf
|
data/lib/numerousapp.rb
CHANGED
@@ -153,12 +153,13 @@ class NumerousClientInternals
|
|
153
153
|
#
|
154
154
|
# and the default policy uses the "data" as a hash of parameters:
|
155
155
|
# :voluntary -- the threshold point for voluntary backoff
|
156
|
+
# :volmaxdelay -- arbitrary maximum *voluntary delay* time
|
156
157
|
#
|
157
158
|
@arbitraryMaximumTries = 10
|
158
|
-
voluntary = { voluntary: 40}
|
159
|
+
voluntary = { voluntary: 40, volmaxdelay: 5}
|
159
160
|
# you can keep the dflt throttle but just alter the voluntary param, this way:
|
160
161
|
if throttleData and not throttle
|
161
|
-
voluntary = throttleData
|
162
|
+
voluntary = voluntary.merge(throttleData)
|
162
163
|
end
|
163
164
|
@throttlePolicy = [ThrottleDefault, voluntary, nil]
|
164
165
|
if throttle
|
@@ -222,7 +223,7 @@ class NumerousClientInternals
|
|
222
223
|
|
223
224
|
protected
|
224
225
|
|
225
|
-
VersionString = '
|
226
|
+
VersionString = '20150713-1.2.3'
|
226
227
|
|
227
228
|
MethMap = {
|
228
229
|
GET: Net::HTTP::Get,
|
@@ -596,16 +597,52 @@ class NumerousClientInternals
|
|
596
597
|
return nil # the subclasses return (should return) their own self
|
597
598
|
end
|
598
599
|
|
600
|
+
|
599
601
|
#
|
600
602
|
# The default throttle policy.
|
601
603
|
# Invoked after the response has been received and we are supposed to
|
602
604
|
# return true to force a retry or false to accept this response as-is.
|
603
605
|
#
|
604
606
|
# The policy this implements:
|
605
|
-
# if
|
607
|
+
# * if "getting close" to the limit, arbitrarily delay ourselves.
|
608
|
+
# See ComputeVoluntaryDelay above for details on that policy
|
609
|
+
#
|
610
|
+
# * if we truly got spanked with "Too Many Requests"
|
611
|
+
# then delay the amount of time the server told us to delay.
|
612
|
+
#
|
613
|
+
# The Voluntary delay policy works like this:
|
614
|
+
#
|
615
|
+
# Given N API calls remaining and T seconds until fresh allotment,
|
616
|
+
# compute a N-per-T rate delay so the hard rate limit probably won't
|
617
|
+
# hit (there is no guarantee bcs multiple clients can be running).
|
618
|
+
#
|
619
|
+
# Example: 20 APIs remaining and 5 seconds until fresh allocation.
|
620
|
+
# A delay of 250msec per API ensures we (approximately) don't hit the
|
621
|
+
# limit. Always remember the point here is just to TRY to be NICE.
|
622
|
+
# It's not important to be fussy about exactness.
|
623
|
+
#
|
624
|
+
# In effect the concept is to "smear" an inevitable rate-limit delay
|
625
|
+
# over the tail end of the API rate allocation rather than hitting
|
626
|
+
# the hard limit and encountering a long (e.g., 30 second) hard delay.
|
627
|
+
#
|
628
|
+
# When there are only a few APIs left and a lot of time, this could
|
629
|
+
# impose long delays. E.g., rateleft 2, but 40 seconds to go until
|
630
|
+
# fresh. Although this "shouldn't" happen if you have a single thread
|
631
|
+
# using this smear algorithm, it can certainly happen with multiple
|
632
|
+
# threads or multiple processes all individually consuming APIs.
|
633
|
+
# In this scenario you're going to inevitably hit the hard cap anyway.
|
634
|
+
# Therefore: voluntary delay is arbitrarily capped to a parameter provided
|
635
|
+
# in the throttledata (set up during initialization)
|
636
|
+
#
|
637
|
+
# This has been stress-tested "in the wild" by running code doing
|
638
|
+
# a metric.read() in a loop; theoretically such code should run at
|
639
|
+
# 300 API calls per minute -- and it does, either with this voluntary
|
640
|
+
# throttling or without it. If you are trying to run faster than 300
|
641
|
+
# per minute, it's all just a question of *how* you want to experience
|
642
|
+
# your (ultimately server-imposed) API throttling, not *if* (or how much).
|
643
|
+
#
|
644
|
+
# Speed Limit: 300 API/minute. It's The Law. :)
|
606
645
|
#
|
607
|
-
# if we truly got spanked with "Too Many Requests"
|
608
|
-
# then delay the amount of time the server told us to delay.
|
609
646
|
#
|
610
647
|
# The arguments supplied to us are:
|
611
648
|
# nr is the Numerous
|
@@ -644,8 +681,8 @@ class NumerousClientInternals
|
|
644
681
|
# All of this seems overly general for what amounts to "sleep sometimes"
|
645
682
|
#
|
646
683
|
|
684
|
+
|
647
685
|
ThrottleDefault = Proc.new do |nr, tparams, td, up|
|
648
|
-
rateleft = tparams[:rateRemaining]
|
649
686
|
attempt = tparams[:attempt] # note: is zero on very first try
|
650
687
|
stats = tparams[:statistics]
|
651
688
|
stats[:throttleDefaultCalls] += 1
|
@@ -657,30 +694,35 @@ class NumerousClientInternals
|
|
657
694
|
end
|
658
695
|
end
|
659
696
|
|
660
|
-
backarray = [ 2, 5, 15, 30, 60 ]
|
661
|
-
if attempt < backarray.length
|
662
|
-
backoff = backarray[attempt]
|
663
|
-
else
|
664
|
-
stats[:throttleMaxed] += 1
|
665
|
-
next false # too many tries
|
666
|
-
end
|
667
|
-
|
668
697
|
# if we weren't told to back off, no need to retry
|
669
698
|
if tparams[:resultCode] != 429
|
699
|
+
#
|
670
700
|
# but if we are closing in on the limit then slow ourselves down
|
671
701
|
# note that some errors don't communicate rateleft so we have to
|
672
702
|
# check for that as well (will be -1 here if wasn't sent to us)
|
673
703
|
#
|
674
|
-
#
|
675
|
-
#
|
676
|
-
|
704
|
+
# the point of this policy is to protect OTHER implementations
|
705
|
+
# that might not be aware of rate-limiting (i.e., we're being
|
706
|
+
# generous here by slowing ourselves down to try to avoid
|
707
|
+
# reaching the actual rate limit)
|
708
|
+
#
|
709
|
+
# at constructor time our "throttle data" (td) was set up with
|
710
|
+
# the 'voluntary' arbitrary limit
|
711
|
+
nAPIs_left = tparams[:rateRemaining]
|
712
|
+
if nAPIs_left >= 0 and nAPIs_left < td[:voluntary]
|
677
713
|
stats[:throttleVoluntaryBackoff] += 1
|
678
|
-
|
679
|
-
if
|
680
|
-
|
714
|
+
t = tparams[:rateReset] # time until fresh allocation
|
715
|
+
if t > 0 and nAPIs_left > 0
|
716
|
+
# force floating point
|
717
|
+
secs_per_API = (t + 0.001) / nAPIs_left
|
681
718
|
else
|
682
|
-
|
719
|
+
secs_per_API = 0.5 # arbitrary when no info
|
683
720
|
end
|
721
|
+
# arbitrary voluntary delay cap
|
722
|
+
dt = [td[:volmaxdelay], secs_per_API].min
|
723
|
+
|
724
|
+
stats[:throttleVoluntaryDelays] += dt
|
725
|
+
sleep(dt)
|
684
726
|
end
|
685
727
|
next false # no retry
|
686
728
|
end
|
@@ -688,6 +730,14 @@ class NumerousClientInternals
|
|
688
730
|
# decide how long to delay ... we just wait for as long as the
|
689
731
|
# server told us to (plus "backoff" seconds slop to really be sure we
|
690
732
|
# aren't back too soon)
|
733
|
+
backarray = [ 0.75, 1.5, 5, 15, 45 ]
|
734
|
+
if attempt < backarray.length
|
735
|
+
backoff = backarray[attempt]
|
736
|
+
else
|
737
|
+
stats[:throttleMaxed] += 1
|
738
|
+
next false # too many tries
|
739
|
+
end
|
740
|
+
|
691
741
|
stats[:throttle429] += 1
|
692
742
|
sleep(tparams[:rateReset] + backoff)
|
693
743
|
next true
|
@@ -1125,9 +1175,8 @@ class NumerousMetric < NumerousClientInternals
|
|
1125
1175
|
#
|
1126
1176
|
# It can also be a metric's web link, e.g.:
|
1127
1177
|
# http://n.numerousapp.com/m/1x8ba7fjg72d
|
1128
|
-
#
|
1129
|
-
#
|
1130
|
-
# encoding of the ID.
|
1178
|
+
# or the "embed" (/e/ vs /m/) variant, in which case we "just know"
|
1179
|
+
# that the tail is a base36 encoding of the ID.
|
1131
1180
|
#
|
1132
1181
|
# The decoding logic here makes the specific assumption that
|
1133
1182
|
# the presence of a '/' indicates a non-naked metric ID. This
|
@@ -1155,7 +1204,7 @@ class NumerousMetric < NumerousClientInternals
|
|
1155
1204
|
fields = id.split('/')
|
1156
1205
|
if fields.length() == 1
|
1157
1206
|
actualId = fields[0]
|
1158
|
-
elsif fields[-2]
|
1207
|
+
elsif [ "m", "e" ].include? fields[-2]
|
1159
1208
|
actualId = fields[-1].to_i(36)
|
1160
1209
|
else
|
1161
1210
|
actualId = fields[-1]
|
@@ -1625,7 +1674,14 @@ class NumerousMetric < NumerousClientInternals
|
|
1625
1674
|
j['action'] = 'ADD'
|
1626
1675
|
end
|
1627
1676
|
if updated
|
1628
|
-
|
1677
|
+
# if you gave us a formattable time try converting it
|
1678
|
+
begin
|
1679
|
+
ts = updated.strftime('%Y-%m-%dT%H:%M:%S.')
|
1680
|
+
ts += ("%03dZ" % ((updated.usec+500)/1000))
|
1681
|
+
rescue NoMethodError # just take your argument
|
1682
|
+
ts = updated # which should be a string already
|
1683
|
+
end
|
1684
|
+
j['updated'] = ts
|
1629
1685
|
end
|
1630
1686
|
|
1631
1687
|
@cachedHash = nil # will need to refresh cache on next access
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: numerousapp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.2.
|
4
|
+
version: 1.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Neil Webber
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-07-13 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description: Classes implementing the NumerousApp REST APIs for metrics. Requires
|
14
14
|
Ruby 2.x
|