api_call_cache 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/lib/api_call_cache/version.rb +3 -0
- data/lib/api_call_cache.rb +281 -0
- metadata +132 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 397acaee7206b32cca9bbe9a53a27bdd4a5834e4
|
4
|
+
data.tar.gz: 7644716942c4850d6d9aaef4f2612ef79c0276da
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: 4813d89d3022de7a31b9a73f2a32497b46bcf631f3dbb0daa98aba10dfc666e616ea4cafc336aa40c1888a12825fac787f40cf08a8054a998e00de64e61991a0
|
7
|
+
data.tar.gz: 5666df2248a6a204c75fea2c81a4510f72a3901d95a685b4ee0ddc5ece4dd5443c1b363150df773c1d138f7d876e5855e08239f82ff06beed62ba466c125a193
|
@@ -0,0 +1,281 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'redis'
|
3
|
+
require 'hashie'
|
4
|
+
require 'active_support/core_ext/object'
|
5
|
+
require 'httpclient'
|
6
|
+
require 'rack/oauth2'
|
7
|
+
|
8
|
+
require 'api_call_cache/version'
|
9
|
+
|
10
|
+
class ApiCallCache
|
11
|
+
|
12
|
+
def self.configure
|
13
|
+
yield self
|
14
|
+
end
|
15
|
+
|
16
|
+
def self.api_cache_redis_instance(redis_inst)
|
17
|
+
|
18
|
+
raise ArgumentError, 'requires a redis instance' unless redis_inst.is_a? Redis
|
19
|
+
@@redis_inst = redis_inst
|
20
|
+
end
|
21
|
+
|
22
|
+
def self.logger_path(log_file = 'log/api_call_cache.log', rotation = 'monthly')
|
23
|
+
@@acc_logger ||= Logger.new(log_file, rotation)
|
24
|
+
end
|
25
|
+
|
26
|
+
def self.base_urls(base_urls_hash)
|
27
|
+
@@acc_base_urls = Hashie.symbolize_keys(base_urls_hash)
|
28
|
+
end
|
29
|
+
|
30
|
+
def self.salt(salt_str)
|
31
|
+
raise ArgumentError, 'requires a string as salt' if salt_str.empty?
|
32
|
+
@@acc_salt = salt_str
|
33
|
+
end
|
34
|
+
|
35
|
+
def self.req_defaults
|
36
|
+
@req_opt_defaults ||= {
|
37
|
+
base_url_key: '',
|
38
|
+
access_token: '', # TODO_FIX -- will need NetHttp etc
|
39
|
+
from_cache: true,
|
40
|
+
override_cache_expiry: nil, # seconds. Overrides Cache-Control max-age value
|
41
|
+
override_params_cache_key: nil, # If provided, this value will be used
|
42
|
+
# as the 'params' part of the cache key,
|
43
|
+
# instead of manually 'hashed_params'
|
44
|
+
#
|
45
|
+
# Useful if the api doesn't accept explicit
|
46
|
+
# params, but is distinguished solely on the
|
47
|
+
# basis of some implicit data like
|
48
|
+
# access_token etc
|
49
|
+
}
|
50
|
+
end
|
51
|
+
|
52
|
+
def self.get_redis
|
53
|
+
@@redis_inst
|
54
|
+
end
|
55
|
+
|
56
|
+
def get_redis
|
57
|
+
self.class.get_redis
|
58
|
+
end
|
59
|
+
|
60
|
+
def self.set_tz_offset(offset = '+05:30')
|
61
|
+
@@tz_offset = offset
|
62
|
+
end
|
63
|
+
|
64
|
+
##############################################################################
|
65
|
+
# Class methods
|
66
|
+
##############################################################################
|
67
|
+
def self.api_get(base_url_key, rel_path, req_params = {}, req_opts = {})
|
68
|
+
req_opts = req_opts.merge({base_url_key: base_url_key})
|
69
|
+
cached_api_call(:get, rel_path, req_params, req_opts)
|
70
|
+
end
|
71
|
+
|
72
|
+
def self.cached_api_call(req_type, rel_path, req_params = {}, req_opts = {})
|
73
|
+
obj = self.new
|
74
|
+
obj.cached_api_call_core(req_type, rel_path, req_params, req_opts)
|
75
|
+
rescue Exception => exp
|
76
|
+
obj.acc_log_entry[:status] = 'exception'
|
77
|
+
obj.acc_log_entry[:desc] = exp.message
|
78
|
+
raise
|
79
|
+
ensure
|
80
|
+
@@acc_logger.info obj.acc_log_entry.to_a.flatten.join("\t") if @@acc_logger
|
81
|
+
end
|
82
|
+
|
83
|
+
def self.make_api_call(req_type, url, access_token, body=nil)
|
84
|
+
self.new.make_api_call(req_type, url, access_token, body)
|
85
|
+
end
|
86
|
+
|
87
|
+
##############################################################################
|
88
|
+
# Instance methods
|
89
|
+
##############################################################################
|
90
|
+
def cached_api_call_core(req_type, rel_path, req_params, req_opts)
|
91
|
+
|
92
|
+
Hashie.symbolize_keys!(req_opts)
|
93
|
+
req_opts = self.class.req_defaults.merge(req_opts)
|
94
|
+
req_type = req_type.to_s.downcase.to_sym
|
95
|
+
|
96
|
+
# Get from cache if required
|
97
|
+
cached_body = nil
|
98
|
+
|
99
|
+
base_url_key = req_opts[:base_url_key].to_s
|
100
|
+
raise 'Base URL KEY not found' if base_url_key.empty?
|
101
|
+
|
102
|
+
acc_log_entry[:api] = [base_url_key.to_s, rel_path.to_s].join('/')
|
103
|
+
|
104
|
+
base_url = @@acc_base_urls[base_url_key.to_sym].to_s
|
105
|
+
raise "Base URL not found for #{base_url_key}" if base_url.empty?
|
106
|
+
|
107
|
+
cache_key = gen_api_call_cache_key(req_type, base_url_key,
|
108
|
+
rel_path, req_params,
|
109
|
+
req_opts[:override_params_cache_key])
|
110
|
+
|
111
|
+
try_from_cache = (req_type == :get) && req_opts[:from_cache]
|
112
|
+
write_to_cache = (req_type == :get)
|
113
|
+
acc_log_entry[:cache_key] = cache_key
|
114
|
+
cache_miss = false
|
115
|
+
|
116
|
+
if try_from_cache
|
117
|
+
cached_info = get_redis.hgetall(cache_key)
|
118
|
+
cached_body = cached_info['body']
|
119
|
+
cached_status = cached_info['status'].to_i
|
120
|
+
|
121
|
+
cache_miss = cached_body.nil? # NOTE: 'nil?' used here on purpose instead of 'blank?'
|
122
|
+
acc_log_entry[:status] = cache_miss ? 'miss' : 'hit'
|
123
|
+
|
124
|
+
ttl = get_redis.ttl(cache_key).to_i.seconds
|
125
|
+
else
|
126
|
+
acc_log_entry[:status] = 'ignore'
|
127
|
+
end
|
128
|
+
|
129
|
+
if cache_miss
|
130
|
+
|
131
|
+
rel_url = gen_api_call_rel_url(rel_path, req_params)
|
132
|
+
|
133
|
+
acc_log_entry[:params_f] = req_params.to_s
|
134
|
+
|
135
|
+
tic = ::Time.now
|
136
|
+
api_resp = make_api_call(req_type, "#{base_url}/#{rel_url}",
|
137
|
+
req_opts[:access_token])
|
138
|
+
toc = ::Time.now
|
139
|
+
|
140
|
+
acc_log_entry[:resp_time] = ((toc - tic)*1000).round.to_s
|
141
|
+
acc_log_entry[:resp_code] = api_resp.status
|
142
|
+
|
143
|
+
result_body = api_resp.body
|
144
|
+
result_status = api_resp.status.to_i
|
145
|
+
|
146
|
+
if write_to_cache && api_resp.try(:ok?)
|
147
|
+
ttl = write_to_api_cache(cache_key, api_resp, result_body, result_status,
|
148
|
+
req_opts[:override_cache_expiry])
|
149
|
+
|
150
|
+
acc_log_entry[:expires_at] = (ttl.to_i.seconds.from_now).to_time.localtime(@@tz_offset).to_s
|
151
|
+
end
|
152
|
+
else
|
153
|
+
# Found in cache. Using it.
|
154
|
+
result_body = cached_body
|
155
|
+
result_status = cached_status
|
156
|
+
acc_log_entry[:expires_at] = (ttl.seconds.from_now).to_time.localtime(@@tz_offset).to_s
|
157
|
+
end
|
158
|
+
|
159
|
+
Hashie::Mash.new(status: result_status,
|
160
|
+
body: result_body,
|
161
|
+
source: (try_from_cache && !cache_miss) ? :cache : :api_call,
|
162
|
+
ok: ::HTTP::Status::SUCCESSFUL_STATUS.include?(result_status))
|
163
|
+
end
|
164
|
+
|
165
|
+
def write_to_api_cache(cache_key, api_resp, cache_body, cache_status, override_cache_expiry)
|
166
|
+
expiry = get_cache_expiry(api_resp, override_cache_expiry)
|
167
|
+
|
168
|
+
acc_log_entry[:write_back] = 'true'
|
169
|
+
|
170
|
+
if !expiry.zero?
|
171
|
+
get_redis.hmset(cache_key, 'body', cache_body,
|
172
|
+
'status', cache_status)
|
173
|
+
end
|
174
|
+
|
175
|
+
get_redis.expire(cache_key, expiry)
|
176
|
+
expiry
|
177
|
+
end
|
178
|
+
|
179
|
+
def make_api_call(req_type, url, access_token, body = nil)
|
180
|
+
access_token = Rack::OAuth2::AccessToken::Bearer.new(access_token: access_token)
|
181
|
+
return access_token.send(req_type, url, body)
|
182
|
+
rescue HTTPClient::ReceiveTimeoutError => exp
|
183
|
+
return Hashie::Mash.new(status: 408,
|
184
|
+
body: {}.to_json,
|
185
|
+
ok?: false,
|
186
|
+
timeout?: true)
|
187
|
+
rescue SocketError => exp
|
188
|
+
return Hashie::Mash.new(status: 400,
|
189
|
+
body: {error: exp.message}.to_json,
|
190
|
+
ok?: false)
|
191
|
+
rescue Errno::ECONNREFUSED => exp
|
192
|
+
return Hashie::Mash.new(status: 403,
|
193
|
+
body: {error: exp.message}.to_json,
|
194
|
+
ok?: false)
|
195
|
+
end
|
196
|
+
|
197
|
+
def gen_api_call_rel_url(rel_path, req_params)
|
198
|
+
rel_path += '.json'
|
199
|
+
req_params.empty? ? rel_path : [rel_path, req_params.to_query].join('?')
|
200
|
+
end
|
201
|
+
|
202
|
+
def gen_api_call_cache_key(_req_type, base_url_key, rel_path, req_params, hashed_params = nil)
|
203
|
+
|
204
|
+
if hashed_params.nil? || hashed_params.empty?
|
205
|
+
# Since params hash key hasn't been overridden using
|
206
|
+
plain = req_params.sort.to_h.to_query # sort req_params alphabetically
|
207
|
+
hashed_params = OpenSSL::HMAC.hexdigest('sha256', @@acc_salt, plain)
|
208
|
+
end
|
209
|
+
|
210
|
+
acc_log_entry[:params_h] = hashed_params
|
211
|
+
|
212
|
+
['api_call_cache', 'api', base_url_key, rel_path, hashed_params].join(':')
|
213
|
+
end
|
214
|
+
|
215
|
+
|
216
|
+
def get_cache_expiry(response, user_expiry = nil)
|
217
|
+
# Extract server defined TTL
|
218
|
+
ext_ttl = nil
|
219
|
+
if user_expiry.nil?
|
220
|
+
cache_ctrl_hdr = response.headers['Api-Cache-Control'].to_s
|
221
|
+
cache_ctrl_hdr = response.headers['Cache-Control'] if cache_ctrl_hdr.empty?
|
222
|
+
cache_ctrl_hdr.split(',').map(&:strip).each do |opt|
|
223
|
+
opts = opt.split('=')
|
224
|
+
next if opts.first != 'max-age'
|
225
|
+
ext_ttl = opts.last.to_i
|
226
|
+
break
|
227
|
+
end
|
228
|
+
end
|
229
|
+
|
230
|
+
expiry = user_expiry || ext_ttl || FALLBACK_TTL
|
231
|
+
|
232
|
+
acc_log_entry[:user_ttl] = user_expiry.to_s
|
233
|
+
acc_log_entry[:src_ttl] = ext_ttl.to_s
|
234
|
+
|
235
|
+
# Dither cache expiry
|
236
|
+
dither_factor = 0.1
|
237
|
+
|
238
|
+
# final expiry = given expiry + 10% variance <max 2 minutes>
|
239
|
+
expiry += rand * ([expiry * dither_factor, 5.minutes].min)
|
240
|
+
expiry = expiry.round
|
241
|
+
|
242
|
+
acc_log_entry[:act_ttl] = expiry.to_s
|
243
|
+
|
244
|
+
expiry
|
245
|
+
end
|
246
|
+
|
247
|
+
attr_accessor :_acc_log_entry
|
248
|
+
|
249
|
+
def acc_log_entry
|
250
|
+
@_acc_log_entry ||= {
|
251
|
+
time: log_time,
|
252
|
+
status: '', # hit/miss/exception/ignore
|
253
|
+
api: '',
|
254
|
+
resp_time: '',
|
255
|
+
resp_code: '',
|
256
|
+
write_back: 'false',
|
257
|
+
user_ttl: '', # Suggested by the internal user for overriding
|
258
|
+
src_ttl: '', # Suggested by the external server
|
259
|
+
act_ttl: '', # Actual TTL after dithering
|
260
|
+
params_h: '', # hashed relative path
|
261
|
+
params_f: '', # full form relative path
|
262
|
+
cache_key: '',
|
263
|
+
desc: '',
|
264
|
+
}
|
265
|
+
end
|
266
|
+
|
267
|
+
def log_time
|
268
|
+
time = Time.now.localtime(@@tz_offset)
|
269
|
+
usec = time.usec.to_s.rjust(6, '0')
|
270
|
+
time.strftime "%Y-%m-%d %H:%M:%S.#{usec} %z"
|
271
|
+
end
|
272
|
+
end
|
273
|
+
|
274
|
+
ApiCallCache.configure do |config|
|
275
|
+
config.base_urls({})
|
276
|
+
|
277
|
+
config.logger_path nil
|
278
|
+
|
279
|
+
config.salt 'api-call-cache'
|
280
|
+
config.set_tz_offset '+05:30'
|
281
|
+
end
|
metadata
ADDED
@@ -0,0 +1,132 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: api_call_cache
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.3
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Akshay Rao
|
8
|
+
- Rupesh Joshi
|
9
|
+
autorequire:
|
10
|
+
bindir: exe
|
11
|
+
cert_chain: []
|
12
|
+
date: 2018-08-01 00:00:00.000000000 Z
|
13
|
+
dependencies:
|
14
|
+
- !ruby/object:Gem::Dependency
|
15
|
+
name: bundler
|
16
|
+
requirement: !ruby/object:Gem::Requirement
|
17
|
+
requirements:
|
18
|
+
- - ">="
|
19
|
+
- !ruby/object:Gem::Version
|
20
|
+
version: '1'
|
21
|
+
type: :runtime
|
22
|
+
prerelease: false
|
23
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
version: '1'
|
28
|
+
- !ruby/object:Gem::Dependency
|
29
|
+
name: rack-oauth2
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
31
|
+
requirements:
|
32
|
+
- - ">="
|
33
|
+
- !ruby/object:Gem::Version
|
34
|
+
version: '1.2'
|
35
|
+
type: :runtime
|
36
|
+
prerelease: false
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
38
|
+
requirements:
|
39
|
+
- - ">="
|
40
|
+
- !ruby/object:Gem::Version
|
41
|
+
version: '1.2'
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: redis
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - ">="
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '3.2'
|
49
|
+
type: :runtime
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - ">="
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '3.2'
|
56
|
+
- !ruby/object:Gem::Dependency
|
57
|
+
name: hashie
|
58
|
+
requirement: !ruby/object:Gem::Requirement
|
59
|
+
requirements:
|
60
|
+
- - ">="
|
61
|
+
- !ruby/object:Gem::Version
|
62
|
+
version: '3.4'
|
63
|
+
type: :runtime
|
64
|
+
prerelease: false
|
65
|
+
version_requirements: !ruby/object:Gem::Requirement
|
66
|
+
requirements:
|
67
|
+
- - ">="
|
68
|
+
- !ruby/object:Gem::Version
|
69
|
+
version: '3.4'
|
70
|
+
- !ruby/object:Gem::Dependency
|
71
|
+
name: activesupport
|
72
|
+
requirement: !ruby/object:Gem::Requirement
|
73
|
+
requirements:
|
74
|
+
- - ">="
|
75
|
+
- !ruby/object:Gem::Version
|
76
|
+
version: '2.1'
|
77
|
+
type: :runtime
|
78
|
+
prerelease: false
|
79
|
+
version_requirements: !ruby/object:Gem::Requirement
|
80
|
+
requirements:
|
81
|
+
- - ">="
|
82
|
+
- !ruby/object:Gem::Version
|
83
|
+
version: '2.1'
|
84
|
+
- !ruby/object:Gem::Dependency
|
85
|
+
name: httpclient
|
86
|
+
requirement: !ruby/object:Gem::Requirement
|
87
|
+
requirements:
|
88
|
+
- - ">="
|
89
|
+
- !ruby/object:Gem::Version
|
90
|
+
version: '2.7'
|
91
|
+
type: :runtime
|
92
|
+
prerelease: false
|
93
|
+
version_requirements: !ruby/object:Gem::Requirement
|
94
|
+
requirements:
|
95
|
+
- - ">="
|
96
|
+
- !ruby/object:Gem::Version
|
97
|
+
version: '2.7'
|
98
|
+
description: Useful for inter-service GET HTTP calls
|
99
|
+
email:
|
100
|
+
- 14akshayrao@gmail.com
|
101
|
+
executables: []
|
102
|
+
extensions: []
|
103
|
+
extra_rdoc_files: []
|
104
|
+
files:
|
105
|
+
- lib/api_call_cache.rb
|
106
|
+
- lib/api_call_cache/version.rb
|
107
|
+
homepage: https://github.com/peerlearning/api-call-cache/
|
108
|
+
licenses:
|
109
|
+
- MIT
|
110
|
+
metadata:
|
111
|
+
allowed_push_host: https://rubygems.org
|
112
|
+
post_install_message:
|
113
|
+
rdoc_options: []
|
114
|
+
require_paths:
|
115
|
+
- lib
|
116
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
117
|
+
requirements:
|
118
|
+
- - ">="
|
119
|
+
- !ruby/object:Gem::Version
|
120
|
+
version: '2.2'
|
121
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
122
|
+
requirements:
|
123
|
+
- - ">="
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
126
|
+
requirements: []
|
127
|
+
rubyforge_project:
|
128
|
+
rubygems_version: 2.6.14
|
129
|
+
signing_key:
|
130
|
+
specification_version: 4
|
131
|
+
summary: Framework to manage caching of http api calls
|
132
|
+
test_files: []
|