chef_stash 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,355 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Author: Stefano Harding <riddopic@gmail.com>
4
+ # License: Apache License, Version 2.0
5
+ # Copyright: (C) 2014-2015 Stefano Harding
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+
20
+ require 'monitor'
21
+
22
+ module ChefStash
23
+ # In-process cache with least-recently used (LRU) and time-to-live (TTL)
24
+ # expiration semantics. This implementation is thread-safe. It does not use
25
+ # a thread to clean up expired values. Instead, an expiration check is
26
+ # performed:
27
+ #
28
+ # 1. Every time you retrieve a value, against that value. If the value has
29
+ # expired, it will be removed and `nil` will be returned.
30
+ #
31
+ # 2. Every `expire_interval` operations as the cache is used to remove all
32
+ # expired values up to that point.
33
+ #
34
+ # For manual expiration call {#expire!}.
35
+ #
36
+ class TimeCache
37
+ # The maximum number of seconds an element can exist in the cache
38
+ # regardless of use. The element expires at this limit and will no longer
39
+ # be returned from the cache. The default value is 3600, or 1 hour. Setting
40
+ # A TTL value of 0 means no TTL eviction takes place (infinite lifetime).
41
+ DEFAULT_TTL_SECONDS = 3600
42
+
43
+ # The maximum number of seconds an element can exist in the cache without
44
+ # being accessed. The element expires at this limit and will no longer be
45
+ # returned from the cache. The default value is 3600, or 1 hour. Setting a
46
+ # TTI value of 0 means no TTI eviction takes place (infinite lifetime).
47
+ DEFAULT_TTI_SECONDS = 3600
48
+
49
+ # The maximum sum total number of elements (cache entries) allowed on the
50
+ # disk tier for the cache. If this target is exceeded, eviction occurs to
51
+ # bring the count within the allowed target. The default value is 1,000. A
52
+ # setting of 0 means that no eviction of the cache's entries takes place
53
+ # (infinite size is allowed), and consequently can cause the node to run
54
+ # out of disk space.
55
+ DEFAULT_MAX_ENTRIES = 1_000
56
+
57
+ # @!attribute [r] :stats
58
+ # @return [CacheStats] The Cache statistics.
59
+ attr_reader :stats
60
+
61
+ # @!attribute [r] :ttl (DEFAULT_TTL_SECONDS)
62
+ # @return [Integer] The time to live for an element before it expires.
63
+ attr_reader :ttl
64
+
65
+ # @!attribute [r] :tti (DEFAULT_TTI_SECONDS)
66
+ # @return [Integer] The time to idle for an element before it expires.
67
+ attr_reader :tti
68
+
69
+ # Initializes the cache.
70
+ #
71
+ # @param [Hash] opts
72
+ # The options to configure the cache.
73
+ #
74
+ # @option opts [Integer] :max_entries
75
+ # Maximum number of elements in the cache.
76
+ #
77
+ # @option opts [Numeric] :ttl
78
+ # Maximum time, in seconds, for a value to stay in the cache.
79
+ #
80
+ # @option opts [Numeric] :tti
81
+ # Maximum time, in seconds, for a value to stay in the cache without
82
+ # being accessed.
83
+ #
84
+ # @option opts [Integer] :interval
85
+ # Number of cache operations between calls to {#expire!}.
86
+ #
87
+ def initialize(opts = {})
88
+ @max_entries = opts.fetch(:max_entries, DEFAULT_MAX_ENTRIES)
89
+ @ttl_seconds = opts.fetch(:ttl_seconds, DEFAULT_TTL_SECONDS)
90
+ @tti_seconds = opts.fetch(:ttl_seconds, DEFAULT_TTI_SECONDS)
91
+ @interval = opts.fetch(:interval, 100)
92
+ @operations = 0
93
+ @monitor = Monitor.new
94
+ @stash = {}
95
+ @expires_at = {}
96
+ end
97
+
98
+ # Loads a hash of data into the stash.
99
+ #
100
+ # @param [Hash] data
101
+ # Hash of data with either String or Symbol keys.
102
+ #
103
+ # @return nothing.
104
+ #
105
+ def load(data)
106
+ @monitor.synchronize do
107
+ data.each do |key, value|
108
+ expire!
109
+ store(key, val)
110
+ end
111
+ end
112
+ end
113
+
114
+ # Retrieves a value from the cache, if available and not expired, or yields
115
+ # to a block that calculates the value to be stored in the cache.
116
+ #
117
+ # @param [Object] key
118
+ # The key to look up or store at.
119
+ #
120
+ # @yield yields when the value is not present.
121
+ #
122
+ # @yieldreturn [Object]
123
+ # The value to store in the cache.
124
+ #
125
+ # @return [Object]
126
+ # The value at the key.
127
+ #
128
+ def fetch(key)
129
+ @monitor.synchronize do
130
+ found, value = get(key)
131
+ found ? value : store(key, yield)
132
+ end
133
+ end
134
+
135
+ # Retrieves a value from the cache.
136
+ #
137
+ # @param [Object] key
138
+ # The key to look up.
139
+ #
140
+ # @return [Object, nil]
141
+ # The value at the key, when present, or `nil`.
142
+ #
143
+ def [](key)
144
+ @monitor.synchronize do
145
+ _, value = get(key)
146
+ value
147
+ end
148
+ end
149
+ alias_method :get, :[]
150
+
151
+ # Stores a value in the cache.
152
+ #
153
+ # @param [Object] key
154
+ # The key to store.
155
+ #
156
+ # @param val [Object]
157
+ # The value to store.
158
+ #
159
+ # @return [Object, nil]
160
+ # The value at the key.
161
+ #
162
+ def []=(key, val)
163
+ @monitor.synchronize do
164
+ expire!
165
+ store(key, val)
166
+ end
167
+ end
168
+ alias_method :set, :[]=
169
+
170
+ # Removes a value from the cache.
171
+ #
172
+ # @param [Object] key
173
+ # The key to remove.
174
+ #
175
+ # @return [Object, nil]
176
+ # The value at the key, when present, or `nil`.
177
+ #
178
+ def delete(key)
179
+ @monitor.synchronize do
180
+ entry = @stash.delete(key)
181
+ if entry
182
+ @expires_at.delete(entry)
183
+ entry.value
184
+ else
185
+ nil
186
+ end
187
+ end
188
+ end
189
+
190
+ # Checks whether the cache is empty.
191
+ #
192
+ # @note calls to {#empty?} do not count against `expire_interval`.
193
+ #
194
+ # @return [Boolean]
195
+ #
196
+ def empty?
197
+ @monitor.synchronize { count == 0 }
198
+ end
199
+
200
+ # Clears the cache.
201
+ #
202
+ # @return [self]
203
+ #
204
+ def clear
205
+ @monitor.synchronize do
206
+ @stash.clear
207
+ @expires_at.clear
208
+ self
209
+ end
210
+ end
211
+
212
+ # Returns the number of elements in the cache.
213
+ #
214
+ # @note
215
+ # Calls to {#empty?} do not count against `expire_interval`. Therefore,
216
+ # the number of elements is that prior to any expiration.
217
+ #
218
+ # @return [Integer]
219
+ # Number of elements in the cache.
220
+ #
221
+ def count
222
+ @monitor.synchronize { @stash.count }
223
+ end
224
+ alias_method :size, :count
225
+ alias_method :length, :count
226
+
227
+ # Allows iteration over the items in the cache. Enumeration is stable: it
228
+ # is not affected by changes to the cache, including value expiration.
229
+ # Expired values are removed first.
230
+ #
231
+ # @note
232
+ # The returned values could have expired by the time the client code gets
233
+ # to accessing them.
234
+ #
235
+ # @note
236
+ # Because of its stability, this operation is very expensive. Use with
237
+ # caution.
238
+ #
239
+ # @yield [Array<key, value>]
240
+ # Key/value pairs, when a block is provided.
241
+ #
242
+ # @return [Enumerator, Array<key, value>]
243
+ # An Enumerator, when no block is provided, or array of key/value pairs.
244
+ #
245
+ def each(&block)
246
+ @monitor.synchronize do
247
+ expire!
248
+ @stash.map { |key, entry| [key, entry.value] }.each(&block)
249
+ end
250
+ end
251
+
252
+ # Removes expired values from the cache.
253
+ #
254
+ # @return [self]
255
+ #
256
+ def expire!
257
+ @monitor.synchronize do
258
+ check_expired(Time.now.to_f)
259
+ self
260
+ end
261
+ end
262
+
263
+ # Return all keys in the store as an array.
264
+ #
265
+ # @return [Array<String, Symbol>] all the keys in store
266
+ #
267
+ def keys
268
+ @monitor.synchronize { @stash.keys }
269
+ end
270
+
271
+ # Returns information about the number of objects in the cache, its
272
+ # maximum size and TTL.
273
+ #
274
+ # @return [String]
275
+ #
276
+ def inspect
277
+ @monitor.synchronize do
278
+ "<#{self.class.name} count=#{count} max_entries=#{@max_entries} " \
279
+ "ttl=#{@ttl_seconds}>"
280
+ end
281
+ end
282
+
283
+ private # P R O P R I E T À P R I V A T A Vietato L'accesso
284
+
285
+ # @private
286
+ class Entry
287
+ attr_reader :value
288
+ attr_reader :expires_at
289
+
290
+ def initialize(value, expires_at)
291
+ @value = value
292
+ @expires_at = expires_at
293
+ end
294
+ end
295
+
296
+ def get(key)
297
+ @monitor.synchronize do
298
+ time = Time.now.to_f
299
+ check_expired(time)
300
+ found = true
301
+ entry = @stash.delete(key) { found = false }
302
+ if found
303
+ if entry.expires_at <= time
304
+ @expires_at.delete(entry)
305
+ return false, nil
306
+ else
307
+ @stash[key] = entry
308
+ return true, entry.value
309
+ end
310
+ else
311
+ return false, nil
312
+ end
313
+ end
314
+ end
315
+
316
+ def store(key, val)
317
+ @monitor.synchronize do
318
+ expires_at = Time.now.to_f + @ttl_seconds
319
+ entry = Entry.new(val, expires_at)
320
+ store_entry(key, entry)
321
+ val
322
+ end
323
+ end
324
+
325
+ def store_entry(key, entry)
326
+ @monitor.synchronize do
327
+ @stash.delete(key)
328
+ @stash[key] = entry
329
+ @expires_at[entry] = key
330
+ shrink_if_needed
331
+ end
332
+ end
333
+
334
+ def shrink_if_needed
335
+ @monitor.synchronize do
336
+ if @stash.length > @max_entries
337
+ entry = delete(@stash.shift)
338
+ @expires_at.delete(entry)
339
+ end
340
+ end
341
+ end
342
+
343
+ def check_expired(time)
344
+ @monitor.synchronize do
345
+ if (@operations += 1) % @interval == 0
346
+ while (key_value_pair = @expires_at.first) &&
347
+ (entry = key_value_pair.first).expires_at <= time
348
+ key = @expires_at.delete(entry)
349
+ @stash.delete(key)
350
+ end
351
+ end
352
+ end
353
+ end
354
+ end
355
+ end
@@ -0,0 +1,97 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Author: Stefano Harding <riddopic@gmail.com>
4
+ # License: Apache License, Version 2.0
5
+ # Copyright: (C) 2014-2015 Stefano Harding
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+
20
+ module ChefStash
21
+ module Response
22
+ HTTP_STATUS_CODES = {
23
+ 200 => 'OK',
24
+ 201 => 'Created',
25
+ 202 => 'Accepted',
26
+ 203 => 'Non-Authoritative Information',
27
+ 204 => 'No Content',
28
+ 205 => 'Reset Content',
29
+ 206 => 'Partial Content',
30
+ 300 => 'Multiple Choices',
31
+ 301 => 'Moved Permanently',
32
+ 302 => 'Found',
33
+ 303 => 'See Other',
34
+ 304 => 'Not Modified',
35
+ 305 => 'Use Proxy',
36
+ 306 => 'Switch Proxy',
37
+ 307 => 'Temporary Redirect',
38
+ 308 => 'Permanent Redirect',
39
+ 400 => 'BadRequest',
40
+ 401 => 'Unauthorized',
41
+ 403 => 'Forbidden',
42
+ 404 => 'NotFound',
43
+ 405 => 'MethodNotAllowed',
44
+ 406 => 'AccessDenied',
45
+ 409 => 'Conflict',
46
+ 410 => 'Gone',
47
+ 500 => 'InternalServerError',
48
+ 501 => 'NotImplemented',
49
+ 502 => 'BadGateway',
50
+ 503 => 'ServiceUnavailable',
51
+ 504 => 'GatewayTimeout',
52
+ }
53
+
54
+ def self.code(code)
55
+ status = HTTP_STATUS_CODES[code]
56
+ code.to_s + ' ' + status
57
+ end
58
+ end
59
+
60
+ class FileSize
61
+ def initialize(size)
62
+ @units = {
63
+ b: 1,
64
+ kb: 1024**1,
65
+ mb: 1024**2,
66
+ gb: 1024**3,
67
+ tb: 1024**4,
68
+ pb: 1024**5,
69
+ eb: 1024**6
70
+ }
71
+
72
+ @size_int = size.partition(/\D{1,2}/).at(0).to_i
73
+ unit = size.partition(/\D{1,2}/).at(1).to_s.downcase
74
+ case
75
+ when unit.match(/[kmgtpe]{1}/)
76
+ @size_unit = unit.concat('b')
77
+ when unit.match(/[kmgtpe]{1}b/)
78
+ @size_unit = unit
79
+ else
80
+ @size_unit = 'b'
81
+ end
82
+ end
83
+
84
+ def to_size(unit, places = 1)
85
+ unit_val = @units[unit.to_s.downcase.to_sym]
86
+ bytes = @size_int * @units[@size_unit.to_sym]
87
+ size = bytes.to_f / unit_val.to_f
88
+ value = sprintf("%.#{places}f", size).to_f
89
+ "#{value} #{unit.upcase}"
90
+ end
91
+
92
+ def from_size(places = 1)
93
+ unit_val = @units[@size_unit.to_s.downcase.to_sym]
94
+ sprintf("%.#{places}f", @size_int * unit_val)
95
+ end
96
+ end
97
+ end
@@ -0,0 +1,42 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Author: Stefano Harding <riddopic@gmail.com>
4
+ # License: Apache License, Version 2.0
5
+ # Copyright: (C) 2014-2015 Stefano Harding
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+
20
+ # The version number of the ChefStash Gem
21
+ #
22
+ # @return [String]
23
+ #
24
+ module ChefStash
25
+ # Contains information about this gem's version
26
+ module Version
27
+ MAJOR = 0
28
+ MINOR = 1
29
+ PATCH = 1
30
+
31
+ # Returns a version string by joining MAJOR, MINOR, and PATCH with '.'
32
+ #
33
+ # @example
34
+ # Version.string # => '1.0.1'
35
+ #
36
+ def self.string
37
+ [MAJOR, MINOR, PATCH].join('.')
38
+ end
39
+ end
40
+
41
+ VERSION = ChefStash::Version.string
42
+ end
data/lib/chef_stash.rb ADDED
@@ -0,0 +1,125 @@
1
+ # encoding: UTF-8
2
+ #
3
+ # Author: Stefano Harding <riddopic@gmail.com>
4
+ # License: Apache License, Version 2.0
5
+ # Copyright: (C) 2014-2015 Stefano Harding
6
+ #
7
+ # Licensed under the Apache License, Version 2.0 (the "License");
8
+ # you may not use this file except in compliance with the License.
9
+ # You may obtain a copy of the License at
10
+ #
11
+ # http://www.apache.org/licenses/LICENSE-2.0
12
+ #
13
+ # Unless required by applicable law or agreed to in writing, software
14
+ # distributed under the License is distributed on an "AS IS" BASIS,
15
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16
+ # See the License for the specific language governing permissions and
17
+ # limitations under the License.
18
+ #
19
+
20
+ require 'chef_stash/version'
21
+ require 'chef_stash/core_ext/hash'
22
+ require 'chef_stash/core_ext/numeric'
23
+ require 'chef_stash/time_cache'
24
+ require 'chef_stash/disk_store'
25
+ require 'chef_stash/memoizable'
26
+ require 'chef_stash/rash'
27
+ require 'chef_stash/os'
28
+ require 'chef_stash/utils'
29
+
30
+ # Chef Key/value stash cache hash objects store.
31
+ #
32
+ module ChefStash
33
+ # Check if we're using a version if Ruby that supports caller_locations.
34
+ NEW_CALL = Kernel.respond_to? 'caller_locations'
35
+
36
+ # Default hash stash cache store type.
37
+ DEFAULT_STORE = DiskStore
38
+
39
+ # insert a helper .new() method for creating a new object
40
+ #
41
+ def self.new(*args)
42
+ self::Cache.new(*args)
43
+ end
44
+
45
+ # helper to get the calling function name
46
+ #
47
+ def self.caller_name
48
+ NEW_CALL ? caller_locations(2, 1).first.label : caller[1][/`([^']*)'/, 1]
49
+ end
50
+
51
+ # Chef Key/value stash cache hash objects store.
52
+ #
53
+ class Cache
54
+ # @return [Hash] of the mem stash cache hash store
55
+ #
56
+ # @!attribute [r] :store
57
+ # @return [ChefStash] The Chef Key/value stash cache hash objects store.
58
+ attr_reader :store
59
+
60
+ # Initializes a new Chef Key/value stash cache hash objects store.
61
+ #
62
+ def initialize(params = {})
63
+ params = { store: params } unless params.is_a? Hash
64
+ @store = params.fetch(:store) { ChefStash::DEFAULT_STORE.new }
65
+ end
66
+
67
+ # Retrieves the value for a given key, if nothing is set,
68
+ # returns KeyError
69
+ #
70
+ # @param key [Symbol, String] representing the key
71
+ #
72
+ # @raise [KeyError] if no such key found
73
+ #
74
+ # @return [Hash, Array, String] value for key
75
+ #
76
+ def [](key = nil)
77
+ key ||= ChefStash.caller_name
78
+ fail KeyError, 'Key not cached' unless include? key.to_sym
79
+ @store[key.to_sym]
80
+ end
81
+
82
+ # Retrieves the value for a given key, if nothing is set,
83
+ # run the code, cache the result, and return it
84
+ #
85
+ # @param key [Symbol, String] representing the key
86
+ # @param block [&block] that returns the value to set (optional)
87
+ #
88
+ # @return [Hash, Array, String] value for key
89
+ #
90
+ def cache(key = nil, &code)
91
+ key ||= ChefStash.caller_name
92
+ @store[key.to_sym] ||= code.call
93
+ end
94
+
95
+ # return a boolean indicating presence of the given key in the store
96
+ #
97
+ # @param key [Symbol, String] a string or symbol representing the key
98
+ #
99
+ # @return [TrueClass, FalseClass]
100
+ #
101
+ def include?(key = nil)
102
+ key ||= ChefStash.caller_name
103
+ @store.include? key.to_sym
104
+ end
105
+
106
+ # Clear the whole stash store or the value of a key
107
+ #
108
+ # @param key [Symbol, String] (optional) representing the key to
109
+ # clear.
110
+ #
111
+ # @return nothing.
112
+ #
113
+ def clear!(key = nil)
114
+ key.nil? ? @store.clear : @store.delete(key)
115
+ end
116
+
117
+ # return the size of the store as an integer
118
+ #
119
+ # @return [Fixnum]
120
+ #
121
+ def size
122
+ @store.size
123
+ end
124
+ end
125
+ end
data/rash.rb ADDED
@@ -0,0 +1,59 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ lib = File.expand_path('../lib', __FILE__)
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+
6
+ require 'anemone'
7
+ require 'chef_stash'
8
+ require 'ap'
9
+ require 'chef_stash/core_ext/hash'
10
+
11
+ url = 'http://winini.mudbox.dev/'
12
+ path = 'packages_3.0'
13
+ options = { threads: 20, depth_limit: 3, discard_page_bodies: true }
14
+ results = []
15
+ regex = /#{path}\/\w+.(\w+.(ini|zip)|sha256.txt)$/i
16
+ seen = []
17
+
18
+ def seen_urls
19
+ @seen_urls ||= []
20
+ end
21
+
22
+ Anemone.crawl(url, options) do |anemone|
23
+ anemone.on_pages_like(regex) do |page|
24
+ url = page.url.to_s
25
+ name = File.basename(url)
26
+ key = File.basename(name, '.*').downcase.to_sym
27
+ type = File.extname(name)[1..-1].downcase.to_sym
28
+
29
+ header = page.headers
30
+ bytes = header['content-length'].first
31
+ modified = header['last-modified'].first
32
+ created = Time.now.utc.httpdate
33
+ content = type == :ini ? 'text/inifile' : header['content-type'].first
34
+ size = ChefStash::FileSize.new(bytes).to_size(:mb).to_s
35
+
36
+ seen_urls << { url: url, modified: modified, created: created }
37
+
38
+ result = { key => { type => {
39
+ code: ChefStash::Response.code(page.code),
40
+ depth: page.depth,
41
+ size: size,
42
+ key: key,
43
+ md5: Digest::MD5.hexdigest(page.body.to_s),
44
+ modified: modified,
45
+ name: name,
46
+ referer: page.referer.to_s,
47
+ response_time: page.response_time.time_humanize,
48
+ sha256: OpenSSL::Digest::SHA256.new(page.body).to_s,
49
+ content_type: content,
50
+ url: url,
51
+ created: created,
52
+ visited: page.visited
53
+ } } }
54
+
55
+ ap result
56
+ end
57
+ end
58
+
59
+ ap seen_urls