chef_stash 0.1.1

data/lib/chef_stash/time_cache.rb

@@ -0,0 +1,355 @@
+# encoding: UTF-8
+#
+# Author:    Stefano Harding <riddopic@gmail.com>
+# License:   Apache License, Version 2.0
+# Copyright: (C) 2014-2015 Stefano Harding
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'monitor'
+
+module ChefStash
+  # In-process cache with least-recently used (LRU) and time-to-live (TTL)
+  # expiration semantics. This implementation is thread-safe. It does not use
+  # a thread to clean up expired values. Instead, an expiration check is
+  # performed:
+  #
+  # 1. Every time you retrieve a value, against that value. If the value has
+  #    expired, it will be removed and `nil` will be returned.
+  #
+  # 2. Every `expire_interval` operations as the cache is used to remove all
+  #    expired values up to that point.
+  #
+  # For manual expiration call {#expire!}.
+  #
+  class TimeCache
+    # The maximum number of seconds an element can exist in the cache
+    # regardless of use. The element expires at this limit and will no longer
+    # be returned from the cache. The default value is 3600, or 1 hour. Setting
+    # A TTL value of 0 means no TTL eviction takes place (infinite lifetime).
+    DEFAULT_TTL_SECONDS = 3600
+
+    # The maximum number of seconds an element can exist in the cache without
+    # being accessed. The element expires at this limit and will no longer be
+    # returned from the cache. The default value is 3600, or 1 hour. Setting a
+    # TTI value of 0 means no TTI eviction takes place (infinite lifetime).
+    DEFAULT_TTI_SECONDS = 3600
+
+    # The maximum sum total number of elements (cache entries) allowed on the
+    # disk tier for the cache. If this target is exceeded, eviction occurs to
+    # bring the count within the allowed target. The default value is 1,000. A
+    # setting of 0 means that no eviction of the cache's entries takes place
+    # (infinite size is allowed), and consequently can cause the node to run
+    # out of disk space.
+    DEFAULT_MAX_ENTRIES = 1_000
+
+    # @!attribute [r] :stats
+    #   @return [CacheStats] The Cache statistics.
+    attr_reader :stats
+
+    # @!attribute [r] :ttl (DEFAULT_TTL_SECONDS)
+    #   @return [Integer] The time to live for an element before it expires.
+    attr_reader :ttl
+
+    # @!attribute [r] :tti (DEFAULT_TTI_SECONDS)
+    #   @return [Integer] The time to idle for an element before it expires.
+    attr_reader :tti
+
+    # Initializes the cache.
+    #
+    # @param [Hash] opts
+    #   The options to configure the cache.
+    #
+    # @option opts [Integer] :max_entries
+    #   Maximum number of elements in the cache.
+    #
+    # @option opts [Numeric] :ttl
+    #   Maximum time, in seconds, for a value to stay in the cache.
+    #
+    # @option opts [Numeric] :tti
+    #   Maximum time, in seconds, for a value to stay in the cache without
+    #   being accessed.
+    #
+    # @option opts [Integer] :interval
+    #   Number of cache operations between calls to {#expire!}.
+    #
+    def initialize(opts = {})
+      @max_entries = opts.fetch(:max_entries, DEFAULT_MAX_ENTRIES)
+      @ttl_seconds = opts.fetch(:ttl_seconds, DEFAULT_TTL_SECONDS)
+      @tti_seconds = opts.fetch(:ttl_seconds, DEFAULT_TTI_SECONDS)
+      @interval    = opts.fetch(:interval, 100)
+      @operations  = 0
+      @monitor     = Monitor.new
+      @stash       = {}
+      @expires_at  = {}
+    end
+
+    # Loads a hash of data into the stash.
+    #
+    # @param [Hash] data
+    #   Hash of data with either String or Symbol keys.
+    #
+    # @return nothing.
+    #
+    def load(data)
+      @monitor.synchronize do
+        data.each do |key, value|
+          expire!
+          store(key, val)
+        end
+      end
+    end
+
+    # Retrieves a value from the cache, if available and not expired, or yields
+    # to a block that calculates the value to be stored in the cache.
+    #
+    # @param [Object] key
+    #   The key to look up or store at.
+    #
+    # @yield yields when the value is not present.
+    #
+    # @yieldreturn [Object]
+    #   The value to store in the cache.
+    #
+    # @return [Object]
+    #   The value at the key.
+    #
+    def fetch(key)
+      @monitor.synchronize do
+        found, value = get(key)
+        found ? value : store(key, yield)
+      end
+    end
+
+    # Retrieves a value from the cache.
+    #
+    # @param [Object] key
+    #   The key to look up.
+    #
+    # @return [Object, nil]
+    #   The value at the key, when present, or `nil`.
+    #
+    def [](key)
+      @monitor.synchronize do
+        _, value = get(key)
+        value
+      end
+    end
+    alias_method :get, :[]
+
+    # Stores a value in the cache.
+    #
+    # @param [Object] key
+    #   The key to store.
+    #
+    # @param val [Object]
+    #   The value to store.
+    #
+    # @return [Object, nil]
+    #   The value at the key.
+    #
+    def []=(key, val)
+      @monitor.synchronize do
+        expire!
+        store(key, val)
+      end
+    end
+    alias_method :set, :[]=
+
+    # Removes a value from the cache.
+    #
+    # @param [Object] key
+    #   The key to remove.
+    #
+    # @return [Object, nil]
+    #   The value at the key, when present, or `nil`.
+    #
+    def delete(key)
+      @monitor.synchronize do
+        entry = @stash.delete(key)
+        if entry
+          @expires_at.delete(entry)
+          entry.value
+        else
+          nil
+        end
+      end
+    end
+
+    # Checks whether the cache is empty.
+    #
+    # @note calls to {#empty?} do not count against `expire_interval`.
+    #
+    # @return [Boolean]
+    #
+    def empty?
+      @monitor.synchronize { count == 0 }
+    end
+
+    # Clears the cache.
+    #
+    # @return [self]
+    #
+    def clear
+      @monitor.synchronize do
+        @stash.clear
+        @expires_at.clear
+        self
+      end
+    end
+
+    # Returns the number of elements in the cache.
+    #
+    # @note
+    #   Calls to {#empty?} do not count against `expire_interval`. Therefore,
+    #   the number of elements is that prior to any expiration.
+    #
+    # @return [Integer]
+    #   Number of elements in the cache.
+    #
+    def count
+      @monitor.synchronize { @stash.count }
+    end
+    alias_method :size,   :count
+    alias_method :length, :count
+
+    # Allows iteration over the items in the cache. Enumeration is stable: it
+    # is not affected by changes to the cache, including value expiration.
+    # Expired values are removed first.
+    #
+    # @note
+    #   The returned values could have expired by the time the client code gets
+    #   to accessing them.
+    #
+    # @note
+    #   Because of its stability, this operation is very expensive. Use with
+    #   caution.
+    #
+    # @yield [Array<key, value>]
+    #   Key/value pairs, when a block is provided.
+    #
+    # @return [Enumerator, Array<key, value>]
+    #   An Enumerator, when no block is provided, or array of key/value pairs.
+    #
+    def each(&block)
+      @monitor.synchronize do
+        expire!
+        @stash.map { |key, entry| [key, entry.value] }.each(&block)
+      end
+    end
+
+    # Removes expired values from the cache.
+    #
+    # @return [self]
+    #
+    def expire!
+      @monitor.synchronize do
+        check_expired(Time.now.to_f)
+        self
+      end
+    end
+
+    # Return all keys in the store as an array.
+    #
+    # @return [Array<String, Symbol>] all the keys in store
+    #
+    def keys
+      @monitor.synchronize { @stash.keys }
+    end
+
+    # Returns information about the number of objects in the cache, its
+    # maximum size and TTL.
+    #
+    # @return [String]
+    #
+    def inspect
+      @monitor.synchronize do
+        "<#{self.class.name} count=#{count} max_entries=#{@max_entries} " \
+        "ttl=#{@ttl_seconds}>"
+      end
+    end
+
+    private #        P R O P R I E T À   P R I V A T A   Vietato L'accesso
+
+    # @private
+    class Entry
+      attr_reader :value
+      attr_reader :expires_at
+
+      def initialize(value, expires_at)
+        @value = value
+        @expires_at = expires_at
+      end
+    end
+
+    def get(key)
+      @monitor.synchronize do
+        time = Time.now.to_f
+        check_expired(time)
+        found = true
+        entry = @stash.delete(key) { found = false }
+        if found
+          if entry.expires_at <= time
+            @expires_at.delete(entry)
+            return false, nil
+          else
+            @stash[key] = entry
+            return true, entry.value
+          end
+        else
+          return false, nil
+        end
+      end
+    end
+
+    def store(key, val)
+      @monitor.synchronize do
+        expires_at = Time.now.to_f + @ttl_seconds
+        entry = Entry.new(val, expires_at)
+        store_entry(key, entry)
+        val
+      end
+    end
+
+    def store_entry(key, entry)
+      @monitor.synchronize do
+        @stash.delete(key)
+        @stash[key] = entry
+        @expires_at[entry] = key
+        shrink_if_needed
+      end
+    end
+
+    def shrink_if_needed
+      @monitor.synchronize do
+        if @stash.length > @max_entries
+          entry = delete(@stash.shift)
+          @expires_at.delete(entry)
+        end
+      end
+    end
+
+    def check_expired(time)
+      @monitor.synchronize do
+        if (@operations += 1) % @interval == 0
+          while (key_value_pair = @expires_at.first) &&
+              (entry = key_value_pair.first).expires_at <= time
+            key = @expires_at.delete(entry)
+            @stash.delete(key)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/chef_stash/utils.rb

@@ -0,0 +1,97 @@
+# encoding: UTF-8
+#
+# Author:    Stefano Harding <riddopic@gmail.com>
+# License:   Apache License, Version 2.0
+# Copyright: (C) 2014-2015 Stefano Harding
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module ChefStash
+  module Response
+    HTTP_STATUS_CODES = {
+      200 => 'OK',
+      201 => 'Created',
+      202 => 'Accepted',
+      203 => 'Non-Authoritative Information',
+      204 => 'No Content',
+      205 => 'Reset Content',
+      206 => 'Partial Content',
+      300 => 'Multiple Choices',
+      301 => 'Moved Permanently',
+      302 => 'Found',
+      303 => 'See Other',
+      304 => 'Not Modified',
+      305 => 'Use Proxy',
+      306 => 'Switch Proxy',
+      307 => 'Temporary Redirect',
+      308 => 'Permanent Redirect',
+      400 => 'BadRequest',
+      401 => 'Unauthorized',
+      403 => 'Forbidden',
+      404 => 'NotFound',
+      405 => 'MethodNotAllowed',
+      406 => 'AccessDenied',
+      409 => 'Conflict',
+      410 => 'Gone',
+      500 => 'InternalServerError',
+      501 => 'NotImplemented',
+      502 => 'BadGateway',
+      503 => 'ServiceUnavailable',
+      504 => 'GatewayTimeout',
+    }
+
+    def self.code(code)
+      status = HTTP_STATUS_CODES[code]
+      code.to_s + ' ' + status
+    end
+  end
+
+  class FileSize
+    def initialize(size)
+      @units = {
+        b:  1,
+        kb: 1024**1,
+        mb: 1024**2,
+        gb: 1024**3,
+        tb: 1024**4,
+        pb: 1024**5,
+        eb: 1024**6
+      }
+
+      @size_int = size.partition(/\D{1,2}/).at(0).to_i
+      unit = size.partition(/\D{1,2}/).at(1).to_s.downcase
+      case
+      when unit.match(/[kmgtpe]{1}/)
+        @size_unit = unit.concat('b')
+      when unit.match(/[kmgtpe]{1}b/)
+        @size_unit = unit
+      else
+        @size_unit = 'b'
+      end
+    end
+
+    def to_size(unit, places = 1)
+      unit_val = @units[unit.to_s.downcase.to_sym]
+      bytes    = @size_int * @units[@size_unit.to_sym]
+      size     = bytes.to_f / unit_val.to_f
+      value    = sprintf("%.#{places}f", size).to_f
+      "#{value} #{unit.upcase}"
+    end
+
+    def from_size(places = 1)
+      unit_val = @units[@size_unit.to_s.downcase.to_sym]
+      sprintf("%.#{places}f", @size_int * unit_val)
+    end
+  end
+end

data/lib/chef_stash/version.rb

@@ -0,0 +1,42 @@
+# encoding: UTF-8
+#
+# Author:    Stefano Harding <riddopic@gmail.com>
+# License:   Apache License, Version 2.0
+# Copyright: (C) 2014-2015 Stefano Harding
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# The version number of the ChefStash Gem
+#
+# @return [String]
+#
+module ChefStash
+  # Contains information about this gem's version
+  module Version
+    MAJOR = 0
+    MINOR = 1
+    PATCH = 1
+
+    # Returns a version string by joining MAJOR, MINOR, and PATCH with '.'
+    #
+    # @example
+    #   Version.string # => '1.0.1'
+    #
+    def self.string
+      [MAJOR, MINOR, PATCH].join('.')
+    end
+  end
+
+  VERSION = ChefStash::Version.string
+end

data/lib/chef_stash.rb

@@ -0,0 +1,125 @@
+# encoding: UTF-8
+#
+# Author:    Stefano Harding <riddopic@gmail.com>
+# License:   Apache License, Version 2.0
+# Copyright: (C) 2014-2015 Stefano Harding
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+require 'chef_stash/version'
+require 'chef_stash/core_ext/hash'
+require 'chef_stash/core_ext/numeric'
+require 'chef_stash/time_cache'
+require 'chef_stash/disk_store'
+require 'chef_stash/memoizable'
+require 'chef_stash/rash'
+require 'chef_stash/os'
+require 'chef_stash/utils'
+
+# Chef Key/value stash cache hash objects store.
+#
+module ChefStash
+  # Check if we're using a version if Ruby that supports caller_locations.
+  NEW_CALL = Kernel.respond_to? 'caller_locations'
+
+  # Default hash stash cache store type.
+  DEFAULT_STORE = DiskStore
+
+  # insert a helper .new() method for creating a new object
+  #
+  def self.new(*args)
+    self::Cache.new(*args)
+  end
+
+  # helper to get the calling function name
+  #
+  def self.caller_name
+    NEW_CALL ? caller_locations(2, 1).first.label : caller[1][/`([^']*)'/, 1]
+  end
+
+  # Chef Key/value stash cache hash objects store.
+  #
+  class Cache
+    # @return [Hash] of the mem stash cache hash store
+    #
+    # @!attribute [r] :store
+    #   @return [ChefStash] The Chef Key/value stash cache hash objects store.
+    attr_reader :store
+
+    # Initializes a new Chef Key/value stash cache hash objects store.
+    #
+    def initialize(params = {})
+      params = { store: params } unless params.is_a? Hash
+      @store = params.fetch(:store) { ChefStash::DEFAULT_STORE.new }
+    end
+
+    # Retrieves the value for a given key, if nothing is set,
+    # returns KeyError
+    #
+    # @param key [Symbol, String] representing the key
+    #
+    # @raise [KeyError] if no such key found
+    #
+    # @return [Hash, Array, String] value for key
+    #
+    def [](key = nil)
+      key ||= ChefStash.caller_name
+      fail KeyError, 'Key not cached' unless include? key.to_sym
+      @store[key.to_sym]
+    end
+
+    # Retrieves the value for a given key, if nothing is set,
+    # run the code, cache the result, and return it
+    #
+    # @param key [Symbol, String] representing the key
+    # @param block [&block] that returns the value to set (optional)
+    #
+    # @return [Hash, Array, String] value for key
+    #
+    def cache(key = nil, &code)
+      key ||= ChefStash.caller_name
+      @store[key.to_sym] ||= code.call
+    end
+
+    # return a boolean indicating presence of the given key in the store
+    #
+    # @param key [Symbol, String] a string or symbol representing the key
+    #
+    # @return [TrueClass, FalseClass]
+    #
+    def include?(key = nil)
+      key ||= ChefStash.caller_name
+      @store.include? key.to_sym
+    end
+
+    # Clear the whole stash store or the value of a key
+    #
+    # @param key [Symbol, String] (optional) representing the key to
+    # clear.
+    #
+    # @return nothing.
+    #
+    def clear!(key = nil)
+      key.nil? ? @store.clear : @store.delete(key)
+    end
+
+    # return the size of the store as an integer
+    #
+    # @return [Fixnum]
+    #
+    def size
+      @store.size
+    end
+  end
+end

data/rash.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/env ruby
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require 'anemone'
+require 'chef_stash'
+require 'ap'
+require 'chef_stash/core_ext/hash'
+
+url     = 'http://winini.mudbox.dev/'
+path    = 'packages_3.0'
+options = { threads: 20, depth_limit: 3, discard_page_bodies: true }
+results = []
+regex   = /#{path}\/\w+.(\w+.(ini|zip)|sha256.txt)$/i
+seen    = []
+
+def seen_urls
+  @seen_urls ||= []
+end
+
+Anemone.crawl(url, options) do |anemone|
+  anemone.on_pages_like(regex) do |page|
+    url  = page.url.to_s
+    name = File.basename(url)
+    key  = File.basename(name, '.*').downcase.to_sym
+    type = File.extname(name)[1..-1].downcase.to_sym
+
+    header   = page.headers
+    bytes    = header['content-length'].first
+    modified = header['last-modified'].first
+    created  = Time.now.utc.httpdate
+    content  = type == :ini ? 'text/inifile' : header['content-type'].first
+    size     = ChefStash::FileSize.new(bytes).to_size(:mb).to_s
+
+    seen_urls << { url: url, modified: modified, created: created }
+
+    result = { key => { type => {
+      code:          ChefStash::Response.code(page.code),
+      depth:         page.depth,
+      size:          size,
+      key:           key,
+      md5:           Digest::MD5.hexdigest(page.body.to_s),
+      modified:      modified,
+      name:          name,
+      referer:       page.referer.to_s,
+      response_time: page.response_time.time_humanize,
+      sha256:        OpenSSL::Digest::SHA256.new(page.body).to_s,
+      content_type:  content,
+      url:           url,
+      created:       created,
+      visited:       page.visited
+    } } }
+
+    ap result
+  end
+end
+
+ap seen_urls