fog-core 1.43.0 → 1.44.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 35b3d7d551ace1e046e9abf0421d9b58ad5706d2
-  data.tar.gz: 7564233003859d297d3f3680a9bed1357ca5bdb0
+  metadata.gz: e7e0a641eb2c6f28ba2026bb5fe19918d003ea7f
+  data.tar.gz: 0e72e5f6ae021ce1476d8b297747faa5da894679
 SHA512:
-  metadata.gz: 85c4fc73f6f1813c86ab69e2cfc9d1e4d55e86f7c4f9222a520daf0ae9a953713aa6292b67a9b44852811aea59af9dfabdbadd19e2ea6dbb6cd6fbc855bf8997
-  data.tar.gz: 30bcb3420001dd9b06fc8b6f5d1ab9fee61dc23d90ed5e1df8aae36f2576a1186045d944447d295d18903e391b2da6e976cfedce52ac8601fb64daf6c1e6dda9
+  metadata.gz: f604ea2b9c97ab0b8e596b51c749b4175b96fc3e70e7eff5ec1cc23471024c8e03774aaeec0523b8fe57211f1e6fb2006430ee422f62b1979610160ad7bd2ce8
+  data.tar.gz: 28abea55b24d79055f9c1f5a89aaa5d2dfd622875a2d7162678a5b4fc4ef21a06760d6341cad772c3846f531274b13483823484641ee474cdf4c1630ff275c11

data/.gitignore

@@ -3,7 +3,7 @@
 .bundle
 .config
 .yardoc
-Gemfile.lock
+Gemfile.*lock
 InstalledFiles
 _yardoc
 coverage
@@ -15,4 +15,4 @@ spec/reports
 test/tmp
 test/version_tmp
 tmp
-.#*
+.#*

data/.travis.yml

@@ -1,26 +1,33 @@
 language: ruby
 
-gemfile:
-  - Gemfile
-
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.2.0
-  - jruby9k
+  - 2.1
+  - 2.2
+  - 2.3
+  - 2.4
   - jruby-head
 
 sudo: false
+dist: trusty
 
 script: bundle exec rake travis
 
 matrix:
+  fast_finish: true
   include:
     - rvm: 1.8.7
-      gemfile: Gemfile18
+      gemfile: Gemfile.1.8.7
+    - rvm: jruby-18mode
+      gemfile: Gemfile.1.8.7
+    - rvm: 1.9.3
+      gemfile: Gemfile.1.9.3
+    - rvm: jruby-19mode
+      gemfile: Gemfile.1.9.3
     - rvm: 2.1.0
       gemfile: Gemfile
       env: COVERAGE=true
+    - rvm: jruby-head
+      gemfile: Gemfile
   allow_failures:
     - rvm: jruby-head
     - rvm: jruby9k

data/Gemfile

@@ -1,4 +1,3 @@
 source 'https://rubygems.org'
-
 gemspec
 

data/Gemfile.1.9.3

@@ -0,0 +1,5 @@
+source "https://rubygems.org"
+
+gem "net-ssh", "< 3.0"
+
+gemspec

data/README.md

@@ -25,6 +25,17 @@ Or install it yourself as:
 
     $ gem install fog-core
 
+### Ruby 1.9.3
+
+Some of `fog-core`'s dependencies have dropped support for Ruby 1.9.3 in later
+versions. Rather than limit all `fog` users to older but compatible versions,
+if you are using 1.9.3 you will need to declare a compatible version in your
+application's `Gemfile` like:
+
+    gem "net-ssh", "< 3.0"
+
+See `Gemfile.1.9.3` for the definitive list as tested by Travis.
+
 ## Usage
 
 TODO: Write usage instructions here

data/changelog.md

@@ -1,3 +1,8 @@
+1.44.0 04/28/2017
+==========================================================
+
+add basic caching support
+
 1.43.0 09/28/2016
 ==========================================================
 

data/fog-core.gemspec

@@ -21,7 +21,9 @@ Gem::Specification.new do |spec|
   spec.add_dependency("builder")
   spec.add_dependency("excon", "~> 0.49")
   spec.add_dependency("formatador", "~> 0.2")
+  spec.add_dependency("xmlrpc") if RUBY_VERSION.to_s >= "2.4"
 
+  spec.add_development_dependency("tins") if RUBY_VERSION.to_s > "2.0"
   spec.add_development_dependency("coveralls")
   spec.add_development_dependency("minitest")
   spec.add_development_dependency("minitest-stub-const")

data/lib/fog/core/cache.rb

@@ -0,0 +1,210 @@
+# coding: utf-8
+require "fileutils"
+require "yaml"
+require "tmpdir"
+
+module Fog
+  # A generic cache mechanism for fog resources. This can be for a server, security group, etc.
+  #
+  # Currently this is a on-disk cache using yml files per-model instance, however
+  # there is nothing in the way of extending this to use various other cache
+  # backends.
+  #
+  # == Basic functionality
+  #
+  # set the namespace where this cache will be stored:
+  #
+  # Fog::Cache.namespace_prefix = "service-account-foo-region-bar"
+  #
+  # cache to disk:
+  #
+  #   # after dumping, there will be a yml file on disk:
+  #   resouce.cache.dump
+  #
+  #   # you can load cached data in from a different session
+  #   Fog::Cache.load(Fog::Compute::AWS::Server, compute)
+  #
+  #   # you can also expire cache (removes cached data assocaited with the resources of this model associated to the service passed in).
+  #   Fog::Cache.expire_cache!(Fog::Compute::AWS::Server, compute)
+  #
+  # == More detailed flow/usage
+  #
+  # Normally, you would have a bunch of resources you want to cache/reload from disk.
+  # Every fog model has a +cache+ object injected to accomplish this. So in order to cache a server for exmaple
+  # you would do something like this:
+  #
+  #   # note this is necessary in order to segregate usage of cache between various providers regions and accounts.
+  #   # if you are using one account/region/etc only, you still must set it. 'default' will do.
+  #   Fog::Cache.namespace_prefix = "prod-emea-eu-west-1"
+  #
+  #   s = security_groups.sample; s.name # => "default"
+  #   s.cache.dump # => 2371
+  #
+  # Now it is on disk:
+  #
+  # shai@adsk-lappy ~   % tree ~/.fog-cache/prod-emea-eu-west-1/
+  #
+  #   /Users/shai/.fog-cache/prod-emea-eu-west-1/
+  #     └── fog_compute_aws_real
+  #       └── fog_compute_aws_securitygroup
+  #        ├── default-90928073d9d5d9b4e7545e88aee7ec4f.yml
+  #
+  # You can do the same with a SecurityGroup, Instances, Elbs, etc.
+  #
+  # Note that when loading cache from disk, you need to pass the appropriate model class, and service associated with it.
+  # +Service+ is passed in is so that the service/connection details can be loaded into the loaded instances so they can be re-queried, etc.
+  # +Model+ is passed in so we can find the cache data associated to that model in the namespace of cache this session is using:
+  # Will try to load all resources associated to those. If you had 1 yml file, or 100, it would load whatever it could find.
+  # As such, the normal usage of dumping would be do it on a collection:
+  #
+  #   load_balancers.each {|elb| elb.cache.dump }
+  #
+  # In order to load the cache into a different session with nothing but the service set up, use like so:
+  # As mentioned, will load all resources associated to the +model_klass+ and +service+ passed in.
+  #
+  #   instances = Fog::Cache.load(Fog::Compute::AWS::Server, compute)
+  #   instances.first.id # => "i-0569a70ae6f47d229"
+  #
+  # Note that if there is no cache located for the +model+ class and +service+ passed to `Fog::Cache.load`
+  # you will get an exception you can handle (for example, to load the resources for the fisrt time):
+  #
+  #   Fog::Cache.expire_cache!(Fog::Compute::AWS::SecurityGroup, compute)
+  #   # ... now there is no SecurityGroup cache data. So, if you tried to load it, you would get an exception:
+  #
+  #   Fog::Cache.load(Fog::Compute::AWS::SecurityGroup, compute)
+  #     rescue Fog::Cache::CacheNotFound => e
+  #       puts "could not find any cache data for security groups on #{compute}"
+  #       get_resources_and_dump
+  #
+  # == Extending cache backends
+  #
+  # Currently this is on-disk using yml. If need be, this could be extended to other cache backends:
+  #
+  # Find references of yaml in this file, split out to strategy objects/diff backends etc.
+  class Cache
+
+    # cache associated not found
+    class CacheNotFound < StandardError; end
+
+    # cache directory problem
+    class CacheDir < StandardError; end
+
+    # where different caches per +service+ api keys, regions etc, are stored
+    # see the +namespace_prefix=+ method.
+    SANDBOX = "~/.fog-cache"
+
+    # when a resource is used such as `server.cache.dump` the model klass is passed in
+    # so that it can be identified from a different session.
+    attr_reader :model
+
+    # Loads cache associated to the +model_klass+ and +service+ into memory.
+    #
+    # If no cache is found, it will raise an error for handling:
+    #
+    #   rescue Fog::Cache::CacheNotFound
+    #     set_initial_cache
+    #
+    def self.load(model_klass, service)
+      cache_files = Dir.glob("#{namespace(model_klass, service)}/*")
+
+      raise CacheNotFound if cache_files.empty?
+
+      loaded = cache_files.map do |path|
+        model_klass = Object.const_get(load_cache(path)[:model_klass])
+        model_klass.new(load_cache(path)[:attrs])
+      end
+
+      collection_klass = load_cache(cache_files.sample)[:collection_klass] &&
+                         Object.const_get(load_cache(cache_files.sample)[:collection_klass])
+
+      loaded.each do |i|
+        # See https://github.com/fog/fog-aws/issues/354#issuecomment-286789702
+        i.collection = collection_klass.new(:service => service) if collection_klass
+        i.instance_variable_set(:@service, service)
+      end
+
+      # uniqe-ify based on the total of attributes. duplicate cache can exist due to
+      # `model#identity` not being unique. but if all attributes match, they are unique
+      # and shouldn't be loaded again.
+      uniq_loaded = loaded.uniq { |i| i.attributes }
+      if uniq_loaded.size != loaded.size
+        Fog::Logger.warning("Found duplicate items in the cache. Expire all & refresh cache soon.")
+      end
+
+      # Fog models created, free memory of cached data used for creation.
+      @memoized = nil
+
+      uniq_loaded
+    end
+
+    # creates on-disk cache of this specific +model_klass+ and +@service+
+    def self.create_namespace(model_klass, service)
+      FileUtils.mkdir_p(self.namespace(model_klass, service))
+    end
+
+    # Expires cache - this does not expire all cache associated.
+    # Instead, this will remove all on-disk cache of this specific +model_klass+ and and +@service+
+    def self.expire_cache!(model_klass, service)
+      FileUtils.rm_rf(namespace(model_klass, service))
+    end
+
+    # loads yml cache from path on disk, used
+    # to initialize Fog models.
+    def self.load_cache(path)
+      @memoized ||= {}
+      return @memoized[path] if @memoized[path]
+      @memoized[path] = YAML.load(File.read(path))
+    end
+
+    def self.namespace_prefix=(name)
+      @namespace_prefix = name
+    end
+
+    def self.namespace_prefix
+      @namespace_prefix
+    end
+
+    # The path/namespace where the cache is stored for a specific +model_klass+ and +@service+.
+    def self.namespace(model_klass, service)
+
+      raise CacheDir.new("Must set an explicit identifier/name for this cache. Example: 'serviceX-regionY'") unless namespace_prefix
+
+      ns = File.join(SANDBOX, namespace_prefix, service.class.to_s, model_klass.to_s)
+      ns = safe_path(ns)
+    end
+
+    def self.safe_path(klass)
+      klass.to_s.gsub("::", "_").downcase
+    end
+
+    def initialize(model)
+      @model = model
+    end
+
+    # Dump a Fog::Model resource. Every fog model/instance now has a +cache+ method/object injected in.
+    # as such you can use the #dump method to save the attributes and metadata of that instance as cache
+    # which can be re-used in some other session.
+    def dump
+      if !File.exist?(self.class.namespace(model.class, model.service))
+        self.class.create_namespace(model.class, model.service)
+      end
+
+      data = { :identity => model.identity,
+                     :model_klass => model.class.to_s,
+                     :collection_klass => model.collection && model.collection.class.to_s,
+                     :attrs => model.attributes }
+
+      File.open(dump_to, "w") { |f| f.write(YAML.dump(data)) }
+    end
+
+    # the location of where to save this fog model/instance to.
+    def dump_to
+      # some fog models have an identity field that is duplicate.
+      # duplicate identities can mean the cache for that already exists.
+      # this means cache duplication is possible.
+      #
+      # see "dumping two models that have duplicate identity" test case.
+      name = "#{self.class.namespace(model.class, model.service)}/#{model.identity}-#{SecureRandom.hex}.yml"
+    end
+  end
+end

data/lib/fog/core/collection.rb

@@ -6,7 +6,6 @@ module Fog
     extend Fog::Attributes::ClassMethods
     include Fog::Attributes::InstanceMethods
     include Fog::Core::DeprecatedConnectionAccessors
-    
 
     attr_reader :service
 

data/lib/fog/core/model.rb

@@ -1,4 +1,5 @@
 require "fog/core/deprecated_connection_accessors"
+require "fog/core/cache"
 
 module Fog
   class Model
@@ -20,6 +21,10 @@ module Fog
       merge_attributes(attribs)
     end
 
+    def cache
+      Fog::Cache.new(self)
+    end
+
     def inspect
       Fog::Formatador.format(self)
     end

data/lib/fog/core/version.rb

@@ -1,5 +1,5 @@
 module Fog
   module Core
-    VERSION = "1.43.0"
+    VERSION = "1.44.0"
   end
 end

data/spec/core/cache_spec.rb

@@ -0,0 +1,123 @@
+require "spec_helper"
+require "securerandom"
+require "tmpdir"
+
+module Fog
+  class SubFogTestModel < Fog::Model
+    identity  :id
+  end
+end
+
+module Fog
+  class SubFogTestService < Fog::Service
+
+    class Mock
+      attr_reader :options
+
+      def initialize(opts = {})
+        @options = opts
+      end
+    end
+  end
+end
+
+describe Fog::Cache do
+  before(:each) do
+    Fog.mock!
+    @service = Fog::SubFogTestService.new
+    Fog::Cache.namespace_prefix = "test-dir"
+  end
+
+  it "has a namespace_prefix configurable" do
+    Fog::Cache.namespace_prefix = "for-service-user-region-foo"
+
+    # Expand path does not downcase. case insensitive platform tests.
+    example_cache = File.expand_path(Fog::Cache.namespace(Fog::SubFogTestModel, @service)).downcase
+    expected_namespace = File.expand_path("~/.fog-cache/for-service-user-region-foo").downcase
+
+    puts example_cache
+    puts expected_namespace
+    assert_equal example_cache.include?(expected_namespace), true
+  end
+
+  it "must have a namespace_prefix configurable" do
+    Fog::Cache.namespace_prefix = nil
+    assert_raises Fog::Cache::CacheDir do
+      Fog::Cache.load(Fog::SubFogTestModel, @service)
+    end
+  end
+
+  it "can create a namespace" do
+    Fog::Cache.expire_cache!(Fog::SubFogTestModel, @service)
+    assert_equal File.exist?(Fog::Cache.namespace(Fog::SubFogTestModel, @service)), false
+
+    Fog::Cache.create_namespace(Fog::SubFogTestModel, @service)
+    assert_equal File.exist?(Fog::Cache.namespace(Fog::SubFogTestModel, @service)), true
+  end
+
+  it "will raise if no cache data found" do
+    Fog::Cache.expire_cache!(Fog::SubFogTestModel, @service)
+
+    assert_raises Fog::Cache::CacheNotFound do
+      Fog::Cache.load(Fog::SubFogTestModel, @service)
+    end
+  end
+
+  it "can be dumped and reloaded back in" do
+
+    Fog::Cache.expire_cache!(Fog::SubFogTestModel, @service)
+
+    id = SecureRandom.hex
+    a = Fog::SubFogTestModel.new(:id => id, :service => @service)
+
+    assert_equal File.exist?(Fog::Cache.namespace(Fog::SubFogTestModel, @service)), false
+    a.cache.dump
+    assert_equal File.exist?(Fog::Cache.namespace(Fog::SubFogTestModel, @service)), true
+
+    instances = Fog::Cache.load(Fog::SubFogTestModel, @service)
+
+    assert_equal instances.first.id, a.id
+    assert_equal instances.first.class, a.class
+  end
+
+  it "dumping two models that have a duplicate identity" do
+    Fog::Cache.expire_cache!(Fog::SubFogTestModel, @service)
+
+    id = SecureRandom.hex
+
+    # security gruops on aws for eg can have the same identity group name 'default'.
+    # there are no restrictions on `identity` fog attributes to be uniq.
+    a = Fog::SubFogTestModel.new(:id => id, :service => @service, :bar => 'bar')
+    b = Fog::SubFogTestModel.new(:id => id, :service => @service, :foo => 'foo')
+
+    a.cache.dump
+    b.cache.dump
+
+    instances = Fog::Cache.load(Fog::SubFogTestModel, @service)
+
+    assert_equal instances.size, 2
+  end
+
+  it "dumping two models that have a duplicate identity twice" do
+    Fog::Cache.expire_cache!(Fog::SubFogTestModel, @service)
+
+    id = SecureRandom.hex
+
+    # security gruops on aws for eg can have the same identity group name 'default'.
+    # there are no restrictions on `identity` fog attributes to be uniq.
+    a = Fog::SubFogTestModel.new(:id => id, :service => @service, :bar => 'bar')
+    b = Fog::SubFogTestModel.new(:id => id, :service => @service, :foo => 'foo')
+
+    a.cache.dump
+    b.cache.dump
+
+    # and then again, w/out expiring cache
+    a.cache.dump
+    b.cache.dump
+
+    instances = Fog::Cache.load(Fog::SubFogTestModel, @service)
+
+    # unique-ify based on the attributes...
+    assert_equal instances.size, 2
+  end
+end

data/spec/service_spec.rb

@@ -171,9 +171,10 @@ describe Fog::Service do
   end
 
   describe "when config object can configure the service itself" do
-    it "ignores the global and it's values" do
+    it "ignores the global and its values" do
       @config = MiniTest::Mock.new
       def @config.config_service?;  true; end
+      def @config.nil?; false; end
       def @config.==(other); object_id == other.object_id; end
 
       unexpected_usage = lambda { raise "Accessing global!" }

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fog-core
 version: !ruby/object:Gem::Version
-  version: 1.43.0
+  version: 1.44.0
 platform: ruby
 authors:
 - Evan Light
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-28 00:00:00.000000000 Z
+date: 2017-04-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: builder
@@ -53,6 +53,34 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: xmlrpc
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: tins
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: coveralls
   requirement: !ruby/object:Gem::Requirement
@@ -179,7 +207,8 @@ files:
 - CONTRIBUTING.md
 - CONTRIBUTORS.md
 - Gemfile
-- Gemfile18
+- Gemfile.1.8.7
+- Gemfile.1.9.3
 - LICENSE.md
 - README.md
 - Rakefile
@@ -207,6 +236,7 @@ files:
 - lib/fog/core/attributes/string.rb
 - lib/fog/core/attributes/time.rb
 - lib/fog/core/attributes/timestamp.rb
+- lib/fog/core/cache.rb
 - lib/fog/core/collection.rb
 - lib/fog/core/connection.rb
 - lib/fog/core/credentials.rb
@@ -263,6 +293,7 @@ files:
 - lib/tasks/test_task.rb
 - spec/compute_spec.rb
 - spec/connection_spec.rb
+- spec/core/cache_spec.rb
 - spec/core/model_spec.rb
 - spec/core/stringify_keys_spec.rb
 - spec/core/whitelist_keys_spec.rb
@@ -305,13 +336,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.6.8
 signing_key: 
 specification_version: 4
 summary: Shared classes and tests for fog providers and services.
 test_files:
 - spec/compute_spec.rb
 - spec/connection_spec.rb
+- spec/core/cache_spec.rb
 - spec/core/model_spec.rb
 - spec/core/stringify_keys_spec.rb
 - spec/core/whitelist_keys_spec.rb
@@ -334,4 +366,3 @@ test_files:
 - spec/utils_spec.rb
 - spec/uuid_spec.rb
 - spec/wait_for_spec.rb
-has_rdoc: