fog-internet-archive 0.0.1

data/lib/fog/internet_archive/version.rb

@@ -0,0 +1,5 @@
+module Fog
+  module InternetArchive
+    VERSION = "0.0.1".freeze
+  end
+end

data/tasks/bundler.rake

@@ -0,0 +1,3 @@
+# This installs the tasks that help with gem creation and
+# publishing.
+Bundler::GemHelper.install_tasks

data/tasks/console.rake

@@ -0,0 +1,17 @@
+# From http://erniemiller.org/2014/02/05/7-lines-every-gems-rakefile-should-have/
+# with some modification.
+
+desc "Project IRB console"
+task :console do
+  require "bundler"
+  Bundler.require(:default, :development)
+
+  # Reload helper to avoid resetting the environment when debugging
+  def reload!
+    files = $LOADED_FEATURES.select { |feat| feat =~ /\/fog-internet-archive\// }
+    files.each { |file| load file }
+  end
+
+  ARGV.clear
+  Pry.start
+end

data/tasks/lint.rake

@@ -0,0 +1,3 @@
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new(:lint)

data/tasks/test.rake

@@ -0,0 +1,7 @@
+namespace :test do
+  mock = ENV["FOG_MOCK"] || "true"
+  desc "Run tests for travis"
+  task :travis do
+    sh("export FOG_MOCK=#{mock} && bundle exec shindont")
+  end
+end

data/tests/helper.rb

@@ -0,0 +1,36 @@
+ENV["FOG_RC"]         = ENV["FOG_RC"] || File.expand_path("../.fog", __FILE__)
+ENV["FOG_CREDENTIAL"] = ENV["FOG_CREDENTIAL"] || "default"
+
+require "fog/internet_archive"
+require "securerandom"
+
+Excon.defaults.merge!(:debug_request => true, :debug_response => true)
+
+require File.expand_path(File.join(File.dirname(__FILE__), "helpers", "mock_helper"))
+
+# This overrides the default 600 seconds timeout during live test runs
+if Fog.mocking?
+  FOG_TESTING_TIMEOUT = ENV["FOG_TEST_TIMEOUT"] || 2000
+  Fog.timeout = 2000
+  Fog::Logger.warning "Setting default fog timeout to #{Fog.timeout} seconds"
+else
+  FOG_TESTING_TIMEOUT = Fog.timeout
+end
+
+def lorem_file
+  File.open(File.dirname(__FILE__) + "/lorem.txt", "r")
+end
+
+def array_differences(array_a, array_b)
+  (array_a - array_b) | (array_b - array_a)
+end
+
+def generate_unique_domain(with_trailing_dot = false)
+  # get time (with 1/100th of sec accuracy)
+  # want unique domain name and if provider is fast, this can be called more than once per second
+  time = (Time.now.to_f * 100).to_i
+  domain = "test-" + time.to_s + ".com"
+  domain += "." if with_trailing_dot
+
+  domain
+end

data/tests/helpers/collection_helper.rb

@@ -0,0 +1,88 @@
+# TODO: REMOVE this testing functionality is now covered in `spec/helpers/model_helper.rb`
+
+def collection_tests(collection, params = {}, mocks_implemented = true)
+  tests("success") do
+    tests("#new(#{params.inspect})").succeeds do
+      pending if Fog.mocking? && !mocks_implemented
+      collection.new(params)
+    end
+
+    tests("#create(#{params.inspect})").succeeds do
+      pending if Fog.mocking? && !mocks_implemented
+      @instance = collection.create(params)
+    end
+    # FIXME: work around for timing issue on AWS describe_instances mocks
+
+    if Fog.mocking? && @instance.respond_to?(:ready?)
+      @instance.wait_for { ready? }
+    end
+
+    tests("#all").succeeds do
+      pending if Fog.mocking? && !mocks_implemented
+      collection.all
+    end
+
+    @identity = @instance.identity if !Fog.mocking? || mocks_implemented
+
+    tests("#get(#{@identity})").succeeds do
+      pending if Fog.mocking? && !mocks_implemented
+      collection.get(@identity)
+    end
+
+    tests("Enumerable") do
+      pending if Fog.mocking? && !mocks_implemented
+
+      methods = [
+        "all?", "any?", "find", "detect", "collect", "map",
+        "find_index", "flat_map", "collect_concat", "group_by",
+        "none?", "one?"
+      ]
+
+      # JRuby 1.7.5+ issue causes a SystemStackError: stack level too deep
+      # https://github.com/jruby/jruby/issues/1265
+      if RUBY_PLATFORM == "java" && JRUBY_VERSION =~ /1\.7\.[5-8]/
+        methods.delete("all?")
+      end
+
+      methods.each do |enum_method|
+        next unless collection.respond_to?(enum_method)
+        tests("##{enum_method}").succeeds do
+          block_called = false
+          collection.send(enum_method) { |_| block_called = true }
+          block_called
+        end
+      end
+
+      %w(
+        max_by min_by).each do |enum_method|
+        next unless collection.respond_to?(enum_method)
+        tests("##{enum_method}").succeeds do
+          block_called = false
+          collection.send(enum_method) do |_|
+            block_called = true
+            0
+          end
+          block_called
+        end
+      end
+    end
+
+    yield(@instance) if block_given?
+
+    @instance.destroy if !Fog.mocking? || mocks_implemented
+  end
+
+  tests("failure") do
+    if !Fog.mocking? || mocks_implemented
+      @identity = @identity.to_s
+      @identity = @identity.gsub(/[a-zA-Z]/) { Fog::Mock.random_letters(1) }
+      @identity = @identity.gsub(/\d/)       { Fog::Mock.random_numbers(1) }
+      @identity
+    end
+
+    tests("#get('#{@identity}')").returns(nil) do
+      pending if Fog.mocking? && !mocks_implemented
+      collection.get(@identity)
+    end
+  end
+end

data/tests/helpers/formats_helper.rb

@@ -0,0 +1,98 @@
+require "fog/schema/data_validator"
+
+# format related hackery
+# allows both true.is_a?(Fog::Boolean) and false.is_a?(Fog::Boolean)
+# allows both nil.is_a?(Fog::Nullable::String) and ''.is_a?(Fog::Nullable::String)
+module Fog
+  module Boolean; end
+  module Nullable
+    module Boolean; end
+    module Integer; end
+    module String; end
+    module Time; end
+    module Float; end
+    module Hash; end
+    module Array; end
+  end
+end
+[FalseClass, TrueClass].each { |klass| klass.send(:include, Fog::Boolean) }
+[FalseClass, TrueClass, NilClass, Fog::Boolean].each { |klass| klass.send(:include, Fog::Nullable::Boolean) }
+[NilClass, String].each { |klass| klass.send(:include, Fog::Nullable::String) }
+[NilClass, Time].each { |klass| klass.send(:include, Fog::Nullable::Time) }
+[Integer, NilClass].each { |klass| klass.send(:include, Fog::Nullable::Integer) }
+[Float, NilClass].each { |klass| klass.send(:include, Fog::Nullable::Float) }
+[Hash, NilClass].each { |klass| klass.send(:include, Fog::Nullable::Hash) }
+[Array, NilClass].each { |klass| klass.send(:include, Fog::Nullable::Array) }
+
+module Shindo
+  class Tests
+    # Generates a Shindo test that compares a hash schema to the result
+    # of the passed in block returning true if they match.
+    #
+    # The schema that is passed in is a Hash or Array of hashes that
+    # have Classes in place of values. When checking the schema the
+    # value should match the Class.
+    #
+    # Strict mode will fail if the data has additional keys. Setting
+    # +strict+ to +false+ will allow additional keys to appear.
+    #
+    # @param [Hash] schema A Hash schema
+    # @param [Hash] options Options to change validation rules
+    # @option options [Boolean] :allow_extra_keys
+    #     If +true+ does not fail when keys are in the data that are
+    #     not specified in the schema. This allows new values to
+    #     appear in API output without breaking the check.
+    # @option options [Boolean] :allow_optional_rules
+    #     If +true+ does not fail if extra keys are in the schema
+    #     that do not match the data. Not recommended!
+    # @yield Data to check with schema
+    #
+    # @example Using in a test
+    #     Shindo.tests("comparing welcome data against schema") do
+    #       data = {:welcome => "Hello" }
+    #       data_matches_schema(:welcome => String) { data }
+    #     end
+    #
+    #     comparing welcome data against schema
+    #     + data matches schema
+    #
+    # @example Example schema
+    #     {
+    #       "id" => String,
+    #       "ram" => Integer,
+    #       "disks" => [
+    #         {
+    #           "size" => Float
+    #         }
+    #       ],
+    #       "dns_name" => Fog::Nullable::String,
+    #       "active" => Fog::Boolean,
+    #       "created" => DateTime
+    #     }
+    #
+    # @return [Boolean]
+    def data_matches_schema(schema, options = {})
+      test("data matches schema") do
+        validator = Fog::Schema::DataValidator.new
+        valid = validator.validate(yield, schema, options)
+        @message = validator.message unless valid
+        valid
+      end
+    end
+
+    # @deprecated #formats is deprecated. Use #data_matches_schema instead
+    def formats(format, strict = true)
+      test("has proper format") do
+        if strict
+          options = { :allow_extra_keys => false, :allow_optional_rules => true }
+        else
+          options = { :allow_extra_keys => true, :allow_optional_rules => true }
+        end
+        validator = Fog::Schema::DataValidator.new
+        valid = validator.validate(yield, format, options)
+        @message = validator.message unless valid
+        valid
+      end
+    end
+  end
+end

data/tests/helpers/formats_helper_tests.rb

@@ -0,0 +1,106 @@
+Shindo.tests("test_helper", "meta") do
+  tests("comparing welcome data against schema") do
+    data = { :welcome => "Hello" }
+    data_matches_schema(:welcome => String) { data }
+  end
+
+  tests('#data_matches_schema') do
+    tests("when value matches schema expectation") do
+      data_matches_schema("key" => String) { { "key" => "Value" } }
+    end
+
+    tests("when values within an array all match schema expectation") do
+      data_matches_schema({ "key" => [Integer] }) { { "key" => [1, 2] } }
+    end
+
+    tests("when nested values match schema expectation") do
+      data_matches_schema("key" => { :nested_key => String }) { { "key" => { :nested_key => "Value" } } }
+    end
+
+    tests("when collection of values all match schema expectation") do
+      data_matches_schema([{ "key" => String }]) { [{ "key" => "Value" }, { "key" => "Value" }] }
+    end
+
+    tests("when collection is empty although schema covers optional members") do
+      data_matches_schema([{ "key" => String }], :allow_optional_rules => true) { [] }
+    end
+
+    tests("when additional keys are passed and not strict") do
+      data_matches_schema({ "key" => String }, :allow_extra_keys => true) { { "key" => "Value", :extra => "Bonus" } }
+    end
+
+    tests("when value is nil and schema expects NilClass") do
+      data_matches_schema("key" => NilClass) { { "key" => nil } }
+    end
+
+    tests("when value and schema match as hashes") do
+      data_matches_schema({}) { {} }
+    end
+
+    tests("when value and schema match as arrays") do
+      data_matches_schema([]) { [] }
+    end
+
+    tests("when value is a Time") do
+      data_matches_schema("time" => Time) { { "time" => Time.now } }
+    end
+
+    tests('when key is missing but value should be NilClass (#1477)') do
+      data_matches_schema({ "key" => NilClass }, :allow_optional_rules => true) { {} }
+    end
+
+    tests('when key is missing but value is nullable (#1477)') do
+      data_matches_schema({ "key" => Fog::Nullable::String }, :allow_optional_rules => true) { {} }
+    end
+  end
+
+  tests('#formats backwards compatible changes') do
+    tests("when value matches schema expectation") do
+      formats("key" => String) { { "key" => "Value" } }
+    end
+
+    tests("when values within an array all match schema expectation") do
+      formats("key" => [Integer]) { { "key" => [1, 2] } }
+    end
+
+    tests("when nested values match schema expectation") do
+      formats("key" => { :nested_key => String }) { { "key" => { :nested_key => "Value" } } }
+    end
+
+    tests("when collection of values all match schema expectation") do
+      formats([{ "key" => String }]) { [{ "key" => "Value" }, { "key" => "Value" }] }
+    end
+
+    tests("when collection is empty although schema covers optional members") do
+      formats([{ "key" => String }]) { [] }
+    end
+
+    tests("when additional keys are passed and not strict") do
+      formats({ "key" => String }, false) { { "key" => "Value", :extra => "Bonus" } }
+    end
+
+    tests("when value is nil and schema expects NilClass") do
+      formats("key" => NilClass) { { "key" => nil } }
+    end
+
+    tests("when value and schema match as hashes") do
+      formats({}) { {} }
+    end
+
+    tests("when value and schema match as arrays") do
+      formats([]) { [] }
+    end
+
+    tests("when value is a Time") do
+      formats("time" => Time) { { "time" => Time.now } }
+    end
+
+    tests('when key is missing but value should be NilClass (#1477)') do
+      formats("key" => NilClass) { {} }
+    end
+
+    tests('when key is missing but value is nullable (#1477)') do
+      formats({ "key" => Fog::Nullable::String }) { {} }
+    end
+  end
+end

data/tests/helpers/mock_helper.rb

@@ -0,0 +1,13 @@
+# Use so you can run in mock mode from the command line
+#
+# FOG_MOCK=true fog
+
+Fog.mock! if ENV["FOG_MOCK"] == "true"
+
+# if in mocked mode, fill in some fake credentials for us
+if Fog.mock?
+  Fog.credentials = {
+    ia_access_key_id: "fake_id",
+    ia_secret_access_key: "fake_key",
+  }.merge(Fog.credentials)
+end

data/tests/helpers/model_helper.rb

@@ -0,0 +1,29 @@
+# TODO: REMOVE this testing functionality is now covered in `spec/helpers/model_helper.rb`
+
+def model_tests(collection, params = {}, mocks_implemented = true)
+  tests("success") do
+    @instance = collection.new(params)
+
+    tests("#save").succeeds do
+      pending if Fog.mocking? && !mocks_implemented
+      @instance.save
+    end
+
+    yield(@instance) if block_given?
+
+    tests("#destroy").succeeds do
+      pending if Fog.mocking? && !mocks_implemented
+      @instance.destroy
+    end
+  end
+end
+
+# Generates a unique identifier with a random differentiator.
+# Useful when rapidly re-running tests, so we don't have to wait
+# serveral minutes for deleted objects to disappear from the API
+# E.g. 'fog-test-1234'
+def uniq_id(base_name = "fog-test")
+  # random_differentiator
+  suffix = rand(65_536).to_s(16).rjust(4, "0")
+  [base_name, suffix].join("-")
+end

data/tests/helpers/responds_to_helper.rb

@@ -0,0 +1,11 @@
+module Shindo
+  class Tests
+    def responds_to(method_names)
+      for method_name in [*method_names]
+        tests("#respond_to?(:#{method_name})").returns(true) do
+          @instance.respond_to?(method_name)
+        end
+      end
+    end
+  end
+end

data/tests/helpers/schema_validator_tests.rb

@@ -0,0 +1,101 @@
+Shindo.tests("Fog::Schema::DataValidator", "meta") do
+  validator = Fog::Schema::DataValidator.new
+
+  tests('#validate') do
+    tests("returns true") do
+      returns(true, "when value matches schema expectation") do
+        validator.validate({ "key" => "Value" }, "key" => String)
+      end
+
+      returns(true, "when values within an array all match schema expectation") do
+        validator.validate({ "key" => [1, 2] }, "key" => [Integer])
+      end
+
+      returns(true, "when nested values match schema expectation") do
+        validator.validate({ "key" => { :nested_key => "Value" } }, "key" => { :nested_key => String })
+      end
+
+      returns(true, "when collection of values all match schema expectation") do
+        validator.validate([{ "key" => "Value" }, { "key" => "Value" }], [{ "key" => String }])
+      end
+
+      returns(true, "when collection is empty although schema covers optional members") do
+        validator.validate([], [{ "key" => String }])
+      end
+
+      returns(true, "when additional keys are passed and not strict") do
+        validator.validate({ "key" => "Value", :extra => "Bonus" }, { "key" => String }, :allow_extra_keys => true)
+      end
+
+      returns(true, "when value is nil and schema expects NilClass") do
+        validator.validate({ "key" => nil }, "key" => NilClass)
+      end
+
+      returns(true, "when value and schema match as hashes") do
+        validator.validate({}, {})
+      end
+
+      returns(true, "when value and schema match as arrays") do
+        validator.validate([], [])
+      end
+
+      returns(true, "when value is a Time") do
+        validator.validate({ "time" => Time.now }, "time" => Time)
+      end
+
+      returns(true, 'when key is missing but value should be NilClass (#1477)') do
+        validator.validate({}, { "key" => NilClass }, :allow_optional_rules => true)
+      end
+
+      returns(true, 'when key is missing but value is nullable (#1477)') do
+        validator.validate({}, { "key" => Fog::Nullable::String }, :allow_optional_rules => true)
+      end
+    end
+
+    tests("returns false") do
+      returns(false, "when value does not match schema expectation") do
+        validator.validate({ "key" => nil }, { "key" => String })
+      end
+
+      returns(false, "when key formats do not match") do
+        validator.validate({ "key" => "Value" }, :key => String)
+      end
+
+      returns(false, "when additional keys are passed and strict") do
+        validator.validate({ "key" => "Missing" }, {})
+      end
+
+      returns(false, "when some keys do not appear") do
+        validator.validate({}, "key" => String)
+      end
+
+      returns(false, "when collection contains a member that does not match schema") do
+        validator.validate([{ "key" => "Value" }, { "key" => 5 }], [{ "key" => String }])
+      end
+
+      returns(false, "when collection has multiple schema patterns") do
+        validator.validate([{ "key" => "Value" }], [{ "key" => Integer }, { "key" => String }])
+      end
+
+      returns(false, "when hash and array are compared") do
+        validator.validate({}, [])
+      end
+
+      returns(false, "when array and hash are compared") do
+        validator.validate([], {})
+      end
+
+      returns(false, "when a hash is expected but another data type is found") do
+        validator.validate({ "key" => { :nested_key => [] } }, "key" => { :nested_key => {} })
+      end
+
+      returns(false, 'when key is missing but value should be NilClass (#1477)') do
+        validator.validate({}, { "key" => NilClass }, :allow_optional_rules => false)
+      end
+
+      returns(false, 'when key is missing but value is nullable (#1477)') do
+        validator.validate({}, { "key" => Fog::Nullable::String }, :allow_optional_rules => false)
+      end
+    end
+  end
+end