distributed_cache 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ea80e92a851ce1d5042dcd3d28263449e2c78f35
-  data.tar.gz: ace19ce97a6d3ca598c0b8c2e9daf3da549bc10a
+  metadata.gz: bae74c33b8445bc66d391fedd024530242ed2bcd
+  data.tar.gz: 2c3e0c5bfbceb79feb61d11e4e499c64b3c56683
 SHA512:
-  metadata.gz: fbfd450ca33835729c58406e6de69914e2b9141378c3e71c48fc50959fcf0c5fc7338926a1fb60197f43e35ca8151d6b2b8a30e730956b5713ed6162d0ec762b
-  data.tar.gz: 6b2aa072d7852ddd52655efb26a9dbc037f8b84b4c85433dbd63e6f7b176564f048e9cef11b0e73bd1612c315f674767d23d39d7fd4f47b6146f30350e5926f8
+  metadata.gz: 923dcf7aeadaa1a6abeaa641b5291cae63387c4123a1d3e79158979ad9cf08667dac1075d46f83929977ca5dee7bccc4b050423735b6f5d4641b2c7334f83a53
+  data.tar.gz: 3a407ee819ddf9807510077e772763a5d10200e863681e943a43fb6f6e0daccf8e2219381cead738be88d1624177bc02506aec37adfd9cff7a61d744c643b075

data/Gemfile

@@ -5,4 +5,5 @@ gem 'autoload'
 group :development do
   gem "jeweler"
   gem "rspec"
+  gem 'fakeweb'
 end

data/Gemfile.lock

@@ -5,6 +5,7 @@ GEM
     autoload (0.3.0)
     builder (3.2.2)
     diff-lcs (1.2.4)
+    fakeweb (1.3.0)
     faraday (0.8.8)
       multipart-post (~> 1.2.0)
     git (1.2.5)
@@ -59,5 +60,6 @@ PLATFORMS
 
 DEPENDENCIES
   autoload
+  fakeweb
   jeweler
   rspec

data/VERSION

@@ -1 +1 @@
-0.1.0
+0.1.1

data/distributed_cache.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "distributed_cache"
-  s.version = "0.1.0"
+  s.version = "0.1.1"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Doug Youch"]
-  s.date = "2013-07-31"
+  s.date = "2013-08-01"
   s.description = "Creates tar files of cache directories for distribution"
   s.email = "doug@sessionm.com"
   s.extra_rdoc_files = [
@@ -28,9 +28,12 @@ Gem::Specification.new do |s|
     "VERSION",
     "distributed_cache.gemspec",
     "lib/distributed_cache.rb",
+    "lib/distributed_cache/bundle.rb",
     "lib/distributed_cache/config.rb",
     "lib/distributed_cache/manifest.rb",
-    "lib/distributed_cache/utils.rb"
+    "lib/distributed_cache/utils.rb",
+    "spec/distributed_cache_spec.rb",
+    "spec/helper.rb"
   ]
   s.homepage = "http://github.com/sessionm/distributed_cache"
   s.licenses = ["MIT"]
@@ -45,15 +48,18 @@ Gem::Specification.new do |s|
       s.add_runtime_dependency(%q<autoload>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
       s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_development_dependency(%q<fakeweb>, [">= 0"])
     else
       s.add_dependency(%q<autoload>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
       s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<fakeweb>, [">= 0"])
     end
   else
     s.add_dependency(%q<autoload>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
     s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<fakeweb>, [">= 0"])
   end
 end
 

data/lib/distributed_cache/bundle.rb

@@ -0,0 +1,76 @@
+require 'open-uri'
+
+module DistributedCache
+  class Bundle
+    attr_reader :config, :cache_name, :new_files
+
+    def initialize(config, cache_name)
+      @config = config
+      @cache_name = cache_name
+
+      @new_files = []
+    end
+
+    def install
+      download_manifest
+
+      manifest.files.each do |file, md5|
+        tar_file = create_tar_file file
+        if ! valid_tar_file?(tar_file, md5)
+          download_file file, tar_file
+          raise "invalid tar file (#{tar_file}) with md5 (#{md5})" unless valid_tar_file?(tar_file, md5)
+          @new_files << tar_file
+        elsif @new_files.size > 0
+          @new_files << tar_file
+        end
+      end
+
+      Dir.chdir(cache_dir) do
+        @new_files.each do |tar_file|
+          DistributedCache::Utils.untar tar_file
+        end
+      end
+    end
+
+    def cache_dir
+      config.cache_dir.tap do |dir|
+        FileUtils.mkdir_p(dir) unless File.exists?(dir)
+      end
+    end
+
+    def valid_tar_file?(file, md5)
+      File.exists?(file) && Digest::MD5.file(file).to_s == md5
+    end
+
+    def create_tar_file(file)
+      "#{config.bundle_dir}/#{file}".tap do |f|
+        dir = File.dirname f
+        FileUtils.mkdir_p(dir) unless File.exists?(dir)
+      end
+    end
+
+    def latest_dir
+      "#{config.bundle_dir}/#{cache_name}".tap do |dir|
+        FileUtils.mkdir_p(dir) unless File.exists?(dir)
+      end
+    end
+
+    def download_manifest
+      download_file "#{cache_name}/latest/manifest.yml", manifest_file
+    end
+
+    def download_file(remote_file, local_file)
+      open(local_file, 'wb+') do |f|
+        f.write open("http://#{config.file_server_with_port}/#{config.remote_bundle_dir}/#{remote_file}", 'rb').read
+      end
+    end
+
+    def manifest_file
+      "#{latest_dir}/manifest.yml"
+    end
+
+    def manifest
+      @manifest ||= DistributedCache::Manifest.load(config, manifest_file)
+    end
+  end
+end

data/lib/distributed_cache/config.rb

@@ -1,7 +1,31 @@
 module DistributedCache
   class Config < Struct.new(:cache_dir,
-                            :bundler_dir,
-                            :file_servers
+                            :bundle_dir,
+                            :file_servers,
+                            :file_server_port,
+                            :remote_bundle_dir
                             )
+
+    def initialize
+      self.file_server_port = 80
+      self.remote_bundle_dir = 'cache/bundles'
+      self.cache_dir = "#{Rails.root}/cache" if defined?(Rails)
+      self.bundle_dir = "#{Rails.root}/bundles" if defined?(Rails)
+    end
+
+    def next_file_server
+      @first_file_server_idx ||= rand(self.file_servers.size)
+      self.file_servers[@first_file_server_idx].tap do
+        @first_file_server_idx = (@first_file_server_idx + 1) % self.file_servers.size
+      end
+    end
+
+    def file_server_with_port
+      if self.file_server_port == 80
+        next_file_server
+      else
+        "#{next_file_server}:#{self.file_server_port}"
+      end
+    end
   end
 end

data/lib/distributed_cache/manifest.rb

@@ -1,31 +1,45 @@
+require 'yaml'
+
 module DistributedCache
   class Manifest
     attr_reader :config
+    attr_accessor :data, :date_format
 
     DEFAULT_DATE_FORMAT = '%Y%m%d'
 
-    module Fieldst
+    module Field
       CACHE_NAME = 'cache_name'
       DATE = 'date'
       DATE_FORMAT = 'date_format'
       CACHE_VERSION = 'cache_version'
       FILES = 'files'
     end
-    include Fields
+    include Field
 
-    def initialize(config, cache_name, cache_version)
+    def initialize(config, name=nil, version=nil)
       @config = config
-      @date = {
-        CACHE_NAME => cache_name,
-        CACHE_VERSION => cache_version,
-        DATE => date,
+      @data = {
+        CACHE_NAME => name,
+        CACHE_VERSION => version,
         FILES => []
       }
       load
     end
 
     def files
-      data[FILES]
+      @data[FILES]
+    end
+
+    def cache_name
+      @data[CACHE_NAME]
+    end
+
+    def cache_version
+      @data[CACHE_VERSION]
+    end
+
+    def add_file(name, file)
+      @data[FILES] << [name, Digest::MD5.file(file).to_s]
     end
 
     def [](key)
@@ -51,7 +65,7 @@ module DistributedCache
     end
 
     def date_format
-      @data[DATE_FORMAT] || DEFAULT_DATE_FORMAT
+      @date_format || DEFAULT_DATE_FORMAT
     end
 
     def date
@@ -89,6 +103,12 @@ module DistributedCache
       FileUtils.symlink bundle_dir, latest_bundle_dir
     end
 
+    def self.load(config, manifest_file)
+      new(config).tap do |manifest|
+        manifest.data = YAML.load_file(manifest_file)
+      end
+    end
+
     private
 
     def load

data/lib/distributed_cache/utils.rb

@@ -15,10 +15,30 @@ module DistributedCache
     end
 
     def self.tar(name, files)
-      if name[-3..-1] == 'tgz'
-        `tar zcfh #{name} #{files}`
-      else
-        `tar cfh #{name} #{files}`
+      cmd =
+        if name[-3..-1] == 'tgz'
+          "tar zcf #{name} #{files}"
+        else
+          "tar cf #{name} #{files}"
+        end
+
+      `#{cmd}`.tap do
+        status = $?
+        raise "failed to (#{cmd}), status #{status}" unless status == 0
+      end
+    end
+
+    def self.untar(file)
+      cmd =
+        if file[-3..-1] == 'tgz'
+          "tar zxf #{file}"
+        else
+          "tar xf #{file}"
+        end
+
+      `#{cmd}`.tap do
+        status = $?
+        raise "failed to (#{cmd}), status #{status}" unless status == 0
       end
     end
   end

data/lib/distributed_cache.rb

@@ -5,14 +5,11 @@ module DistributedCache
   autoload :Config, 'distributed_cache/config'
   autoload :Utils, 'distributed_cache/utils'
   autoload :Manifest, 'distributed_cache/manifest'
+  autoload :Bundle, 'distributed_cache/bundle'
 
-  @@config
+  @@config = nil
   def self.config
-    @@config ||= DistributedCache::Config.new.tap do |c|
-      if defined?(Rails)
-        c.cache_dir = "#{Rails.root}/cache"
-      end
-    end
+    @@config ||= DistributedCache::Config.new
   end
 
   def self.configure
@@ -24,7 +21,7 @@ module DistributedCache
   # cache_version - this is the version of the klass being saved, guards against changes to model where new fields are added or removed ex: 20130731
   # update_cache - this method caches data to disk, it is passed the manifest object.  This can be used to store things like the last_id.  So on later runs you can just pull the new records out of the database.
   #                 ex models = self.where("id > #{manifest['last_id'].to_i").all.each { |m| m.cache_model }; manifest['last_id'] = models.map(&:id).max
-  def self.update_cache(klass)
+  def self.update_remote_cache(klass)
     raise "cache_dir not configured" if self.config.cache_dir.nil?
     raise "bundle_dir not configured" if self.config.bundle_dir.nil?
 
@@ -35,16 +32,20 @@ module DistributedCache
     return unless klass.update_cache(manifest)
 
     tar_name = "#{manifest.cache_path}/#{klass.cache_name}-#{manifest.files.size + 1}.#{manifest.full? ? 'tgz' : 'tar'}"
-    manifest.files << tar_name
     tar_file = "#{config.bundle_dir}/#{tar_name}"
     Dir.chdir(config.cache_dir) do
       DistributedCache::Utils.tar tar_file, klass.cache_name
     end
+    manifest.add_file tar_name, tar_file
 
     manifest.save
     manifest.update_latest_bundle_dir
   end
 
+  def self.update_local_cache(cache_name)
+    DistributedCache::Bundle.new(self.config, cache_name).install
+  end
+
   def self.sync
     self.config.file_servers.each do |server|
       DistributedCache::Utils.rsync config.bundle_dir, server

data/spec/distributed_cache_spec.rb

@@ -0,0 +1,158 @@
+require 'helper'
+
+describe DistributedCache do
+  context 'config' do
+    it "should be able to configure itself" do
+      configure_test_distributed_cache
+
+      DistributedCache.config.file_servers.should == ['localhost']
+      DistributedCache.config.cache_dir = "#{BASE_TMP_DIR}/cache"
+      DistributedCache.config.bundle_dir = "#{BASE_TMP_DIR}/bundles"
+    end
+  end
+
+  context "update_remote_cache" do
+    before(:each) do
+      configure_test_distributed_cache
+    end
+
+    it "should create the full cache tar file" do
+      DistributedCache.update_remote_cache CachableKlass
+      DistributedCache::Manifest.new(DistributedCache.config, CachableKlass.cache_name, CachableKlass.cache_version).tap do |manifest|
+        manifest.cache_name.should == CachableKlass.cache_name
+        manifest.cache_version.should == CachableKlass.cache_version
+        manifest.files.size.should == 1
+        manifest.files[0][0].should == "#{CachableKlass.cache_name}/#{Time.now.strftime('%Y%m%d')}/#{CachableKlass.cache_version}/test-1.tgz"
+      end
+    end
+
+    it "should add the partial data as another tar file" do
+      DistributedCache.update_remote_cache CachableKlass
+      DistributedCache.update_remote_cache CachableKlass
+      DistributedCache::Manifest.new(DistributedCache.config, CachableKlass.cache_name, CachableKlass.cache_version).tap do |manifest|
+        manifest.cache_name.should == CachableKlass.cache_name
+        manifest.cache_version.should == CachableKlass.cache_version
+        manifest.files.size.should == 2
+        manifest.files[0][0].should == "#{CachableKlass.cache_name}/#{Time.now.strftime('%Y%m%d')}/#{CachableKlass.cache_version}/test-1.tgz"
+        manifest.files[1][0].should == "#{CachableKlass.cache_name}/#{Time.now.strftime('%Y%m%d')}/#{CachableKlass.cache_version}/test-2.tar"
+      end
+    end
+  end
+
+  context "update_local_cache" do
+    def update_remote_cache_for_context_update_local_cache
+      old_bundle_dir = DistributedCache.config.bundle_dir
+      old_cache_dir = DistributedCache.config.cache_dir
+
+      DistributedCache.configure do |config|
+        config.bundle_dir = "#{BASE_TMP_DIR}/remote/cache/bundles"
+        config.cache_dir = "#{BASE_TMP_DIR}/remote/cache/cache"
+      end
+
+      DistributedCache.update_remote_cache CachableKlass
+
+      DistributedCache.configure do |config|
+        config.bundle_dir = old_bundle_dir
+        config.cache_dir = old_cache_dir
+      end
+    end
+
+    before(:each) do
+      configure_test_distributed_cache
+      update_remote_cache_for_context_update_local_cache
+    end
+
+    it "should be able to install the full tar" do
+      cache_path = "#{CachableKlass.cache_name}/#{Time.now.strftime('%Y%m%d')}/#{CachableKlass.cache_version}"
+      manifest_file = "#{BASE_TMP_DIR}/remote/cache/bundles/test/latest/manifest.yml"
+      tar_file_1 = "#{BASE_TMP_DIR}/remote/cache/bundles/#{cache_path}/test-1.tgz"
+
+      File.exists?(manifest_file).should be_true
+      File.exists?(tar_file_1).should be_true
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/test/latest/manifest.yml", :body => manifest_file
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/#{cache_path}/test-1.tgz", :body => tar_file_1
+      DistributedCache.update_local_cache CachableKlass.cache_name
+
+      [['full1', 'test1'], ['full2', 'test2'], ['full3', 'test3']].each do |name, data|
+        file = "#{DistributedCache.config.cache_dir}/#{CachableKlass.cache_name}/#{name}"
+        File.exists?(file).should be_true
+        File.read(file).should == data
+      end
+    end
+
+    it "should use the partial tar override existing cache files" do
+      cache_path = "#{CachableKlass.cache_name}/#{Time.now.strftime('%Y%m%d')}/#{CachableKlass.cache_version}"
+      manifest_file = "#{BASE_TMP_DIR}/remote/cache/bundles/test/latest/manifest.yml"
+      tar_file_1 = "#{BASE_TMP_DIR}/remote/cache/bundles/#{cache_path}/test-1.tgz"
+      tar_file_2 = "#{BASE_TMP_DIR}/remote/cache/bundles/#{cache_path}/test-2.tar"
+
+      File.exists?(manifest_file).should be_true
+      File.exists?(tar_file_1).should be_true
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/test/latest/manifest.yml", :body => manifest_file
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/#{cache_path}/test-1.tgz", :body => tar_file_1
+
+      DistributedCache.update_local_cache CachableKlass.cache_name
+
+      [['full1', 'test1'], ['full2', 'test2'], ['full3', 'test3']].each do |name, data|
+        file = "#{DistributedCache.config.cache_dir}/#{CachableKlass.cache_name}/#{name}"
+        File.exists?(file).should be_true
+        File.read(file).should == data
+      end
+
+      update_remote_cache_for_context_update_local_cache
+
+      FakeWeb.clean_registry
+      File.exists?(tar_file_2).should be_true
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/test/latest/manifest.yml", :body => manifest_file
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/#{cache_path}/test-2.tar", :body => tar_file_2
+
+      DistributedCache.update_local_cache CachableKlass.cache_name
+
+      [['full1', 'test1'], ['full2', 'changed2'], ['full3', 'test3'], ['partial1', '1'], ['partial2', '2']].each do |name, data|
+        file = "#{DistributedCache.config.cache_dir}/#{CachableKlass.cache_name}/#{name}"
+        File.exists?(file).should be_true
+        File.read(file).should == data
+      end
+    end
+
+    it "should download the tar file if it is missing locally and untar all the files after it" do
+      update_remote_cache_for_context_update_local_cache
+
+      cache_path = "#{CachableKlass.cache_name}/#{Time.now.strftime('%Y%m%d')}/#{CachableKlass.cache_version}"
+      manifest_file = "#{BASE_TMP_DIR}/remote/cache/bundles/test/latest/manifest.yml"
+      tar_file_1 = "#{BASE_TMP_DIR}/remote/cache/bundles/#{cache_path}/test-1.tgz"
+      tar_file_2 = "#{BASE_TMP_DIR}/remote/cache/bundles/#{cache_path}/test-2.tar"
+
+      File.exists?(manifest_file).should be_true
+      File.exists?(tar_file_1).should be_true
+      File.exists?(tar_file_2).should be_true
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/test/latest/manifest.yml", :body => manifest_file
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/#{cache_path}/test-1.tgz", :body => tar_file_1
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/#{cache_path}/test-2.tar", :body => tar_file_2
+
+      DistributedCache.update_local_cache CachableKlass.cache_name
+
+      [['full1', 'test1'], ['full2', 'changed2'], ['full3', 'test3'], ['partial1', '1'], ['partial2', '2']].each do |name, data|
+        file = "#{DistributedCache.config.cache_dir}/#{CachableKlass.cache_name}/#{name}"
+        File.exists?(file).should be_true
+        File.read(file).should == data
+      end
+
+      local_tar_file_1 = "#{DistributedCache.config.bundle_dir}/#{cache_path}/test-1.tgz"
+      File.exists?(local_tar_file_1).should be_true
+      File.unlink local_tar_file_1
+
+      FakeWeb.clean_registry
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/test/latest/manifest.yml", :body => manifest_file
+      FakeWeb.register_uri :get, "http://localhost/cache/bundles/#{cache_path}/test-1.tgz", :body => tar_file_1
+
+      DistributedCache.update_local_cache CachableKlass.cache_name
+
+      [['full1', 'test1'], ['full2', 'changed2'], ['full3', 'test3'], ['partial1', '1'], ['partial2', '2']].each do |name, data|
+        file = "#{DistributedCache.config.cache_dir}/#{CachableKlass.cache_name}/#{name}"
+        File.exists?(file).should be_true
+        File.read(file).should == data
+      end
+    end
+  end
+end

data/spec/helper.rb

@@ -0,0 +1,60 @@
+require 'rubygems'
+require 'bundler'
+
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+
+require 'fakeweb'
+
+class CachableKlass
+  def self.cache_name
+    'test'
+  end
+
+  def self.cache_version
+    19871009
+  end
+
+  def self.cache_file(name, data="test")
+    File.open("#{DistributedCache.config.cache_dir}/#{self.cache_name}/#{name}", 'wb+') { |f| f.write data }
+  end
+
+  def self.update_cache(manifeset)
+    if manifeset.full?
+      cache_file 'full1', 'test1'
+      cache_file 'full2', 'test2'
+      cache_file 'full3', 'test3'
+    else
+      cache_file 'partial1', '1'
+      cache_file 'partial2', '2'
+      cache_file 'full2', 'changed2'
+    end
+  end
+end
+
+BASE_TMP_DIR = "#{Dir.pwd}/tmp/rspec"
+
+def configure_test_distributed_cache
+  FileUtils.rm_rf BASE_TMP_DIR
+
+  DistributedCache.configure do |config|
+    config.file_servers = ['localhost']
+    config.cache_dir = "#{BASE_TMP_DIR}/cache"
+    config.bundle_dir = "#{BASE_TMP_DIR}/bundles"
+  end
+end
+
+RSpec.configure do |config|
+  config.before(:each) do
+    FakeWeb.allow_net_connect = false
+    FakeWeb.clean_registry
+  end
+end
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'distributed_cache'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: distributed_cache
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Doug Youch
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-07-31 00:00:00.000000000 Z
+date: 2013-08-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: autoload
@@ -52,6 +52,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: fakeweb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Creates tar files of cache directories for distribution
 email: doug@sessionm.com
 executables: []
@@ -71,9 +85,12 @@ files:
 - VERSION
 - distributed_cache.gemspec
 - lib/distributed_cache.rb
+- lib/distributed_cache/bundle.rb
 - lib/distributed_cache/config.rb
 - lib/distributed_cache/manifest.rb
 - lib/distributed_cache/utils.rb
+- spec/distributed_cache_spec.rb
+- spec/helper.rb
 homepage: http://github.com/sessionm/distributed_cache
 licenses:
 - MIT