distributed_cache 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ea80e92a851ce1d5042dcd3d28263449e2c78f35
+  data.tar.gz: ace19ce97a6d3ca598c0b8c2e9daf3da549bc10a
+SHA512:
+  metadata.gz: fbfd450ca33835729c58406e6de69914e2b9141378c3e71c48fc50959fcf0c5fc7338926a1fb60197f43e35ca8151d6b2b8a30e730956b5713ed6162d0ec762b
+  data.tar.gz: 6b2aa072d7852ddd52655efb26a9dbc037f8b84b4c85433dbd63e6f7b176564f048e9cef11b0e73bd1612c315f674767d23d39d7fd4f47b6146f30350e5926f8

data/.document

@@ -0,0 +1,5 @@
+lib/**/*.rb
+bin/*
+- 
+features/**/*.feature
+LICENSE.txt

data/.ruby-gemset

@@ -0,0 +1 @@
+distributed_cache

data/.ruby-version

@@ -0,0 +1 @@
+ruby-2.0.0-p247

data/Gemfile

@@ -0,0 +1,8 @@
+source "http://rubygems.org"
+
+gem 'autoload'
+
+group :development do
+  gem "jeweler"
+  gem "rspec"
+end

data/Gemfile.lock

@@ -0,0 +1,63 @@
+GEM
+  remote: http://rubygems.org/
+  specs:
+    addressable (2.3.5)
+    autoload (0.3.0)
+    builder (3.2.2)
+    diff-lcs (1.2.4)
+    faraday (0.8.8)
+      multipart-post (~> 1.2.0)
+    git (1.2.5)
+    github_api (0.10.1)
+      addressable
+      faraday (~> 0.8.1)
+      hashie (>= 1.2)
+      multi_json (~> 1.4)
+      nokogiri (~> 1.5.2)
+      oauth2
+    hashie (2.0.5)
+    highline (1.6.19)
+    httpauth (0.2.0)
+    jeweler (1.8.6)
+      builder
+      bundler (~> 1.0)
+      git (>= 1.2.5)
+      github_api (= 0.10.1)
+      highline (>= 1.6.15)
+      nokogiri (= 1.5.10)
+      rake
+      rdoc
+    json (1.8.0)
+    jwt (0.1.8)
+      multi_json (>= 1.5)
+    multi_json (1.7.7)
+    multi_xml (0.5.4)
+    multipart-post (1.2.0)
+    nokogiri (1.5.10)
+    oauth2 (0.9.2)
+      faraday (~> 0.8)
+      httpauth (~> 0.2)
+      jwt (~> 0.1.4)
+      multi_json (~> 1.0)
+      multi_xml (~> 0.5)
+      rack (~> 1.2)
+    rack (1.5.2)
+    rake (10.1.0)
+    rdoc (4.0.1)
+      json (~> 1.4)
+    rspec (2.14.1)
+      rspec-core (~> 2.14.0)
+      rspec-expectations (~> 2.14.0)
+      rspec-mocks (~> 2.14.0)
+    rspec-core (2.14.4)
+    rspec-expectations (2.14.0)
+      diff-lcs (>= 1.1.3, < 2.0)
+    rspec-mocks (2.14.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  autoload
+  jeweler
+  rspec

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2013 Doug Youch
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,19 @@
+= distributed_cache
+
+Description goes here.
+
+== Contributing to distributed_cache
+ 
+* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet.
+* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it.
+* Fork the project.
+* Start a feature/bugfix branch.
+* Commit and push until you are happy with your contribution.
+* Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
+* Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
+
+== Copyright
+
+Copyright (c) 2013 Doug Youch. See LICENSE.txt for
+further details.
+

data/Rakefile

@@ -0,0 +1,26 @@
+# encoding: utf-8
+
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'rake'
+
+require 'jeweler'
+Jeweler::Tasks.new do |gem|
+  # gem is a Gem::Specification... see http://docs.rubygems.org/read/chapter/20 for more options
+  gem.name = "distributed_cache"
+  gem.homepage = "http://github.com/sessionm/distributed_cache"
+  gem.license = "MIT"
+  gem.summary = %Q{Distributed file cache bundler}
+  gem.description = %Q{Creates tar files of cache directories for distribution}
+  gem.email = "doug@sessionm.com"
+  gem.authors = ["Doug Youch"]
+  # dependencies defined in Gemfile
+end
+Jeweler::RubygemsDotOrgTasks.new

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/distributed_cache.gemspec

@@ -0,0 +1,59 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE DIRECTLY
+# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = "distributed_cache"
+  s.version = "0.1.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Doug Youch"]
+  s.date = "2013-07-31"
+  s.description = "Creates tar files of cache directories for distribution"
+  s.email = "doug@sessionm.com"
+  s.extra_rdoc_files = [
+    "LICENSE.txt",
+    "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+    ".ruby-gemset",
+    ".ruby-version",
+    "Gemfile",
+    "Gemfile.lock",
+    "LICENSE.txt",
+    "README.rdoc",
+    "Rakefile",
+    "VERSION",
+    "distributed_cache.gemspec",
+    "lib/distributed_cache.rb",
+    "lib/distributed_cache/config.rb",
+    "lib/distributed_cache/manifest.rb",
+    "lib/distributed_cache/utils.rb"
+  ]
+  s.homepage = "http://github.com/sessionm/distributed_cache"
+  s.licenses = ["MIT"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = "2.0.3"
+  s.summary = "Distributed file cache bundler"
+
+  if s.respond_to? :specification_version then
+    s.specification_version = 4
+
+    if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
+      s.add_runtime_dependency(%q<autoload>, [">= 0"])
+      s.add_development_dependency(%q<jeweler>, [">= 0"])
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+    else
+      s.add_dependency(%q<autoload>, [">= 0"])
+      s.add_dependency(%q<jeweler>, [">= 0"])
+      s.add_dependency(%q<rspec>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<autoload>, [">= 0"])
+    s.add_dependency(%q<jeweler>, [">= 0"])
+    s.add_dependency(%q<rspec>, [">= 0"])
+  end
+end
+

data/lib/distributed_cache/config.rb

@@ -0,0 +1,7 @@
+module DistributedCache
+  class Config < Struct.new(:cache_dir,
+                            :bundler_dir,
+                            :file_servers
+                            )
+  end
+end

data/lib/distributed_cache/manifest.rb

@@ -0,0 +1,98 @@
+module DistributedCache
+  class Manifest
+    attr_reader :config
+
+    DEFAULT_DATE_FORMAT = '%Y%m%d'
+
+    module Fieldst
+      CACHE_NAME = 'cache_name'
+      DATE = 'date'
+      DATE_FORMAT = 'date_format'
+      CACHE_VERSION = 'cache_version'
+      FILES = 'files'
+    end
+    include Fields
+
+    def initialize(config, cache_name, cache_version)
+      @config = config
+      @date = {
+        CACHE_NAME => cache_name,
+        CACHE_VERSION => cache_version,
+        DATE => date,
+        FILES => []
+      }
+      load
+    end
+
+    def files
+      data[FILES]
+    end
+
+    def [](key)
+      @data[key]
+    end
+
+    def []=(key, value)
+      @data[key] = value
+    end
+
+    # if the manifest file exists we only need todo a partial update of the cache
+    def partial?
+      File.exists? manifest_file
+    end
+
+    def full?
+      ! partial?
+    end
+
+    # this will be used as the remote file path and the local bundle directory
+    def cache_path
+      "#{cache_name}/#{date}/#{cache_version}"
+    end
+
+    def date_format
+      @data[DATE_FORMAT] || DEFAULT_DATE_FORMAT
+    end
+
+    def date
+      Time.now.strftime date_format
+    end
+
+    def bundle_dir
+      "#{config.bundle_dir}/#{cache_path}".tap do |d|
+        FileUtils.mkdir_p(d) unless File.exists?(d)
+      end
+    end
+
+    def cache_dir
+      "#{config.cache_dir}/#{cache_name}".tap do |d|
+        FileUtils.mkdir_p(d) unless File.exists?(d)
+      end
+    end
+
+    def manifest_file
+      "#{bundle_dir}/manifest.yml"
+    end
+
+    def latest_bundle_dir
+      "#{config.bundle_dir}/#{cache_name}/latest"
+    end
+
+    def save
+      File.open(manifest_file, 'w+') do |f|
+        f.write @data.to_yaml
+      end
+    end
+
+    def update_latest_bundle_dir
+      File.unlink(latest_bundle_dir) if File.exists?(latest_bundle_dir)
+      FileUtils.symlink bundle_dir, latest_bundle_dir
+    end
+
+    private
+
+    def load
+      @data = YAML.load_file(manifest_file) if File.exists?(manifest_file)
+    end
+  end
+end

data/lib/distributed_cache/utils.rb

@@ -0,0 +1,25 @@
+require 'fileutils'
+
+module DistributedCache
+  module Utils
+    # these folders could contain 100s of thousands of files
+    # faster to just mv the folder out of the way and delete it in a forked process
+    def self.rm_rf(dir)
+      return unless File.exists?(dir)
+      time = Time.now.to_i
+      dest_dir = "#{dir}.#{time}"
+      FileUtils.mv dir, dest_dir
+      Process.fork do
+        FileUtils.rm_rf dest_dir
+      end
+    end
+
+    def self.tar(name, files)
+      if name[-3..-1] == 'tgz'
+        `tar zcfh #{name} #{files}`
+      else
+        `tar cfh #{name} #{files}`
+      end
+    end
+  end
+end

data/lib/distributed_cache.rb

@@ -0,0 +1,53 @@
+require 'autoload'
+require 'fileutils'
+
+module DistributedCache
+  autoload :Config, 'distributed_cache/config'
+  autoload :Utils, 'distributed_cache/utils'
+  autoload :Manifest, 'distributed_cache/manifest'
+
+  @@config
+  def self.config
+    @@config ||= DistributedCache::Config.new.tap do |c|
+      if defined?(Rails)
+        c.cache_dir = "#{Rails.root}/cache"
+      end
+    end
+  end
+
+  def self.configure
+    yield self.config
+  end
+
+  # in order to cache a klass it must respond to
+  # cache_name - this is the directory in the base cache directory where the new cache files are created, ex: applications
+  # cache_version - this is the version of the klass being saved, guards against changes to model where new fields are added or removed ex: 20130731
+  # update_cache - this method caches data to disk, it is passed the manifest object.  This can be used to store things like the last_id.  So on later runs you can just pull the new records out of the database.
+  #                 ex models = self.where("id > #{manifest['last_id'].to_i").all.each { |m| m.cache_model }; manifest['last_id'] = models.map(&:id).max
+  def self.update_cache(klass)
+    raise "cache_dir not configured" if self.config.cache_dir.nil?
+    raise "bundle_dir not configured" if self.config.bundle_dir.nil?
+
+    manifest = DistributedCache::Manifest.new self.config, klass.cache_name, klass.cache_version
+    DistributedCache::Utils.rm_rf manifest.cache_dir
+    FileUtils.mkdir_p manifest.cache_dir
+
+    return unless klass.update_cache(manifest)
+
+    tar_name = "#{manifest.cache_path}/#{klass.cache_name}-#{manifest.files.size + 1}.#{manifest.full? ? 'tgz' : 'tar'}"
+    manifest.files << tar_name
+    tar_file = "#{config.bundle_dir}/#{tar_name}"
+    Dir.chdir(config.cache_dir) do
+      DistributedCache::Utils.tar tar_file, klass.cache_name
+    end
+
+    manifest.save
+    manifest.update_latest_bundle_dir
+  end
+
+  def self.sync
+    self.config.file_servers.each do |server|
+      DistributedCache::Utils.rsync config.bundle_dir, server
+    end
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: distributed_cache
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Doug Youch
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: autoload
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: jeweler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Creates tar files of cache directories for distribution
+email: doug@sessionm.com
+executables: []
+extensions: []
+extra_rdoc_files:
+- LICENSE.txt
+- README.rdoc
+files:
+- .document
+- .ruby-gemset
+- .ruby-version
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.rdoc
+- Rakefile
+- VERSION
+- distributed_cache.gemspec
+- lib/distributed_cache.rb
+- lib/distributed_cache/config.rb
+- lib/distributed_cache/manifest.rb
+- lib/distributed_cache/utils.rb
+homepage: http://github.com/sessionm/distributed_cache
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Distributed file cache bundler
+test_files: []