buildcache 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 30a10ca2c786761a31c76f72b11a13d1f5b0cbdd
+  data.tar.gz: 1088fe6a3088038d904d1dff0d9e403486d7479b
+SHA512:
+  metadata.gz: f013dd48f41f66bac7ecdc0f6d9cbcc58df8205766791f5fd86b2b0f58d0ea6cbb702948c1e89384ef2927a1b037e0e4f77f85f51c40746805473512fdfee15a
+  data.tar.gz: a92855f76154bbd1ff65b39c25d474b9cb3811bc2979e835a6e79e67dee6a93460994b8d52c699c69241401d1fb164d51aab13ffc4ffd3c46a471f02cc5c4aef

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,5 @@
+sudo: false
+language: ruby
+rvm:
+  - 2.1.1
+before_install: gem install bundler -v 1.12.5

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in buildcache.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Victor Lyuboslavsky
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,78 @@
+[![Build Status](https://travis-ci.org/getvictor/buildcache.svg?branch=master)](https://travis-ci.org/getvictor/buildcache)
+
+# BuildCache
+
+A simple cache for files. It is currently used in production to cache the results of frequently run built steps (using a custom build flow).
+
+The cache receives the input files and some metadata. It generates a hash key and checks the cache. If the cache is hit, the files are copied from the cache to the provided location. If we have a cache miss, the provided block is run, and the resulting files are copied to the cache.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'buildcache'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install buildcache
+
+## Usage
+
+### Block Style
+
+    :::ruby
+    require 'buildcache'
+    @my_cache = BuildCache::DiskCache.new('/tmp/cache')
+    @input_files = ['./input.txt']
+    @metadata = {:cmd => 'parse input'}
+    @dest_dir = './build'
+    @my_cache.cache(@input_files, @metadata, @dest_dir) do
+      # This is time consuming code where you generate result files.
+      # The block is only run on a cache miss
+      ...
+      # You must return resulting files so they can be cached
+      next files
+    end
+
+### Use API
+
+The cache uses 2 keys. The `first_key` corresponds to directory name in the cache. `second_key` (optional) is held in a file and is used to resolve hash conflicts on first_key.
+
+The following methods are available:
+
+    :::ruby
+    # Generate first_key
+    first_key = BuildCache.key_gen(input_files_array, metadata)
+    
+    :::ruby
+    # Check for a cache hit
+    @my_cache.hit?(first_key, second_key)
+    
+    :::ruby
+    # Set the cache
+    @my_cache.set(first_key, second_key, files_array)
+    
+    :::ruby
+    # Retrieve from cache. The result is a cache directory where the files are stored
+    contents_dir = @my_cache.get(first_key, second_key)
+    
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/getvictor/buildcache.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,5 @@
+require "bundler/gem_tasks"
+
+Dir.glob('tasks/**/*.rake').each(&method(:import))
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "buildcache"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/buildcache.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'buildcache/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "buildcache"
+  spec.version       = BuildCache::VERSION
+  spec.authors       = ["Victor Lyuboslavsky"]
+
+  spec.summary       = %q{Build cache for any build system.}
+  spec.homepage      = "https://github.com/getvictor/buildcache"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "simplecov", "~> 0.11.2"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/lib/buildcache.rb

@@ -0,0 +1,233 @@
+require 'buildcache/version'
+require 'digest'
+
+module BuildCache
+
+  def self.key_gen files=[], metadata={}
+    # TODO: Validate inputs.
+    flat_sig = ''
+    unless files.empty?
+      files.each { |file|
+        flat_sig << Digest::MD5.file(file).hexdigest
+      }
+    end
+    unless metadata.empty?
+      flat_sig << Digest::MD5.hexdigest(metadata.to_s)
+    end
+    return Digest::MD5.hexdigest(flat_sig)
+  end
+
+  class DiskCache
+
+    # The directory containing cached files
+    attr_reader :dir
+
+    # The Linux permissions that cached files should have
+    attr_reader :permissions
+
+    # logger to use, if logger is not set, then messages will not be logged
+    attr_accessor :logger
+
+    # Percent of time to check the cache size
+    attr_accessor :check_size_percent
+
+    # The maximum number of entries in the cache.
+    # Note that since we don't check the cache size every time, the actual size might exceed this number
+    attr_accessor :max_cache_size
+
+    # The percent of entries to evict (delete) if size is exceeded
+    attr_accessor :evict_percent
+
+    def initialize dir='/tmp/cache', permissions=0666
+      # Make sure 'dir' is not a file
+      if (File.exist?(dir) && !File.directory?(dir))
+        raise "DiskCache dir #{dir} should be a directory."
+      end
+      @dir = dir
+      @permissions = permissions
+      @enable_logging = false
+      @check_size_percent = 20
+      @max_cache_size = 5000
+      @evict_percent = 20.0
+      mkdir
+    end
+
+    def set first_key, second_key='', files=[]
+      # TODO: validate inputs
+
+      # If cache exists already, overwrite it.
+      content_dir = get first_key, second_key
+      second_key_file = nil
+
+      begin
+        if (content_dir.nil?)
+
+          # Check the size of cache, and evict entries if too large
+          check_cache_size if (rand(100) < check_size_percent)
+
+          # Make sure cache dir doesn't exist already
+          first_cache_dir = File.join(dir, first_key)
+          if (File.exist?first_cache_dir)
+            raise "BuildCache directory #{first_cache_dir} should be a directory" unless File.directory?(first_cache_dir)
+          else
+            FileUtils.mkpath(first_cache_dir)
+          end
+          num_second_dirs = Dir[first_cache_dir + '/*'].length
+          cache_dir = File.join(first_cache_dir, num_second_dirs.to_s)
+          # If cache directory already exists, then a directory must have been evicted here, so we pick another name
+          while File.directory?cache_dir
+            cache_dir = File.join(first_cache_dir, rand(num_second_dirs).to_s)
+          end
+          content_dir = File.join(cache_dir, '/content')
+          FileUtils.mkpath(content_dir)
+
+          # Create 'last_used' file
+          last_used_filename = File.join(cache_dir, 'last_used')
+          FileUtils.touch last_used_filename
+          FileUtils.chmod(permissions, last_used_filename)
+
+          # Copy second key
+          second_key_file = File.open(cache_dir + '/second_key', 'w+')
+          second_key_file.flock(File::LOCK_EX)
+          second_key_file.write(second_key)
+
+        else
+          log "overwriting cache #{content_dir}"
+
+          FileUtils.touch content_dir + '/../last_used'
+          second_key_file = File.open(content_dir + '/../second_key', 'r')
+          second_key_file.flock(File::LOCK_EX)
+          # Clear any existing files out of cache directory
+          FileUtils.rm_rf(content_dir + '/.')
+        end
+
+        # Copy files into content_dir
+        files.each do |filename|
+          FileUtils.cp(filename, content_dir)
+        end
+        FileUtils.chmod(permissions, Dir[content_dir + '/*'])
+
+        # Release the lock
+        second_key_file.close
+        return content_dir
+      rescue => e
+        # Something went wrong, like a full disk or some other error.
+        # Delete any work so we don't leave cache in corrupted state
+        unless content_dir.nil?
+          # Delete parent of content directory
+          FileUtils.rm_rf(File.expand_path('..', content_dir))
+        end
+        log "ERROR: Could not set cache entry. #{e.to_s}"
+        return 'ERROR: !NOT CACHED!'
+      end
+      
+    end
+
+    # Get the cache directory containing the contents corresponding to the keys
+    def get first_key, second_key=''
+      # TODO: validate inputs
+
+      begin
+        cache_dirs = Dir[File.join(@dir, first_key + '/*')]
+        cache_dirs.each do |cache_dir|
+          second_key_filename = cache_dir + '/second_key'
+          # If second key filename is bad, we skip this directory
+          if (!File.exist?(second_key_filename) || File.directory?(second_key_filename))
+            next
+          end
+          second_key_file = File.open(second_key_filename, "r" )
+          second_key_file.flock(File::LOCK_SH)
+          out = second_key_file.read
+          if (second_key.to_s == out)
+            FileUtils.touch cache_dir + '/last_used'
+            cache_dir = File.join(cache_dir, 'content')
+            second_key_file.close
+            return cache_dir if File.directory?(cache_dir)
+          end
+          second_key_file.close
+        end
+      rescue => e
+        log "ERROR: Could not get cache entry. #{e.to_s}"
+      end
+      return nil
+    end
+
+    def hit? first_key, second_key=''
+      return get(first_key, second_key) != nil
+    end
+
+    def cache input_files, metadata, dest_dir
+      # Create the cache keys
+      first_key = BuildCache.key_gen input_files, metadata
+      second_key = metadata.to_s
+
+      # If cache hit, copy the files to the dest_dir
+      if (hit?first_key, second_key)
+        begin
+          cache_dir = get first_key, second_key
+          log "cache hit #{cache_dir}"
+          mkdir dest_dir
+          FileUtils.cp_r(cache_dir + '/.', dest_dir)
+          return Dir[cache_dir + '/*'].map { |pathname| File.basename pathname }
+        rescue => e
+          # Since we don't return, error counts as a cache miss
+          log "ERROR: Could not retrieve cache entry contents. #{e.to_s}"
+        end
+      end
+
+      # If cache miss, run the block and put the results in the cache
+      files = yield
+      output_files = files.map { |filename| File.join(dest_dir, filename) }
+      # Check the cache again in case someone else populated it already
+      unless (hit?first_key, second_key)
+        cache_dir = set(first_key, second_key, output_files)
+        log "cache miss, caching results to #{cache_dir}"
+      end
+      return files
+    end
+
+    def check_cache_size
+      log "checking cache size"
+      entries = Dir[@dir + '/*/*']
+      if entries.length > max_cache_size
+        # If cache is locked for maintainance (lock exists and less than 8 hours old), then we skip the check
+        lock_filename = File.join(@dir, 'cache_maintenance')
+        return if (File.exist?(lock_filename) && (File.mtime(lock_filename) > Time.now - (8 * 60 * 60)))
+        FileUtils.touch(lock_filename)
+
+        log "evicting old cache entries"
+        # evict some entries
+        entries = entries.sort do |a,b|
+          # evict entries that don't have a last_used file
+          a_file = File.join(a, 'last_used')
+          next -1 if !File.exist?a_file
+          b_file = File.join(b, 'last_used')
+          next 1 if !File.exist?b_file
+          next File.mtime(a_file) <=> File.mtime(b_file)
+        end
+
+        entries_to_delete = (entries.length * evict_percent / 100).ceil
+        entries[0..(entries_to_delete-1)].each { |entry| FileUtils.rm_rf(entry) }
+        # Delete empty directories
+        Dir[@dir + '/*'].each { |d| Dir.rmdir d if (File.directory?(d) && (Dir.entries(d) - %w[ . .. ]).empty?) }
+
+        # Delete lock file
+        FileUtils.rm lock_filename, :force => true
+      end
+    end
+    
+    private
+      
+    def mkdir dir=@dir
+      FileUtils.mkpath(dir) unless File.directory?(dir)
+    end
+
+    def log message
+      unless (@logger.nil?)
+        @logger.info { "[BuildCache::DiskCache] #{message.to_s}" }
+      end
+    end
+
+  end
+  
+end

data/lib/buildcache/version.rb

@@ -0,0 +1,3 @@
+module BuildCache
+  VERSION = "0.1.0"
+end

data/tasks/rspec.rake

@@ -0,0 +1,3 @@
+require 'rspec/core/rake_task'
+
+RSpec::Core::RakeTask.new(:spec)

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: buildcache
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Victor Lyuboslavsky
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-06-30 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.11.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: 
+email: 
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- buildcache.gemspec
+- lib/buildcache.rb
+- lib/buildcache/version.rb
+- tasks/rspec.rake
+homepage: https://github.com/getvictor/buildcache
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Build cache for any build system.
+test_files: []
+has_rdoc: