travis-lock 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4c983c384c38f7bb07cddd92b3ea775570ff2aec
+  data.tar.gz: 21b952b4ce5c7eeed44d21d4db04e6b4e920f428
+SHA512:
+  metadata.gz: 490a9cb44c09b52426876d131d30c118e3e279a82dd75d7c9cc30d0d6553d5278ba4e9b8d686d34683a18e370727e09a6e2e1ea124365f359a4aaf23a2c2f3c0
+  data.tar.gz: afccd06931aaa156e0e6fbafa590d3929e0a0fc903f3386554499d15277e862f6ab6d1ee09822ae9cf0042e91c134e1a07959def5a58f3522b8439aaa48d590d

data/Gemfile

@@ -0,0 +1,14 @@
+source 'https://rubygems.org'
+
+gemspec
+
+gem 'redlock'
+
+platform :ruby do
+  gem 'pg'
+end
+
+group :test do
+  gem 'rspec', '~> 3.0'
+  gem 'mocha', '~> 1.1'
+end

data/Gemfile.lock

@@ -0,0 +1,61 @@
+PATH
+  remote: .
+  specs:
+    travis-lock (0.1.4)
+      activerecord (~> 4.0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activemodel (4.2.4)
+      activesupport (= 4.2.4)
+      builder (~> 3.1)
+    activerecord (4.2.4)
+      activemodel (= 4.2.4)
+      activesupport (= 4.2.4)
+      arel (~> 6.0)
+    activesupport (4.2.4)
+      i18n (~> 0.7)
+      json (~> 1.7, >= 1.7.7)
+      minitest (~> 5.1)
+      thread_safe (~> 0.3, >= 0.3.4)
+      tzinfo (~> 1.1)
+    arel (6.0.3)
+    builder (3.2.2)
+    diff-lcs (1.2.5)
+    i18n (0.7.0)
+    json (1.8.3)
+    metaclass (0.0.4)
+    minitest (5.8.0)
+    mocha (1.1.0)
+      metaclass (~> 0.0.1)
+    pg (0.18.3)
+    redis (3.2.1)
+    redlock (0.1.1)
+      redis (~> 3, >= 3.0.5)
+    rspec (3.3.0)
+      rspec-core (~> 3.3.0)
+      rspec-expectations (~> 3.3.0)
+      rspec-mocks (~> 3.3.0)
+    rspec-core (3.3.2)
+      rspec-support (~> 3.3.0)
+    rspec-expectations (3.3.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-mocks (3.3.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.3.0)
+    rspec-support (3.3.0)
+    thread_safe (0.3.5)
+    tzinfo (1.2.2)
+      thread_safe (~> 0.1)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  mocha (~> 1.1)
+  pg
+  redlock
+  rspec (~> 3.0)
+  travis-lock!

data/README.md

@@ -0,0 +1,32 @@
+# Travis Lock
+
+Application-level locks for use in, e.g. travis-hub.
+
+At the moment it seems the Redlock strategy works fine as well as the
+Postgresql advisory locks strategy when used with the options in the
+example below.
+
+Usage:
+
+```
+options = {
+  strategy: :postgresql,
+  try: true,
+  transactional: false
+}
+Travis::Lock.exclusive('build-1', options) do
+  # update build
+end
+```
+
+### Doing a Rubygem release
+
+Any tool works. The current releases were done with
+[`gem-release`](https://github.com/svenfuchs/gem-release) which allows creating
+a Git tag, pushing it to GitHub, building the gem and pushing it to Rubygems in
+one go:
+
+```bash
+$ gem install gem-release
+$ gem bump --push --tag --release
+```

data/lib/travis/lock/none.rb

@@ -0,0 +1,9 @@
+module Travis
+  module Lock
+    class None < Struct.new(:name, :options)
+      def exclusive
+        yield
+      end
+    end
+  end
+end

data/lib/travis/lock/postgresql.rb

@@ -0,0 +1,76 @@
+# http://hashrocket.com/blog/posts/advisory-locks-in-postgres
+# https://github.com/mceachen/with_advisory_lock
+# 13.3.4. Advisory Locks : http://www.postgresql.org/docs/9.3/static/explicit-locking.html
+# http://www.postgresql.org/docs/9.3/static/functions-admin.html#FUNCTIONS-ADVISORY-LOCKS
+
+require 'zlib'
+require 'active_record'
+require 'travis/lock/support/retry'
+
+module Travis
+  module Lock
+    class Postgresql < Struct.new(:name, :options)
+      def initialize(*)
+        super
+        fail 'lock name cannot be blank' if name.nil? || name.empty?
+      end
+
+      def exclusive(&block)
+        with_timeout { obtain_lock }
+        transactional? ? connection.transaction(&block) : with_release(&block)
+      end
+
+      private
+
+        def with_timeout(&block)
+          try? ? Retry.new(name, options).run(&block) : with_statement_timeout(&block)
+        end
+
+        def obtain_lock
+          result = connection.select_value("select #{pg_function}(#{key});")
+          try? ? result == 't' : true
+        end
+
+        def with_release
+          yield
+        ensure
+          connection.execute("select pg_advisory_unlock(#{key});")
+        end
+
+        def try?
+          !!options[:try]
+        end
+
+        def timeout
+          options[:timeout] || 30
+        end
+
+        def transactional?
+          !!options[:transactional]
+        end
+
+        def with_statement_timeout
+          connection.execute("set statement_timeout to #{Integer(timeout * 1000)};")
+          yield
+        rescue ActiveRecord::StatementInvalid => e
+          retry if defined?(PG) && e.original_exception.is_a?(PG::QueryCanceled)
+          timeout!
+        end
+
+        def pg_function
+          func = ['pg', 'advisory', 'lock']
+          func.insert(2, 'xact') if transactional?
+          func.insert(1, 'try')  if try?
+          func.join('_')
+        end
+
+        def connection
+          ActiveRecord::Base.connection
+        end
+
+        def key
+          Zlib.crc32(name).to_i & 0x7fffffff
+        end
+    end
+  end
+end

data/lib/travis/lock/redis.rb

@@ -0,0 +1,60 @@
+require 'monitor'
+begin
+  require 'redlock'
+rescue LoadError
+end
+
+module Travis
+  module Lock
+    class Redis
+      class LockError < StandardError
+        def initialize(key)
+          super("Could not obtain lock for #{key.inspect} on Redis.")
+        end
+      end
+
+      extend MonitorMixin
+
+      DEFAULTS = {
+        ttl:      5 * 60,
+        retries:  5,
+        interval: 0.1
+      }
+
+      attr_reader :name, :config, :retried
+
+      def initialize(name, config)
+        @name    = name
+        @config  = DEFAULTS.merge(config)
+        @retried = 0
+      end
+
+      def exclusive
+        retrying do
+          client.lock(name, config[:ttl]) do |lock|
+            lock ? yield : raise(LockError.new(name))
+          end
+        end
+      end
+
+      private
+
+        def client
+          Redlock::Client.new([url])
+        end
+
+        def url
+          config[:url] || fail("No Redis URL specified")
+        end
+
+        def retrying
+          yield
+        rescue LockError
+          raise if retried.to_i >= config[:retries]
+          sleep config[:interval]
+          @retries = retried + 1
+          retry
+        end
+    end
+  end
+end

data/lib/travis/lock/support/retry.rb

@@ -0,0 +1,33 @@
+module Travis
+  module Lock
+    class Retry < Struct.new(:name, :options)
+      WAIT = 0.0001..0.0009
+
+      def run
+        wait until result = yield
+        result
+      end
+
+      def wait
+        sleep(rand(options[:wait] || WAIT))
+        timeout! if timeout?
+      end
+
+      def started
+        @started ||= Time.now
+      end
+
+      def timeout?
+        started + timeout < Time.now
+      end
+
+      def timeout
+        options[:timeout] || 30
+      end
+
+      def timeout!
+        fail Timeout.new(name, options)
+      end
+    end
+  end
+end

data/lib/travis/lock/version.rb

@@ -0,0 +1,5 @@
+module Travis
+  module Lock
+    VERSION = '0.1.0'
+  end
+end

data/lib/travis/lock.rb

@@ -0,0 +1,28 @@
+require 'travis/lock/none'
+require 'travis/lock/postgresql'
+require 'travis/lock/redis'
+
+module Travis
+  module Lock
+    class Timeout < StandardError
+      def initialize(name, options)
+        super("Could not obtain lock for #{name}: #{options.map { |*pair| pair.join('=') }.join(' ')}")
+      end
+    end
+
+    extend self
+
+    attr_reader :default_strategy
+
+    def exclusive(name, options = {}, &block)
+      options[:strategy] ||= Lock.default_strategy || :none
+      const_get(camelize(options[:strategy])).new(name, options).exclusive(&block)
+    end
+
+    private
+
+      def camelize(object)
+        object.to_s.split('_').collect(&:capitalize).join
+      end
+  end
+end

data/spec/integration.rb

@@ -0,0 +1,55 @@
+require 'active_record'
+require 'travis/lock'
+
+ActiveRecord::Base.establish_connection(
+  adapter: 'postgresql',
+  database: 'travis_development',
+  pool: 30
+)
+
+class Lock
+  class Redis
+    def exclusive(&block)
+      options = {
+        strategy: :redis,
+        url: 'redis://localhost:6379'
+      }
+      Travis::Lock.exclusive('test', options, &block)
+    end
+  end
+
+  class Postgresql
+    def exclusive(&block)
+      options = {
+        strategy: :postgresql,
+        # try: true,
+        try: false,
+        transactional: false
+      }
+      Travis::Lock.exclusive('test', options, &block)
+    end
+  end
+end
+
+number_of_runs = Integer(ARGV[0] || 1)
+concurrency    = Integer(ARGV[1] || 20)
+lock_types     = ARGV[2] ? [ARGV[2].to_sym] : Lock.constants
+
+1.upto(number_of_runs) do |ix|
+  lock_types.each do |strategy|
+    puts "#{ix} Using strategy #{strategy}"
+    lock  = Lock.const_get(strategy).new
+    count = 0
+
+    threads = (1..concurrency).to_a.map do
+      Thread.new do
+        lock.exclusive do
+          count = count.tap { sleep(rand(0.001..0.009)) } + 1
+        end
+      end
+    end
+    threads.map(&:join)
+
+    puts "  #{count == concurrency ? "\033[32;1m" : "\033[31;1m" }Expected count to be #{concurrency}. Actually is #{count}.\033[0m\n\n"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+ENV['ENV'] = 'test'
+
+require 'mocha'
+require 'travis/lock'
+require 'support/database'
+
+RSpec.configure do |config|
+  config.mock_with :mocha
+end

data/spec/support/database.rb

@@ -0,0 +1,7 @@
+require 'active_record'
+
+ActiveRecord::Base.establish_connection(
+  adapter:  'postgresql',
+  database: 'travis_test',
+  pool:     50
+)

data/spec/travis/lock/none_spec.rb

@@ -0,0 +1,8 @@
+describe Travis::Lock::None do
+  let(:lock) { described_class.new }
+
+  it 'yields' do
+    lock.exclusive { @called = true }
+    expect(@called).to eq(true)
+  end
+end

data/spec/travis/lock/postgresql_spec.rb

@@ -0,0 +1,138 @@
+describe Travis::Lock::Postgresql do
+  let(:lock) { described_class.new(name, config) }
+  let(:name) { 'name' }
+  let(:key)  { 1579384326 }
+
+  def rescueing
+    yield
+  rescue Travis::Lock::Timeout
+  end
+
+  shared_examples_for 'yields' do
+    it 'yields' do
+      lock.exclusive { @called = true }
+      expect(@called).to eq(true)
+    end
+  end
+
+  context do
+    let(:conn) { stub('connection', select_value: 't', execute: nil) }
+    before     { def conn.transaction; yield end }
+    before     { ActiveRecord::Base.stubs(:connection).returns(conn) }
+
+    shared_examples_for 'locks_with' do |method|
+      it "locks_with #{method}" do
+        conn.expects(:select_value).with("select #{method}(#{key});").returns('t')
+        lock.exclusive { }
+      end
+    end
+
+    shared_examples_for 'retries until timeout' do
+      it 'retries until timeout' do
+        conn.expects(:select_value).returns('f').at_least(50)
+        rescueing { lock.exclusive { } }
+      end
+    end
+
+    shared_examples_for 'sets a statement level timeout' do
+      it 'sets a statement level timeout' do
+        conn.expects(:execute).with('set statement_timeout to 100;')
+        lock.exclusive { }
+      end
+    end
+
+    shared_examples_for 'raises Travis::Lock::Timeout when timed out' do
+      it 'raises Travis::Lock::Timeout when timed out' do
+        conn.stubs(:select_value).returns('f')
+        expect { lock.exclusive { } }.to raise_error(Travis::Lock::Timeout)
+      end
+    end
+
+    describe 'using try_*' do
+      describe 'not using transactions' do
+        let(:config) { { try: true, transactional: false, timeout: 0.1 } }
+
+        include_examples 'yields'
+        include_examples 'locks_with', 'pg_try_advisory_lock'
+        include_examples 'retries until timeout'
+        include_examples 'raises Travis::Lock::Timeout when timed out'
+      end
+
+      describe 'using transactions' do
+        let(:config) { { try: true, transactional: true, timeout: 0.1 } }
+
+        include_examples 'yields'
+        include_examples 'locks_with', 'pg_try_advisory_xact_lock'
+        include_examples 'retries until timeout'
+        include_examples 'raises Travis::Lock::Timeout when timed out'
+      end
+    end
+
+    describe 'not using try_*' do
+      describe 'not using transactions' do
+        let(:config) { { try: false, transactional: false, timeout: 0.1 } }
+
+        include_examples 'yields'
+        include_examples 'locks_with', 'pg_advisory_lock'
+        include_examples 'sets a statement level timeout'
+        # include_examples 'raises Travis::Lock::Timeout when timed out'
+      end
+
+      describe 'using transactions' do
+        let(:config) { { try: false, transactional: true, timeout: 0.1 } }
+
+        include_examples 'yields'
+        include_examples 'locks_with', 'pg_advisory_xact_lock'
+        include_examples 'sets a statement level timeout'
+        # include_examples 'raises Travis::Lock::Timeout when timed out'
+      end
+    end
+  end
+
+  # describe 'integration' do
+  #   shared_examples_for 'no race condition' do
+  #     runs    = ENV['RUNS']    || 1
+  #     threads = ENV['THREADS'] || 10
+
+  #     1.upto(runs) do |ix|
+  #       it "does not see a race condition on #{threads} threads (run #{ix})" do
+  #         counter = 0
+
+  #         Array(1..threads).map do
+  #           Thread.new do
+  #             lock.exclusive do
+  #               counter = counter.tap { sleep(rand(0.001)) } + 1
+  #             end
+  #           end
+  #         end.map(&:join)
+
+  #         expect(counter).to eq(threads)
+  #       end
+  #     end
+  #   end
+
+  #   describe 'using try_*' do
+  #     describe 'not using transactions' do
+  #       let(:config) { { try: true, transactional: false, timeout: 0.1 } }
+  #       include_examples 'no race condition'
+  #     end
+
+  #     describe 'using transactions' do
+  #       let(:config) { { try: true, transactional: true, timeout: 0.1 } }
+  #       include_examples 'no race condition'
+  #     end
+  #   end
+
+  #   describe 'not using try_*' do
+  #     describe 'not using transactions' do
+  #       let(:config) { { try: false, transactional: false, timeout: 0.1 } }
+  #       include_examples 'no race condition'
+  #     end
+
+  #     describe 'using transactions' do
+  #       let(:config) { { try: false, transactional: true, timeout: 0.1 } }
+  #       include_examples 'no race condition'
+  #     end
+  #   end
+  # end
+end

data/spec/travis/lock/redis_spec.rb

@@ -0,0 +1,17 @@
+describe Travis::Lock::Redis do
+  let(:config) { { url: 'redis://localhost' } }
+  let(:client) { stub('redlock', lock: nil)  }
+  let(:lock)   { described_class.new(name, config) }
+  let(:name)   { 'name' }
+
+  it 'yields' do
+    lock.exclusive { @called = true }
+    expect(@called).to eq(true)
+  end
+
+  it 'delegates to a Redlock instance' do
+    Redlock::Client.stubs(:new).returns(client)
+    client.expects(:lock).with(name, 300)
+    lock.exclusive {}
+  end
+end

data/travis-lock.gemspec

@@ -0,0 +1,22 @@
+# encoding: utf-8
+
+$:.unshift File.expand_path('../lib', __FILE__)
+require 'travis/lock/version'
+
+Gem::Specification.new do |s|
+  s.name         = "travis-lock"
+  s.version      = Travis::Lock::VERSION
+  s.authors      = ["Travis CI"]
+  s.email        = "contact@travis-ci.org"
+  s.homepage     = "https://github.com/travis-ci/travis-lock"
+  s.summary      = "Travis CI config"
+  s.description  = "#{s.summary}."
+  s.license      = "MIT"
+
+  s.files        = Dir['{lib/**/*,spec/**/*,[A-Z]*}']
+  s.platform     = Gem::Platform::RUBY
+  s.require_path = 'lib'
+  s.rubyforge_project = '[none]'
+
+  s.add_dependency 'activerecord' , '~> 4.0'
+end

metadata

@@ -0,0 +1,74 @@
+--- !ruby/object:Gem::Specification
+name: travis-lock
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Travis CI
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-09-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+description: Travis CI config.
+email: contact@travis-ci.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- README.md
+- lib/travis/lock.rb
+- lib/travis/lock/none.rb
+- lib/travis/lock/postgresql.rb
+- lib/travis/lock/redis.rb
+- lib/travis/lock/support/retry.rb
+- lib/travis/lock/version.rb
+- spec/integration.rb
+- spec/spec_helper.rb
+- spec/support/database.rb
+- spec/travis/lock/none_spec.rb
+- spec/travis/lock/postgresql_spec.rb
+- spec/travis/lock/redis_spec.rb
+- travis-lock.gemspec
+homepage: https://github.com/travis-ci/travis-lock
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: "[none]"
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: Travis CI config
+test_files: []
+has_rdoc: